You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ok...@apache.org on 2017/06/16 20:57:38 UTC
[01/34] incubator-madlib git commit: Graph:
Repository: incubator-madlib
Updated Branches:
refs/heads/latest_release a3863b6c2 -> 8e2778a39
Graph:
- Create generic graph validation and help message to standardize
future graph algorithm development.
- Expand the design document with more detail on the graph
representation as well as the SSSP implementation.
Closes #105
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/01586c0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/01586c0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/01586c0d
Branch: refs/heads/latest_release
Commit: 01586c0d05761a794efa953c09fa568f27c84cb7
Parents: a3863b6
Author: Orhan Kislal <ok...@pivotal.io>
Authored: Mon Mar 13 16:38:34 2017 -0700
Committer: Orhan Kislal <ok...@pivotal.io>
Committed: Mon Mar 13 16:38:34 2017 -0700
----------------------------------------------------------------------
doc/design/figures/graph_example.pdf | Bin 0 -> 23083 bytes
doc/design/modules/graph.tex | 208 ++++++++++++++++++-
.../postgres/modules/graph/graph_utils.py_in | 107 ++++++++++
.../postgres/modules/graph/graph_utils.sql_in | 0
src/ports/postgres/modules/graph/sssp.py_in | 62 +-----
src/ports/postgres/modules/graph/sssp.sql_in | 1 -
6 files changed, 315 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/01586c0d/doc/design/figures/graph_example.pdf
----------------------------------------------------------------------
diff --git a/doc/design/figures/graph_example.pdf b/doc/design/figures/graph_example.pdf
new file mode 100644
index 0000000..fd29e5f
Binary files /dev/null and b/doc/design/figures/graph_example.pdf differ
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/01586c0d/doc/design/modules/graph.tex
----------------------------------------------------------------------
diff --git a/doc/design/modules/graph.tex b/doc/design/modules/graph.tex
index 758f407..5c3910c 100644
--- a/doc/design/modules/graph.tex
+++ b/doc/design/modules/graph.tex
@@ -1,4 +1,5 @@
-% When using TeXShop on the Mac, let it know the root document. The following must be one of the first 20 lines.
+% When using TeXShop on the Mac, let it know the root document. The following
+% must be one of the first 20 lines.
% !TEX root = ../design.tex
% Licensed to the Apache Software Foundation (ASF) under one
@@ -25,31 +26,99 @@
\item[History]
\begin{modulehistory}
\item[v0.1] Initial version, SSSP only.
+ \item[v0.2] Graph Framework, SSSP implementation details.
\end{modulehistory}
\end{moduleinfo}
% Abstract. What is the problem we want to solve?
-This module implements various graph algorithms that are used in a number of applications such as social networks, telecommunications and road networks.
+This module implements various graph algorithms that are used in a number of
+applications such as social networks, telecommunications and road networks.
-% \section{Graph Representation} \label{sec:graph:rep}
+\section{Graph Framework} \label{sec:graph:fw}
+
+MADlib graph representation depends on two structures, a \emph{vertex} table
+and an \emph{edge} table. The vertex table has to have a column of vertex ids.
+The edge table has to have 2 columns: source vertex id, destination vertex id.
+For most algorithms an edge weight column is required as well. The
+representation assumes a directed graph, an edge from $x$ to $y$ does
+\emph{not} guarantee the existence of an edge from $y$ to $x$. Both of the
+tables may have additional columns as required. Multi-edges (multiple edges
+from a vertex to the same destination) and loops (edge from a vertex to
+itself) are allowed. This representation does not impose any ordering of
+vertices or edges. An example graph is given in Figure~\ref{sssp:example} and
+its representative tables are given in Table~\ref{sssp:rep}.
+
+\begin{figure}[h]
+ \centering
+ \includegraphics[width=0.9\textwidth]{figures/graph_example.pdf}
+\caption{A sample graph}
+\label{sssp:example}
+\end{figure}
+
+\begin{table}
+ \begin{tabular}{| c | }
+ \hline
+ vid \\ \hline
+ 0 \\ \hline
+ 1 \\ \hline
+ 2 \\ \hline
+ 3 \\ \hline
+ 4 \\ \hline
+ 5 \\ \hline
+ 6 \\ \hline
+ 7 \\
+ \hline
+ \end{tabular}
+ \quad
+ \begin{tabular}{| c | c | c |}
+ \hline
+ src & dest & weight \\ \hline
+ 0 & 1 & 1 \\ \hline
+ 0 & 2 & 1 \\ \hline
+ 0 & 4 & 10 \\ \hline
+ 1 & 2 & 2 \\ \hline
+ 1 & 3 & 10 \\ \hline
+ 1 & 5 & 1 \\ \hline
+ 2 & 3 & 1 \\ \hline
+ 2 & 5 & 1 \\ \hline
+ 2 & 6 & 3 \\ \hline
+ 3 & 0 & 1 \\ \hline
+ 5 & 6 & 1 \\ \hline
+ 6 & 7 & 1 \\
+ \hline
+ \end{tabular}
+ \caption{Graph representation of vertices (left) and edges(right) in the
+ database}
+ \label{sssp:rep}
+\end{table}
-% Our graph representation depends on two structures, a \emph{vertex} table and an \emph{edge} table.
\section{Single Source Shortest Path} \label{sec:graph:sssp}
-Given a graph and a source vertex, single source shortest path (SSSP) algorithm finds a path for every vertex such that the sum of the weights of its constituent edges is minimized.
+Given a graph and a source vertex, single source shortest path (SSSP)
+algorithm finds a path for every vertex such that the sum of the weights of
+its constituent edges is minimized.
-Shortest path is defined as follows. Let $e_{i,j}$ be the edge from vertex $i$ to vertex $j$ and $w_{i,j}$ be its weight. Given a graph G, the shortest path from $s$ to $d$ is $P = (v_1, v_2 \dots, v_n)$ (where $v_1=s$ and $v_n=d$) that over all possible $n$ minimizes the sum $ \sum _{i=1}^{n-1}f(e_{i,i+1})$.
+Shortest path is defined as follows. Let $e_{i,j}$ be the edge from vertex $i$
+to vertex $j$ and $w_{i,j}$ be its weight. Given a graph G, the shortest path
+from $s$ to $d$ is $P = (v_1, v_2 \dots, v_n)$ (where $v_1=s$ and $v_n=d$)
+that over all possible $n$ minimizes the sum $ \sum _{i=1}^{n-1}f(e_{i,i+1})$.
% \subsection{Bellman Ford Algorithm}
-Bellman-Ford Algorithm \cite{bellman1958routing,ford1956network} is based on the following idea: We start with a naive approximation for the cost of reaching every vertex. At each iteration, these values are refined based on the edge list and the existing approximations. If there are no refinements at any given step, the algorithm returns the calculated results. If the algorithm does not converge in $|V|-1$ iterations, this indicates the existence of a negative cycle in the graph.
+Bellman-Ford Algorithm \cite{bellman1958routing,ford1956network} is based on
+the following idea: We start with a naive approximation for the cost of
+reaching every vertex. At each iteration, these values are refined based on
+the edge list and the existing approximations. If there are no refinements at
+any given step, the algorithm returns the calculated results. If the algorithm
+does not converge in $|V|-1$ iterations, this indicates the existence of a
+negative cycle in the graph.
\begin{algorithm}[SSSP$(V,E,start)$] \label{alg:sssp}
-\alginput{Vertex set $V$, edge set $E$, starting vertex $start$}
+\alginput{Vertex set $v$, edge set $E$, starting vertex $start$}
\algoutput{Distance and parent set for every vertex $cur$}
\begin{algorithmic}[1]
\State $toupdate(0) \set (start,0,start)$
@@ -80,9 +149,126 @@ Bellman-Ford Algorithm \cite{bellman1958routing,ford1956network} is based on the
Changes from the standard Bellman-Ford algorithm:
\begin{description}
-\item Line~\ref{alg:sssp:update}: We only check the vertices that have been updated in the last iteration.
-\item Line~\ref{alg:sssp:single}: At each iteration, we update a given vertex only one time. This means the toupdate set cannot contain multiple records for the same vertex which requires the comparison with the existing value.
+\item Line~\ref{alg:sssp:update}: We only check the vertices that have been
+updated in the last iteration.
+\item Line~\ref{alg:sssp:single}: At each iteration, we update a given vertex
+only one time. This means the toupdate set cannot contain multiple records
+for the same vertex which requires the comparison with the existing value.
\end{description}
-This is not a 1-to-1 pseudocode for the implementation since we don't compare the `toupdate` table records one by one but calculate the overall minimum. In addition, the comparison with `cur` values take place earlier to reduce the number of tuples in the `toupdate` table.
+This is not a 1-to-1 pseudocode for the implementation since we don't compare
+the `toupdate` table records one by one but calculate the overall minimum. In
+addition, the comparison with `cur` values take place earlier to reduce the
+number of tuples in the `toupdate` table.
+
+\subsection{Implementation Details}
+
+In this section, we discuss the MADlib implementation of the SSSP algorithm
+in depth.
+
+\begin{algorithm}[SSSP$(V,E,start)$] \label{alg:sssp:high}
+\begin{algorithmic}[1]
+ \Repeat
+ \State Find Updates
+ \State Apply updates to the output table
+ \Until {There are no updates}
+\end{algorithmic}
+\end{algorithm}
+
+The implementation consists of two SQL blocks that are called sequentially
+inside a loop. We will follow the example graph at Figure~\ref{sssp:example}
+with the starting point as $v_0$. The very first update on the output table is
+the source vertex. Its weight is $0$ and its parent is itself ($v_0$). After
+this initialization step, the loop starts with Find Updates (the individual
+updates will be represented with <dest,value,parent> format). Looking at the
+example, it is clear that the updates should be <1,1,0>, <2,1,0> and <4,10,0>.
+We will assume this iteration is already completed and look how the next
+iteration of the algorithm works to explain the implementation details.
+
+\begin{algorithm}[Find Updates$(E,old\_update,out\_table)$]
+\label{alg:sssp:findu}
+\begin{lstlisting}
+INSERT INTO new_update
+ SELECT DISTINCT ON (y.id) y.id AS id,
+ y.val AS val,
+ y.parent AS parent
+ FROM out_table INNER JOIN (
+ SELECT edge_table.dest AS id, x.val AS val, old_update.id AS parent
+ FROM old_update
+ INNER JOIN edge_table
+ ON (edge_table.src = old_update.id)
+ INNER JOIN (
+ SELECT edge_table.dest AS id,
+ min(old_update.val + edge_table.weight) AS val
+ FROM old_update INNER JOIN
+ edge_table AS edge_table ON
+ (edge_table.src=old_update.id)
+ GROUP BY edge_table.dest
+ ) x
+ ON (edge_table.dest = x.id)
+ WHERE ABS(old_update.val + edge_table.weight - x.val) < EPSILON
+ ) AS y ON (y.id = out_table.vertex_id)
+ WHERE y.val<out_table.weight
+\end{lstlisting}
+\end{algorithm}
+
+The Find Updates query is constructed in 4 levels of subqueries: \emph{find
+values, find parents, eliminate duplicates and ensure improvement}.
+
+\begin{itemize}
+
+\item We begin our analysis at the innermost subquery, emph{find values}
+(lines 11-16). This subquery takes a set of vertices (in the table
+$old_update$) and finds the reachable vertices. In case a vertex is reachable
+by multiple vertices, only the path that has the minimum cost is considered
+(hence the name find values). There are two important points to note:
+ \begin{itemize}
+ \item The input vertices need the value of their path as well.
+ \begin{itemize}
+ \item In our example, both $v_1$ and $v_2$ can reach $v_3$. We would
+ have to use $v_2$ -> $v_3$ edge since that gives the lowest possible
+ path value.
+ \end{itemize}
+ \item The subquery is aggregating the rows using the $min$ operator for
+ each destination vertex and unable to return the source vertex at the
+ same time to use as the parent value.
+ \begin{itemize}
+ \item We know the value of $v_3$ should be $2$ but we cannot know
+ its parent ($v_2$) at the same time.
+ \end{itemize}
+ \end{itemize}
+
+\item The \emph{find parents} subquery is designed to solve the
+aforementioned limitation. We combine the result of \emph{find values} with
+$edge$ and $old\_update$ tables (lines 7-10) and get the rows that has the
+same minimum value.
+ \begin{itemize}
+ \item Note that, we would have to tackle the problem of tie-breaking.
+ \begin{itemize}
+ \item Vertex $v_5$ has two paths leading into: <5,2,1> and <5,2,2>.
+ The inner subquery will return <5,2> and it will match both of these
+ edges.
+ \end{itemize}
+ \item It is redundant to keep both of them in the update list as that
+ would require updating the same vertex multiple times in a given
+ iteration.
+ \end{itemize}
+
+\item At this level, we employ the \emph{eliminate duplicates} subquery. By
+using the $DISTINCT$ clause at line 2, we allow the underlying system to
+accept only a single one of them.
+
+\item Finally, we introduce the \emph{ensure improvement} subquery to make
+sure these updates are actually leading us to shortest paths. Line 21 ensures
+that the values stored in the $out\_table$ does not increase and the solution
+does not regress throughout the iterations.
+\end{itemize}
+
+Applying updates is straightforward as the values and the associated parent
+values are replaced using the $new\_update$ table. After this operation is
+completed the $new\_update$ table becomes $old\_update$ for the next iteration
+of the algorithm.
+
+Please note that, for ideal performance, \emph{vertex} and \emph{edge} tables
+should be distributed on \emph{vertex id} and \emph{source id} respectively.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/01586c0d/src/ports/postgres/modules/graph/graph_utils.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/graph_utils.py_in b/src/ports/postgres/modules/graph/graph_utils.py_in
new file mode 100644
index 0000000..fb43491
--- /dev/null
+++ b/src/ports/postgres/modules/graph/graph_utils.py_in
@@ -0,0 +1,107 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Graph Methods
+
+# Please refer to the graph.sql_in file for the documentation
+
+"""
+@file graph.py_in
+
+@namespace graph
+"""
+
+import plpy
+from utilities.control import MinWarning
+from utilities.utilities import _assert
+from utilities.utilities import extract_keyvalue_params
+from utilities.utilities import unique_string
+from utilities.validate_args import get_cols
+from utilities.validate_args import unquote_ident
+from utilities.validate_args import table_exists
+from utilities.validate_args import columns_exist_in_table
+from utilities.validate_args import table_is_empty
+
+
+def validate_graph_coding(vertex_table, vertex_id, edge_table, edge_params,
+ out_table, func_name, **kwargs):
+ """
+ Validates graph tables (vertex and edge) as well as the output table.
+ """
+ _assert(out_table and out_table.strip().lower() not in ('null', ''),
+ "Graph {func_name}: Invalid output table name!".format(**locals()))
+ _assert(not table_exists(out_table),
+ "Graph {func_name}: Output table already exists!".format(**locals()))
+
+ _assert(vertex_table and vertex_table.strip().lower() not in ('null', ''),
+ "Graph {func_name}: Invalid vertex table name!".format(**locals()))
+ _assert(table_exists(vertex_table),
+ "Graph {func_name}: Vertex table ({vertex_table}) is missing!".format(
+ **locals()))
+ _assert(not table_is_empty(vertex_table),
+ "Graph {func_name}: Vertex table ({vertex_table}) is empty!".format(
+ **locals()))
+
+ _assert(edge_table and edge_table.strip().lower() not in ('null', ''),
+ "Graph {func_name}: Invalid edge table name!".format(**locals()))
+ _assert(table_exists(edge_table),
+ "Graph {func_name}: Edge table ({edge_table}) is missing!".format(
+ **locals()))
+ _assert(not table_is_empty(edge_table),
+ "Graph {func_name}: Edge table ({edge_table}) is empty!".format(
+ **locals()))
+
+ existing_cols = set(unquote_ident(i) for i in get_cols(vertex_table))
+ _assert(vertex_id in existing_cols,
+ """Graph {func_name}: The vertex column {vertex_id} is not present in
+ vertex table ({vertex_table}) """.format(**locals()))
+ _assert(columns_exist_in_table(edge_table, edge_params.values()),
+ """Graph {func_name}: Not all columns from {cols} present in edge
+ table ({edge_table})""".format(cols=edge_params.values(), **locals()))
+
+ return None
+
+def get_graph_usage(schema_madlib, func_name, other_text):
+
+ usage = """
+----------------------------------------------------------------------------
+ USAGE
+----------------------------------------------------------------------------
+ SELECT {schema_madlib}.{func_name}(
+ vertex_table TEXT, -- Name of the table that contains the vertex data.
+ vertex_id TEXT, -- Name of the column containing the vertex ids.
+ edge_table TEXT, -- Name of the table that contains the edge data.
+ edge_args TEXT{comma} -- A comma-delimited string containing multiple
+ -- named arguments of the form "name=value".
+ {other_text}
+);
+
+The following parameters are supported for edge table arguments ('edge_args'
+ above):
+
+src (default = 'src') : Name of the column containing the source
+ vertex ids in the edge table.
+dest (default = 'dest') : Name of the column containing the destination
+ vertex ids in the edge table.
+weight (default = 'weight') : Name of the column containing the weight of
+ edges in the edge table.
+""".format(schema_madlib=schema_madlib, func_name=func_name,
+ other_text=other_text, comma = ',' if other_text is not None else ' ')
+
+ return usage
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/01586c0d/src/ports/postgres/modules/graph/graph_utils.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/graph_utils.sql_in b/src/ports/postgres/modules/graph/graph_utils.sql_in
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/01586c0d/src/ports/postgres/modules/graph/sssp.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/sssp.py_in b/src/ports/postgres/modules/graph/sssp.py_in
index 558ec3d..4d27761 100644
--- a/src/ports/postgres/modules/graph/sssp.py_in
+++ b/src/ports/postgres/modules/graph/sssp.py_in
@@ -28,6 +28,7 @@
"""
import plpy
+from graph_utils import *
from utilities.control import MinWarning
from utilities.utilities import _assert
from utilities.utilities import extract_keyvalue_params
@@ -84,7 +85,7 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
local_distribution = m4_ifdef(<!__POSTGRESQL__!>, <!''!>,
<!"DISTRIBUTED BY (id)"!>)
- validate_graph_coding(vertex_table, vertex_id, edge_table,
+ validate_sssp(vertex_table, vertex_id, edge_table,
edge_params, source_vertex, out_table)
plpy.execute(" DROP TABLE IF EXISTS {0},{1},{2}".format(
@@ -284,35 +285,11 @@ def graph_sssp_get_path(schema_madlib, sssp_table, dest_vertex, **kwargs):
return None
-def validate_graph_coding(vertex_table, vertex_id, edge_table, edge_params,
+def validate_sssp(vertex_table, vertex_id, edge_table, edge_params,
source_vertex, out_table, **kwargs):
- _assert(out_table and out_table.strip().lower() not in ('null', ''),
- "Graph SSSP: Invalid output table name!")
- _assert(not table_exists(out_table),
- "Graph SSSP: Output table already exists!")
-
- _assert(vertex_table and vertex_table.strip().lower() not in ('null', ''),
- "Graph SSSP: Invalid vertex table name!")
- _assert(table_exists(vertex_table),
- "Graph SSSP: Vertex table ({0}) is missing!".format(vertex_table))
- _assert(not table_is_empty(vertex_table),
- "Graph SSSP: Vertex table ({0}) is empty!".format(vertex_table))
-
- _assert(edge_table and edge_table.strip().lower() not in ('null', ''),
- "Graph SSSP: Invalid edge table name!")
- _assert(table_exists(edge_table),
- "Graph SSSP: Edge table ({0}) is missing!".format(edge_table))
- _assert(not table_is_empty(edge_table),
- "Graph SSSP: Edge table ({0}) is empty!".format(edge_table))
-
- existing_cols = set(unquote_ident(i) for i in get_cols(vertex_table))
- _assert(vertex_id in existing_cols,
- """Graph SSSP: The vertex column {vertex_id} is not present in vertex
- table ({vertex_table}) """.format(**locals()))
- _assert(columns_exist_in_table(edge_table, edge_params.values()),
- "Graph SSSP: Not all columns from {0} present in edge table ({1})".
- format(edge_params.values(), edge_table))
+ validate_graph_coding(vertex_table, vertex_id, edge_table, edge_params,
+ out_table,'SSSP')
_assert(isinstance(source_vertex,int),
"""Graph SSSP: Source vertex {source_vertex} has to be an integer """.
@@ -377,28 +354,7 @@ For more details on function usage:
"""
elif message in ['usage', 'help', '?']:
help_string = """
-----------------------------------------------------------------------------
- USAGE
-----------------------------------------------------------------------------
- SELECT {schema_madlib}.graph_sssp(
- vertex_table TEXT, -- Name of the table that contains the vertex data.
- vertex_id TEXT, -- Name of the column containing the vertex ids.
- edge_table TEXT, -- Name of the table that contains the edge data.
- edge_args TEXT, -- A comma-delimited string containing multiple
- -- named arguments of the form "name=value".
- source_vertex INT, -- The source vertex id for the algorithm to start.
- out_table TEXT -- Name of the table to store the result of SSSP.
-);
-
-The following parameters are supported for edge table arguments ('edge_args'
- above):
-
-src (default = 'src') : Name of the column containing the source
- vertex ids in the edge table.
-dest (default = 'dest') : Name of the column containing the destination
- vertex ids in the edge table.
-weight (default = 'weight') : Name of the column containing the weight of
- edges in the edge table.
+{graph_usage}
To retrieve the path for a specific vertex:
@@ -428,5 +384,9 @@ shortest path from the initial source vertex to the desired destination vertex.
else:
help_string = "No such option. Use {schema_madlib}.graph_sssp()"
- return help_string.format(schema_madlib=schema_madlib)
+ return help_string.format(schema_madlib=schema_madlib,
+ graph_usage=get_graph_usage(schema_madlib, 'graph_sssp',
+ """source_vertex INT, -- The source vertex id for the algorithm to start.
+ out_table TEXT -- Name of the table to store the result of SSSP."""))
# ---------------------------------------------------------------------
+
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/01586c0d/src/ports/postgres/modules/graph/sssp.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/sssp.sql_in b/src/ports/postgres/modules/graph/sssp.sql_in
index 7534a75..7f89823 100644
--- a/src/ports/postgres/modules/graph/sssp.sql_in
+++ b/src/ports/postgres/modules/graph/sssp.sql_in
@@ -286,4 +286,3 @@ RETURNS VARCHAR AS $$
$$ LANGUAGE sql IMMUTABLE
m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `');
--------------------------------------------------------------------------------
-
[23/34] incubator-madlib git commit: DT: Assign memory only for
reachable nodes
Posted by ok...@apache.org.
DT: Assign memory only for reachable nodes
JIRA: MADLIB-1057
TreeAccumulator assigns a matrix to track the statistics of rows
reaching the last layer of nodes. This matrix assumes a complete
tree and assigns memory for all nodes. As the tree gets deeper,
most of the nodes are unreachable, resulting in excessive wasted
memory. This commit reduces that waste by only assigning memory
for nodes that are reachable and accessing them through a lookup
table.
Closes #120
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/20b11580
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/20b11580
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/20b11580
Branch: refs/heads/latest_release
Commit: 20b115800e8e984553d3239c81c8ff62c64efaa3
Parents: 0cdd644
Author: Rahul Iyer <ri...@apache.org>
Authored: Tue Apr 25 15:00:40 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Tue Apr 25 15:00:40 2017 -0700
----------------------------------------------------------------------
src/modules/recursive_partitioning/DT_impl.hpp | 125 +++++++++++--------
src/modules/recursive_partitioning/DT_proto.hpp | 20 ++-
.../recursive_partitioning/decision_tree.cpp | 62 +++++++--
3 files changed, 143 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/20b11580/src/modules/recursive_partitioning/DT_impl.hpp
----------------------------------------------------------------------
diff --git a/src/modules/recursive_partitioning/DT_impl.hpp b/src/modules/recursive_partitioning/DT_impl.hpp
index 64d2b88..6d15db5 100644
--- a/src/modules/recursive_partitioning/DT_impl.hpp
+++ b/src/modules/recursive_partitioning/DT_impl.hpp
@@ -475,7 +475,7 @@ DecisionTree<Container>::expand(const Accumulator &state,
const uint16_t &min_split,
const uint16_t &min_bucket,
const uint16_t &max_depth) {
- uint16_t n_non_leaf_nodes = static_cast<uint16_t>(state.n_leaf_nodes - 1);
+ uint32_t n_non_leaf_nodes = static_cast<uint32_t>(state.n_leaf_nodes - 1);
bool children_not_allocated = true;
bool children_wont_split = true;
@@ -483,8 +483,11 @@ DecisionTree<Container>::expand(const Accumulator &state,
for (Index i=0; i < state.n_leaf_nodes; i++) {
Index current = n_non_leaf_nodes + i;
if (feature_indices(current) == IN_PROCESS_LEAF) {
+ Index stats_i = static_cast<Index>(state.stats_lookup(i));
+ assert(stats_i >= 0);
+
// 1. Set the prediction for current node from stats of all rows
- predictions.row(current) = state.node_stats.row(i);
+ predictions.row(current) = state.node_stats.row(stats_i);
// 2. Compute the best feature to split current node by
@@ -502,14 +505,14 @@ DecisionTree<Container>::expand(const Accumulator &state,
// each value of feature
Index fv_index = state.indexCatStats(f, v, true);
double gain = impurityGain(
- state.cat_stats.row(i).segment(fv_index, sps * 2), sps);
+ state.cat_stats.row(stats_i).
+ segment(fv_index, sps * 2), sps);
if (gain > max_impurity_gain){
max_impurity_gain = gain;
max_feat = f;
max_bin = v;
max_is_cat = true;
- max_stats = state.cat_stats.row(i).segment(fv_index,
- sps * 2);
+ max_stats = state.cat_stats.row(stats_i).segment(fv_index, sps * 2);
}
}
}
@@ -519,14 +522,13 @@ DecisionTree<Container>::expand(const Accumulator &state,
// each bin of feature
Index fb_index = state.indexConStats(f, b, true);
double gain = impurityGain(
- state.con_stats.row(i).segment(fb_index, sps * 2), sps);
+ state.con_stats.row(stats_i).segment(fb_index, sps * 2), sps);
if (gain > max_impurity_gain){
max_impurity_gain = gain;
max_feat = f;
max_bin = b;
max_is_cat = false;
- max_stats = state.con_stats.row(i).segment(fb_index,
- sps * 2);
+ max_stats = state.con_stats.row(stats_i).segment(fb_index, sps * 2);
}
}
}
@@ -548,7 +550,8 @@ DecisionTree<Container>::expand(const Accumulator &state,
}
children_wont_split &=
updatePrimarySplit(
- current, static_cast<int>(max_feat),
+ current,
+ static_cast<int>(max_feat),
max_threshold, max_is_cat,
min_split,
max_stats.segment(0, sps), // true_stats
@@ -626,8 +629,8 @@ DecisionTree<Container>::pickSurrogates(
Matrix cat_stats_counts(state.cat_stats * cat_agg_matrix);
Matrix con_stats_counts(state.con_stats * con_agg_matrix);
- // cat_stats_counts size = n_nodes x n_cats*2
- // con_stats_counts size = n_nodes x n_cons*2
+ // cat_stats_counts size = n_reachable_leaf_nodes x n_cats*2
+ // con_stats_counts size = n_reachable_leaf_nodes x n_cons*2
// *_stats_counts now contains the agreement count for each split where
// each even col represents forward surrogate split count and
// each odd col represents reverse surrogate split count.
@@ -635,12 +638,14 @@ DecisionTree<Container>::pickSurrogates(
// Number of nodes in a last layer = 2^(tree_depth-1). (since depth starts from 1)
// For n_surr_nodes, we need number of nodes in 2nd last layer,
// so we use 2^(tree_depth-2)
- uint16_t n_surr_nodes = static_cast<uint16_t>(pow(2, tree_depth - 2));
- uint16_t n_ancestors = static_cast<uint16_t>(n_surr_nodes - 1);
+ uint32_t n_surr_nodes = static_cast<uint32_t>(pow(2, tree_depth - 2));
+ uint32_t n_ancestors = static_cast<uint32_t>(n_surr_nodes - 1);
for (Index i=0; i < n_surr_nodes; i++){
Index curr_node = n_ancestors + i;
assert(curr_node >= 0 && curr_node < feature_indices.size());
+ Index stats_i = static_cast<Index>(state.stats_lookup(i));
+ assert(stats_i >= 0);
if (feature_indices(curr_node) >= 0){
// 1. Compute the max count and corresponding split threshold for
@@ -652,11 +657,11 @@ DecisionTree<Container>::pickSurrogates(
for (Index each_cat=0; each_cat < n_cats; each_cat++){
Index n_levels = state.cat_levels_cumsum(each_cat) - prev_cum_levels;
Index max_label;
- (cat_stats_counts.row(i).segment(
+ (cat_stats_counts.row(stats_i).segment(
prev_cum_levels * 2, n_levels * 2)).maxCoeff(&max_label);
cat_max_thres(each_cat) = static_cast<double>(max_label / 2);
cat_max_count(each_cat) =
- cat_stats_counts(i, prev_cum_levels*2 + max_label);
+ cat_stats_counts(stats_i, prev_cum_levels*2 + max_label);
// every odd col is for reverse, hence i % 2 == 1 for reverse index i
cat_max_is_reverse(each_cat) = (max_label % 2 == 1) ? 1 : 0;
prev_cum_levels = state.cat_levels_cumsum(each_cat);
@@ -667,11 +672,11 @@ DecisionTree<Container>::pickSurrogates(
IntegerVector con_max_is_reverse = IntegerVector::Zero(n_cons);
for (Index each_con=0; each_con < n_cons; each_con++){
Index max_label;
- (con_stats_counts.row(i).segment(
+ (con_stats_counts.row(stats_i).segment(
each_con*n_bins*2, n_bins*2)).maxCoeff(&max_label);
con_max_thres(each_con) = con_splits(each_con, max_label / 2);
con_max_count(each_con) =
- con_stats_counts(i, each_con*n_bins*2 + max_label);
+ con_stats_counts(stats_i, each_con*n_bins*2 + max_label);
con_max_is_reverse(each_con) = (max_label % 2 == 1) ? 1 : 0;
}
@@ -740,7 +745,7 @@ DecisionTree<Container>::expand_by_sampling(const Accumulator &state,
const uint16_t &max_depth,
const int &n_random_features) {
- uint16_t n_non_leaf_nodes = static_cast<uint16_t>(state.n_leaf_nodes - 1);
+ uint32_t n_non_leaf_nodes = static_cast<uint32_t>(state.n_leaf_nodes - 1);
bool children_not_allocated = true;
bool children_wont_split = true;
@@ -756,9 +761,12 @@ DecisionTree<Container>::expand_by_sampling(const Accumulator &state,
for (Index i=0; i < state.n_leaf_nodes; i++) {
Index current = n_non_leaf_nodes + i;
+ Index stats_i = static_cast<Index>(state.stats_lookup(i));
+ assert(stats_i >= 0);
+
if (feature_indices(current) == IN_PROCESS_LEAF) {
// 1. Set the prediction for current node from stats of all rows
- predictions.row(current) = state.node_stats.row(i);
+ predictions.row(current) = state.node_stats.row(stats_i);
for (int j=0; j<total_cat_con_features; j++) {
cat_con_feature_indices[j] = j;
@@ -785,14 +793,16 @@ DecisionTree<Container>::expand_by_sampling(const Accumulator &state,
// each value of feature
Index fv_index = state.indexCatStats(f, v, true);
double gain = impurityGain(
- state.cat_stats.row(i).segment(fv_index, sps * 2), sps);
+ state.cat_stats.row(stats_i).
+ segment(fv_index, sps * 2),
+ sps);
if (gain > max_impurity_gain){
max_impurity_gain = gain;
max_feat = f;
max_bin = v;
max_is_cat = true;
- max_stats = state.cat_stats.row(i).segment(fv_index,
- sps * 2);
+ max_stats = state.cat_stats.row(stats_i).
+ segment(fv_index, sps * 2);
}
}
@@ -804,14 +814,16 @@ DecisionTree<Container>::expand_by_sampling(const Accumulator &state,
// each bin of feature
Index fb_index = state.indexConStats(f, b, true);
double gain = impurityGain(
- state.con_stats.row(i).segment(fb_index, sps * 2), sps);
+ state.con_stats.row(stats_i).
+ segment(fb_index, sps * 2),
+ sps);
if (gain > max_impurity_gain){
max_impurity_gain = gain;
max_feat = f;
max_bin = b;
max_is_cat = false;
- max_stats = state.con_stats.row(i).segment(fb_index,
- sps * 2);
+ max_stats = state.con_stats.row(stats_i).
+ segment(fb_index, sps * 2);
}
}
}
@@ -1061,7 +1073,7 @@ DecisionTree<Container>::recomputeTreeDepth() const{
return tree_depth;
for(uint16_t depth_counter = 2; depth_counter <= tree_depth; depth_counter++){
- uint32_t n_leaf_nodes = static_cast<uint16_t>(pow(2, depth_counter - 1));
+ uint32_t n_leaf_nodes = static_cast<uint32_t>(pow(2, depth_counter - 1));
uint32_t leaf_start_index = n_leaf_nodes - 1;
bool all_non_existing = true;
for (uint32_t leaf_index=0; leaf_index < n_leaf_nodes; leaf_index++){
@@ -1125,7 +1137,7 @@ DecisionTree<Container>::displayLeafNode(
n_elem = NUM_PER_LINE;
} else {
// less than NUM_PER_LINE left, avoid reading past the end
- n_elem = pred_size - i;
+ n_elem = static_cast<uint16_t>(pred_size - i);
}
display_str << predictions.row(id).segment(i, n_elem) << "\n";
}
@@ -1169,7 +1181,7 @@ DecisionTree<Container>::displayInternalNode(
size_t to_skip = 0;
for (Index i=0; i < feature_indices(id); i++)
to_skip += cat_n_levels[i];
- const size_t index = to_skip + feature_thresholds(id);
+ const size_t index = to_skip + static_cast<size_t>(feature_thresholds(id));
label_str << get_text(cat_levels_text, index);
}
@@ -1195,7 +1207,7 @@ DecisionTree<Container>::displayInternalNode(
// not overflowing the vector
n_elem = NUM_PER_LINE;
} else {
- n_elem = pred_size - i;
+ n_elem = static_cast<uint16_t>(pred_size - i);
}
display_str << predictions.row(id).segment(i, n_elem) << "\n";
}
@@ -1520,8 +1532,8 @@ TreeAccumulator<Container, DTree>::TreeAccumulator(
* there is no guarantee yet that the element can indeed be accessed. It is
* cruicial to first check this.
*
- * Provided that this methods correctly lists all member variables, all other
- * methods can, however, rely on that fact that all variables are correctly
+ * Provided that this method correctly lists all member variables, all other
+ * methods can rely on that fact that all variables are correctly
* initialized and accessible.
*/
template <class Container, class DTree>
@@ -1536,6 +1548,7 @@ TreeAccumulator<Container, DTree>::bind(ByteStream_type& inStream) {
>> n_con_features
>> total_n_cat_levels
>> n_leaf_nodes
+ >> n_reachable_leaf_nodes
>> stats_per_split
>> weights_as_rows ;
@@ -1543,7 +1556,8 @@ TreeAccumulator<Container, DTree>::bind(ByteStream_type& inStream) {
uint16_t n_cat = 0;
uint16_t n_con = 0;
uint32_t tot_levels = 0;
- uint16_t n_leafs = 0;
+ uint32_t n_leaves = 0;
+ uint32_t n_reachable_leaves = 0;
uint16_t n_stats = 0;
if (!n_rows.isNull()){
@@ -1551,15 +1565,17 @@ TreeAccumulator<Container, DTree>::bind(ByteStream_type& inStream) {
n_cat = n_cat_features;
n_con = n_con_features;
tot_levels = total_n_cat_levels;
- n_leafs = n_leaf_nodes;
+ n_leaves = n_leaf_nodes;
+ n_reachable_leaves = n_reachable_leaf_nodes;
n_stats = stats_per_split;
}
inStream
>> cat_levels_cumsum.rebind(n_cat)
- >> cat_stats.rebind(n_leafs, tot_levels * n_stats * 2)
- >> con_stats.rebind(n_leafs, n_con * n_bins_tmp * n_stats * 2)
- >> node_stats.rebind(n_leafs, n_stats);
+ >> cat_stats.rebind(n_reachable_leaves, tot_levels * n_stats * 2)
+ >> con_stats.rebind(n_reachable_leaves, n_con * n_bins_tmp * n_stats * 2)
+ >> node_stats.rebind(n_reachable_leaves, n_stats)
+ >> stats_lookup.rebind(n_leaves);
}
// -------------------------------------------------------------------------
@@ -1574,7 +1590,8 @@ void
TreeAccumulator<Container, DTree>::rebind(
uint16_t in_n_bins, uint16_t in_n_cat_feat,
uint16_t in_n_con_feat, uint32_t in_n_total_levels,
- uint16_t tree_depth, uint16_t in_n_stats, bool in_weights_as_rows) {
+ uint16_t tree_depth, uint16_t in_n_stats,
+ bool in_weights_as_rows, uint32_t n_reachable_leaves) {
n_bins = in_n_bins;
n_cat_features = in_n_cat_feat;
@@ -1582,9 +1599,13 @@ TreeAccumulator<Container, DTree>::rebind(
total_n_cat_levels = in_n_total_levels;
weights_as_rows = in_weights_as_rows;
if (tree_depth > 0)
- n_leaf_nodes = static_cast<uint16_t>(pow(2, tree_depth - 1));
+ n_leaf_nodes = static_cast<uint32_t>(pow(2, tree_depth - 1));
else
n_leaf_nodes = 1;
+ if (n_reachable_leaves >= n_leaf_nodes)
+ n_reachable_leaf_nodes = n_leaf_nodes;
+ else
+ n_reachable_leaf_nodes = n_reachable_leaves;
stats_per_split = in_n_stats;
this->resize();
}
@@ -1618,7 +1639,7 @@ TreeAccumulator<Container, DTree>::operator<<(const tuple_type& inTuple) {
} else if (n_con_features != static_cast<uint16_t>(con_features.size())) {
warning("Inconsistent numbers of continuous independent variables.");
} else{
- uint16_t n_non_leaf_nodes = static_cast<uint16_t>(n_leaf_nodes - 1);
+ uint32_t n_non_leaf_nodes = static_cast<uint32_t>(n_leaf_nodes - 1);
Index dt_search_index = dt.search(cat_features, con_features);
if (dt.feature_indices(dt_search_index) != dt.FINISHED_LEAF &&
dt.feature_indices(dt_search_index) != dt.NODE_NON_EXISTING) {
@@ -1687,8 +1708,8 @@ TreeAccumulator<Container, DTree>::operator<<(const surr_tuple_type& inTuple) {
} else{
// the accumulator is setup to train for the 2nd last layer
// hence the n_leaf_nodes is same as n_surr_nodes
- uint16_t n_surr_nodes = n_leaf_nodes;
- uint16_t n_non_surr_nodes = static_cast<uint16_t>(n_surr_nodes - 1);
+ uint32_t n_surr_nodes = n_leaf_nodes;
+ uint32_t n_non_surr_nodes = static_cast<uint32_t>(n_surr_nodes - 1);
Index dt_parent_index = dt.parentIndex(dt.search(cat_features, con_features));
@@ -1710,8 +1731,7 @@ TreeAccumulator<Container, DTree>::operator<<(const surr_tuple_type& inTuple) {
if (dt.feature_indices(dt_parent_index) >= 0){
Index row_index = dt_parent_index - n_non_surr_nodes;
- assert(row_index >= 0 && row_index < cat_stats.rows() &&
- row_index < con_stats.rows());
+ assert(row_index >= 0 && row_index < stats_lookup.rows());
for (Index i=0; i < n_cat_features; ++i){
if (is_primary_cat && i == primary_index)
@@ -1800,7 +1820,8 @@ TreeAccumulator<Container, DTree>::updateNodeStats(bool is_regression,
stats(static_cast<uint16_t>(response)) = weight;
stats.tail(1)(0) = n_rows;
}
- node_stats.row(node_index) += stats;
+ assert(stats_lookup(node_index) >= 0);
+ node_stats.row(stats_lookup(node_index)) += stats;
}
// -------------------------------------------------------------------------
@@ -1826,11 +1847,12 @@ TreeAccumulator<Container, DTree>::updateStats(bool is_regression,
stats(static_cast<uint16_t>(response)) = weight;
stats.tail(1)(0) = n_rows;
}
-
+ Index stats_i = stats_lookup(row_index);
+ assert(stats_i >= 0);
if (is_cat) {
- cat_stats.row(row_index).segment(stats_index, stats_per_split) += stats;
+ cat_stats.row(stats_i).segment(stats_index, stats_per_split) += stats;
} else {
- con_stats.row(row_index).segment(stats_index, stats_per_split) += stats;
+ con_stats.row(stats_i).segment(stats_index, stats_per_split) += stats;
}
}
// -------------------------------------------------------------------------
@@ -1854,10 +1876,12 @@ TreeAccumulator<Container, DTree>::updateSurrStats(
else
stats << 0, dup_count;
+ Index stats_i = stats_lookup(row_index);
+ assert(stats_i >= 0);
if (is_cat) {
- cat_stats.row(row_index).segment(stats_index, stats_per_split) += stats;
+ cat_stats.row(stats_i).segment(stats_index, stats_per_split) += stats;
} else {
- con_stats.row(row_index).segment(stats_index, stats_per_split) += stats;
+ con_stats.row(stats_i).segment(stats_index, stats_per_split) += stats;
}
}
// -------------------------------------------------------------------------
@@ -1881,7 +1905,8 @@ TreeAccumulator<Container, DTree>::indexCatStats(Index feature_index,
int cat_value,
bool is_split_true) const {
// cat_stats is a matrix
- // size = (n_leaf_nodes) x (total_n_cat_levels * stats_per_split * 2)
+ // size = (n_reachable_leaf_nodes) x
+ // (total_n_cat_levels * stats_per_split * 2)
assert(feature_index < n_cat_features);
unsigned int cat_cumsum_value = (feature_index == 0) ? 0 : cat_levels_cumsum(feature_index - 1);
return computeSubIndex(static_cast<Index>(cat_cumsum_value),
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/20b11580/src/modules/recursive_partitioning/DT_proto.hpp
----------------------------------------------------------------------
diff --git a/src/modules/recursive_partitioning/DT_proto.hpp b/src/modules/recursive_partitioning/DT_proto.hpp
index a2881a5..272fec4 100644
--- a/src/modules/recursive_partitioning/DT_proto.hpp
+++ b/src/modules/recursive_partitioning/DT_proto.hpp
@@ -245,7 +245,8 @@ public:
void bind(ByteStream_type& inStream);
void rebind(uint16_t n_bins, uint16_t n_cat_feat,
uint16_t n_con_feat, uint32_t n_total_levels,
- uint16_t tree_depth, uint16_t n_stats, bool weights_as_rows);
+ uint16_t tree_depth, uint16_t n_stats, bool weights_as_rows,
+ uint32_t n_reachable_leaf_nodes);
TreeAccumulator& operator<<(const tuple_type& inTuple);
TreeAccumulator& operator<<(const surr_tuple_type& inTuple);
@@ -284,7 +285,13 @@ public:
// sum of num of levels in each categorical variable
uint32_type total_n_cat_levels;
// n_leaf_nodes = 2^{dt.tree_depth-1} for dt.tree_depth > 0
- uint16_type n_leaf_nodes;
+ uint32_type n_leaf_nodes;
+
+ // Not all "leaf" nodes at a tree level are reachable. A leaf becomes
+ // non-reachable when one of its ancestor is itself a leaf.
+ // For a full tree, n_leaf_nodes = n_reachable_leaf_nodes
+ uint32_type n_reachable_leaf_nodes;
+
// For regression, stats_per_split = 4, i.e. (w, w*y, w*y^2, 1)
// For classification, stats_per_split = (number of class labels + 1)
// i.e. (w_1, w_2, ..., w_c, 1)
@@ -305,10 +312,11 @@ public:
// con_stats and cat_stats are matrices that contain the statistics used
// during training.
// cat_stats is a matrix of size:
- // (n_leaf_nodes) x (total_n_cat_levels * stats_per_split * 2)
+ // (n_reachable_leaf_nodes) x (total_n_cat_levels * stats_per_split * 2)
Matrix_type cat_stats;
+
// con_stats is a matrix:
- // (n_leaf_nodes) x (n_con_features * n_bins * stats_per_split * 2)
+ // (n_reachable_leaf_nodes) x (n_con_features * n_bins * stats_per_split * 2)
Matrix_type con_stats;
// node_stats is used to keep a statistic of all the rows that land on a
@@ -317,6 +325,10 @@ public:
// cat_stats/con_stats. In the presence of NULL value, the stats could be
// different.
Matrix_type node_stats;
+
+ // Above stats matrices are used as pseudo-sparse matrices since not all
+ // leaf nodes are reachable (esp. as tree gets deeper).
+ IntegerVector_type stats_lookup;
};
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/20b11580/src/modules/recursive_partitioning/decision_tree.cpp
----------------------------------------------------------------------
diff --git a/src/modules/recursive_partitioning/decision_tree.cpp b/src/modules/recursive_partitioning/decision_tree.cpp
index b298df8..b85923a 100644
--- a/src/modules/recursive_partitioning/decision_tree.cpp
+++ b/src/modules/recursive_partitioning/decision_tree.cpp
@@ -154,6 +154,24 @@ compute_leaf_stats_transition::run(AnyType & args){
}
if (state.empty()){
+ // To initialize the accumulator, first find which of the leaf nodes
+ // in current tree are actually reachable.
+ // The lookup vector maps the leaf node index in a (fictional) complete
+ // tree to the index in the actual tree.
+ ColumnVector leaf_feature_indices =
+ dt.feature_indices.tail(dt.feature_indices.size()/2 + 1).cast<double>();
+ ColumnVector leaf_node_lookup(leaf_feature_indices.size());
+ size_t n_leaves_not_finished = 0;
+ for (Index i=0; i < leaf_feature_indices.size(); i++){
+ if ((leaf_feature_indices(i) != dt.NODE_NON_EXISTING) &&
+ (leaf_feature_indices(i) != dt.FINISHED_LEAF)){
+ leaf_node_lookup(i) = n_leaves_not_finished++; // increment after assigning
+ }
+ else{
+ leaf_node_lookup(i) = -1;
+ }
+ }
+
// For classification, we store for each split the number of weighted
// tuples for each possible response value and the number of unweighted
// tuples landing on that node.
@@ -167,22 +185,27 @@ compute_leaf_stats_transition::run(AnyType & args){
static_cast<uint32_t>(cat_levels.sum()),
static_cast<uint16_t>(dt.tree_depth),
stats_per_split,
- weights_as_rows
+ weights_as_rows,
+ static_cast<uint32_t>(n_leaves_not_finished)
);
+ for (Index i=0; i < state.stats_lookup.size(); i++)
+ state.stats_lookup(i) = leaf_node_lookup(i);
+
// compute cumulative sum of the levels of the categorical variables
int current_sum = 0;
for (Index i=0; i < state.n_cat_features; ++i){
- // We assume that the levels of each categorical variable are sorted
- // by the entropy for predicting the response. We then create splits
- // of the form 'A <= t', where A has N levels and t in [0, N-2].
+ // Assuming that the levels of each categorical variable are ordered,
+ // create splits of the form 'A <= t', where A has N levels
+ // and t in [0, N-2].
// This split places all levels <= t on true node and
- // others on false node. We only check till N-2 since we want at
- // least 1 level falling to the false node.
- // We keep a variable with just 1 level to ensure alignment,
- // even though that variable will not be used as a split feature.
+ // others on false node. Checking till N-2 instead of N-1
+ // since at least 1 level should go to false node.
+ // Variable with just 1 level is maintained to ensure alignment,
+ // even though the variable will not be used as a split feature.
current_sum += cat_levels(i);
state.cat_levels_cumsum(i) = current_sum;
}
+
}
state << MutableLevelState::tuple_type(dt, cat_features, con_features,
@@ -236,6 +259,7 @@ dt_apply::run(AnyType & args){
return_code = TERMINATED; // indicates termination due to error
}
+
AnyType output_tuple;
output_tuple << dt.storage()
<< return_code
@@ -292,6 +316,21 @@ compute_surr_stats_transition::run(AnyType & args){
// the root be an internal node i.e. we need the tree_depth to be more than 1.
if (dt.tree_depth > 1){
if (state.empty()){
+ // To initialize the accumulator, first find which of the last
+ // level of internal nodes are actually reachable.
+ ColumnVector final_internal_feature_indices =
+ dt.feature_indices.segment(dt.feature_indices.size()/4,
+ dt.feature_indices.size()/4 + 1).cast<double>();
+ ColumnVector index_lookup(final_internal_feature_indices.size());
+ Index n_internal_nodes_reachable = 0;
+ for (Index i=0; i < final_internal_feature_indices.size(); i++){
+ if (final_internal_feature_indices(i) >= 0){
+ index_lookup(i) = n_internal_nodes_reachable++; // increment after assigning
+ }
+ else{
+ index_lookup(i) = -1;
+ }
+ }
// 1. We need to compute stats for parent of each leaf.
// Hence the tree_depth is decremented by 1.
// 2. We store 2 values for each surrogate split
@@ -303,11 +342,14 @@ compute_surr_stats_transition::run(AnyType & args){
static_cast<uint32_t>(cat_levels.sum()),
static_cast<uint16_t>(dt.tree_depth - 1),
2,
- false // dummy, only used in compute_leaf_stat
+ false, // dummy, only used in compute_leaf_stat
+ n_internal_nodes_reachable
);
+ for (Index i = 0; i < state.stats_lookup.size(); i++)
+ state.stats_lookup(i) = index_lookup(i);
// compute cumulative sum of the levels of the categorical variables
int current_sum = 0;
- for (Index i=0; i < state.n_cat_features; ++i){
+ for (Index i=0; i < state.n_cat_features; i++){
current_sum += cat_levels(i);
state.cat_levels_cumsum(i) = current_sum;
}
[03/34] incubator-madlib git commit: Build: Avoid downloading mathjax
during make doc
Posted by ok...@apache.org.
Build: Avoid downloading mathjax during make doc
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/7be68936
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/7be68936
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/7be68936
Branch: refs/heads/latest_release
Commit: 7be68936f2cf09e44fd9a8ae3e893db73dc99b26
Parents: 8679cbd
Author: Rahul Iyer <ri...@apache.org>
Authored: Tue Mar 7 13:11:23 2017 -0800
Committer: Rahul Iyer <ri...@apache.org>
Committed: Wed Mar 15 11:08:51 2017 -0700
----------------------------------------------------------------------
doc/CMakeLists.txt | 54 +++++++++++++++++++++++++++-----------
doc/etc/developer.doxyfile.in | 3 ++-
doc/etc/user.doxyfile.in | 28 +++++++++++++++++++-
3 files changed, 67 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/7be68936/doc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt
index e92aab4..aa969dc 100644
--- a/doc/CMakeLists.txt
+++ b/doc/CMakeLists.txt
@@ -54,20 +54,40 @@ set(_DOXYGEN_INPUT_DEVELOPER
)
join_strings(DOXYGEN_INPUT_DEVELOPER " " "${_DOXYGEN_INPUT_DEVELOPER}")
-set(DOXYGEN_USE_MATHJAX NO CACHE BOOL "In user documentation, render LaTeX formulas using MathJax")
+if(NOT DEFINED DOXYGEN_USE_MATHJAX)
+ set(DOXYGEN_USE_MATHJAX YES CACHE BOOL
+ "In user documentation, render LaTeX formulas using MathJax")
+endif(NOT DEFINED DOXYGEN_USE_MATHJAX)
+
+if(DOXYGEN_USE_MATHJAX)
+ if(NOT DEFINED MATHJAX_DIR)
+ find_path(MATHJAX_DIR
+ NAMES MathJax.js
+ PATHS "$ENV{MATHJAX_DIR}" "/usr/share/javascript/mathjax/"
+ DOC "Path to local MathJax.js")
+ endif(NOT DEFINED MATHJAX_DIR)
+ if(MATHJAX_DIR)
+ set(MATHJAX_RELPATH_CONFIG "MATHJAX_RELPATH = ${MATHJAX_DIR}")
+ message(STATUS "Using local MathJax: " ${MATHJAX_DIR})
+ else(MATHJAX_DIR)
+ set(MATHJAX_RELPATH_CONFIG "")
+ message(STATUS "Using default web-based MathJax")
+ endif(MATHJAX_DIR)
+endif(DOXYGEN_USE_MATHJAX)
+
+# set(MATHJAX_INSTALLATION "${CMAKE_BINARY_DIR}/third_party/downloads/mathjax" CACHE PATH
+# "Path to MathJax installation (used to clone MathJax repository; absolute or relative to \${CMAKE_BINARY_DIR}/doc)"
+# )
+# set(DOXYGEN_MATHJAX_RELPATH "${CMAKE_BINARY_DIR}/third_party/downloads/mathjax" CACHE STRING
+# "Path to MathJax installation (used by Doxygen; absolute or relative to \${DOXYGEN_HTML_OUTPUT})"
+# )
+
set(DOXYGEN_INCLUDE_PATH "\"${CMAKE_SOURCE_DIR}/src\" \"${CMAKE_SOURCE_DIR}/src/ports/postgres\"")
# Note: Type PATH implies that the value is either a relative path to
# ${CMAKE_CURRENT_BINARY_DIR} (and CMake generates the full path) or as an
# absolute path. Therefore, paths not relative to ${CMAKE_CURRENT_BINARY_DIR}
# must be of type STRING!
-
-set(MATHJAX_INSTALLATION "${CMAKE_BINARY_DIR}/third_party/downloads/mathjax" CACHE PATH
- "Path to MathJax installation (used to clone MathJax repository; absolute or relative to \${CMAKE_BINARY_DIR}/doc)"
-)
-set(DOXYGEN_MATHJAX_RELPATH "${CMAKE_BINARY_DIR}/third_party/downloads/mathjax" CACHE STRING
- "Path to MathJax installation (used by Doxygen; absolute or relative to \${DOXYGEN_HTML_OUTPUT})"
-)
set(DOXYGEN_OUTPUT_DEVELOPER "${CMAKE_CURRENT_BINARY_DIR}/developer" CACHE PATH
"Base path where the documentation generated by Doxygen will be put (abolsute or relative to \${CMAKE_BINARY_DIR}/doc/etc)"
)
@@ -130,14 +150,16 @@ if(FLEX_FOUND AND BISON_FOUND AND DOXYGEN_FOUND)
# -- Update MathJax ------------------------------------------------------------
-
- add_custom_target(update_mathjax
- COMMAND bin/update_mathjax.sh
- WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
- )
- if(USE_MATHJAX)
- set(_MATHJAX_DEPENDENCY_USER update_mathjax)
- endif(USE_MATHJAX)
+# NOTE: Below has been disabled since Mathjax CDN is used to obtain the
+# appropriate files
+
+# add_custom_target(update_mathjax
+# COMMAND bin/update_mathjax.sh
+# WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+# )
+# if(DOXYGEN_USE_MATHJAX)
+# set(_MATHJAX_DEPENDENCY_USER update_mathjax)
+# endif(DOXYGEN_USE_MATHJAX)
# -- Run doxygen ---------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/7be68936/doc/etc/developer.doxyfile.in
----------------------------------------------------------------------
diff --git a/doc/etc/developer.doxyfile.in b/doc/etc/developer.doxyfile.in
index 2c9b11d..02558c9 100644
--- a/doc/etc/developer.doxyfile.in
+++ b/doc/etc/developer.doxyfile.in
@@ -1282,7 +1282,8 @@ MATHJAX_FORMAT = HTML-CSS
# However, it is strongly recommended to install a local
# copy of MathJax from http://www.mathjax.org before deployment.
-MATHJAX_RELPATH = @DOXYGEN_MATHJAX_RELPATH@
+# MATHJAX_RELPATH = @DOXYGEN_MATHJAX_RELPATH@
+@MATHJAX_RELPATH_CONFIG@
# The MATHJAX_EXTENSIONS tag can be used to specify one or MathJax extension
# names that should be enabled during MathJax rendering.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/7be68936/doc/etc/user.doxyfile.in
----------------------------------------------------------------------
diff --git a/doc/etc/user.doxyfile.in b/doc/etc/user.doxyfile.in
index fc0648e..230aae8 100644
--- a/doc/etc/user.doxyfile.in
+++ b/doc/etc/user.doxyfile.in
@@ -127,12 +127,38 @@ INPUT = @DOXYGEN_INPUT_USER@
# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax
# (see http://www.mathjax.org) which uses client side Javascript for the
# rendering instead of using prerendered bitmaps. Use this if you do not
-# have LaTeX installed or if you want to formulas look prettier in the HTML
+# have LaTeX installed or if you want formulas to look prettier in the HTML
# output. When enabled you also need to install MathJax separately and
# configure the path to it using the MATHJAX_RELPATH option.
USE_MATHJAX = @DOXYGEN_USE_MATHJAX@
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. Supported types are HTML-CSS, NativeMML (i.e. MathML) and
+# SVG. The default value is HTML-CSS, which is slower, but has the best
+# compatibility.
+
+MATHJAX_FORMAT = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the
+# HTML output directory using the MATHJAX_RELPATH option. The destination
+# directory should contain the MathJax.js script. For instance, if the mathjax
+# directory is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to
+# the MathJax Content Delivery Network so you can quickly see the result without
+# installing MathJax.
+# However, it is strongly recommended to install a local
+# copy of MathJax from http://www.mathjax.org before deployment.
+
+# MATHJAX_RELPATH = @DOXYGEN_MATHJAX_RELPATH@
+@MATHJAX_RELPATH_CONFIG@
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or MathJax extension
+# names that should be enabled during MathJax rendering.
+
+MATHJAX_EXTENSIONS = TeX/AMSmath \
+ TeX/AMSsymbols
+
#---------------------------------------------------------------------------
# configuration options related to source browsing
#---------------------------------------------------------------------------
[12/34] incubator-madlib git commit: Feautre: Add grouping support
for PageRank
Posted by ok...@apache.org.
Feautre: Add grouping support for PageRank
MADLIB-1082
- Add grouping support for pagerank, which will compute a PageRank
probability distribution for the graph represented by each group.
- Add convergence test, so that PageRank computation terminates
if the pagerank value of no node changes beyond a threshold across
two consecutive iterations (or max_iters number of iterations are
done, whichever happens first). In case of grouping, the algorithm
terminates only after all groups have converged.
- Create a summary table apart from the output table that records
the number of iterations required for convergence. Iterations
required for convergence of each group is recorded when grouping
is used. This implementation also ensures that we don't compute
PageRank for groups that have already converged.
- Update design doc with PageRank module.
Closes #112
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/c6948930
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/c6948930
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/c6948930
Branch: refs/heads/latest_release
Commit: c6948930661344c629e8a1398040f3b0a80f2136
Parents: f3b906e
Author: Nandish Jayaram <nj...@apache.org>
Authored: Fri Mar 31 17:03:50 2017 -0700
Committer: Nandish Jayaram <nj...@apache.org>
Committed: Thu Apr 13 15:39:35 2017 -0700
----------------------------------------------------------------------
doc/design/figures/pagerank_example.pdf | 208 +++++++
doc/design/modules/graph.tex | 147 ++++-
doc/literature.bib | 7 +
src/ports/postgres/modules/graph/pagerank.py_in | 583 ++++++++++++++++---
.../postgres/modules/graph/pagerank.sql_in | 183 ++++--
.../postgres/modules/graph/test/pagerank.sql_in | 63 +-
6 files changed, 1039 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c6948930/doc/design/figures/pagerank_example.pdf
----------------------------------------------------------------------
diff --git a/doc/design/figures/pagerank_example.pdf b/doc/design/figures/pagerank_example.pdf
new file mode 100644
index 0000000..bc1b19c
--- /dev/null
+++ b/doc/design/figures/pagerank_example.pdf
@@ -0,0 +1,208 @@
+%PDF-1.4
+%����
+1 0 obj
+ <<
+ /Title ()
+ /Author ()
+ /Subject ()
+ /Keywords ()
+ /Creator (yExport 1.5)
+ /Producer (org.freehep.graphicsio.pdf.YPDFGraphics2D 1.5)
+ /CreationDate (D:20170407154415-07'00')
+ /ModDate (D:20170407154415-07'00')
+ /Trapped /False
+ >>
+endobj
+2 0 obj
+ <<
+ /Type /Catalog
+ /Pages 3 0 R
+ /ViewerPreferences 4 0 R
+ /OpenAction [5 0 R /Fit]
+ >>
+endobj
+4 0 obj
+ <<
+ /FitWindow true
+ /CenterWindow false
+ >>
+endobj
+5 0 obj
+ <<
+ /Parent 3 0 R
+ /Type /Page
+ /Contents 6 0 R
+ >>
+endobj
+6 0 obj
+ <<
+ /Length 7 0 R
+ /Filter [/ASCII85Decode /FlateDecode]
+ >>
+stream
+Gb!SpbDml/DVaJWj:BJBhN+L7"q5`2p;1Si!RE=&%6sh6Yg,6S,9j2HC82/+[q?K,-RDV(78\F\DLSGR
+^\@Bqk>1mS9>c7lJ,Jm2+)fJc^V&*S^5bDugQ9aj-]-0Y^V9[<gV4aiJ+L.As8LW"amX`TY:h0gTE"i4
+p](5RJ+o8Dp\4.UnSnEhs22C]b3&$D^WP]>lm/'m-/nk=li0oH5@ouV;jki$i$QkLkaOr6hu,V^.Ukm5
+Qa\mE%=^G,fs(/LMPL2Yhu<<KCkI4@kN=%<PPed?SR1W?r4i,Q/9jr%Bt6n9N_=ecoq$BE5HStG]F*DH
+52Pk3Y?@TB4@Y>mh:'tp)AqHYdt;OY\i-Usm`3EIV#),*42([m&U*IFrPMdaIarmJE5^3K$c@8)eqU?0
+r#4TW35+GWpAXkfgE!1n*peAEPKukpX/$]Kkk/"1hdZ+#h(E*&r#^ZC_nJ>Nr`]lV+'sHrl)2Y1as_rp
+*,A"9W5U'@Rd*6UA[e-[b.<XUj?,NTML8r\cuiLdXO`j/,G4o5gp9uR;=]"'--&Nb-ZgFI>TX_N5?t,<
+!1L*_&'D)h%iQ:*er\$i:%HsQY.W7j$0S2!g3<kfBW\Jh/lZ/ZEie6tRZ":j&Re1iX>j"%mEE<;Cb9XF
+P2mmgi39lZG(dU*eb@U_YRI17&?rT,I$TgPa627H6N5:6MjPJsc75lFGt-ElnMG0d=q*shKDMV_cC-#V
+r`Vs"J>XRVOs,ZI>R($C5)WZ^9oWg5d[cP>WCE4PCTqm<@1MZoTp4-d?KH&%EhA-\)=&I-q3kK&eZKk6
+eTF2@jaQHMj573Aqr==ET4e2HU`HVgHo]5aE(-O#,b</F:-3:UXKO4]B<CPUZJFc>YAg0AnWDp![l@Z_
+eQV-GjF#DSWV5d3jIF`RNenX8L#[b8rRemA?:4LpQelFs:@ES]gB[^4FOkF#0i67I,O$H9CE)*L^5L&1
+i1/\UN^9]4(or+Z"EUTW_"H?G-G"oZ3!@#ol&IsGDfbi%i<ai4grVAA,!Q$h*Fkt(f's[O6>&hD5)>9$
+`5`FhS[(6K`6\bPl02r"1j,)#KoD]3[FsCu/m,"@>_tJLCV9n;(%[NB,.e5S@%t=,R-r+Bd_8P1\OBlC
+"83?U(TtEHfWBe'pZ4LFk-p9p.;b)%K`0k93mJko=ZUbMMMG*/'.k0<e;CU+)LHa+D';7iPMs#T-,YAN
+d6hh:I[;teO)JZ[U<bj_6Pb]iU,pat(3Pjs/?56OCMf,pD*E2fX;1QPnt><Sn6)l95"<in72A,6k(8XG
+]r'*nf]'5`p2#0_T@8!hY0Z<]cFj)!d;ZV0`EP75Situk>9*:;s#7Hu5?%HFFX%'/9V:oGd;Y:LrB6al
+5J@@$][PNY`hf.`kZC#S_dsJ;?n8me_AgFdb8Z+mIiFgDFAH8]o'(O4HI-eQctBc;qA<A-FFWFeCaC6a
+6194h(5=VhiU[KfLmcP\c]-II@>o2&?<fBt0;5_,K3,`i)k7I93l_\C.C/<.I8@55L_ErDVm6kL"e+od
+e@/)$]9#o*8d[Y'Fn?i+a1[`m;&>uF@2)P`ei-`b\%(!T>$pgI\Q+KE<qI,.GSm&X58Y(=f1GKL_S:`3
+U:&8E5."rI0*j>V7.LZS=K`N%L9IndiPcoL,F<SP&Z%g^f4D8YLgIARr)f?P!l&M3qB5fL?`b2p(]GlD
+rM^UJ:ClKHIu]#iLXF[tf_9PUd*'/tP](`t:tTu1D3'hH:,hNd46/'lDWuYGCFJP]\'E%8q5!md4cFGh
+O"K)V0G;^r3p.PhB'+GA6S]$3\)HP'GPO8sdFr91%YDlCd-d\cQuP-X7+XO9FEPG:%r7hL)'[+jc36ZY
+9)C<.8/HGjk>u8MkhosDJ.YBnNq/TL(S"CGGWGWYg/a6bZgsrTWt;,P=dJdW6u\>P+@C6I&r4Pf3>sTZ
+l:]nLU**c*qn2!9_&n7Kc7Idr!)G+"(d%:OJL]#T)uRuH-Js-W2Hlc+C8Fm;fiRaX0Ur_;b;1Z6\4\G.
+>:[<j7Nh4X<j]G-'Sbf^rIF<UT*JrTV(m]P_gh$nAH0>&mhBN[<eNQsBE_TJp[Xt[M/mZj^)8B9BMH-H
+0X;?h'8P]K&4qc&6P*pSEl?c@[&<RC?AQidEbV/PkS@o!9s-3L>-YVBQkd1[m\su)2`b$hY?%dtN5R7o
+LJ>3OCSb<@4%fmc\FPmQn+/I5j'0k`\&kskG4ldkR8&J#QO=TT00TnA4Qr^A*2D<R>h$*h7o+\Dd`u0B
+BLPB+j","$b62q4E':dW>`D!SISSgUoQcCk+pHes%(=Knka"5!YS?]bq4/K)Hj'I".FX"-3$r;)+UJ:Z
+KB6c7LsO@/F87rXin,+kjN,-q\5;[5GTt6M,8ZA3d,"2N5Mfl#N<A;_D,qUIo#GY00kgJDlPS9:cI'N)
+S/>do%QZKt&*E=@e)X1;:*o[+i1khU$;6L7e4ls'.rMI+am9r1:<JDuEE:2oJPsO:!LY%=QM`X2WqG&*
+YZ9;2O//[pUUur-nl]\!>lKaEoLF=dC%l)t6Fnf5DL&'B/#5"(NBAOJ,^>Yo83BP+.j[BA_D;U(>r,@T
+%!>^i^pX:`8H\CD&c1E\VL#X.?>^T?/ekp-Gel>taDu9BkEd]d4=XH3frTT0T@&;!l</DKIDtLO8IC5b
+^Jb\-F(F!UK=NaOpouQVrBO"gCaaa!ja9VJ>%&^>#rcn>(clW&gP#teWfOV.#%`Uj:g[fZU6Wfmqa%g4
+liO]$1FVK/"@jYHIs>e=<ec\QH+Yi4D+Z=tKB%VQ/(Qo'ImUu0P]l9b<(8jS+"g3C<aQ:\?[Hbm1oR7o
+G"Gr?U,/*L,odZPFS-r_+:hFo;U4fF"^SRD<g2L*]N=*oAT0TnU^Z#=?DV7\"XcZDb9uDiP<X%XJ;SIK
+"HPer2bGI9;k#]d0mqHM6akdXb4$9G31C2'MmFg"IkKr>'2j=qS*p'Yg8s>-\$N0S(OY7^fB<+qf^HRr
+=jqQXrnF!L[/9;7[hH+PHa'G`pYS8UZg@[4jS+F>h1%UdpRfq-qp/l^qp.7-qp5%sqp06:J",;]hg\k\
+!k\IBg@]XhmB/N9p@k4@ir4!5r#=5ck9FOYS?Cb!5-+*9j2gZ'L]"pI!C#gi9b's:g9&.r]>\u8L$`N^
+MFq@nFn'd>l=IM>[t'["]Vq3k`,j5NR29u[6YU8BJS>C;A7Es.kpEal4tE3bXEWhU<@ll"9guZ/\6<1@
+.H!3rcYe<^l9njW6#?kgrT3X,MmHVQ\V\4DU#98pDjgI6O$"pW]B+-q>a"]>=Yl7.cI%atkW\'gh("Bk
+JTc>iV)=5Z#!1-Ud.(./X4/$21p'*^U,IuV&FSYu:c6TTWB+&hdRGRp7\i-EH,p590U]CD*amBUbW"fP
+SO&,EqG,o[[SBEL7P<ug7<_+H0e:_(/q6AahFc^cX3jF(i[*gN,&bG5Kp3]iKp+.*]6m!*=aF=%n]=U.
+(poK=U@nm6Y#L=&Cb*k'4J\sCA?'N@aK='pPrW*0*#FLg&]XY[-'#U[3P+h*(%f^(pT04\6JsE`"BA.j
+2rEQY<Z<nn"K<1@_aS_ip!d9]`Mn]GpJ4NX9&Q^r"h!(t3cj8Q2jbce(1/%&O)=S0LQ5l_A?t;I%M#Uj
+E9K8@<1WmF:cR[0cnP4_i/>KUT=37SgJ[g&.r=J2]-4Y[Z\Qh'Y@a-6AP4CX-b1fd'Xl)fCoNCN>AA8t
+QJ/EP98t1&V8dH+;TEZJ%j!NCl(O=p><87...@D5>i_UDpYkAi3e-*18C,
+aT"ah+2aA>4:cTcnBu4%#q="ja\?DkD=Crl=rtm8EOTn$GNCTr/0#AbPPCY*:tB@`RgZE^k4BSJBi`#^
+705en+uhX<D:^rGSR&iBG\FSM*)qs+PQGm1#OU[4/`rJEk7JLt:<-0XOj<XjI%TqI\8ig1jB4Kq"4:C[
+]B-@?]]>;4@lHsQ+b/VYO3c:>BAq*:O'ffP`"d_qp08fsae^$)>JGefr&K/?ESk:B=icj]PYJ04Z.f'B
+,Gt^TWn^;.MR,q;0UUd3;W]rJi3t^5hr-Z6egZI1aQ49cUW\Fa1$`9idH#3Or+>7f>a4Bt84^Ar?-%T-
+9UcbpJrGJ*lmBQ\HpcmrH<_n=`h\>TX(s%qT-MZa)YOnZjCBDe,VsQ66#J)cH>A3>`/u:k[6'*^CC$E$
+7rWS5(Fs/J.oSTQ.g0b$kuoR:#(4D-X`ccQX7\OX_d+u60P*Q[7tJdi+)"9AMq'g$dfE25oI&s-`/3#6
+(o"-<DAQK6O`-VO)1XFDfna;fVW)\]Il4ig^34h)W%%j$Hgo[*XZUtJ=\pa:HqfN%aFiNH31[a:nLt]B
+[d,m76&s[,+$?6O@'.-]52EQ#^d*k-q%U=C(C,)bFj1:G`T_qB5-)tVd*e-;'NGUaE(SnZ&!p.$7;g<j
+#uBt(j)je:)O%jHiEf:I/.Tcf7CnO(Djm]<GWd"to"=4Cj%HGYe[&M]*63k)8Xic>naI$jb!QmYf;^s0
+Ofh\4hY0[D=GdBd1)*n,R$:<DUfI*o+LEUS2<J(r7fdUAJ^WpRK?)]KekXudqq:bY41*/8CK8pl1Tb[i
+VDhGBiEDN2BLf,*`jBJr<nF?\3[ZOVW1Tb`Na/V)9</Q^XJT!eZWp\AAT:GFR%//u3ZQ[je#L=c8iNlp
+.S-%0iE:C4XhllPHIJmWAGKJ#*+)lLb/7iuCd&6LlEp<X=nj=P\Bi$9+mni'n:%&GpK<%9`n`hpEKpD2
+Kjjg/5#OS&drh;%B9gg&*6Di5IbeWCCd&5QkNP/<cj67tDa*IJZZ)MqG:)4I%/3m+"oj=X*mck?'lYLY
+Y59b6X*b]G83Y3qr]fs-q>&#^',K4U@=3_@Pp!eaLe85NcJ0m84@;sPi_u2B"9l));W+Fu2qkAH'Qbo?
+k-H[eYVU0^+Qq?4@?*JaL>GuXM!$`Q@6jQZ&EemB"OL-T$5EF4>9t[e0"6i78l3aCFa&dK-0QfX%m*#M
+8e<**6mW-%mA^<ZZa3j^#l(D&d1Sko-r>,eY$32#7[EgsmT'YRXAK9J7o9R:^UHfH[.1.!;60ROZ9Kg)
+^fMh<YG*ZF40hITi1XdprDj43\lK..=]Gd6o,SGp?ReA>bH2DqQYTG)!j/L9'WCmZ[s"eT`CQ9b9ep%7
+i:Epo[AI.S.$3nX@pO;)^_IiS09r22etWl=fIK1glpiACnWa])GauNe'F]<u]iBJAoF87!IIt"=I:17O
+[lTo@n=72PMr%pGoddarfZKth!*h,64;,NO'U(%PKkP26KsuD=.^J=C#7tW1d%n"CNULX*H%$XXg2Pkd
+p`d)QlrBf.VYpO%$8TBUkc^?MMs:/,N>Y*&ZBDDJ45'>V*]nOlgs`A\d<`N,-@kNLM:TkP1')Js-3C_@
+F"D\),@uq,Y1mHUea0P\AG4[N%=;nNlu>h1aop'kR/jg"-9+e#-@i8A7%iI&h^T9o;YJc-\ioaiAWMfq
+Zc<UPs"H[d`ZprgfWo5!oil!ZT5-i+!hX8`.hf^"A\pVKZ##jRcUJ(=M6bZ9A]!\X2U,l72U+6[2U2'j
+Cg(5iVP3`aRAcG5=#LNO?CB_6OVnD8.MIAbs)tG:pdeuFS??MRHrah-S0MUoT(6O&!.0-p9[pLhPmO+a
+1/ZeP<$q)MkGh2U?'U73T(R-Jb(PgTL7NF)PmP_SPmMIF=_@5OX%_#LDNtO$X"o7X62/MimHmrB!4jWn
+JZLU=GjG<1]2^2JY#crt<`LL33D3Fbi[NcHpoQ<&d8sn2T*[ns_N@nH!o"()XYLKOZ-!41G#]k3j/XpI
+.!B7Z.MEe)/rnAHb!bXW;4.]+PmG*Jbhp7!<QD_^HY"D*jl,_jmTu<#m!4*Lms4rKQM8GSa*VC7nO7@[
+.MIAO*,aUi(Pda'i]+Y(95t.Hg9Nn1Kb+0G]`8&hg-s$X'cHknZOZdT)r@BGm6Y)up'm4kfo/B*Iobr]
+;lgKlV!g:(#N]b$BCQMHV&`KYPdb7rF3$:[2+dHDj2#\7XO#G^P\#h[E?[+JF<SpWGZn1LXbL2.ee9fZ
+euDgo\F5=`-('l'PE*4`rP1c28b=;#+Lq0PCR1r%+#lpMfYYPD:o/k@H0&OC4Ap+;Dj;'dpJ@'AAF(kk
+f;)%1j5W^AFU[$fY#?#q@eeA$rLCuPs#KfMDm!;%#u8<:9nH#WY9m7rojd`g'$Gb()2e=/EBMXZW_#rE
+mQ[2kmQ]aQmQ]_jmQ\a@ruHJeP<eB4-3;r'8Xq0.8QBIC^],96s+]q?Z]T_1p7n);ZeG@j*81;bOT,jK
+;,#\h\k6<+lZ2^#p;bcSBP6^nc`c7JZ1]Q#X_k9?l1YaX^-&!/?sB,#r>Fm,4#>Ykqm2Db[6LEj]_87+
+Rk!+2;+k6mlhZPaC4C@(PLog+g:CkGrY1O'XP+aLH-Bd+5Q8.q\7"]"mW<6Ed2<;j%8YORCueL]cff8M
+rRn8V[6JZ\PLoddq6ME.=(R6UH:K\i)%f?_'Y2-,=<CPkV&doEMQ'sGBT2U]F%!^?1"HkWX/HXeerPsi
+VHT8jEmf;i3F(]k%0;J9AWE)i;q]f<Z$qh)n#\*Sm2EP1C?2,$o^/$j]'*/(r#JqA+.0<^N+B"3F=l)<
+=Z_U(3,7cBi*F&?nLQic3sqLrjqtJ:AqK_:Z<0>qaj!:1\8]PHMtJ5P<0Ot;0MC+:lN`='1!Eal,>3"J
+i0PrrqX1%]*:11<CgWLB/bt9N7+)5Rc[D]6gar.ORn<NrZ,QRehdZQ"(@.;#XNo1\$X`NEU6fffNPiG7
+o'o@0=Vc[&?_:8"n9iTSNq=hT4=A]`A8UBp_!Kf9Vp0sKQ*h9G#k3fWJr4qGnh;%Z^He/Y=RMi[G3ooS
+D'j>nOSSc-h0u+G.<IM>j=ecBcrd?=+t-/RV@36SK0:OFKhEEl!B;,Voa>njJJ"9a%g,5uUl,\n[(8rl
+I8JcoL!V=M8)$eHIb=r4"!NkH^06qK_8mllW!QK(-(H=:deM'^14,U\"A*4k'qX/5bJX_@>FM`/j;Ln&
+o"0@tLJPm+FC>mCGj+mVoNbPa\qUTVL96r;REVBa3kb=%Rmb2m3IJ8Nk_$VP`3I8=&]rMA0>dL28YW9C
+Ck,K]?2XX;eQj-[68q=40C\&+@UNa+5*u6ig:qU@9OjB\+.e5S\d*mfd%uR:Q[?*n8m,!u3@s&jPEMLu
+?P;2fr@))F]Ka&c^ueA,&SfM&#rk7_\2Q_*[9_XBKbENB!:a#`GP^Zbb'.m]'HGarH4rTRpobfS<d>A(
+Oh;"`f=_L2mS32;N2Cq8FR-f5!8@,:-9+fkgPh0hE,Qpsn8EQ_'5t?>Hp=NEhF,T!4ZLO3]nrZbJ)pKT
+rQR:VAJb?af0YO*0D^C3;Gr(74`K<77sKBcp)hQ*=5j[o&1r3,bb_ZLkAU:R)56q<O/eXFYHY6W-&_\>
+BB8j0=1%n@d([9oJaYZ=c7sqN7Gi=\q0gd<CX0[4Znrd1oVha@7+&,+97Y&(9t-^#K]0JdrpfnnIfH5/
+lX.Pss7tCUkb4G7Xm0=(0uD[LrAF.Y&;LK&<F>Gebdca'1G`3!ZTa;hP%+sJK=;VBE.+>Jl;PHN_5p8\
+-aWDGbl?1qVN9bTqY0p5qA6m%\a,l;7-u53rR^9U@p:8:#4q,+X[449o:lN#.A9pI#u1?1Fo1q3HAd>E
+7*<GP#^IgP3rZD^6,!'*0tb@dg^gHC9E)Hi-@[174+n2u<VoSSAs&b+LK;:9`.:8PZ2S6U\6TKW:=f/u
+^h<;dq.&8kErO0%X0UA7\Em"OJ<tHA<t5S`*Q2=`(&JMie!HoR'&p:a4(dRi.UN:ec1pC.]I0]I*S7E2
+'.-_*.V=,$1(UnZV<c'6`&F$2prcffS][R0ddK`cXj!Zir8cVoU9@WKqcA-F=#GKTc]Bse>mG#fho;nj
+M-i?Oc:#pT^?hf2!VH$r?L/h5i_'KnM7?.7nC7mjX0Q\nIc4?Z;pd,_$V''.9c]C8hFmUncnF:(,O[ks
+J,`[55G.uU2e3Y@ao~>
+endstream
+endobj
+7 0 obj
+ 8038
+endobj
+3 0 obj
+ <<
+ /Parent null
+ /Type /Pages
+ /MediaBox [0.0000 0.0000 331.00 240.00]
+ /Resources 8 0 R
+ /Kids [5 0 R]
+ /Count 1
+ >>
+endobj
+9 0 obj
+ [/PDF /Text /ImageC]
+endobj
+10 0 obj
+ <<
+ /S /Transparency
+ /CS /DeviceRGB
+ /I true
+ /K false
+ >>
+endobj
+11 0 obj
+ <<
+ /Alpha1
+ <<
+ /ca 1.0000
+ /CA 1.0000
+ /BM /Normal
+ /AIS false
+ >>
+ >>
+endobj
+8 0 obj
+ <<
+ /ProcSet 9 0 R
+ /ExtGState 11 0 R
+ >>
+endobj
+xref
+0 12
+0000000000 65535 f
+0000000015 00000 n
+0000000315 00000 n
+0000008780 00000 n
+0000000445 00000 n
+0000000521 00000 n
+0000000609 00000 n
+0000008757 00000 n
+0000009234 00000 n
+0000008950 00000 n
+0000008989 00000 n
+0000009091 00000 n
+trailer
+<<
+ /Size 12
+ /Root 2 0 R
+ /Info 1 0 R
+>>
+startxref
+9307
+%%EOF
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c6948930/doc/design/modules/graph.tex
----------------------------------------------------------------------
diff --git a/doc/design/modules/graph.tex b/doc/design/modules/graph.tex
index 5c3910c..1d0233c 100644
--- a/doc/design/modules/graph.tex
+++ b/doc/design/modules/graph.tex
@@ -22,11 +22,12 @@
\chapter[Graph]{Graph}
\begin{moduleinfo}
-\item[Author] \href{mailto:okislal@pivotal.io}{Orhan Kislal}
+\item[Authors] \href{mailto:okislal@pivotal.io}{Orhan Kislal}, \href{mailto:njayaram@pivotal.io}{Nandish Jayaram}
\item[History]
\begin{modulehistory}
\item[v0.1] Initial version, SSSP only.
\item[v0.2] Graph Framework, SSSP implementation details.
+ \item[v0.3] PageRank
\end{modulehistory}
\end{moduleinfo}
@@ -272,3 +273,147 @@ of the algorithm.
Please note that, for ideal performance, \emph{vertex} and \emph{edge} tables
should be distributed on \emph{vertex id} and \emph{source id} respectively.
+\section{PageRank} \label{sec:graph:pagerank}
+\begin{figure}[h]
+ \centering
+ \includegraphics[width=0.5\textwidth]{figures/pagerank_example.pdf}
+\caption{An example graph for PageRank}
+\label{pagerank:example}
+\end{figure}
+
+PageRank is a link analysis algorithm that assigns a score to every vertex
+measuring the relative importance of vertices within the set of all
+vertices. PageRank~\cite{pagerank} was first used by Google to measure the
+importance of website pages where the World Wide Web was modeled as a directed
+graph. Figure~\ref{pagerank:example} shows an example graph with the PageRank
+value of each vertex. The intuition behind the algorithm is that the number and
+quality of links to a vertex determine the authoritativeness of the vertex,
+which is reflected in the PageRank scores as shown in the figure.
+
+The pagerank module in MADlib implements the model of a random surfer who
+follows the edges of a graph to traverse it, and jumps to a random vertex
+after several clicks. The random surfer is modeled using a damping factor
+that represents the probability with which the surfer will continue to follow
+links in the graph rather than jumping to a random vertex. MADlib's pagerank
+module outputs a probability distribution that represents the likelihood that
+the random surfer arrives at a particular vertex in the graph.
+
+PageRank is an iterative algorithm where the PageRank scores of vertices from
+the previous iteration are used to compute the new PageRank scores. The
+PageRank score of a vertex $v$, at the $i^{th}$ iteration, denoted by $PR(v_i)$
+is computed as:
+
+\begin{equation}
+PR(v_i) = \frac{1-d}{N} + d \sum_{u \in M(v)}(\frac{PR(u_{i-1})}{L(u)})
+\label{eq:pagerank}
+\end{equation}
+
+where $N$ is the number of vertices in the graph, $d$ is the damping factor,
+$M(v)$ represents the set of vertices that have an edge to vertex $v$,
+$L(u)$ represents the out-degree of vertex $u$, i.e., the number of
+out-going edges from vertex $u$, and $PR(u_{i-1})$ represents the PageRank
+score of vertex $u$ in the $(i-1)^{st}$ iteration.
+
+$\frac{1-d}{N}$ represents the tiny probability with which the surfer
+would randomly jump to vertex $v$, rather than arriving at $v$ following
+links in the graph. This ensures that there is some probability of visiting
+every vertex in the graph even if they do not have any incoming edges. Note
+that the PageRank score computed for a vertex $v$ using~\ref{eq:pagerank}
+in the $i^{th}$ iteration is not updated until the new score is computed for
+all the vertices in the graph. The computation terminates either when the
+PageRank score of no vertex changes beyond a threshold across two consecutive
+iterations, or when a pre-set number of iterations are completed.
+
+\subsection{Implementation Details} \label{sec:pagerank:implementation}
+
+In this section, we discuss the MADlib implementation of PageRank in depth.
+We maintain two tables at every iteration: $previous$ and $cur$. The
+$previous$ table maintains the PageRank scores of all vertices computed in
+the previous iteration, while $cur$ maintains the updated scores of all
+vertices in the current iteration.
+
+\begin{algorithm}[PageRank$(V,E)$] \label{alg:pagerank:high}
+\begin{algorithmic}[1]
+ \State Create $previous$ table with a default PageRank score of
+ $\frac{1}{N}$ for every vertex
+ \Repeat
+ \State Create empty table $cur$.
+ \State Update $cur$ using PageRank scores of vertices in $previous$
+ \State Update PageRank scores of vertices without incoming edges
+ \State Drop $previous$ and rename $cur$ to $previous$
+ \Until {PageRank scores have converged or \emph{max} iterations have elapsed}
+\end{algorithmic}
+\end{algorithm}
+
+The implementation consists of updating the PageRank scores of all vertices
+at every iteration, using the PageRank scores of vertices from the previous
+iteration. The PageRank score of every vertex is initialized to $\frac{1}{N}$
+where $N$ is the total number of vertices in the graph. The out-degree of
+every vertex in the graph (represented by $L(u)$ in eq.~\ref{eq:pagerank}),
+is captured in table $out\_cnts$. The following query is used to create and
+update the PageRank scores in $cur$ table using the PageRank scores in
+$previous$ table.
+
+\begin{algorithm}[Update PageRank scores$(previous,out\_cnts,d,N)$]
+\label{alg:pagerank:update}
+\begin{lstlisting}
+CREATE TABLE cur AS
+ SELECT edge_table.dest AS id,
+ SUM(previous1.pagerank/out_cnts.cnt)*d + (1-d)/N AS pagerank
+ FROM edge_table
+ INNER JOIN previous ON edge_table.dest = previous.id
+ INNER JOIN out_cnts ON edge_table.src = out_cnts.id
+ INNER JOIN previous AS previous1 ON edge_table.src = previous1.id
+ GROUP BY edge_table.dest
+
+-- Update PageRank scores of vertices without any incoming edges:
+INSERT INTO cur
+ SELECT id, (1-d)/N AS pagerank
+ FROM previous
+ WHERE id NOT IN (
+ SELECT id
+ FROM cur
+ )
+\end{lstlisting}
+\end{algorithm}
+
+The PageRank computation is terminated either when a fixed number of iterations
+are completed, or when the PageRank scores of all vertices have converged. The
+PageRank score of a vertex is deemed converged if the absolute difference in
+its PageRank scores from $previous$ and $cur$ is less than a specified threshold.
+The following query is used to find all the vertices whose PageRank scores have
+not converged yet.
+
+\begin{algorithm}[Update PageRank scores$(previous,cur,threshold)$]
+\label{alg:pagerank:update}
+\begin{lstlisting}
+SELECT id
+FROM cur
+INNER JOIN previous ON cur.id = previous.id
+WHERE ABS(previous.pagerank - cur.pagerank) > threshold
+\end{lstlisting}
+\end{algorithm}
+
+\subsection{Best Practices} \label{sec:pagerank:bestpractices}
+
+The pagerank module in MADlib has a few optional parameters: damping factor
+$d$, number of iterations $max$, and the threshold for convergence $threshold$.
+The default values for these parameters when not specified by the user are
+$0.85$, $100$ and $\frac{1}{N*100}$ respectively.
+
+The damping factor denotes the probability with which the surfer uses the edges
+to traverse the graph. If set to $0$, it implies that the only way a surfer
+would visit a vertex in the graph is by randomly jumping to it. If set to
+$1$, it implies that the only way the surfer can reach a vertex is by following
+the edges in the graph, thus precluding the surfer from reaching a vertex
+that has no incoming edges. It is common practice to set damping factor
+to $0.85$~\cite{pagerank}, and the maximum number of iterations to $100$.
+The convergence test for PageRank in MADlib checks for the delta between
+the PageRank scores of a vertex across two consecutive iterations. Since
+the initial value of the PageRank score is set to $\frac{1}{N}$, the delta
+will be small in the initial iterations when $N$ is large (say over 100
+million). We thus set the threshold to $\frac{1}{N*100}$, and it is to be
+noted that this is not based on any experimental study. Users of MADlib are
+encouraged to consider this factor when setting a value for threshold, since
+a high $threshold$ value would lead to early termination of PageRank
+computation, thus resulting in incorrect PageRank values.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c6948930/doc/literature.bib
----------------------------------------------------------------------
diff --git a/doc/literature.bib b/doc/literature.bib
index 0353c23..c98a131 100644
--- a/doc/literature.bib
+++ b/doc/literature.bib
@@ -907,3 +907,10 @@ Applied Survival Analysis},
year={1956},
institution={DTIC Document}
}
+
+@inproceedings{pagerank,
+ booktitle = {Seventh International World-Wide Web Conference (WWW)},
+ title = {The Anatomy of a Large-Scale Hypertextual Web Search Engine},
+ author = {S. Brin and L. Page},
+ year = {1998}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c6948930/src/ports/postgres/modules/graph/pagerank.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/pagerank.py_in b/src/ports/postgres/modules/graph/pagerank.py_in
index 13cdcc5..202e536 100644
--- a/src/ports/postgres/modules/graph/pagerank.py_in
+++ b/src/ports/postgres/modules/graph/pagerank.py_in
@@ -31,33 +31,37 @@ import plpy
from utilities.control import MinWarning
from utilities.utilities import _assert
from utilities.utilities import extract_keyvalue_params
-from utilities.utilities import unique_string
-from utilities.control import IterationController2S
+from utilities.utilities import unique_string, split_quoted_delimited_str
+from utilities.validate_args import columns_exist_in_table, get_cols_and_types
from graph_utils import *
-import time
-
m4_changequote(`<!', `!>')
-def validate_pagerank_args(vertex_table, vertex_id, edge_table, edge_params,
- out_table, damping_factor, max_iter, threshold, module_name):
+def validate_pagerank_args(schema_madlib, vertex_table, vertex_id, edge_table,
+ edge_params, out_table, damping_factor, max_iter, threshold,
+ grouping_cols_list, module_name):
"""
Function to validate input parameters for PageRank
"""
validate_graph_coding(vertex_table, vertex_id, edge_table, edge_params,
out_table, module_name)
_assert(damping_factor >= 0.0 and damping_factor <= 1.0,
- """PageRank: Invalid damping factor value ({0}), must be between 0 and 1."""
- .format(damping_factor))
- _assert(threshold >= 0.0 and threshold <= 1.0,
- """PageRank: Invalid threshold value ({0}), must be between 0 and 1."""
- .format(threshold))
+ """PageRank: Invalid damping factor value ({0}), must be between 0 and 1.""".
+ format(damping_factor))
+ _assert(not threshold or (threshold >= 0.0 and threshold <= 1.0),
+ """PageRank: Invalid threshold value ({0}), must be between 0 and 1.""".
+ format(threshold))
_assert(max_iter > 0,
- """PageRank: Invalid max_iter value ({0}), must be a positive integer. """
- .format(max_iter))
+ """PageRank: Invalid max_iter value ({0}), must be a positive integer.""".
+ format(max_iter))
+ if grouping_cols_list:
+ # validate the grouping columns. We currently only support grouping_cols
+ # to be column names in the edge_table, and not expressions!
+ _assert(columns_exist_in_table(edge_table, grouping_cols_list, schema_madlib),
+ "PageRank error: One or more grouping columns specified do not exist!")
def pagerank(schema_madlib, vertex_table, vertex_id, edge_table, edge_args,
- out_table, damping_factor, max_iter, threshold, **kwargs):
+ out_table, damping_factor, max_iter, threshold, grouping_cols, **kwargs):
"""
Function that computes the PageRank
@@ -87,66 +91,278 @@ def pagerank(schema_madlib, vertex_table, vertex_id, edge_table, edge_args,
damping_factor = 0.85
if max_iter is None:
max_iter = 100
- if threshold is None:
- threshold = 0.00001
if vertex_id is None:
vertex_id = "id"
- validate_pagerank_args(vertex_table, vertex_id, edge_table, edge_params,
- out_table, damping_factor, max_iter, threshold, 'PageRank')
+ if not grouping_cols:
+ grouping_cols = ''
+
+ grouping_cols_list = split_quoted_delimited_str(grouping_cols)
+ validate_pagerank_args(schema_madlib, vertex_table, vertex_id, edge_table,
+ edge_params, out_table, damping_factor, max_iter, threshold,
+ grouping_cols_list, 'PageRank')
+ summary_table = out_table + "_summary"
+ _assert(not table_exists(summary_table),
+ "Graph PageRank: Output summary table ({summary_table}) already exists."
+ .format(**locals()))
src = edge_params["src"]
dest = edge_params["dest"]
+ nvertices = plpy.execute("""
+ SELECT COUNT({0}) AS cnt
+ FROM {1}
+ """.format(vertex_id, vertex_table))[0]["cnt"]
+ # A fixed threshold value, of say 1e-5, might not work well when the
+ # number of vertices is a billion, since the initial pagerank value
+ # of all nodes would then be 1/1e-9. So, assign default threshold
+ # value based on number of nodes in the graph.
+ # NOTE: The heuristic below is not based on any scientific evidence.
+ if threshold is None:
+ threshold = 1.0/(nvertices*100)
+
+ # table/column names used when grouping_cols is set.
+ distinct_grp_table = ''
+ vertices_per_group = ''
+ vpg = ''
+ grouping_where_clause = ''
+ group_by_clause = ''
+ random_prob = ''
edge_temp_table = unique_string(desp='temp_edge')
distribution = m4_ifdef(<!__POSTGRESQL__!>, <!''!>,
<!"DISTRIBUTED BY ({0})".format(dest)!>)
- plpy.execute("""
- DROP TABLE IF EXISTS {edge_temp_table};
- CREATE TEMP TABLE {edge_temp_table} AS
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(edge_temp_table))
+ plpy.execute("""CREATE TEMP TABLE {edge_temp_table} AS
SELECT * FROM {edge_table}
{distribution}
""".format(**locals()))
# GPDB and HAWQ have distributed by clauses to help them with indexing.
- # For Postgres we add the indices manually.
+ # For Postgres we add the index explicitly.
sql_index = m4_ifdef(<!__POSTGRESQL__!>,
<!"""CREATE INDEX ON {edge_temp_table} ({src});
""".format(**locals())!>,
<!''!>)
plpy.execute(sql_index)
- nvertices = plpy.execute("""
- SELECT COUNT({0}) AS cnt
- FROM {1}
- """.format(vertex_id, vertex_table))[0]["cnt"]
- init_value = 1.0/nvertices
- random_prob = (1.0-damping_factor)/nvertices
+ # Intermediate tables required.
cur = unique_string(desp='cur')
message = unique_string(desp='message')
- plpy.execute("""
- CREATE TEMP TABLE {cur} AS
- SELECT {vertex_id}, {init_value}::DOUBLE PRECISION AS pagerank
- FROM {vertex_table}
- """.format(**locals()))
- v1 = unique_string(desp='v1')
-
+ cur_unconv = unique_string(desp='cur_unconv')
+ message_unconv = unique_string(desp='message_unconv')
out_cnts = unique_string(desp='out_cnts')
out_cnts_cnt = unique_string(desp='cnt')
- # Compute the out-degree of every node in the graph.
+ v1 = unique_string(desp='v1')
+
cnts_distribution = m4_ifdef(<!__POSTGRESQL__!>, <!''!>,
- <!"DISTRIBUTED BY ({0})".format(vertex_id)!>)
+ <!"DISTRIBUTED BY ({0})".format(vertex_id)!>)
+ cur_join_clause = """{edge_temp_table}.{dest}={cur}.{vertex_id}
+ """.format(**locals())
+ out_cnts_join_clause = """{out_cnts}.{vertex_id}={edge_temp_table}.{src}
+ """.format(**locals())
+ v1_join_clause = """{v1}.{vertex_id}={edge_temp_table}.{src}
+ """.format(**locals())
- plpy.execute("""
- DROP TABLE IF EXISTS {out_cnts};
- CREATE TEMP TABLE {out_cnts} AS
- SELECT {src} AS {vertex_id}, COUNT({dest}) AS {out_cnts_cnt}
- FROM {edge_table}
- GROUP BY {src}
- {cnts_distribution}
- """.format(**locals()))
+ random_probability = (1.0-damping_factor)/nvertices
+ ######################################################################
+ # Create several strings that will be used to construct required
+ # queries. These strings will be required only during grouping.
+ random_jump_prob = random_probability
+ ignore_group_clause_first = ''
+ limit = ' LIMIT 1 '
+
+ grouping_cols_select_pr = ''
+ vertices_per_group_inner_join_pr = ''
+ ignore_group_clause_pr= ''
+
+ grouping_cols_select_ins = ''
+ vpg_from_clause_ins = ''
+ vpg_where_clause_ins = ''
+ message_grp_where_ins = ''
+ ignore_group_clause_ins = ''
+
+ grouping_cols_select_conv = '{0}.{1}'.format(cur, vertex_id)
+ group_by_grouping_cols_conv = ''
+ message_grp_clause_conv = ''
+ ignore_group_clause_conv = ''
+ ######################################################################
+
+ # Queries when groups are involved need a lot more conditions in
+ # various clauses, so populating the required variables. Some intermediate
+ # tables are unnecessary when no grouping is involved, so create some
+ # tables and certain columns only during grouping.
+ if grouping_cols:
+ distinct_grp_table = unique_string(desp='grp')
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(distinct_grp_table))
+ plpy.execute("""CREATE TEMP TABLE {distinct_grp_table} AS
+ SELECT DISTINCT {grouping_cols} FROM {edge_table}
+ """.format(**locals()))
+ vertices_per_group = unique_string(desp='nvert_grp')
+ init_pr = unique_string(desp='init')
+ random_prob = unique_string(desp='rand')
+ subq = unique_string(desp='subquery')
+ rand_damp = 1-damping_factor
+ grouping_where_clause = ' AND '.join(
+ [distinct_grp_table+'.'+col+'='+subq+'.'+col
+ for col in grouping_cols_list])
+ group_by_clause = ', '.join([distinct_grp_table+'.'+col
+ for col in grouping_cols_list])
+ # Find number of vertices in each group, this is the normalizing factor
+ # for computing the random_prob
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(vertices_per_group))
+ plpy.execute("""CREATE TEMP TABLE {vertices_per_group} AS
+ SELECT {distinct_grp_table}.*,
+ 1/COUNT(__vertices__)::DOUBLE PRECISION AS {init_pr},
+ {rand_damp}/COUNT(__vertices__)::DOUBLE PRECISION AS {random_prob}
+ FROM {distinct_grp_table} INNER JOIN (
+ SELECT {grouping_cols}, {src} AS __vertices__
+ FROM {edge_table}
+ UNION
+ SELECT {grouping_cols}, {dest} FROM {edge_table}
+ ){subq}
+ ON {grouping_where_clause}
+ GROUP BY {group_by_clause}
+ """.format(**locals()))
+
+ grouping_where_clause = ' AND '.join(
+ [vertices_per_group+'.'+col+'='+subq+'.'+col
+ for col in grouping_cols_list])
+ group_by_clause = ', '.join([vertices_per_group+'.'+col
+ for col in grouping_cols_list])
+ plpy.execute("""
+ CREATE TEMP TABLE {cur} AS
+ SELECT {group_by_clause}, {subq}.__vertices__ as {vertex_id},
+ {init_pr} AS pagerank
+ FROM {vertices_per_group} INNER JOIN (
+ SELECT {grouping_cols}, {src} AS __vertices__
+ FROM {edge_table}
+ UNION
+ SELECT {grouping_cols}, {dest} FROM {edge_table}
+ ){subq}
+ ON {grouping_where_clause}
+ """.format(**locals()))
+ vpg = unique_string(desp='vpg')
+ # Compute the out-degree of every node in the group-based subgraphs.
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(out_cnts))
+ plpy.execute("""CREATE TEMP TABLE {out_cnts} AS
+ SELECT {grouping_cols_select} {src} AS {vertex_id},
+ COUNT({dest}) AS {out_cnts_cnt}
+ FROM {edge_table}
+ GROUP BY {grouping_cols_select} {src}
+ {cnts_distribution}
+ """.format(grouping_cols_select=grouping_cols+','
+ if grouping_cols else '', **locals()))
+
+ message_grp = ' AND '.join(
+ ["{cur}.{col}={message}.{col}".format(**locals())
+ for col in grouping_cols_list])
+ cur_join_clause = cur_join_clause + ' AND ' + ' AND '.join(
+ ["{edge_temp_table}.{col}={cur}.{col}".format(**locals())
+ for col in grouping_cols_list])
+ out_cnts_join_clause = out_cnts_join_clause + ' AND ' + ' AND '.join(
+ ["{edge_temp_table}.{col}={out_cnts}.{col}".format(**locals())
+ for col in grouping_cols_list])
+ v1_join_clause = v1_join_clause + ' AND ' + ' AND '.join(
+ ["{edge_temp_table}.{col}={v1}.{col}".format(**locals())
+ for col in grouping_cols_list])
+ vpg_join_clause = ' AND '.join(
+ ["{edge_temp_table}.{col}={vpg}.{col}".format(**locals())
+ for col in grouping_cols_list])
+ vpg_cur_join_clause = ' AND '.join(
+ ["{cur}.{col}={vpg}.{col}".format(**locals())
+ for col in grouping_cols_list])
+ # join clause specific to populating random_prob for nodes without any
+ # incoming edges.
+ edge_grouping_cols_select = ', '.join(
+ ["{edge_temp_table}.{col}".format(**locals())
+ for col in grouping_cols_list])
+ cur_grouping_cols_select = ', '.join(
+ ["{cur}.{col}".format(**locals()) for col in grouping_cols_list])
+ # Create output summary table:
+ cols_names_types = get_cols_and_types(edge_table)
+ grouping_cols_clause = ', '.join([c_name+" "+c_type
+ for (c_name, c_type) in cols_names_types
+ if c_name in grouping_cols_list])
+ plpy.execute("""
+ CREATE TABLE {summary_table} (
+ {grouping_cols_clause},
+ __iterations__ INTEGER
+ )
+ """.format(**locals()))
+ # Create output table. This will be updated whenever a group converges
+ # Note that vertex_id is assumed to be an integer (as described in
+ # documentation)
+ plpy.execute("""
+ CREATE TABLE {out_table} (
+ {grouping_cols_clause},
+ {vertex_id} INTEGER,
+ pagerank DOUBLE PRECISION
+ )
+ """.format(**locals()))
+ temp_summary_table = unique_string(desp='temp_summary')
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(temp_summary_table))
+ plpy.execute("""
+ CREATE TABLE {temp_summary_table} (
+ {grouping_cols_clause}
+ )
+ """.format(**locals()))
+ ######################################################################
+ # Strings required for the main PageRank computation query
+ grouping_cols_select_pr = edge_grouping_cols_select+', '
+ random_jump_prob = 'MIN({vpg}.{random_prob})'.format(**locals())
+ vertices_per_group_inner_join_pr = """INNER JOIN {vertices_per_group}
+ AS {vpg} ON {vpg_join_clause}""".format(**locals())
+ ignore_group_clause_pr=' WHERE '+get_ignore_groups(summary_table,
+ edge_temp_table, grouping_cols_list)
+ # Strings required for updating PageRank scores of vertices that have
+ # no incoming edges
+ grouping_cols_select_ins = cur_grouping_cols_select+','
+ vpg_from_clause_ins = ', {vertices_per_group} AS {vpg}'.format(
+ **locals())
+ vpg_where_clause_ins = '{vpg_cur_join_clause} AND '.format(
+ **locals())
+ message_grp_where_ins = 'WHERE {message_grp}'.format(**locals())
+ ignore_group_clause_ins = ' AND '+get_ignore_groups(summary_table,
+ cur, grouping_cols_list)
+ # Strings required for convergence test query
+ grouping_cols_select_conv = cur_grouping_cols_select
+ group_by_grouping_cols_conv = ' GROUP BY {0}'.format(
+ cur_grouping_cols_select)
+ message_grp_clause_conv = '{0} AND '.format(message_grp)
+ ignore_group_clause_conv = ' AND '+get_ignore_groups(summary_table,
+ cur, grouping_cols_list)
+ limit = ''
+ else:
+ # cur and out_cnts tables can be simpler when no grouping is involved.
+ init_value = 1.0/nvertices
+ plpy.execute("""
+ CREATE TEMP TABLE {cur} AS
+ SELECT {vertex_id}, {init_value}::DOUBLE PRECISION AS pagerank
+ FROM {vertex_table}
+ """.format(**locals()))
- for i in range(max_iter):
+ # Compute the out-degree of every node in the graph.
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(out_cnts))
+ plpy.execute("""CREATE TEMP TABLE {out_cnts} AS
+ SELECT {src} AS {vertex_id}, COUNT({dest}) AS {out_cnts_cnt}
+ FROM {edge_table}
+ GROUP BY {src}
+ {cnts_distribution}
+ """.format(**locals()))
+
+ # The summary table when there is no grouping will contain only
+ # the iteration column. We don't need to create the out_table
+ # when no grouping is used since the 'cur' table will be renamed
+ # to out_table after pagerank computation is completed.
+ plpy.execute("""
+ CREATE TABLE {summary_table} (
+ __iterations__ INTEGER
+ )
+ """.format(**locals()))
+ unconverged = 0
+ iteration_num = 0
+ for iteration_num in range(max_iter):
#####################################################################
# PageRank for node 'A' at any given iteration 'i' is given by:
- # PR_i(A) = damping_factor(PR_i-1(B)/degree(B) + PR_i-1(C)/degree(C) + ...) + (1-damping_factor)/N
+ # PR_i(A) = damping_factor(PR_i-1(B)/degree(B) +
+ # PR_i-1(C)/degree(C) + ...) + (1-damping_factor)/N
# where 'N' is the number of vertices in the graph,
# B, C are nodes that have edges to node A, and
# degree(node) represents the number of outgoing edges from 'node'
@@ -157,45 +373,183 @@ def pagerank(schema_madlib, vertex_table, vertex_id, edge_table, edge_args,
# More information can be found at:
# https://en.wikipedia.org/wiki/PageRank#Damping_factor
- # The query below computes the PageRank of each node using the above formula.
+ # The query below computes the PageRank of each node using the above
+ # formula. A small explanatory note on ignore_group_clause:
+ # This is used only when grouping is set. This essentially will have
+ # the condition that will help skip the PageRank computation on groups
+ # that have converged.
plpy.execute("""
CREATE TABLE {message} AS
- SELECT {edge_temp_table}.{dest} AS {vertex_id},
- SUM({v1}.pagerank/{out_cnts}.{out_cnts_cnt})*{damping_factor}+{random_prob} AS pagerank
+ SELECT {grouping_cols_select_pr} {edge_temp_table}.{dest} AS {vertex_id},
+ SUM({v1}.pagerank/{out_cnts}.{out_cnts_cnt})*{damping_factor}+{random_jump_prob} AS pagerank
FROM {edge_temp_table}
- INNER JOIN {cur} ON {edge_temp_table}.{dest}={cur}.{vertex_id}
- INNER JOIN {out_cnts} ON {out_cnts}.{vertex_id}={edge_temp_table}.{src}
- INNER JOIN {cur} AS {v1} ON {v1}.{vertex_id}={edge_temp_table}.{src}
- GROUP BY {edge_temp_table}.{dest}
- """.format(**locals()))
- # If there are nodes that have no incoming edges, they are not captured in the message table.
- # Insert entries for such nodes, with random_prob.
+ INNER JOIN {cur} ON {cur_join_clause}
+ INNER JOIN {out_cnts} ON {out_cnts_join_clause}
+ INNER JOIN {cur} AS {v1} ON {v1_join_clause}
+ {vertices_per_group_inner_join_pr}
+ {ignore_group_clause}
+ GROUP BY {grouping_cols_select_pr} {edge_temp_table}.{dest}
+ """.format(ignore_group_clause=ignore_group_clause_pr
+ if iteration_num>0 else ignore_group_clause_first,
+ **locals()))
+ # If there are nodes that have no incoming edges, they are not
+ # captured in the message table. Insert entries for such nodes,
+ # with random_prob.
plpy.execute("""
INSERT INTO {message}
- SELECT {vertex_id}, {random_prob}::DOUBLE PRECISION AS pagerank
- FROM {cur}
- WHERE {vertex_id} NOT IN (
+ SELECT {grouping_cols_select_ins} {cur}.{vertex_id},
+ {random_jump_prob} AS pagerank
+ FROM {cur} {vpg_from_clause_ins}
+ WHERE {vpg_where_clause_ins} {vertex_id} NOT IN (
SELECT {vertex_id}
FROM {message}
+ {message_grp_where_ins}
)
- """.format(**locals()))
- # Check for convergence will be done as part of grouping support for pagerank:
- # https://issues.apache.org/jira/browse/MADLIB-1082. So, the threshold parameter
- # is a dummy variable at the moment, the PageRank computation happens for
- # {max_iter} number of times.
+ {ignore_group_clause}
+ GROUP BY {grouping_cols_select_ins} {cur}.{vertex_id}
+ """.format(ignore_group_clause=ignore_group_clause_ins
+ if iteration_num>0 else ignore_group_clause_first,
+ **locals()))
+
+ # Check for convergence:
+ ## Check for convergence only if threshold != 0.
+ if threshold != 0:
+ # message_unconv and cur_unconv will contain the unconverged groups
+ # after current # and previous iterations respectively. Groups that
+ # are missing in message_unconv but appear in cur_unconv are the
+ # groups that have converged after this iteration's computations.
+ # If no grouping columns are specified, then we check if there is
+ # at least one unconverged node (limit 1 is used in the query).
+ plpy.execute("""
+ CREATE TEMP TABLE {message_unconv} AS
+ SELECT {grouping_cols_select_conv}
+ FROM {message}
+ INNER JOIN {cur}
+ ON {cur}.{vertex_id}={message}.{vertex_id}
+ WHERE {message_grp_clause_conv}
+ ABS({cur}.pagerank-{message}.pagerank) > {threshold}
+ {ignore_group_clause}
+ {group_by_grouping_cols_conv}
+ {limit}
+ """.format(ignore_group_clause=ignore_group_clause_ins
+ if iteration_num>0 else ignore_group_clause_conv,
+ **locals()))
+ unconverged = plpy.execute("""SELECT COUNT(*) AS cnt FROM {0}
+ """.format(message_unconv))[0]["cnt"]
+ if iteration_num > 0 and grouping_cols:
+ # Update result and summary tables for groups that have
+ # converged
+ # since the last iteration.
+ update_result_tables(temp_summary_table, iteration_num,
+ summary_table, out_table, message, grouping_cols_list,
+ cur_unconv, message_unconv)
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(cur_unconv))
+ plpy.execute("""ALTER TABLE {message_unconv} RENAME TO
+ {cur_unconv} """.format(**locals()))
+ else:
+ # Do not run convergence test if threshold=0, since that implies
+ # the user wants to run max_iter iterations.
+ unconverged = 1
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(cur))
+ plpy.execute("""ALTER TABLE {message} RENAME TO {cur}
+ """.format(**locals()))
+ if unconverged == 0:
+ break
+
+ # If there still are some unconverged groups/(entire table),
+ # update results.
+ if grouping_cols:
+ if unconverged > 0:
+ if threshold != 0:
+ # We completed max_iters, but there are still some unconverged
+ # groups # Update the result and summary tables for unconverged
+ # groups.
+ update_result_tables(temp_summary_table, iteration_num,
+ summary_table, out_table, cur, grouping_cols_list,
+ cur_unconv)
+ else:
+ # No group has converged. List of all group values are in
+ # distinct_grp_table.
+ update_result_tables(temp_summary_table, iteration_num,
+ summary_table, out_table, cur, grouping_cols_list,
+ distinct_grp_table)
+ else:
+ plpy.execute("""ALTER TABLE {table_name} RENAME TO {out_table}
+ """.format(table_name=cur, **locals()))
plpy.execute("""
- DROP TABLE IF EXISTS {cur};
- ALTER TABLE {message} RENAME TO {cur}
+ INSERT INTO {summary_table} VALUES
+ ({iteration_num}+1);
""".format(**locals()))
- plpy.execute("ALTER TABLE {cur} RENAME TO {out_table}".format(**locals()))
-
## Step 4: Cleanup
- plpy.execute("""
- DROP TABLE IF EXISTS {0},{1},{2},{3};
- """.format(out_cnts, edge_temp_table, cur, message))
+ plpy.execute("""DROP TABLE IF EXISTS {0},{1},{2},{3},{4},{5};
+ """.format(out_cnts, edge_temp_table, cur, message, cur_unconv,
+ message_unconv))
+ if grouping_cols:
+ plpy.execute("""DROP TABLE IF EXISTS {0},{1},{2};
+ """.format(vertices_per_group, temp_summary_table,
+ distinct_grp_table))
plpy.execute("SET client_min_messages TO %s" % old_msg_level)
+def update_result_tables(temp_summary_table, i, summary_table, out_table,
+ res_table, grouping_cols_list, cur_unconv, message_unconv=None):
+ """
+ This function updates the summary and output tables only for those
+ groups that have converged. This is found out by looking at groups
+ that appear in cur_unvonv but not in message_unconv: message_unconv
+ consists of groups that have not converged in the current iteration,
+ while cur_unconv contains groups that had not converged in the
+ previous iterations. The entries in cur_unconv is a superset of the
+ entries in message_unconv. So the difference in the groups across
+ the two tables represents the groups that converged in the current
+ iteration.
+ """
+ plpy.execute("TRUNCATE TABLE {0}".format(temp_summary_table))
+ if message_unconv is None:
+ # If this function is called after max_iter is completed, without
+ # convergence, all the unconverged groups from cur_unconv is used
+ # (note that message_unconv is renamed to cur_unconv before checking
+ # for unconverged==0 in the pagerank function's for loop)
+ plpy.execute("""
+ INSERT INTO {temp_summary_table}
+ SELECT * FROM {cur_unconv}
+ """.format(**locals()))
+ else:
+ plpy.execute("""
+ INSERT INTO {temp_summary_table}
+ SELECT {cur_unconv}.*
+ FROM {cur_unconv}
+ WHERE {join_condition}
+ """.format(join_condition=get_ignore_groups(
+ message_unconv, cur_unconv, grouping_cols_list), **locals()))
+ plpy.execute("""
+ INSERT INTO {summary_table}
+ SELECT *, {i}+1 AS __iteration__
+ FROM {temp_summary_table}
+ """.format(**locals()))
+ plpy.execute("""
+ INSERT INTO {out_table}
+ SELECT {res_table}.*
+ FROM {res_table}
+ INNER JOIN {temp_summary_table}
+ ON {join_condition}
+ """.format(join_condition=' AND '.join(
+ ["{res_table}.{col}={temp_summary_table}.{col}".format(
+ **locals())
+ for col in grouping_cols_list]), **locals()))
+
+def get_ignore_groups(first_table, second_table, grouping_cols_list):
+ """
+ This function generates the necessary clause to only select the
+ groups that appear in second_table and not in first_table.
+ """
+ return """({second_table_cols}) NOT IN (SELECT {grouping_cols} FROM
+ {first_table}) """.format(second_table_cols=', '.join(
+ ["{second_table}.{col}".format(**locals())
+ for col in grouping_cols_list]),
+ grouping_cols=', '.join([col for col in grouping_cols_list]),
+ **locals())
+
def pagerank_help(schema_madlib, message, **kwargs):
"""
Help function for pagerank
@@ -212,12 +566,20 @@ def pagerank_help(schema_madlib, message, **kwargs):
message.lower() in ("usage", "help", "?"):
help_string = "Get from method below"
help_string = get_graph_usage(schema_madlib, 'PageRank',
- """out_table TEXT, -- Name of the output table for PageRank
- damping_factor, DOUBLE PRECISION, -- Damping factor in random surfer model
- -- (DEFAULT = 0.85)
- max_iter, INTEGER, -- Maximum iteration number (DEFAULT = 100)
- threshold DOUBLE PRECISION -- Stopping criteria (DEFAULT = 1e-5)
-""")
+ """out_table TEXT, -- Name of the output table for PageRank
+ damping_factor DOUBLE PRECISION, -- Damping factor in random surfer model
+ -- (DEFAULT = 0.85)
+ max_iter INTEGER, -- Maximum iteration number (DEFAULT = 100)
+ threshold DOUBLE PRECISION, -- Stopping criteria (DEFAULT = 1/(N*100),
+ -- N is number of vertices in the graph)
+ grouping_col TEXT -- Comma separated column names to group on
+ -- (DEFAULT = NULL, no grouping)
+""") + """
+
+A summary table is also created that contains information regarding the
+number of iterations required for convergence. It is named by adding the
+suffix '_summary' to the 'out_table' parameter.
+"""
else:
if message is not None and \
message.lower() in ("example", "examples"):
@@ -232,7 +594,8 @@ CREATE TABLE vertex(
);
CREATE TABLE edge(
src INTEGER,
- dest INTEGER
+ dest INTEGER,
+ user_id INTEGER
);
INSERT INTO vertex VALUES
(0),
@@ -243,30 +606,62 @@ INSERT INTO vertex VALUES
(5),
(6);
INSERT INTO edge VALUES
-(0, 1),
-(0, 2),
-(0, 4),
-(1, 2),
-(1, 3),
-(2, 3),
-(2, 5),
-(2, 6),
-(3, 0),
-(4, 0),
-(5, 6),
-(6, 3);
+(0, 1, 1),
+(0, 2, 1),
+(0, 4, 1),
+(1, 2, 1),
+(1, 3, 1),
+(2, 3, 1),
+(2, 5, 1),
+(2, 6, 1),
+(3, 0, 1),
+(4, 0, 1),
+(5, 6, 1),
+(6, 3, 1),
+(0, 1, 2),
+(0, 2, 2),
+(0, 4, 2),
+(1, 2, 2),
+(1, 3, 2),
+(2, 3, 2),
+(3, 0, 2),
+(4, 0, 2),
+(5, 6, 2),
+(6, 3, 2);
-- Compute the PageRank:
-DROP TABLE IF EXISTS pagerank_out;
+DROP TABLE IF EXISTS pagerank_out, pagerank_out_summary;
+SELECT madlib.pagerank(
+ 'vertex', -- Vertex table
+ 'id', -- Vertix id column
+ 'edge', -- Edge table
+ 'src=src, dest=dest', -- Comma delimted string of edge arguments
+ 'pagerank_out'); -- Output table of PageRank
+
+-- View the PageRank of all vertices, sorted by their scores.
+SELECT * FROM pagerank_out ORDER BY pagerank DESC;
+-- View the summary table to find the number of iterations required for
+-- convergence.
+SELECT * FROM pagerank_out_summary;
+
+-- Compute PageRank of nodes associated with each user:
+DROP TABLE IF EXISTS pagerank_out, pagerank_out_summary;
SELECT madlib.pagerank(
'vertex', -- Vertex table
'id', -- Vertix id column
'edge', -- Edge table
'src=src, dest=dest', -- Comma delimted string of edge arguments
- 'pagerank_out') -- Output table of PageRank
+ 'pagerank_out', -- Output table of PageRank
+ NULL, -- Default damping factor
+ NULL, -- Default max_iter
+ 0.00000001, -- Threshold
+ 'user_id'); -- Grouping column
-- View the PageRank of all vertices, sorted by their scores.
-SELECT * FROM pagerank_out ORDER BY pagerank desc;
+SELECT * FROM pagerank_out ORDER BY user_id, pagerank DESC;
+-- View the summary table to find the number of iterations required for
+-- convergence for each group.
+SELECT * FROM pagerank_out_summary;
"""
else:
help_string = """
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c6948930/src/ports/postgres/modules/graph/pagerank.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/pagerank.sql_in b/src/ports/postgres/modules/graph/pagerank.sql_in
index 712d146..6531bb5 100644
--- a/src/ports/postgres/modules/graph/pagerank.sql_in
+++ b/src/ports/postgres/modules/graph/pagerank.sql_in
@@ -58,7 +58,8 @@ pagerank( vertex_table,
out_table,
damping_factor,
max_iter,
- threshold
+ threshold,
+ grouping_cols
)
</pre>
@@ -91,7 +92,13 @@ this string argument:
It will contain a row for every vertex from 'vertex_table' with
the following columns:
- vertex_id : The id of a vertex. Will use the input parameter 'vertex_id' for column naming.
- - pagerank : The vertex's PageRank.</dd>
+ - pagerank : The vertex's PageRank.
+ - grouping_cols : Grouping column (if any) values associated with the vertex_id.</dd>
+
+A summary table is also created that contains information
+regarding the number of iterations required for convergence.
+It is named by adding the suffix '_summary' to the 'out_table'
+parameter.
<dt>damping_factor</dt>
<dd>FLOAT8, default 0.85. The probability, at any step, that a user will continue following the links in a random surfer model.</dd>
@@ -100,9 +107,18 @@ the following columns:
<dd>INTEGER, default: 100. The maximum number of iterations allowed.</dd>
<dt>threshold</dt>
-<dd>FLOAT8, default: 1e-5. If the difference between the PageRank of every vertex of two consecutive
+<dd>FLOAT8, default: (1/number of vertices * 100). If the difference between the PageRank of every vertex of two consecutive
iterations is smaller than 'threshold', or the iteration number is larger than 'max_iter', the
-computation stops. If you set the threshold to zero, then you will force the algorithm to run for the full number of iterations specified in 'max_iter'.</dd>
+computation stops. If you set the threshold to zero, then you will force the algorithm to run for the full number of iterations specified in 'max_iter'.
+It is advisable to set threshold to a value lower than 1/(number of vertices in the graph) since the PageRank value of nodes is initialized to that
+value.</dd>
+
+<dt>grouping_cols (optional)</dt>
+<dd>TEXT, default: NULL. A single column or a list of comma-separated
+columns that divides the input data into discrete groups, resulting in one
+distribution per group. When this value is NULL, no grouping is used and
+a single model is generated for all data.
+@note Expressions are not currently supported for 'grouping_cols'.</dd>
</dl>
@@ -122,7 +138,8 @@ CREATE TABLE vertex(
);
CREATE TABLE edge(
src INTEGER,
- dest INTEGER
+ dest INTEGER,
+ user_id INTEGER
);
INSERT INTO vertex VALUES
(0),
@@ -133,47 +150,66 @@ INSERT INTO vertex VALUES
(5),
(6);
INSERT INTO edge VALUES
-(0, 1),
-(0, 2),
-(0, 4),
-(1, 2),
-(1, 3),
-(2, 3),
-(2, 5),
-(2, 6),
-(3, 0),
-(4, 0),
-(5, 6),
-(6, 3);
+(0, 1, 1),
+(0, 2, 1),
+(0, 4, 1),
+(1, 2, 1),
+(1, 3, 1),
+(2, 3, 1),
+(2, 5, 1),
+(2, 6, 1),
+(3, 0, 1),
+(4, 0, 1),
+(5, 6, 1),
+(6, 3, 1),
+(0, 1, 2),
+(0, 2, 2),
+(0, 4, 2),
+(1, 2, 2),
+(1, 3, 2),
+(2, 3, 2),
+(3, 0, 2),
+(4, 0, 2),
+(5, 6, 2),
+(6, 3, 2);
</pre>
-# Compute the PageRank:
<pre class="syntax">
-DROP TABLE IF EXISTS pagerank_out;
+DROP TABLE IF EXISTS pagerank_out, pagerank_out_summary;
SELECT madlib.pagerank(
'vertex', -- Vertex table
'id', -- Vertix id column
'edge', -- Edge table
'src=src, dest=dest', -- Comma delimted string of edge arguments
'pagerank_out'); -- Output table of PageRank
-SELECT * FROM pagerank_out ORDER BY pagerank desc;
+SELECT * FROM pagerank_out ORDER BY pagerank DESC;
</pre>
<pre class="result">
id | pagerank
-----+--------------------
- 0 | 0.278256122055856
- 3 | 0.201882680839737
- 2 | 0.142878491945534
- 6 | 0.114538731993905
- 1 | 0.100266150276761
- 4 | 0.100266150276761
- 5 | 0.061911672611445
+----+-------------------
+ 0 | 0.28753749341184
+ 3 | 0.21016988901855
+ 2 | 0.14662683454062
+ 4 | 0.10289614384217
+ 1 | 0.10289614384217
+ 6 | 0.09728637768887
+ 5 | 0.05258711765692
(7 rows)
</pre>
+<pre class="syntax">
+SELECT * FROM pagerank_out_summary;
+</pre>
+<pre class="result">
+ __iterations__
+ ----------------+
+ 16
+(1 row)
+</pre>
--# Run PageRank with a damping factor of 0.5 results in different final values:
+-# Running PageRank with a damping factor of 0.5 results in different final values:
<pre class="syntax">
-DROP TABLE IF EXISTS pagerank_out;
+DROP TABLE IF EXISTS pagerank_out, pagerank_out_summary;
SELECT madlib.pagerank(
'vertex', -- Vertex table
'id', -- Vertix id column
@@ -181,21 +217,67 @@ SELECT madlib.pagerank(
'src=src, dest=dest', -- Comma delimted string of edge arguments
'pagerank_out', -- Output table of PageRank
0.5); -- Damping factor
-SELECT * FROM pagerank_out ORDER BY pagerank desc;
+SELECT * FROM pagerank_out ORDER BY pagerank DESC;
</pre>
<pre class="result">
- id | pagerank
-----+-------------------
- 0 | 0.221378135793372
- 3 | 0.191574922960784
- 6 | 0.140994575864846
- 2 | 0.135406336658892
- 4 | 0.108324751971412
- 1 | 0.108324751971412
- 5 | 0.093996524779681
+ id | pagerank
+----+--------------------
+ 0 | 0.225477161441199
+ 3 | 0.199090328586664
+ 2 | 0.136261327206477
+ 6 | 0.132691559968224
+ 4 | 0.109009291409508
+ 1 | 0.109009291409508
+ 5 | 0.0884610399788161
(7 rows)
</pre>
+-# Now compute the PageRank of vertices associated with each user
+using the grouping feature:
+<pre class="syntax">
+DROP TABLE IF EXISTS pagerank_out, pagerank_out_summary;
+SELECT madlib.pagerank(
+ 'vertex', -- Vertex table
+ 'id', -- Vertix id column
+ 'edge', -- Edge table
+ 'src=src, dest=dest', -- Comma delimted string of edge arguments
+ 'pagerank_out', -- Output table of PageRank
+ NULL, -- Default damping factor (0.85)
+ NULL, -- Default max iters (100)
+ 0.00000001, -- Threshold
+ 'user_id'); -- Grouping column name
+SELECT * FROM pagerank_out ORDER BY user_id, pagerank DESC;
+</pre>
+<pre class="result">
+ user_id | id | pagerank
+---------+----+--------------------
+ 1 | 0 | 0.27825488388552
+ 1 | 3 | 0.20188114667075
+ 1 | 2 | 0.14288112346059
+ 1 | 6 | 0.11453637832147
+ 1 | 1 | 0.10026745615438
+ 1 | 4 | 0.10026745615438
+ 1 | 5 | 0.06191155535288
+ 2 | 0 | 0.31854625004173
+ 2 | 3 | 0.23786686773343
+ 2 | 2 | 0.15914876489397
+ 2 | 1 | 0.11168334437971
+ 2 | 4 | 0.11168334437971
+ 2 | 6 | 0.03964285714285
+ 2 | 5 | 0.02142857142857
+(14 rows)
+</pre>
+<pre class="syntax">
+SELECT * FROM pagerank_out_summary ORDER BY user_id;
+</pre>
+<pre class="result">
+ user_id | __iterations__
+---------+----------------
+ 1 | 27
+ 2 | 31
+(2 rows)
+</pre>
+
@anchor literature
@par Literature
@@ -210,7 +292,8 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank(
out_table TEXT,
damping_factor FLOAT8,
max_iter INTEGER,
- threshold FLOAT8
+ threshold FLOAT8,
+ grouping_cols VARCHAR
) RETURNS VOID AS $$
PythonFunction(graph, pagerank, pagerank)
$$ LANGUAGE plpythonu VOLATILE
@@ -223,9 +306,23 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank(
edge_args TEXT,
out_table TEXT,
damping_factor FLOAT8,
+ max_iter INTEGER,
+ threshold FLOAT8
+) RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.pagerank($1, $2, $3, $4, $5, $6, $7, $8, NULL)
+$$ LANGUAGE SQL
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
+-------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank(
+ vertex_table TEXT,
+ vertex_id TEXT,
+ edge_table TEXT,
+ edge_args TEXT,
+ out_table TEXT,
+ damping_factor FLOAT8,
max_iter INTEGER
) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.pagerank($1, $2, $3, $4, $5, $6, $7, 0.00001)
+ SELECT MADLIB_SCHEMA.pagerank($1, $2, $3, $4, $5, $6, $7, 0.00001, NULL)
$$ LANGUAGE SQL
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-------------------------------------------------------------------------
@@ -237,7 +334,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank(
out_table TEXT,
damping_factor FLOAT8
) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.pagerank($1, $2, $3, $4, $5, $6, 100, 0.00001)
+ SELECT MADLIB_SCHEMA.pagerank($1, $2, $3, $4, $5, $6, 100, 0.00001, NULL)
$$ LANGUAGE SQL
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-------------------------------------------------------------------------
@@ -248,7 +345,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank(
edge_args TEXT,
out_table TEXT
) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.pagerank($1, $2, $3, $4, $5, 0.85, 100, 0.00001)
+ SELECT MADLIB_SCHEMA.pagerank($1, $2, $3, $4, $5, 0.85, 100, 0.00001, NULL)
$$ LANGUAGE SQL
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c6948930/src/ports/postgres/modules/graph/test/pagerank.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/test/pagerank.sql_in b/src/ports/postgres/modules/graph/test/pagerank.sql_in
index 1d695e2..2e84f35 100644
--- a/src/ports/postgres/modules/graph/test/pagerank.sql_in
+++ b/src/ports/postgres/modules/graph/test/pagerank.sql_in
@@ -19,13 +19,14 @@
*
*//* ----------------------------------------------------------------------- */
-DROP TABLE IF EXISTS vertex, edge, pagerank_out;
+DROP TABLE IF EXISTS vertex, edge;
CREATE TABLE vertex(
id INTEGER
);
CREATE TABLE edge(
src INTEGER,
- dest INTEGER
+ dest INTEGER,
+ user_id INTEGER
);
INSERT INTO vertex VALUES
(0),
@@ -36,19 +37,30 @@ INSERT INTO vertex VALUES
(5),
(6);
INSERT INTO edge VALUES
-(0, 1),
-(0, 2),
-(0, 4),
-(1, 2),
-(1, 3),
-(2, 3),
-(2, 5),
-(2, 6),
-(3, 0),
-(4, 0),
-(5, 6),
-(6, 3);
+(0, 1, 1),
+(0, 2, 1),
+(0, 4, 1),
+(1, 2, 1),
+(1, 3, 1),
+(2, 3, 1),
+(2, 5, 1),
+(2, 6, 1),
+(3, 0, 1),
+(4, 0, 1),
+(5, 6, 1),
+(6, 3, 1),
+(0, 1, 2),
+(0, 2, 2),
+(0, 4, 2),
+(1, 2, 2),
+(1, 3, 2),
+(2, 3, 2),
+(3, 0, 2),
+(4, 0, 2),
+(5, 6, 2),
+(6, 3, 2);
+DROP TABLE IF EXISTS pagerank_out, pagerank_out_summary;
SELECT madlib.pagerank(
'vertex', -- Vertex table
'id', -- Vertix id column
@@ -60,3 +72,26 @@ SELECT madlib.pagerank(
SELECT assert(relative_error(SUM(pagerank), 1) < 0.00001,
'PageRank: Scores do not sum up to 1.'
) FROM pagerank_out;
+
+DROP TABLE IF EXISTS pagerank_out, pagerank_out_summary;
+SELECT madlib.pagerank(
+ 'vertex', -- Vertex table
+ 'id', -- Vertix id column
+ 'edge', -- Edge table
+ 'src=src, dest=dest', -- Edge args
+ 'pagerank_out', -- Output table of PageRank
+ NULL,
+ NULL,
+ 0.00000001,
+ 'user_id');
+
+-- View the PageRank of all vertices, sorted by their scores.
+SELECT assert(relative_error(SUM(pagerank), 1) < 0.00001,
+ 'PageRank: Scores do not sum up to 1 for group 1.'
+ ) FROM pagerank_out WHERE user_id=1;
+SELECT assert(relative_error(__iterations__, 27) = 0,
+ 'PageRank: Incorrect iterations for group 1.'
+ ) FROM pagerank_out_summary WHERE user_id=1;
+SELECT assert(relative_error(__iterations__, 31) = 0,
+ 'PageRank: Incorrect iterations for group 2.'
+ ) FROM pagerank_out_summary WHERE user_id=2;
[07/34] incubator-madlib git commit: README: Add build status badge
icon
Posted by ok...@apache.org.
README: Add build status badge icon
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/1392c5d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/1392c5d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/1392c5d6
Branch: refs/heads/latest_release
Commit: 1392c5d6a701a31303702521a5905e6ff79fdcf4
Parents: aaf5f82
Author: Rahul Iyer <ri...@apache.org>
Authored: Mon Mar 27 14:06:47 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Mon Mar 27 14:06:47 2017 -0700
----------------------------------------------------------------------
README.md | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/1392c5d6/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index cd4f155..eac06c2 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,9 @@
It provides data-parallel implementations of mathematical, statistical and
machine learning methods for structured and unstructured data.
+[](https://builds.apache.org/job/madlib-master-build)
+
+
Installation and Contribution
==============================
See the project webpage [`MADlib Home`](http://madlib.incubator.apache.org/) for links to the
[26/34] incubator-madlib git commit: DT: Include rows with NULL
features in training
Posted by ok...@apache.org.
DT: Include rows with NULL features in training
JIRA: MADLIB-1095
This commit enables decision tree to include rows with NULL feature
values in the training dataset. Features that have NULL values are not
used during the training, but the features with non-null values are
used.
Note: Training of a level requires each row to go through the tree above
the level. If a row contains NULL value for a feature used for splitting
a node in the tree, the path for that row (either left or right) will
be determined by
1. using a surrogate feature (if surrogates are enabled) or
2. using the branch that had majority of rows assigned to it.
Closes #125
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/a3d54be6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/a3d54be6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/a3d54be6
Branch: refs/heads/latest_release
Commit: a3d54be66cd868b1e8f0fa98c8a8c97a7aa17601
Parents: 8bd4947
Author: Rahul Iyer <ri...@apache.org>
Authored: Wed Apr 26 17:07:22 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Wed Apr 26 17:18:48 2017 -0700
----------------------------------------------------------------------
.../recursive_partitioning/decision_tree.py_in | 119 +++++++------------
.../recursive_partitioning/random_forest.py_in | 5 +-
2 files changed, 47 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/a3d54be6/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
index f7c4bd8..dbf7db7 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
@@ -90,6 +90,7 @@ def _tree_validate_args(
_assert(max_depth >= 0 and max_depth < 100,
"Decision tree error: maximum tree depth must be positive and less than 100.")
+ _assert(cp >= 0, "Decision tree error: cp must be non-negative.")
_assert(min_split > 0, "Decision tree error: min_split must be positive.")
_assert(min_bucket > 0, "Decision tree error: min_bucket must be positive.")
_assert(n_bins > 1, "Decision tree error: number of bins must be at least 2.")
@@ -370,9 +371,7 @@ def _get_tree_states(schema_madlib, is_classification, split_criterion,
for each group. For the no grouping case, the
key is ''
"""
- filter_null = _get_filter_str(schema_madlib, cat_features, con_features,
- boolean_cats, dependent_variable,
- grouping_cols, max_n_surr)
+ filter_dep = _get_filter_str(dependent_variable, grouping_cols)
# 3)
if is_classification:
if split_criterion.lower().strip() == "mse":
@@ -381,11 +380,11 @@ def _get_tree_states(schema_madlib, is_classification, split_criterion,
# For classifications, we also need to map dependent_variable to integers
n_rows, dep_list = _get_n_and_deplist(training_table_name,
dependent_variable,
- filter_null)
+ filter_dep)
dep_list.sort()
if dep_is_bool:
- dep_col_str = ("case when " + dependent_variable +
- " then 'True' else 'False' end")
+ dep_col_str = ("CASE WHEN {0} THEN 'True' ELSE 'False' END".
+ format(dependent_variable))
else:
dep_col_str = dependent_variable
dep_var_str = ("(CASE " +
@@ -397,10 +396,11 @@ def _get_tree_states(schema_madlib, is_classification, split_criterion,
if split_criterion.lower().strip() != "mse":
plpy.warning("Decision tree: Using MSE as split criterion as it "
"is the only one supported for regression trees.")
- n_rows = long(plpy.execute(
- "SELECT count(*)::bigint FROM {source_table} WHERE {filter_null}".
- format(source_table=training_table_name,
- filter_null=filter_null))[0]['count'])
+ n_rows = long(plpy.execute("SELECT count(*)::bigint "
+ "FROM {src} "
+ "WHERE {filter}".
+ format(src=training_table_name,
+ filter=filter_dep))[0]['count'])
dep_var_str = dependent_variable
dep_list = []
@@ -411,8 +411,8 @@ def _get_tree_states(schema_madlib, is_classification, split_criterion,
# categorical bins and continuous bins
bins = _get_bins(schema_madlib, training_table_name, cat_features,
ordered_cat_features, con_features, n_bins,
- dep_var_str, boolean_cats,
- n_rows, is_classification, dep_n_levels, filter_null)
+ dep_var_str, boolean_cats, n_rows, is_classification,
+ dep_n_levels, filter_dep)
# some features may be dropped if they have only one value
cat_features = bins['cat_features']
@@ -439,7 +439,7 @@ def _get_tree_states(schema_madlib, is_classification, split_criterion,
boolean_cats, grouping_cols,
grouping_array_str, n_rows,
is_classification, dep_n_levels,
- filter_null)
+ filter_dep)
cat_features = bins['cat_features']
# 3b) Load each group's tree state in memory and set to the initial tree
@@ -704,8 +704,8 @@ def _is_dep_categorical(training_table_name, dependent_variable):
def _get_bins(schema_madlib, training_table_name,
cat_features, ordered_cat_features,
- con_features, n_bins, dependent_variable, boolean_cats,
- n_rows, is_classification, dep_n_levels, filter_null):
+ con_features, n_bins, dependent_variable, boolean_cats, n_rows,
+ is_classification, dep_n_levels, filter_null):
""" Compute the bins of all features
@param training_table_name Data source table
@@ -715,7 +715,6 @@ def _get_bins(schema_madlib, training_table_name,
@param dependent_variable Will be needed when sorting the levels of
categorical variables
@param boolean_cats The categorical variables that are of boolean type
- @param n_rows The total number of rows in the data table
return one dictionary containing two arrays: categorical and continuous
"""
@@ -743,12 +742,6 @@ def _get_bins(schema_madlib, training_table_name,
# _compute_splits function in CoxPH module, but deal with
# multiple columns together.
con_features_str = py_list_to_sql_string(con_features, "double precision")
- con_split_str = ("{schema_madlib}._dst_compute_con_splits(" +
- con_features_str +
- ", {sample_size}::integer, {n_bins}::smallint)"
- ).format(schema_madlib=schema_madlib,
- sample_size=actual_sample_size,
- n_bins=n_bins)
sample_table_name = unique_string()
plpy.execute("""
@@ -764,6 +757,11 @@ def _get_bins(schema_madlib, training_table_name,
""".format(**locals()))
# The splits for continuous variables
+ con_split_str = ("""{schema_madlib}._dst_compute_con_splits(
+ {con_features_str},
+ {actual_sample_size}::integer,
+ {n_bins}::smallint)""".
+ format(**locals()))
con_splits = plpy.execute("""
SELECT {con_split_str} as con_splits
FROM {sample_table_name}
@@ -990,32 +988,31 @@ def _get_bins_grps(
con_split_str = """{schema_madlib}._dst_compute_con_splits(
{con_features_str},
{n_per_seg}::integer,
- {n_bins}::smallint)""".format(
- con_features_str=con_features_str,
- schema_madlib=schema_madlib,
- n_per_seg=n_per_seg_str,
- n_bins=n_bins)
- sql = """
- SELECT
+ {n_bins}::smallint)""".format(con_features_str=con_features_str,
+ schema_madlib=schema_madlib,
+ n_per_seg=n_per_seg_str,
+ n_bins=n_bins)
+ con_splits_all = plpy.execute(
+ """ SELECT
{con_split_str} AS con_splits,
{grouping_array_str} AS grp_key
FROM {sample_table_name}
GROUP BY {grouping_cols}
""".format(**locals()) # multiple rows
-
- con_splits_all = plpy.execute(sql)
+ )
plpy.execute("DROP TABLE {sample_table_name}".format(**locals()))
if cat_features:
if is_classification:
# For classifications
- order_fun = "{schema_madlib}._dst_compute_entropy({dependent_variable}, {n})".format(
- schema_madlib=schema_madlib,
- dependent_variable=dependent_variable,
- n=dep_n_levels)
+ order_fun = ("{schema_madlib}._dst_compute_entropy("
+ "{dependent_variable}, {n})".
+ format(schema_madlib=schema_madlib,
+ dependent_variable=dependent_variable,
+ n=dep_n_levels))
else:
- order_fun = "avg({dependent_variable})".format(dependent_variable=dependent_variable)
+ order_fun = "avg({0})".format(dependent_variable)
sql_cat_levels = """
SELECT
@@ -1106,10 +1103,9 @@ def get_feature_str(schema_madlib, boolean_cats,
"(coalesce(" + col + "::text,'{0}')".format(unique_val) +
")::text")
- cat_features_str = (
- "{0}._map_catlevel_to_int(array[" +
- ", ".join(cat_features_cast) + "], {1}, {2})"
- ).format(schema_madlib, levels_str, n_levels_str)
+ cat_features_str = ("{0}._map_catlevel_to_int(array[" +
+ ", ".join(cat_features_cast) + "], {1}, {2})"
+ ).format(schema_madlib, levels_str, n_levels_str)
else:
cat_features_str = "NULL"
@@ -1582,38 +1578,17 @@ def _create_summary_table(
# ------------------------------------------------------------
-def _get_filter_str(schema_madlib, cat_features, con_features,
- boolean_cats, dependent_variable,
- grouping_cols, max_n_surr=0):
+def _get_filter_str(dependent_variable, grouping_cols):
""" Return a 'WHERE' clause string that filters out all rows that contain a
NULL.
"""
if grouping_cols:
- g_filter = ' and '.join('(' + s.strip() + ') is not NULL' for s in grouping_cols.split(','))
- else:
- g_filter = None
-
- if cat_features and max_n_surr == 0:
- cat_filter = \
- 'NOT {schema_madlib}.array_contains_null({cat_features_array})'.format(
- schema_madlib=schema_madlib,
- cat_features_array='array[' + ','.join(
- '(' + cat + ')::text' if cat not in boolean_cats else
- "(case when " + cat + " then 'True' else 'False' end)::text"
- for cat in cat_features) + ']')
+ group_filter = ' and '.join('({0}) is not NULL'.format(g.strip())
+ for g in grouping_cols.split(','))
else:
- cat_filter = None
-
- if con_features and max_n_surr == 0:
- con_filter = \
- 'NOT {schema_madlib}.array_contains_null({con_features_array})'.format(
- schema_madlib=schema_madlib,
- con_features_array='array[' + ','.join(con_features) + ']')
- else:
- con_filter = None
-
+ group_filter = None
dep_filter = '(' + dependent_variable + ") is not NULL"
- return ' and '.join(filter(None, [g_filter, cat_filter, con_filter, dep_filter]))
+ return ' and '.join(filter(None, [group_filter, dep_filter]))
# -------------------------------------------------------------------------
@@ -1814,7 +1789,7 @@ def _get_display_header(table_name, dep_levels, is_regression, dot_format=True):
""".format(str(dep_levels))
return_str += "\n-------------------------------------"
return return_str
-#------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
def tree_display(schema_madlib, model_table, dot_format=True, verbose=False,
@@ -2008,8 +1983,6 @@ def _prune_and_cplist(schema_madlib, tree, cp, compute_cp_list=False):
cp_list: list of cp values at which tree can be pruned
(returned only if compute_cp_list=True)
"""
- if cp <= 0 and not compute_cp_list:
- return tree
sql = """
SELECT (pruned_tree).*
FROM (
@@ -2198,7 +2171,7 @@ def _xvalidate(schema_madlib, tree_states, training_table_name, output_table_nam
def _tree_train_using_bins(
schema_madlib, bins, training_table_name,
cat_features, con_features, boolean_cats, n_bins, weights,
- dep_var_str, min_split, min_bucket, max_depth, filter_null,
+ dep_var_str, min_split, min_bucket, max_depth, filter_dep,
dep_n_levels, is_classification, split_criterion,
subsample=False, n_random_features=1, max_n_surr=0, **kwargs):
"""Trains a tree without grouping columns"""
@@ -2225,7 +2198,7 @@ def _tree_train_using_bins(
schema_madlib, training_table_name,
cat_features, con_features, boolean_cats, bins,
n_bins, tree_state, weights, dep_var_str,
- min_split, min_bucket, max_depth, filter_null,
+ min_split, min_bucket, max_depth, filter_dep,
dep_n_levels, subsample, n_random_features, max_n_surr)
plpy.notice("Completed training of level {0}".format(tree_depth))
@@ -2236,7 +2209,7 @@ def _tree_train_using_bins(
def _tree_train_grps_using_bins(
schema_madlib, bins, training_table_name, cat_features, con_features,
boolean_cats, n_bins, weights, grouping_cols, grouping_array_str, dep_var_str,
- min_split, min_bucket, max_depth, filter_null, dep_n_levels,
+ min_split, min_bucket, max_depth, filter_dep, dep_n_levels,
is_classification, split_criterion, subsample=False,
n_random_features=1, tree_terminated=None, max_n_surr=0, **kwargs):
@@ -2281,7 +2254,7 @@ def _tree_train_grps_using_bins(
con_features, boolean_cats, bins, n_bins,
tree_states, weights, grouping_cols,
grouping_array_str, dep_var_str, min_split, min_bucket,
- max_depth, filter_null, dep_n_levels, subsample,
+ max_depth, filter_dep, dep_n_levels, subsample,
n_random_features, max_n_surr)
level += 1
plpy.notice("Finished training for level " + str(level))
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/a3d54be6/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
index 930d916..1226591 100644
--- a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
@@ -332,10 +332,7 @@ def forest_train(
cat_features, ordered_cat_features, con_features, boolean_cats = \
_classify_features(all_cols_types, features)
- filter_null = _get_filter_str(schema_madlib, cat_features,
- con_features, boolean_cats,
- dependent_variable, grouping_cols,
- max_n_surr)
+ filter_null = _get_filter_str(dependent_variable, grouping_cols)
# the total number of records
n_all_rows = plpy.execute("SELECT count(*) FROM {0}".
format(training_table_name))[0]['count']
[05/34] incubator-madlib git commit: PGXN: Fix license and images in
metadata
Posted by ok...@apache.org.
PGXN: Fix license and images in metadata
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/63f59e2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/63f59e2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/63f59e2b
Branch: refs/heads/latest_release
Commit: 63f59e2bcf7bf61d7ae22a87d70a598efe94bb0a
Parents: 5984d82
Author: Rahul Iyer <ri...@apache.org>
Authored: Mon Mar 20 14:11:32 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Mon Mar 20 14:11:32 2017 -0700
----------------------------------------------------------------------
README.md | 4 ++--
deploy/PGXN/META.json.in | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/63f59e2b/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index eaff324..cd4f155 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-  
+  
=================================================
**MADlib<sup>®</sup>** is an open-source library for scalable in-database analytics.
It provides data-parallel implementations of mathematical, statistical and
@@ -66,7 +66,7 @@ The following block-diagram gives a high-level overview of MADlib's
architecture.
-
+
Third Party Components
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/63f59e2b/deploy/PGXN/META.json.in
----------------------------------------------------------------------
diff --git a/deploy/PGXN/META.json.in b/deploy/PGXN/META.json.in
index 47dd9cb..914454d 100644
--- a/deploy/PGXN/META.json.in
+++ b/deploy/PGXN/META.json.in
@@ -3,8 +3,8 @@
"abstract": "An open-source in-database analytics library",
"description": "An open-source in-database analytics library",
"version": "@MADLIB_PGXN_VERSION_STR@",
- "maintainer": "MADlib development team",
- "license": "bsd",
+ "maintainer": "MADlib contributors <de...@madlib.incubator.apache.org>",
+ "license": "apache_2_0",
"provides": {
"madlib": {
"file": "madlib--@MADLIB_VERSION_MAJOR@.@MADLIB_VERSION_MINOR@.@MADLIB_VERSION_PATCH@.sql",
[08/34] incubator-madlib git commit: Build: Update pom version + add
rat check script
Posted by ok...@apache.org.
Build: Update pom version + add rat check script
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/d344f1f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/d344f1f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/d344f1f1
Branch: refs/heads/latest_release
Commit: d344f1f1948aecd06af7ffbb494570bf1726fc4a
Parents: 1392c5d
Author: Rahul Iyer <ri...@apache.org>
Authored: Mon Mar 27 14:12:15 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Mon Mar 27 14:12:15 2017 -0700
----------------------------------------------------------------------
README.md | 21 ++++++++------
pom.xml | 2 +-
tool/jenkins/rat_check.sh | 62 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 76 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/d344f1f1/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index eac06c2..1d507a3 100644
--- a/README.md
+++ b/README.md
@@ -18,19 +18,24 @@ Development with Docker
We provide a Docker image with necessary dependencies required to compile and test MADlib on PostgreSQL 9.6. You can view the dependency Docker file at ./tool/docker/base/Dockerfile_postgres_9_6. The image is hosted on Docker Hub at madlib/postgres_9.6:latest. Later we will provide a similar Docker image for Greenplum Database.
Some useful commands to use the docker file:
+
```
## 1) Pull down the `madlib/postgres_9.6:latest` image from docker hub:
docker pull madlib/postgres_9.6:latest
-## 2) Launch a container corresponding to the MADlib image, mounting the source code folder to the container:
-docker run -d -it --name madlib -v (path to incubator-madlib directory):/incubator-madlib/ madlib/postgres_9.6
+## 2) Launch a container corresponding to the MADlib image, mounting the
+## source code folder to the container:
+docker run -d -it --name madlib \
+ -v (path to incubator-madlib directory):/incubator-madlib/ madlib/postgres_9.6
# where incubator-madlib is the directory where the MADlib source code resides.
-############################################## * WARNING * ##################################################
-# Please be aware that when mounting a volume as shown above, any changes you make in the "incubator-madlib"
-# folder inside the Docker container will be reflected on your local disk (and vice versa). This means that
-# deleting data in the mounted volume from a Docker container will delete the data from your local disk also.
-#############################################################################################################
+################################# * WARNING * #################################
+# Please be aware that when mounting a volume as shown above, any changes you
+# make in the "incubator-madlib" folder inside the Docker container will be
+# reflected on your local disk (and vice versa). This means that deleting data
+# in the mounted volume from a Docker container will delete the data from your
+# local disk also.
+###############################################################################
## 3) When the container is up, connect to it and build MADlib:
docker exec -it madlib bash
@@ -44,7 +49,7 @@ make install
## 4) Install MADlib:
src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install
-## 5) Several other commands, apart from the ones above can now be run, such as:
+## 5) Several other commands can now be run, such as:
# Run install check, on all modules:
src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check
# Run install check, on a specific module, say svm:
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/d344f1f1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5defa28..f033334 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,7 @@
<groupId>org.apache.madlib</groupId>
<artifactId>madlib</artifactId>
- <version>1.10</version>
+ <version>1.11-dev</version>
<packaging>pom</packaging>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/d344f1f1/tool/jenkins/rat_check.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/rat_check.sh b/tool/jenkins/rat_check.sh
new file mode 100644
index 0000000..7c12673
--- /dev/null
+++ b/tool/jenkins/rat_check.sh
@@ -0,0 +1,62 @@
+# ----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------
+# This file captures the Apache Jenkins rat check script
+# ----------------------------------------------------------------------
+
+set -exu
+
+workdir=`pwd`
+
+# Check if NOTICE file year is current
+grep "Copyright 2016-$(date +"%Y") The Apache Software Foundation" "${workdir}/incubator-madlib/NOTICE"
+
+# Check if pom.xml file version is current
+# With below grep, it's possible to get a "False Positive" (i.e. no error when it should have)
+# but won't give a "False Negative" (i.e. if it fails then there's definitely a problem)
+grep "<version>$(cat "${workdir}/incubator-madlib/src/config/Version.yml" | cut -d" " -f2)</version>" \
+ "${workdir}/incubator-madlib/pom.xml"
+
+set +x
+
+badfile_extentions="class jar tar tgz zip"
+badfiles_found=false
+
+for extension in ${badfile_extentions}; do
+ echo "Searching for ${extension} files:"
+ badfile_count=$(find . -name "${workdir}/incubator-madlib/*.${extension}" | wc -l)
+ if [ ${badfile_count} != 0 ]; then
+ echo "----------------------------------------------------------------------"
+ echo "FATAL: ${extension} files should not exist"
+ echo "For ASF compatibility: the source tree should not contain"
+ echo "binary (jar) files as users have a hard time verifying their"
+ echo "contents."
+
+ find . -name "${workdir}/incubator-madlib/*.${extension}"
+ echo "----------------------------------------------------------------------"
+ badfiles_found=true
+ else
+ echo "PASSED: No ${extension} files found."
+ fi
+done
+
+if [ ${badfiles_found} = "true" ]; then
+ exit 1
+fi
+
+set -x
\ No newline at end of file
[22/34] incubator-madlib git commit: Multiple: Fix PCA IC bug
Posted by ok...@apache.org.
Multiple: Fix PCA IC bug
- Add comments for disabled IC tests.
- Simplify SSSP code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/0cdd644a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/0cdd644a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/0cdd644a
Branch: refs/heads/latest_release
Commit: 0cdd644a9d07fdbd0f9ddfdf84964aac8d8fd791
Parents: 0d815f2
Author: Orhan Kislal <ok...@pivotal.io>
Authored: Fri Apr 21 13:18:58 2017 -0700
Committer: Orhan Kislal <ok...@pivotal.io>
Committed: Fri Apr 21 13:18:58 2017 -0700
----------------------------------------------------------------------
.../test/elastic_net_install_check.sql_in | 2 +
src/ports/postgres/modules/graph/sssp.py_in | 14 +++---
.../postgres/modules/graph/test/pagerank.sql_in | 4 ++
src/ports/postgres/modules/pca/test/pca.sql_in | 52 ++++++++++----------
.../validation/test/cross_validation.sql_in | 3 ++
5 files changed, 42 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0cdd644a/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in b/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
index cda7549..077afbb 100644
--- a/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
+++ b/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
@@ -840,6 +840,8 @@ SELECT elastic_net_train(
SELECT * FROM house_en;
SELECT * FROM house_en_summary;
+-- This test has been temporarily removed for GPDB5 alpha support
+
-- DROP TABLE if exists house_en, house_en_summary, house_en_cv;
-- SELECT elastic_net_train(
-- 'lin_housing_wi',
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0cdd644a/src/ports/postgres/modules/graph/sssp.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/sssp.py_in b/src/ports/postgres/modules/graph/sssp.py_in
index 4dbd1b1..a8339a8 100644
--- a/src/ports/postgres/modules/graph/sssp.py_in
+++ b/src/ports/postgres/modules/graph/sssp.py_in
@@ -316,10 +316,9 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
SELECT {grp_comma} id, {weight}, parent FROM {oldupdate};
"""
plpy.execute(sql.format(**locals()))
- sql = "DROP TABLE {out_table}"
- plpy.execute(sql.format(**locals()))
- sql = "ALTER TABLE {temp_table} RENAME TO {out_table}"
- plpy.execute(sql.format(**locals()))
+ plpy.execute("DROP TABLE {0}".format(out_table))
+ plpy.execute("ALTER TABLE {0} RENAME TO {1}".
+ format(temp_table,out_table))
sql = """ CREATE TABLE {temp_table} AS (
SELECT * FROM {out_table} LIMIT 0)
{distribution};"""
@@ -435,10 +434,9 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
WHERE {checkg_oo_sub}
);"""
plpy.execute(sql_del.format(**locals()))
- sql_del = "DROP TABLE {out_table}"
- plpy.execute(sql_del.format(**locals()))
- sql_del = "ALTER TABLE {temp_table} RENAME TO {out_table};"
- plpy.execute(sql_del.format(**locals()))
+ plpy.execute("DROP TABLE {0}".format(out_table))
+ plpy.execute("ALTER TABLE {0} RENAME TO {1}".
+ format(temp_table,out_table))
else:
sql_del = """ DELETE FROM {out_table}
USING {oldupdate} AS oldupdate
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0cdd644a/src/ports/postgres/modules/graph/test/pagerank.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/test/pagerank.sql_in b/src/ports/postgres/modules/graph/test/pagerank.sql_in
index 4c02df3..3ccfdd1 100644
--- a/src/ports/postgres/modules/graph/test/pagerank.sql_in
+++ b/src/ports/postgres/modules/graph/test/pagerank.sql_in
@@ -93,6 +93,10 @@ SELECT assert(relative_error(SUM(pagerank), 1) < 0.00001,
SELECT assert(relative_error(SUM(pagerank), 1) < 0.00001,
'PageRank: Scores do not sum up to 1 for group 2.'
) FROM pagerank_gr_out WHERE user_id=2;
+
+
+-- These tests have been temporarily removed for GPDB5 alpha support
+
-- SELECT assert(relative_error(__iterations__, 27) = 0,
-- 'PageRank: Incorrect iterations for group 1.'
-- ) FROM pagerank_gr_out_summary WHERE user_id=1;
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0cdd644a/src/ports/postgres/modules/pca/test/pca.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/pca/test/pca.sql_in b/src/ports/postgres/modules/pca/test/pca.sql_in
index fe397fc..f2e5192 100644
--- a/src/ports/postgres/modules/pca/test/pca.sql_in
+++ b/src/ports/postgres/modules/pca/test/pca.sql_in
@@ -119,31 +119,33 @@ select * from result_table_214712398172490837;
select * from result_table_214712398172490838;
-- Test dense data with grouping
--- DROP TABLE IF EXISTS mat;
--- CREATE TABLE mat (
--- id integer,
--- row_vec double precision[],
--- grp integer
--- );
-
--- COPY mat (id, row_vec, grp) FROM stdin delimiter '|';
--- 1|{396,840,353,446,318,886,15,584,159,383}|1
--- 2|{691,58,899,163,159,533,604,582,269,390}|1
--- 3|{293,742,298,75,404,857,941,662,846,2}|1
--- 4|{462,532,787,265,982,306,600,608,212,885}|1
--- 5|{304,151,337,387,643,753,603,531,459,652}|1
--- 6|{327,946,368,943,7,516,272,24,591,204}|1
--- 7|{877,59,260,302,891,498,710,286,864,675}|1
--- 8|{458,959,774,376,228,354,300,669,718,565}|2
--- 9|{824,390,818,844,180,943,424,520,65,913}|2
--- 10|{882,761,398,688,761,405,125,484,222,873}|2
--- 11|{528,1,860,18,814,242,314,965,935,809}|2
--- 12|{492,220,576,289,321,261,173,1,44,241}|2
--- 13|{415,701,221,503,67,393,479,218,219,916}|2
--- 14|{350,192,211,633,53,783,30,444,176,932}|2
--- 15|{909,472,871,695,930,455,398,893,693,838}|2
--- 16|{739,651,678,577,273,935,661,47,373,618}|2
--- \.
+DROP TABLE IF EXISTS mat;
+CREATE TABLE mat (
+ id integer,
+ row_vec double precision[],
+ grp integer
+);
+
+COPY mat (id, row_vec, grp) FROM stdin delimiter '|';
+1|{396,840,353,446,318,886,15,584,159,383}|1
+2|{691,58,899,163,159,533,604,582,269,390}|1
+3|{293,742,298,75,404,857,941,662,846,2}|1
+4|{462,532,787,265,982,306,600,608,212,885}|1
+5|{304,151,337,387,643,753,603,531,459,652}|1
+6|{327,946,368,943,7,516,272,24,591,204}|1
+7|{877,59,260,302,891,498,710,286,864,675}|1
+8|{458,959,774,376,228,354,300,669,718,565}|2
+9|{824,390,818,844,180,943,424,520,65,913}|2
+10|{882,761,398,688,761,405,125,484,222,873}|2
+11|{528,1,860,18,814,242,314,965,935,809}|2
+12|{492,220,576,289,321,261,173,1,44,241}|2
+13|{415,701,221,503,67,393,479,218,219,916}|2
+14|{350,192,211,633,53,783,30,444,176,932}|2
+15|{909,472,871,695,930,455,398,893,693,838}|2
+16|{739,651,678,577,273,935,661,47,373,618}|2
+\.
+
+-- This test has been temporarily removed for GPDB5 alpha support
-- Learn individaul PCA models based on grouping column (grp)
-- drop table if exists result_table_214712398172490837;
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0cdd644a/src/ports/postgres/modules/validation/test/cross_validation.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/validation/test/cross_validation.sql_in b/src/ports/postgres/modules/validation/test/cross_validation.sql_in
index 3548178..e50b5de 100644
--- a/src/ports/postgres/modules/validation/test/cross_validation.sql_in
+++ b/src/ports/postgres/modules/validation/test/cross_validation.sql_in
@@ -1365,6 +1365,9 @@ select check_cv0();
-- select check_cv_ridge();
+
+-- This test has been temporarily removed for GPDB5 alpha support
+
-- m4_ifdef(<!__HAWQ__!>, <!!>, <!
-- CREATE TABLE houses (
-- id SERIAL NOT NULL,
[32/34] incubator-madlib git commit: MADLIB-1098. Corrections for
MADlib naming consistency
Posted by ok...@apache.org.
MADLIB-1098. Corrections for MADlib naming consistency
Closes #130
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/ef4101e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/ef4101e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/ef4101e6
Branch: refs/heads/latest_release
Commit: ef4101e6ef460cd95a1e2ad4faf3c1a5bba9220c
Parents: d54be2b
Author: Roman Shaposhnik <rv...@apache.org>
Authored: Thu May 4 11:16:42 2017 -0700
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Thu May 4 11:58:18 2017 -0700
----------------------------------------------------------------------
deploy/CMakeLists.txt | 4 ++--
deploy/PGXN/ReadMe.txt | 15 ++++++++++++++-
deploy/PackageMaker/Welcome.html | 15 ++++++++++++++-
deploy/description.txt | 19 ++++++++++++++++---
4 files changed, 46 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/ef4101e6/deploy/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/deploy/CMakeLists.txt b/deploy/CMakeLists.txt
index 90e925f..0bd17a1 100644
--- a/deploy/CMakeLists.txt
+++ b/deploy/CMakeLists.txt
@@ -26,12 +26,12 @@ endif(PACKAGE_SUFFIX)
set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/description.txt")
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
- "Open-Source Library for Scalable in-Database Analytics")
+ "Apache MADlib (incubating) is an Open-Source Library for Scalable in-Database Analytics")
set(CPACK_PACKAGE_FILE_NAME
"madlib${_PACKAGE_SUFFIX}-${MADLIB_VERSION_STRING_NO_HYPHEN}-${CMAKE_SYSTEM_NAME}")
set(CPACK_PACKAGE_INSTALL_DIRECTORY "madlib")
set(CPACK_PACKAGE_NAME "MADlib${_PACKAGE_SUFFIX}")
-set(CPACK_PACKAGE_VENDOR "MADlib")
+set(CPACK_PACKAGE_VENDOR "Apache MADlib (incubating)")
set(CPACK_PACKAGE_VERSION ${MADLIB_VERSION_STRING_NO_HYPHEN})
set(CPACK_PACKAGE_VERSION_MAJOR ${MADLIB_VERSION_MAJOR})
set(CPACK_PACKAGE_VERSION_MINOR ${MADLIB_VERSION_MINOR})
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/ef4101e6/deploy/PGXN/ReadMe.txt
----------------------------------------------------------------------
diff --git a/deploy/PGXN/ReadMe.txt b/deploy/PGXN/ReadMe.txt
index 2982477..2478d6f 100644
--- a/deploy/PGXN/ReadMe.txt
+++ b/deploy/PGXN/ReadMe.txt
@@ -1,4 +1,4 @@
-MADlib Read Me
+Apache MADlib (incubating) Read Me
--------------
MADlib is an open-source library for scalable in-database analytics.
@@ -60,3 +60,16 @@ upgrading to the next major version:
individual optimizer parameters (max_iter, optimizer, tolerance). These
parameters have been replaced with a single optimizer parameter.
- All overloaded functions 'margins_logregr'.
+
+
+Apache MADlib is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/ef4101e6/deploy/PackageMaker/Welcome.html
----------------------------------------------------------------------
diff --git a/deploy/PackageMaker/Welcome.html b/deploy/PackageMaker/Welcome.html
index b900fe2..725cec4 100644
--- a/deploy/PackageMaker/Welcome.html
+++ b/deploy/PackageMaker/Welcome.html
@@ -5,8 +5,21 @@
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>Welcome to MADlib</title>
<body>
-<h2>Welcome to MADlib!</h2>
+<h2>Welcome to Apache MADlib (incubating)!</h2>
<p>This installer will guide you through the process of installing MADlib onto
your computer.</p>
+<p>
+Apache MADlib is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
+</p>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/ef4101e6/deploy/description.txt
----------------------------------------------------------------------
diff --git a/deploy/description.txt b/deploy/description.txt
index 77175ac..2df5724 100644
--- a/deploy/description.txt
+++ b/deploy/description.txt
@@ -1,6 +1,7 @@
-MADlib is an open-source library for scalable in-database analytics. It
-provides data-parallel implementations of mathematical, statistical and
-machine learning methods for structured and unstructured data.
+Apache MADlib (incubating) is an open-source library for scalable
+in-database analytics. It provides data-parallel implementations
+of mathematical, statistical and machine learning methods for
+structured and unstructured data.
The MADlib mission: to foster widespread development of scalable
analytic skills, by harnessing efforts from commercial practice,
@@ -8,3 +9,15 @@ academic research, and open-source development.
To more information, please see the MADlib wiki at
https://cwiki.apache.org/confluence/display/MADLIB
+
+Apache MADlib is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further
+review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other
+successful ASF projects.
+
+While incubation status is not necessarily a reflection of the
+completeness or stability of the code, it does indicate that the
+project has yet to be fully endorsed by the ASF.
[14/34] incubator-madlib git commit: Decision Tree: Multiple fixes -
pruning, tree_depth, viz
Posted by ok...@apache.org.
Decision Tree: Multiple fixes - pruning, tree_depth, viz
Commit includes following changes:
- Pruning is not performed when cp = 0 (default behavior)
- A particular bug is fixed: User input of max_depth starts from 0 and
the internal tree_depth starts from 1. This discrepancy was not taken into
account when tree train termination was checked leading to trees
containing only two leaf nodes on last level.
- Integer categorical variable is treated as ordered and hence is not
re-ordered. If the original ordering method is desired, then integer
column needs to be cast to TEXT.
- Visualization is improved: nodes with categorical feature splits only
provide the last value in the split, instead of the complete list.
This is consistent with the visualization in scikit-learn.
Closes #111
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/c82b9d0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/c82b9d0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/c82b9d0a
Branch: refs/heads/latest_release
Commit: c82b9d0ad844a03ef5f16111fa73e441a03850d5
Parents: 975d34e
Author: Rahul Iyer <ri...@apache.org>
Authored: Fri Apr 14 17:21:19 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Fri Apr 14 17:23:30 2017 -0700
----------------------------------------------------------------------
src/modules/recursive_partitioning/DT_impl.hpp | 88 ++++++--
.../recursive_partitioning/decision_tree.cpp | 4 +-
.../recursive_partitioning/feature_encoding.cpp | 6 +-
.../recursive_partitioning/decision_tree.py_in | 203 +++++++++++--------
.../recursive_partitioning/decision_tree.sql_in | 19 +-
.../recursive_partitioning/random_forest.py_in | 25 ++-
.../test/decision_tree.sql_in | 1 +
.../modules/validation/cross_validation.py_in | 1 -
8 files changed, 217 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c82b9d0a/src/modules/recursive_partitioning/DT_impl.hpp
----------------------------------------------------------------------
diff --git a/src/modules/recursive_partitioning/DT_impl.hpp b/src/modules/recursive_partitioning/DT_impl.hpp
index f622f94..64d2b88 100644
--- a/src/modules/recursive_partitioning/DT_impl.hpp
+++ b/src/modules/recursive_partitioning/DT_impl.hpp
@@ -574,7 +574,6 @@ DecisionTree<Container>::expand(const Accumulator &state,
feature_indices(i) = FINISHED_LEAF;
}
}
-
return training_finished;
}
// -------------------------------------------------------------------------
@@ -1025,11 +1024,10 @@ DecisionTree<Container>::shouldSplit(const ColumnVector &combined_stats,
uint64_t thresh_min_bucket = (min_bucket == 0) ? 1u : min_bucket;
uint64_t true_count = statCount(combined_stats.segment(0, stats_per_split));
uint64_t false_count = statCount(combined_stats.segment(stats_per_split, stats_per_split));
-
return ((true_count + false_count) >= min_split &&
true_count >= thresh_min_bucket &&
false_count >= thresh_min_bucket &&
- tree_depth <= max_depth);
+ tree_depth <= max_depth + 1);
}
// ------------------------------------------------------------------------
@@ -1107,11 +1105,32 @@ DecisionTree<Container>::displayLeafNode(
display_str << "\"" << id_prefix << id << "\" [label=\"" << predict_str.str();
if(verbose){
- display_str << "\\n samples = " << statCount(predictions.row(id)) << "\\n value = ";
+ display_str << "\\n impurity = "<< impurity(predictions.row(id))
+ << "\\n samples = " << statCount(predictions.row(id))
+ << "\\n value = ";
if (is_regression)
display_str << statPredict(predictions.row(id));
else{
- display_str << "[" << predictions.row(id).head(n_y_labels)<< "]";
+ display_str << "[";
+ // NUM_PER_LINE: inserting new lines at fixed intervals
+ // avoids a really long 'value' line
+ const uint16_t NUM_PER_LINE = 10;
+ // note: last element of predictions is 'statCount' and
+ // can be ignored
+ const Index pred_size = predictions.row(id).size() - 1;
+ for (Index i = 0; i < pred_size; i += NUM_PER_LINE){
+ uint16_t n_elem;
+ if (i + NUM_PER_LINE <= pred_size) {
+ // not overflowing the vector
+ n_elem = NUM_PER_LINE;
+ } else {
+ // less than NUM_PER_LINE left, avoid reading past the end
+ n_elem = pred_size - i;
+ }
+ display_str << predictions.row(id).segment(i, n_elem) << "\n";
+ }
+ display_str << "]";
+
}
}
display_str << "\",shape=box]" << ";";
@@ -1143,23 +1162,46 @@ DecisionTree<Container>::displayInternalNode(
label_str << escape_quotes(feature_name) << " <= " << feature_thresholds(id);
} else {
feature_name = get_text(cat_features_str, feature_indices(id));
- label_str << escape_quotes(feature_name) << " in "
- << getCatLabels(feature_indices(id),
- static_cast<Index>(0),
- static_cast<Index>(feature_thresholds(id)),
- cat_levels_text, cat_n_levels);
+ label_str << escape_quotes(feature_name) << " <= ";
+
+ // Text for all categoricals are stored in a flat array (cat_levels_text);
+ // find the appropriate index for this node
+ size_t to_skip = 0;
+ for (Index i=0; i < feature_indices(id); i++)
+ to_skip += cat_n_levels[i];
+ const size_t index = to_skip + feature_thresholds(id);
+ label_str << get_text(cat_levels_text, index);
}
std::stringstream display_str;
display_str << "\"" << id_prefix << id << "\" [label=\"" << label_str.str();
if(verbose){
+ display_str << "\\n impurity = "<< impurity(predictions.row(id)) << "\\n samples = " << statCount(predictions.row(id));
- display_str << "\\n impurity = "<< impurity(predictions.row(id)) << "\\n samples = " << statCount(predictions.row(id)) << "\\n value = ";
+ display_str << "\\n value = ";
if (is_regression)
display_str << statPredict(predictions.row(id));
else{
- display_str << "[" << predictions.row(id).head(n_y_labels)<< "]";
+ display_str << "[";
+ // NUM_PER_LINE: inserting new lines at fixed interval
+ // avoids really long 'value' line
+ const uint16_t NUM_PER_LINE = 10;
+ // note: last element of predictions is just 'statCount' and needs to
+ // be ignored
+ const Index pred_size = predictions.row(id).size() - 1;
+ for (Index i = 0; i < pred_size; i += NUM_PER_LINE){
+ uint16_t n_elem;
+ if (i + NUM_PER_LINE <= pred_size) {
+ // not overflowing the vector
+ n_elem = NUM_PER_LINE;
+ } else {
+ n_elem = pred_size - i;
+ }
+ display_str << predictions.row(id).segment(i, n_elem) << "\n";
+ }
+ display_str << "]";
}
+
std::stringstream predict_str;
if (static_cast<bool>(is_regression)){
predict_str << predict_response(id);
@@ -1360,20 +1402,24 @@ DecisionTree<Container>::getCatLabels(Index cat_index,
Index end_value,
ArrayHandle<text*> &cat_levels_text,
ArrayHandle<int> &cat_n_levels) {
+ Index MAX_LABELS = 5;
size_t to_skip = 0;
for (Index i=0; i < cat_index; i++) {
to_skip += cat_n_levels[i];
}
std::stringstream cat_levels;
- size_t start_index;
+ size_t index;
cat_levels << "{";
- for (start_index = to_skip + start_value;
- start_index < to_skip + end_value &&
- start_index < cat_levels_text.size();
- start_index++) {
- cat_levels << get_text(cat_levels_text, start_index) << ",";
+ for (index = to_skip + start_value;
+ index < to_skip + end_value && index < cat_levels_text.size();
+ index++) {
+ cat_levels << get_text(cat_levels_text, index) << ",";
+ if (index > to_skip + start_value + MAX_LABELS){
+ cat_levels << " ... ";
+ break;
+ }
}
- cat_levels << get_text(cat_levels_text, start_index) << "}";
+ cat_levels << get_text(cat_levels_text, index) << "}";
return cat_levels.str();
}
// -------------------------------------------------------------------------
@@ -1575,7 +1621,7 @@ TreeAccumulator<Container, DTree>::operator<<(const tuple_type& inTuple) {
uint16_t n_non_leaf_nodes = static_cast<uint16_t>(n_leaf_nodes - 1);
Index dt_search_index = dt.search(cat_features, con_features);
if (dt.feature_indices(dt_search_index) != dt.FINISHED_LEAF &&
- dt.feature_indices(dt_search_index) != dt.NODE_NON_EXISTING) {
+ dt.feature_indices(dt_search_index) != dt.NODE_NON_EXISTING) {
Index row_index = dt_search_index - n_non_leaf_nodes;
assert(row_index >= 0);
// add this row into the stats for the node
@@ -1651,7 +1697,7 @@ TreeAccumulator<Container, DTree>::operator<<(const surr_tuple_type& inTuple) {
double primary_val = is_primary_cat ? cat_features(primary_index) :
con_features(primary_index);
- // We only capture statistics for rows that:
+ // Only capture statistics for rows that:
// 1. lead to leaf nodes in the last layer. Surrogates for other nodes
// have already been trained.
// 2. have non-null values for the primary split.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c82b9d0a/src/modules/recursive_partitioning/decision_tree.cpp
----------------------------------------------------------------------
diff --git a/src/modules/recursive_partitioning/decision_tree.cpp b/src/modules/recursive_partitioning/decision_tree.cpp
index 7a8ec95..b298df8 100644
--- a/src/modules/recursive_partitioning/decision_tree.cpp
+++ b/src/modules/recursive_partitioning/decision_tree.cpp
@@ -237,7 +237,9 @@ dt_apply::run(AnyType & args){
}
AnyType output_tuple;
- output_tuple << dt.storage() << return_code << static_cast<uint16_t>(dt.tree_depth - 1);
+ output_tuple << dt.storage()
+ << return_code
+ << static_cast<uint16_t>(dt.tree_depth - 1);
return output_tuple;
} // apply function
// -------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c82b9d0a/src/modules/recursive_partitioning/feature_encoding.cpp
----------------------------------------------------------------------
diff --git a/src/modules/recursive_partitioning/feature_encoding.cpp b/src/modules/recursive_partitioning/feature_encoding.cpp
index 0b868df..20856e2 100644
--- a/src/modules/recursive_partitioning/feature_encoding.cpp
+++ b/src/modules/recursive_partitioning/feature_encoding.cpp
@@ -156,11 +156,11 @@ p_log2_p(const double &p) {
AnyType
dst_compute_entropy_final::run(AnyType &args){
MappedIntegerVector state = args[0].getAs<MappedIntegerVector>();
- double sum = static_cast<double>(state.sum());
- ColumnVector probilities = state.cast<double>() / sum;
+ double sum_of_dep_counts = static_cast<double>(state.sum());
+ ColumnVector probs = state.cast<double>() / sum_of_dep_counts;
// usage of unaryExpr with functor:
// http://eigen.tuxfamily.org/dox/classEigen_1_1MatrixBase.html#a23fc4bf97168dee2516f85edcfd4cfe7
- return -(probilities.unaryExpr(std::ptr_fun(p_log2_p))).sum();
+ return -(probs.unaryExpr(std::ptr_fun(p_log2_p))).sum();
}
// ------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c82b9d0a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
index 40f4b7e..fb18278 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
@@ -27,7 +27,6 @@ from utilities.validate_args import table_is_empty
from utilities.validate_args import columns_exist_in_table
from utilities.validate_args import is_var_valid
from utilities.validate_args import unquote_ident
-from utilities.validate_args import quote_ident
from utilities.utilities import _assert
from utilities.utilities import extract_keyvalue_params
from utilities.utilities import unique_string
@@ -137,14 +136,14 @@ def _get_features_to_use(schema_madlib, training_table_name,
# ------------------------------------------------------------
-def _get_col_value(input_dict, col_name):
+def _dict_get_quoted(input_dict, col_name):
"""Return value from dict where key could be quoted or unquoted name"""
return input_dict.get(
col_name, input_dict.get(unquote_ident(col_name)))
# -------------------------------------------------------------------------
-def _classify_features(all_feature_names_types, features):
+def _classify_features(feature_to_type, features):
""" Returns
1) an array of categorical features (all casted to string)
2) an array of continuous features
@@ -155,24 +154,28 @@ def _classify_features(all_feature_names_types, features):
text_types = ['text', 'varchar', 'character varying', 'char', 'character']
boolean_types = ['boolean']
cat_types = int_types + text_types + boolean_types
+ ordered_cat_types = int_types
+
+ cat_features = [c for c in features
+ if _dict_get_quoted(feature_to_type, c) in cat_types]
+ ordered_cat_features = [c for c in features if _dict_get_quoted(
+ feature_to_type, c) in ordered_cat_types]
- cat_features = [col for col in features
- if _get_col_value(all_feature_names_types, col) in cat_types]
cat_features_set = set(cat_features)
# continuous types - 'real' is cast to 'double precision' for uniformity
con_types = ['real', 'float8', 'double precision']
- con_features = [col for col in features
- if (not col in cat_features_set and
- _get_col_value(all_feature_names_types, col) in con_types)]
+ con_features = [c for c in features
+ if (c not in cat_features_set and
+ _dict_get_quoted(feature_to_type, c) in con_types)]
# In order to be able to form an array, all categorical variables
- # will be casted into TEXT type, but GPDB cannot cast a boolean
- # directly into a text. Thus, boolean type categorical variables
- # needs special treatment: cast them into integers before casting
+ # will be cast into TEXT type, but GPDB cannot cast a boolean
+ # directly into a text. Thus, boolean categorical variables
+ # need special treatment: cast them into integers before casting
# into text.
- boolean_cats = [col for col in features
- if _get_col_value(all_feature_names_types, col) in boolean_types]
- return cat_features, con_features, boolean_cats
+ boolean_cats = [c for c in features
+ if _dict_get_quoted(feature_to_type, c) in boolean_types]
+ return cat_features, ordered_cat_features, con_features, boolean_cats
# ------------------------------------------------------------
@@ -357,8 +360,9 @@ def _extract_pruning_params(pruning_params_str):
def _get_tree_states(schema_madlib, is_classification, split_criterion,
training_table_name, output_table_name, id_col_name,
dependent_variable, dep_is_bool,
- grouping_cols, cat_features, con_features,
- n_bins, boolean_cats, min_split, min_bucket, weights,
+ grouping_cols, cat_features, ordered_cat_features,
+ con_features, n_bins, boolean_cats,
+ min_split, min_bucket, weights,
max_depth, grp_key_to_cp, compute_cp_list=False,
max_n_surr=0, **kwargs):
"""
@@ -407,8 +411,9 @@ def _get_tree_states(schema_madlib, is_classification, split_criterion,
# 3) Find the splitting bins, one dict containing two arrays:
# categorical bins and continuous bins
bins = _get_bins(schema_madlib, training_table_name, cat_features,
- con_features, n_bins, dep_var_str, boolean_cats, n_rows,
- is_classification, dep_n_levels, filter_null)
+ ordered_cat_features, con_features, n_bins,
+ dep_var_str, boolean_cats,
+ n_rows, is_classification, dep_n_levels, filter_null)
# some features may be dropped if they have only one value
cat_features = bins['cat_features']
@@ -429,7 +434,8 @@ def _get_tree_states(schema_madlib, is_classification, split_criterion,
# result in excessive memory usage.
plpy.notice("Analyzing data to compute split boundaries for variables")
bins = _get_bins_grps(schema_madlib, training_table_name,
- cat_features, con_features, n_bins,
+ cat_features, ordered_cat_features,
+ con_features, n_bins,
dep_var_str,
boolean_cats, grouping_cols,
grouping_array_str, n_rows,
@@ -451,10 +457,14 @@ def _get_tree_states(schema_madlib, is_classification, split_criterion,
# cp values if cross-validation is required (cp_list = [] if not)
for tree in tree_states:
if 'cp' in tree:
- pruned_tree = _prune_and_cplist(schema_madlib, tree['tree_state'],
- tree['cp'], compute_cp_list=compute_cp_list)
+ pruned_tree = _prune_and_cplist(schema_madlib, tree,
+ tree['cp'],
+ compute_cp_list=compute_cp_list)
tree['tree_state'] = pruned_tree['tree_state']
- tree['pruned_depth'] = pruned_tree['pruned_depth']
+ if 'pruned_depth' in pruned_tree:
+ tree['pruned_depth'] = pruned_tree['pruned_depth']
+ else:
+ tree['pruned_depth'] = pruned_tree['tree_depth']
if 'cp_list' in pruned_tree:
tree['cp_list'] = pruned_tree['cp_list']
@@ -474,7 +484,7 @@ def get_grouping_array_str(table_name, grouping_cols, qualifier=None):
all_cols_types = dict(get_cols_and_types(table_name))
grouping_cols_list = [col.strip() for col in grouping_cols.split(',')]
- grouping_cols_and_types = [(col, _get_col_value(all_cols_types, col))
+ grouping_cols_and_types = [(col, _dict_get_quoted(all_cols_types, col))
for col in grouping_cols_list]
grouping_array_str = 'array_to_string(array[' + \
','.join("(case when " + col + " then 'True' else 'False' end)::text"
@@ -489,7 +499,8 @@ def get_grouping_array_str(table_name, grouping_cols, qualifier=None):
def _build_tree(schema_madlib, is_classification, split_criterion,
training_table_name, output_table_name, id_col_name,
dependent_variable, dep_is_bool,
- cat_features, boolean_cats, con_features, grouping_cols,
+ cat_features, ordered_cat_features,
+ boolean_cats, con_features, grouping_cols,
weights, max_depth, min_split, min_bucket, n_bins,
cp_table, max_n_surr=0, msg_level="warning", k=0, **kwargs):
@@ -556,13 +567,13 @@ def tree_train(schema_madlib, training_table_name, output_table_name,
"""
msg_level = "notice" if verbose_mode else "warning"
- #### Set default values for optional arguments
+ # Set default values for optional arguments
min_split = 20 if (min_split is None and min_bucket is None) else min_split
min_bucket = min_split // 3 if min_bucket is None else min_bucket
min_split = min_bucket * 3 if min_split is None else min_split
n_bins = 100 if n_bins is None else n_bins
split_criterion = 'gini' if not split_criterion else split_criterion
- plpy.notice("split_criterion:"+split_criterion)
+ plpy.notice("split_criterion:" + split_criterion)
pruning_param_dict = _extract_pruning_params(pruning_params)
cp = pruning_param_dict['cp']
n_folds = pruning_param_dict['n_folds']
@@ -591,8 +602,8 @@ def tree_train(schema_madlib, training_table_name, output_table_name,
# 2)
all_cols_types = dict(get_cols_and_types(training_table_name))
- cat_features, con_features, boolean_cats = _classify_features(
- all_cols_types, features)
+ cat_features, ordered_cat_features, con_features, boolean_cats = \
+ _classify_features(all_cols_types, features)
# get all rows
n_all_rows = plpy.execute("SELECT count(*) FROM {source_table}".
format(source_table=training_table_name)
@@ -630,7 +641,8 @@ def _create_output_tables(schema_madlib, training_table_name, output_table_name,
if not grouping_cols:
_create_result_table(schema_madlib, tree_states[0],
bins['cat_origin'], bins['cat_n'], cat_features,
- con_features, output_table_name, use_existing_tables, running_cv, k)
+ con_features, output_table_name,
+ use_existing_tables, running_cv, k)
else:
_create_grp_result_table(
schema_madlib, tree_states, bins, cat_features,
@@ -687,7 +699,8 @@ def _is_dep_categorical(training_table_name, dependent_variable):
# ------------------------------------------------------------
-def _get_bins(schema_madlib, training_table_name, cat_features,
+def _get_bins(schema_madlib, training_table_name,
+ cat_features, ordered_cat_features,
con_features, n_bins, dependent_variable, boolean_cats,
n_rows, is_classification, dep_n_levels, filter_null):
""" Compute the bins of all features
@@ -757,39 +770,37 @@ def _get_bins(schema_madlib, training_table_name, cat_features,
else:
con_splits = {'con_splits': ''} # no continuous features present
- # For categorical variables, different from the continuous
- # variable case, we scan the whole table to extract all the
+ # For categorical variables, scan the whole table to extract all the
# levels of the categorical variables, and at the same time
# sort the levels according to the entropy of the dependent
# variable.
# So this aggregate returns a composite type with two columns:
# col 1 is the array of ordered levels; col 2 is the number of
- # levels in col1.
+ # levels in col 1.
# TODO When n_bins is larger than 2^k - 1, where k is the number
# of levels of a given categrical feature, we can actually compute
# all combinations of levels and obtain a complete set of splits
- # instead of using sorting to get an approximate set of splits. This
- # can also be done in the following aggregate, but we may not need it
- # in the initial draft. Implement this optimization only if it is
- # necessary.
-
+ # instead of using sorting to get an approximate set of splits.
+ #
# We will use integer to represent levels of categorical variables.
# So before everything, we need to create a mapping from categorical
# variable levels to integers, and keep this mapping in the memory.
if len(cat_features) > 0:
if is_classification:
# For classifications
- order_fun = ("{schema_madlib}._dst_compute_entropy("
- "{dependent_variable}, {n})".
- format(schema_madlib=schema_madlib,
- dependent_variable=dependent_variable,
+ order_fun = ("{madlib}._dst_compute_entropy({dep}, {n})".
+ format(madlib=schema_madlib,
+ dep=dependent_variable,
n=dep_n_levels))
else:
# For regressions
- order_fun = \
- "AVG({dependent_variable})".format(dependent_variable=dependent_variable)
+ order_fun = "AVG({0})".format(dependent_variable)
+ # Note that 'sql_cat_levels' goes through two levels of formatting
+ # Try to obtain all the levels in one scan of the table.
+ # () are needed when casting the categorical variables because
+ # they can be expressions.
sql_cat_levels = """
SELECT
'{{col_name}}'::text AS colname,
@@ -801,7 +812,7 @@ def _get_bins(schema_madlib, training_table_name, cat_features,
FROM (
SELECT
({{col}})::text AS levels,
- {order_fun} AS dep_avg
+ {{order_fun}} AS dep_avg
FROM {training_table_name}
WHERE {filter_null}
AND {{col}} is not NULL
@@ -810,22 +821,23 @@ def _get_bins(schema_madlib, training_table_name, cat_features,
) s1
WHERE array_upper(levels, 1) > 1
""".format(training_table_name=training_table_name,
- order_fun=order_fun, filter_null=filter_null)
+ filter_null=filter_null)
- # Try to obtain all the levels in one scan of the table.
- # () are needed when casting the categorical variables because
- # they can be expressions.
- sql_all_cats = ' UNION ALL '.join(
- sql_cat_levels.format(col="(CASE WHEN " + col + " THEN 'True' ELSE 'False' END)"
- if col in boolean_cats else col,
- col_name=col) for col in cat_features)
+ sql_all_cats = ' UNION '.join(
+ sql_cat_levels.format(
+ col="(CASE WHEN " + col + " THEN 'True' ELSE 'False' END)"
+ if col in boolean_cats else col,
+ col_name=col,
+ order_fun=col if col in ordered_cat_features else order_fun)
+ for col in cat_features)
all_levels = plpy.execute(sql_all_cats)
if len(all_levels) != len(cat_features):
plpy.warning("Decision tree warning: Categorical columns with only "
"one value are dropped from the tree model.")
use_cat_features = [row['colname'] for row in all_levels]
- cat_features = [feature for feature in cat_features if feature in use_cat_features]
+ cat_features = [feature for feature in cat_features
+ if feature in use_cat_features]
col_to_row = dict((row['colname'], i) for i, row in enumerate(all_levels))
@@ -863,6 +875,8 @@ def _create_result_table(schema_madlib, tree_state,
header = "insert into " + output_table_name + " "
else:
header = "create table " + output_table_name + " as "
+ depth = (tree_state['pruned_depth'] if 'pruned_depth' in tree_state
+ else tree_state['tree_depth'])
if len(cat_features) > 0:
sql = header + """
SELECT
@@ -870,10 +884,11 @@ def _create_result_table(schema_madlib, tree_state,
$1 as tree,
$2 as cat_levels_in_text,
$3 as cat_n_levels,
- {tree_depth} as tree_depth
+ {depth} as tree_depth
{fold}
- """.format(tree_depth=tree_state['pruned_depth'],
- cp=tree_state['cp'], fold=fold)
+ """.format(depth=depth,
+ cp=tree_state['cp'],
+ fold=fold)
sql_plan = plpy.prepare(sql, ['{0}.bytea8'.format(schema_madlib),
'text[]', 'integer[]'])
plpy.execute(sql_plan, [tree_state['tree_state'], cat_origin, cat_n])
@@ -884,10 +899,11 @@ def _create_result_table(schema_madlib, tree_state,
$1 as tree,
NULL::text[] as cat_levels_in_text,
NULL::integer[] as cat_n_levels,
- {tree_depth} as tree_depth
+ {depth} as tree_depth
{fold}
- """.format(tree_depth=tree_state['pruned_depth'],
- cp=tree_state['cp'], fold=fold)
+ """.format(depth=depth,
+ cp=tree_state['cp'],
+ fold=fold)
sql_plan = plpy.prepare(sql, ['{0}.bytea8'.format(schema_madlib)])
plpy.execute(sql_plan, [tree_state['tree_state']])
@@ -895,7 +911,7 @@ def _create_result_table(schema_madlib, tree_state,
def _get_bins_grps(
- schema_madlib, training_table_name, cat_features,
+ schema_madlib, training_table_name, cat_features, ordered_cat_features,
con_features, n_bins, dependent_variable, boolean_cats,
grouping_cols, grouping_array_str, n_rows, is_classification,
dep_n_levels, filter_null):
@@ -1012,7 +1028,7 @@ def _get_bins_grps(
SELECT
{grouping_array_str} as grp_key,
({{col}})::text as levels,
- {order_fun} as dep_avg
+ {{order_fun}} as dep_avg
FROM {training_table_name}
WHERE {filter_null}
AND {{col}} is not NULL
@@ -1027,7 +1043,8 @@ def _get_bins_grps(
sql_cat_levels.format(
col=("(CASE WHEN " + col + " THEN 'True' ELSE 'False' END)"
if col in boolean_cats else col),
- col_name=col)
+ col_name=col,
+ order_fun=col if col in ordered_cat_features else order_fun)
for col in cat_features)
all_levels = list(plpy.execute(sql_all_cats))
@@ -1454,7 +1471,8 @@ def _create_grp_result_table(
plpy.execute(sql_plan, [
[t['grp_key'] for t in tree_states],
[t['tree_state'] for t in tree_states],
- [t['pruned_depth'] for t in tree_states],
+ [t['pruned_depth'] if 'pruned_depth' in t else t['tree_depth']
+ for t in tree_states],
[t['cp'] for t in tree_states],
bins['grp_key_cat'],
bins['cat_n'],
@@ -1469,9 +1487,10 @@ def _create_grp_result_table(
plpy.execute(sql_plan, [
[t['grp_key'] for t in tree_states],
[t['tree_state'] for t in tree_states],
- [t['pruned_depth'] for t in tree_states],
+ [t['pruned_depth'] if 'pruned_depth' in t else t['tree_depth']
+ for t in tree_states],
[t['cp'] for t in tree_states]
- ])
+ ])
# ------------------------------------------------------------
@@ -1511,7 +1530,7 @@ def _create_summary_table(
"$dep_list$")
else:
dep_list_str = "NULL"
- indep_type = ', '.join(_get_col_value(all_cols_types, col)
+ indep_type = ', '.join(_dict_get_quoted(all_cols_types, col)
for col in cat_features + con_features)
dep_type = _get_dep_type(training_table_name, dependent_variable)
cat_features_str = ','.join(cat_features)
@@ -1560,8 +1579,12 @@ def _create_summary_table(
# ------------------------------------------------------------
-def _get_filter_str(schema_madlib, cat_features, con_features, boolean_cats,
- dependent_variable, grouping_cols, max_n_surr=0):
+def _get_filter_str(schema_madlib, cat_features, con_features,
+ boolean_cats, dependent_variable,
+ grouping_cols, max_n_surr=0):
+ """ Return a 'WHERE' clause string that filters out all rows that contain a
+ NULL.
+ """
if grouping_cols:
g_filter = ' and '.join('(' + s.strip() + ') is not NULL' for s in grouping_cols.split(','))
else:
@@ -1592,7 +1615,7 @@ def _get_filter_str(schema_madlib, cat_features, con_features, boolean_cats,
def _validate_predict(schema_madlib, model, source, output, use_existing_tables):
- # validations for inputs
+ # validations for inputs
_assert(source and source.strip().lower() not in ('null', ''),
"Decision tree error: Invalid data table name: {0}".format(source))
_assert(table_exists(source),
@@ -1961,13 +1984,13 @@ SELECT * FROM tree_predict_out;
# ------------------------------------------------------------
-def _prune_and_cplist(schema_madlib, tree_state, cp, compute_cp_list=False):
+def _prune_and_cplist(schema_madlib, tree, cp, compute_cp_list=False):
""" Prune tree with given cost-complexity parameters
and return a list of cp values at which tree can be pruned
Args:
@param schema_madlib: str, MADlib schema name
- @param tree_state: schema_madlib.bytea8, tree to prune
+ @param tree: Tree data to prune
@param cp: float, cost-complexity parameter, all splits that have a
complexity lower than 'cp' will be pruned
@param compute_cp_list: bool, optionally return a list of cp values that
@@ -1982,20 +2005,22 @@ def _prune_and_cplist(schema_madlib, tree_state, cp, compute_cp_list=False):
cp_list: list of cp values at which tree can be pruned
(returned only if compute_cp_list=True)
"""
+ if cp <= 0 and not compute_cp_list:
+ return tree
sql = """
SELECT (pruned_tree).*
FROM (
- SELECT {schema_madlib}._prune_and_cplist(
+ SELECT {madlib}._prune_and_cplist(
$1,
({cp})::double precision,
({compute_cp_list})::boolean
) as pruned_tree
) q
- """.format(schema_madlib=schema_madlib, cp=cp,
+ """.format(madlib=schema_madlib, cp=cp,
compute_cp_list=bool(compute_cp_list))
- sql_plan = plpy.prepare(sql, ['{schema_madlib}.bytea8'.format(schema_madlib=schema_madlib)])
- pruned_tree = plpy.execute(sql_plan, [tree_state])[0]
+ sql_plan = plpy.prepare(sql, [schema_madlib + '.bytea8'])
+ pruned_tree = plpy.execute(sql_plan, [tree['tree_state']])[0]
return pruned_tree
# -------------------------------------------------------------------------
@@ -2003,7 +2028,7 @@ def _prune_and_cplist(schema_madlib, tree_state, cp, compute_cp_list=False):
def _xvalidate(schema_madlib, tree_states, training_table_name, output_table_name,
id_col_name, dependent_variable,
list_of_features, list_of_features_to_exclude,
- cat_features, con_features, boolean_cats,
+ cat_features, ordered_cat_features, boolean_cats, con_features,
split_criterion, grouping_cols, weights, max_depth,
min_split, min_bucket, n_bins, is_classification,
dep_is_bool, dep_n_levels, n_folds, n_rows,
@@ -2068,24 +2093,26 @@ def _xvalidate(schema_madlib, tree_states, training_table_name, output_table_nam
plpy.execute(plan, [grp_list, cp_list])
# 2) call CV function to actually cross-validate _build_tree
- # expect output table model_cv({grouping_cols), cp, avg, stddev)
+ # expects output table model_cv({grouping_cols), cp, avg, stddev)
model_cv = output_table_name + "_cv"
metric_function = "_tree_misclassified" if is_classification else "_tree_rmse"
pred_name = '"estimated_{0}"'.format(dependent_variable.strip(' "'))
grouping_str = 'NULL' if not grouping_cols else '"' + grouping_cols + '"'
cat_feature_str = _array_to_string(cat_features)
- con_feature_str = _array_to_string(con_features)
+ ordered_cat_feature_str = _array_to_string(ordered_cat_features)
boolean_cat_str = _array_to_string(boolean_cats)
+ con_feature_str = _array_to_string(con_features)
modeling_params = [str(i) for i in
(is_classification,
split_criterion, "%data%", "%model%", id_col_name,
dependent_variable, dep_is_bool,
- cat_feature_str, boolean_cat_str, con_feature_str,
+ cat_feature_str, ordered_cat_feature_str,
+ boolean_cat_str, con_feature_str,
grouping_str, weights, max_depth,
min_split, min_bucket, n_bins,
"%explore%", max_n_surr, msg_level)]
modeling_param_types = (["BOOLEAN"] + ["TEXT"] * 5 + ["BOOLEAN"] +
- ["VARCHAR[]"] * 3 + ["TEXT"] * 2 + ["INTEGER"] * 4 +
+ ["VARCHAR[]"] * 4 + ["TEXT"] * 2 + ["INTEGER"] * 4 +
["TEXT", "SMALLINT", "TEXT"])
cross_validation_grouping_w_params(
@@ -2141,7 +2168,6 @@ def _xvalidate(schema_madlib, tree_states, training_table_name, output_table_nam
grp_key_to_best_cp = dict((row['grp_key'], row['cp']) for row in validation_result)
- plpy.notice("Finished cross validation, final pruning ...")
# 4) update tree_states to have the best cp cross-validated
for tree in tree_states:
best_cp = grp_key_to_best_cp[tree['grp_key']]
@@ -2151,11 +2177,16 @@ def _xvalidate(schema_madlib, tree_states, training_table_name, output_table_nam
# giving the optimal pruned tree.
# This time we don't need the cp_list.
pruned_tree = _prune_and_cplist(schema_madlib,
- tree['tree_state'],
+ tree,
tree['cp'],
compute_cp_list=False)
tree['tree_state'] = pruned_tree['tree_state']
- tree['pruned_depth'] = pruned_tree['pruned_depth']
+ if 'pruned_depth' in pruned_tree:
+ tree['pruned_depth'] = pruned_tree['pruned_depth']
+ elif 'tree_depth' in pruned_tree:
+ tree['pruned_depth'] = pruned_tree['tree_depth']
+ else:
+ tree['pruned_depth'] = 0
plpy.execute("DROP TABLE {group_to_param_list_table}".format(**locals()))
# ------------------------------------------------------------
@@ -2184,9 +2215,9 @@ def _tree_train_using_bins(
max_n_surr=max_n_surr))[0]
plpy.notice("Starting tree building")
tree_depth = -1
- while not tree_state['finished']:
- tree_depth += 1
+ while tree_state['finished'] == 0:
# finished: 0 = running, 1 = finished training, 2 = terminated prematurely
+ tree_depth += 1
tree_state = _one_step(
schema_madlib, training_table_name,
cat_features, con_features, boolean_cats, bins,
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c82b9d0a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
index b5ed4a2..97e8471 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
@@ -225,11 +225,11 @@ tree_train(
double precision columns are considered continuous. The categorical variables
are not encoded and used as is for the training.
- There are no limitations on the number of levels in a categorical variable.
- It is, however, important to note that we don't test for every combination of
+ It is important to note that we don't test for every combination of
levels of a categorical variable when evaluating a split. We order the levels
- of the variable by the entropy of the variable in predicting the response. The
- split at each node is evaluated between these ordered levels.
+ of the non-integer categorical variable by the entropy of the variable in
+ predicting the response. The split at each node is evaluated between these
+ ordered levels. Integer categorical variables are ordered by their value.
</DD>
<DT>list_of_features_to_exclude</DT>
@@ -337,7 +337,10 @@ tree_train(
- Many of the parameters are designed to be similar to the popular R package 'rpart'.
An important distinction between rpart and the MADlib function is that
for both response and feature variables, MADlib considers integer values as
-categorical values, while rpart considers them as continuous.
+categorical values, while rpart considers them as continuous. To use integers as
+continuous, please cast them to double precision.
+- Integer values are ordered by value for computing the split boundaries. Please
+cast to TEXT if the entropy-based ordering method is desired.
- When using no surrogates (<em>max_surrogates</em>=0), all rows containing NULL values
for any of the features used for training will be ignored from training and prediction.
- When cross-validation is not used (<em>n_folds</em>=0), each tree output
@@ -349,8 +352,9 @@ to compute the optimal sub-tree. The optimal sub-tree and the 'cp' corresponding
to this optimal sub-tree is placed in the <em>output_table</em>, with the
columns named as <em>tree</em> and <em>pruning_cp</em> respectively.
- The main parameters that affect memory usage are: depth of tree, number
-of features, and number of values per feature. If you are hitting VMEM limits,
-consider reducing one or more of these parameters.
+of features, number of values per categorical feature, and number of bins for
+continuous features. If you are hitting VMEM limits, consider reducing one or
+more of these parameters.
@anchor predict
@par Prediction Function
@@ -986,6 +990,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__build_tree(
dependent_variable TEXT,
dep_is_bool BOOLEAN,
cat_features VARCHAR[],
+ ordered_cat_features VARCHAR[],
boolean_cats VARCHAR[],
con_features VARCHAR[],
grouping_cols TEXT,
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c82b9d0a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
index 0eb5985..930d916 100644
--- a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
@@ -34,7 +34,7 @@ from decision_tree import _is_dep_categorical
from decision_tree import _get_n_and_deplist
from decision_tree import _classify_features
from decision_tree import _get_filter_str
-from decision_tree import _get_col_value
+from decision_tree import _dict_get_quoted
from decision_tree import _get_display_header
from decision_tree import get_feature_str
# ------------------------------------------------------------
@@ -329,8 +329,8 @@ def forest_train(
"is more than the actual number of features.")
all_cols_types = dict(get_cols_and_types(training_table_name))
- cat_features, con_features, boolean_cats = _classify_features(
- all_cols_types, features)
+ cat_features, ordered_cat_features, con_features, boolean_cats = \
+ _classify_features(all_cols_types, features)
filter_null = _get_filter_str(schema_madlib, cat_features,
con_features, boolean_cats,
@@ -382,7 +382,8 @@ def forest_train(
# bins, and continuous bins
num_groups = 1
bins = _get_bins(schema_madlib, training_table_name,
- cat_features, con_features, num_bins, dep,
+ cat_features, ordered_cat_features,
+ con_features, num_bins, dep,
boolean_cats, n_rows, is_classification,
dep_n_levels, filter_null)
# some features may be dropped because they have only one value
@@ -390,13 +391,14 @@ def forest_train(
bins['grp_key_cat'] = ['']
else:
grouping_cols_list = [col.strip() for col in grouping_cols.split(',')]
- grouping_cols_and_types = [(col, _get_col_value(all_cols_types, col))
+ grouping_cols_and_types = [(col, _dict_get_quoted(all_cols_types, col))
for col in grouping_cols_list]
- grouping_array_str = "array_to_string(array[" + \
- ','.join("(case when " + col + " then 'True' else 'False' end)::text"
+ grouping_array_str = (
+ "array_to_string(array[" +
+ ','.join("(case when " + col + " then 'True' else 'False' end)::text"
if col_type == 'boolean' else '(' + col + ')::text'
- for col, col_type in grouping_cols_and_types) + \
- "], ',')"
+ for col, col_type in grouping_cols_and_types) +
+ "], ',')")
grouping_cols_str = ('' if grouping_cols is None
else grouping_cols + ",")
sql_grp_key_to_grp_cols = """
@@ -417,7 +419,8 @@ def forest_train(
""".format(**locals()))[0]['count']
plpy.notice("Analyzing data to compute split boundaries for variables")
bins = _get_bins_grps(schema_madlib, training_table_name,
- cat_features, con_features, num_bins, dep,
+ cat_features, ordered_cat_features,
+ con_features, num_bins, dep,
boolean_cats, grouping_cols,
grouping_array_str, n_rows,
is_classification, dep_n_levels, filter_null)
@@ -1198,7 +1201,7 @@ def _create_summary_table(**kwargs):
else:
kwargs['dep_list_str'] = "NULL"
- kwargs['indep_type'] = ', '.join(_get_col_value(kwargs['all_cols_types'], col)
+ kwargs['indep_type'] = ', '.join(_dict_get_quoted(kwargs['all_cols_types'], col)
for col in kwargs['cat_features'] + kwargs['con_features'])
kwargs['dep_type'] = _get_dep_type(kwargs['training_table_name'],
kwargs['dependent_variable'])
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c82b9d0a/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
index 8f9168c..1863b64 100644
--- a/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
@@ -370,6 +370,7 @@ select __build_tree(
FALSE,
ARRAY['"OUTLOOK"']::text[],
'{}',
+ '{}',
'{humidity}',
'class',
'1',
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c82b9d0a/src/ports/postgres/modules/validation/cross_validation.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/validation/cross_validation.py_in b/src/ports/postgres/modules/validation/cross_validation.py_in
index 7b39c90..f157ffa 100644
--- a/src/ports/postgres/modules/validation/cross_validation.py_in
+++ b/src/ports/postgres/modules/validation/cross_validation.py_in
@@ -402,7 +402,6 @@ def cross_validation_grouping_w_params(
with MinWarning("warning"):
if not data_cols:
data_cols = get_cols(data_tbl, schema_madlib)
-
n_rows = _validate_cv_args(**locals())
explore_type_str = "::INTEGER"
[04/34] incubator-madlib git commit: Build: Update docker files to
enable Jenkins
Posted by ok...@apache.org.
Build: Update docker files to enable Jenkins
This commit updates the Jenkins script to enable automatic creation of
builds.
Add JUnit export format for Jenkins
Add sys import
Duration in seconds not millisec
Add cast to float
Update jenkins build script
Comments in build script
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/5984d827
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/5984d827
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/5984d827
Branch: refs/heads/latest_release
Commit: 5984d8277556c6271b516f7726640e923152c90f
Parents: 7be6893
Author: Rahul Iyer <ri...@apache.org>
Authored: Fri Mar 3 17:31:00 2017 -0800
Committer: Rahul Iyer <ri...@apache.org>
Committed: Wed Mar 15 14:27:01 2017 -0700
----------------------------------------------------------------------
.../docker/base/Dockerfile_postgres_9_6_Jenkins | 2 +-
tool/jenkins/jenkins_build.sh | 81 +++++++++++++----
tool/jenkins/junit_export.py | 96 ++++++++++++++++++++
3 files changed, 160 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/5984d827/tool/docker/base/Dockerfile_postgres_9_6_Jenkins
----------------------------------------------------------------------
diff --git a/tool/docker/base/Dockerfile_postgres_9_6_Jenkins b/tool/docker/base/Dockerfile_postgres_9_6_Jenkins
index 137842e..fe6a95a 100644
--- a/tool/docker/base/Dockerfile_postgres_9_6_Jenkins
+++ b/tool/docker/base/Dockerfile_postgres_9_6_Jenkins
@@ -27,7 +27,7 @@ RUN apt-get update && apt-get install -y wget \
libssl-dev \
libboost-all-dev \
m4 \
- wget
+ rpm
### Build custom CMake with SSQL support
RUN wget https://cmake.org/files/v3.6/cmake-3.6.1.tar.gz && \
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/5984d827/tool/jenkins/jenkins_build.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_build.sh b/tool/jenkins/jenkins_build.sh
index 72ada55..f03bc78 100644
--- a/tool/jenkins/jenkins_build.sh
+++ b/tool/jenkins/jenkins_build.sh
@@ -1,4 +1,4 @@
-#
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,28 +16,73 @@
# specific language governing permissions and limitations
# under the License.
-#!/bin/sh
-
-#####################################################################################
-### If this bash script is executed as a stand-alone file, assuming this
-### is not part of the MADlib source code, then the following two commands
-### may have to be used:
-# git clone https://github.com/apache/incubator-madlib.git
-# pushd incubator-madlib
#####################################################################################
+workdir=`pwd`
+user_name=`whoami`
+echo "Build by user $user_name in directory $workdir"
+echo "-------------------------------"
+echo "ls -la"
+ls -la
+echo "-------------------------------"
+echo "rm -rf build"
+rm -rf build
+echo "-------------------------------"
+echo "rm -rf logs"
+rm -rf logs
+echo "mkdir logs"
+mkdir logs
+echo "-------------------------------"
+echo "docker kill madlib"
+docker kill madlib
+echo "docker rm madlib"
+docker rm madlib
+
+echo "Creating docker container"
# Pull down the base docker images
-docker pull madlib/postgres_9_6:jenkins
-# Assuming git clone of incubator-madlib has been done, launch a container with the volume mounted
-docker run -d --name madlib -v incubator-madlib:/incubator-madlib madlib/postgres_9.6:jenkins
+docker pull madlib/postgres_9.6:jenkins
+# Launch docker container with volume mounted from workdir
+echo "-------------------------------"
+cat <<EOF
+docker run -d --name madlib -v "${workdir}/incubator-madlib":/incubator-madlib madlib/postgres_9.6:jenkins | tee logs/docker_setup.log
+EOF
+docker run -d --name madlib -v "${workdir}/incubator-madlib":/incubator-madlib madlib/postgres_9.6:jenkins | tee logs/docker_setup.log
+echo "-------------------------------"
+
## This sleep is required since it takes a couple of seconds for the docker
## container to come up, which is required by the docker exec command that follows.
sleep 5
-# cmake, make and make install MADlib
-docker exec madlib bash -c 'mkdir /incubator-madlib/build ; cd /incubator-madlib/build ; cmake .. ; make ; make install'
+
+echo "---------- Building package -----------"
+# cmake, make, make install, and make package
+cat <<EOF
+docker exec madlib bash -c 'rm -rf /build; mkdir /build; cd /build; cmake ../incubator-madlib; make clean; make; make install; make package' | tee $workdir/logs/madlib_compile.log
+EOF
+docker exec madlib bash -c 'rm -rf /build; mkdir /build; cd /build; cmake ../incubator-madlib; make clean; make; make install; make package' | tee $workdir/logs/madlib_compile.log
+
+echo "---------- Installing and running install-check --------------------"
# Install MADlib and run install check
-docker exec -it madlib /incubator-madlib/build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install
-docker exec -it madlib /incubator-madlib/build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check
+cat <<EOF
+docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install | tee $workdir/logs/madlib_install.log
+docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check | tee $workdir/logs/madlib_install_check.log
+EOF
+docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install | tee $workdir/logs/madlib_install.log
+docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check | tee $workdir/logs/madlib_install_check.log
-docker kill madlib
-docker rm madlib
+echo "--------- Copying packages -----------------"
+echo "docker cp madlib:build $workdir"
+docker cp madlib:build $workdir
+
+echo "-------------------------------"
+echo "ls -la"
+ls -la
+echo "-------------------------------"
+echo "ls -la build"
+ls -la build/
+echo "-------------------------------"
+
+# convert install-check test results to junit format for reporting
+cat <<EOF
+python incubator-madlib/tool/jenkins/junit_export.py $workdir/logs/madlib_install_check.log $workdir/logs/madlib_install_check.xml
+EOF
+python incubator-madlib/tool/jenkins/junit_export.py $workdir/logs/madlib_install_check.log $workdir/logs/madlib_install_check.xml
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/5984d827/tool/jenkins/junit_export.py
----------------------------------------------------------------------
diff --git a/tool/jenkins/junit_export.py b/tool/jenkins/junit_export.py
new file mode 100644
index 0000000..ce30320
--- /dev/null
+++ b/tool/jenkins/junit_export.py
@@ -0,0 +1,96 @@
+#!/usr/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import re
+import sys
+from collections import namedtuple
+
+""" Convert install-check results into a standardized JUnit XML format
+
+Example of JUnit output:
+
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite tests="3">
+ <testcase classname="foo1" name="ASuccessfulTest"/>
+ <testcase classname="foo2" name="AnotherSuccessfulTest"/>
+ <testcase classname="foo3" name="AFailingTest">
+ <failure type="NotEnoughFoo"> details about failure </failure>
+ </testcase>
+</testsuite>
+"""
+
+
+TestResult = namedtuple("TestResult", 'name suite status duration')
+
+
+def _test_result_factory(install_check_log):
+ """
+ Args:
+ @param install_check_log: File name containing results from install-check
+
+ Returns:
+ Next result of type test_result
+ """
+ with open(install_check_log, 'r') as ic_log:
+ for line in ic_log:
+ m = re.match(r"^TEST CASE RESULT\|Module: (.*)\|(.*)\|(.*)\|Time: ([0-9]+)(.*)", line)
+ if m:
+ yield TestResult(name=m.group(2), suite=m.group(1),
+ status=m.group(3), duration=m.group(4))
+# ----------------------------------------------------------------------
+
+
+def _add_header(out_log, n_tests):
+ header = ['<?xml version="1.0" encoding="UTF-8"?>',
+ '<testsuite tests="{0}">'.format(n_tests), '']
+ out_log.write('\n'.join(header))
+
+
+def _add_footer(out_log):
+ header = ['', '</testsuite>']
+ out_log.write('\n'.join(header))
+
+
+def _add_test_case(out_log, test_results):
+ for res in test_results:
+ try:
+ # convert duration from milliseconds to seconds
+ duration = float(res.duration)/1000
+ except TypeError:
+ duration = 0.0
+ output = ['<testcase classname="{t.suite}" name="{t.name}" '
+ 'status="{t.status}" time="{d}">'.
+ format(t=res, d=duration)]
+ output.append('</testcase>')
+ out_log.write('\n'.join(output))
+
+
+def main(install_check_log, test_output_log):
+
+ # need number of test results - so have to create the iterable
+ all_test_results = [i for i in _test_result_factory(install_check_log)]
+
+ with open(test_output_log, 'w') as out_log:
+ _add_header(out_log, len(all_test_results))
+ _add_test_case(out_log, all_test_results)
+ _add_footer(out_log)
+
+
+if __name__ == "__main__":
+ main(sys.argv[1], sys.argv[2])
[17/34] incubator-madlib git commit: Doc: Update documentation
Posted by ok...@apache.org.
Doc: Update documentation
Minor corrections and changes in elastic net, decision tree, random
forest, pivot.
Closes #118
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/206e1269
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/206e1269
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/206e1269
Branch: refs/heads/latest_release
Commit: 206e1269edfef4589639021d27fa5072b9297339
Parents: 3eec0a8
Author: Frank McQuillan <fm...@pivotal.io>
Authored: Tue Apr 18 13:07:05 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Tue Apr 18 17:28:05 2017 -0700
----------------------------------------------------------------------
.../modules/elastic_net/elastic_net.sql_in | 3 +-
.../recursive_partitioning/decision_tree.sql_in | 17 ++++---
.../recursive_partitioning/random_forest.sql_in | 49 +++++++++++++-------
.../postgres/modules/utilities/pivot.sql_in | 4 +-
4 files changed, 48 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/206e1269/src/ports/postgres/modules/elastic_net/elastic_net.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/elastic_net.sql_in b/src/ports/postgres/modules/elastic_net/elastic_net.sql_in
index 2949fc5..f3a8980 100644
--- a/src/ports/postgres/modules/elastic_net/elastic_net.sql_in
+++ b/src/ports/postgres/modules/elastic_net/elastic_net.sql_in
@@ -735,7 +735,8 @@ The two queries above will result in same residuals:
-# Reuse the houses table above.
Here we use 3-fold cross validation with 3 automatically generated
lambda values and 3 specified alpha values. (This can take some time to
-run since elastic net is effectively being called 27 times.)
+run since elastic net is effectively being called 27 times for
+these combinations, then a 28th time for the whole dataset.)
<pre class="example">
DROP TABLE IF EXISTS houses_en3, houses_en3_summary, houses_en3_cv;
SELECT madlib.elastic_net_train( 'houses', -- Source table
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/206e1269/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
index ef671fc..7251a9c 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
@@ -259,7 +259,9 @@ tree_train(
<DT>max_depth (optional)</DT>
<DD>INTEGER, default: 7. Maximum depth of any node of the final tree,
- with the root node counted as depth 0.</DD>
+ with the root node counted as depth 0. A deeper tree can
+ lead to better prediction but will also result in
+ longer processing time and higher memory usage.</DD>
<DT>min_split (optional)</DT>
<DD>INTEGER, default: 20. Minimum number of observations that must exist
@@ -276,7 +278,7 @@ tree_train(
discrete quantiles to compute split boundaries. This global parameter
is used to compute the resolution of splits for continuous features.
Higher number of bins will lead to better prediction,
- but will also result in longer processing.</DD>
+ but will also result in longer processing time and higher memory usage.</DD>
<DT>pruning_params (optional)</DT>
<DD>TEXT. Comma-separated string of key-value pairs giving
@@ -351,9 +353,10 @@ provided <em>cp</em> and explore all possible sub-trees (up to a single-node tre
to compute the optimal sub-tree. The optimal sub-tree and the 'cp' corresponding
to this optimal sub-tree is placed in the <em>output_table</em>, with the
columns named as <em>tree</em> and <em>pruning_cp</em> respectively.
-- The main parameters that affect memory usage are: depth of tree, number
-of features, number of values per categorical feature, and number of bins for
-continuous features. If you are hitting VMEM limits, consider reducing one or
+- The main parameters that affect memory usage are: depth of
+tree (‘max_depth’), number of features, number of values per
+categorical feature, and number of bins for continuous features (‘num_splits’).
+If you are hitting memory limits, consider reducing one or
more of these parameters.
@anchor predict
@@ -922,7 +925,9 @@ File decision_tree.sql_in documenting the training function
* each observation.
* @param max_depth OPTIONAL (Default = 7). Set the maximum depth
* of any node of the final tree, with the root node counted
- * as depth 0.
+ * as depth 0. A deeper tree can lead to better prediction
+ * but will also result in longer processing time and higher
+ * memory usage.
* @param min_split OPTIONAL (Default = 20). Minimum number of
* observations that must exist in a node for a split to
* be attempted.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/206e1269/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
index 3d4da87..f263cf9 100644
--- a/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
@@ -34,6 +34,9 @@ constructed using bootstrapped samples from the input data. The results of these
models are then combined to yield a single prediction, which, at the
expense of some loss in interpretation, have been found to be highly accurate.
+Please also refer to the decision tree user documentation for
+information relevant to the implementation of random forests in MADlib.
+
@anchor train
@par Training Function
Random Forest training function has the following format:
@@ -276,8 +279,16 @@ forest_train(training_table_name,
<DT>list_of_features</DT>
<DD>text. Comma-separated string of column names to use as predictors. Can
also be a '*' implying all columns are to be used as predictors (except the
- ones included in the next argument). Boolean, integer and text columns are
- considered categorical columns.</DD>
+ ones included in the next argument). The types of the features can be mixed
+ where boolean, integer, and text columns are considered categorical and
+ double precision columns are considered continuous. The categorical variables
+ are not encoded and used as is for the training.
+
+ It is important to note that we don't test for every combination of
+ levels of a categorical variable when evaluating a split. We order the levels
+ of the non-integer categorical variable by the entropy of the variable in
+ predicting the response. The split at each node is evaluated between these
+ ordered levels. Integer categorical variables are ordered by their value.</DD>
<DT>list_of_features_to_exclude</DT>
<DD>text. Comma-separated string of column names to exclude from the predictors
@@ -317,9 +328,11 @@ forest_train(training_table_name,
the default value of 1 is sufficient to compute the importance.
</DD>
- <DT>max_depth (optional)</DT>
- <DD>integer, default: 10. Maximum depth of any node of a tree,
- with the root node counted as depth 0.</DD>
+ <DT>max_tree_depth (optional)</DT>
+ <DD>integer, default: 7. Maximum depth of any node of a tree,
+ with the root node counted as depth 0. A deeper tree can
+ lead to better prediction but will also result in
+ longer processing time and higher memory usage.</DD>
<DT>min_split (optional)</DT>
<DD>integer, default: 20. Minimum number of observations that must exist
@@ -331,11 +344,11 @@ forest_train(training_table_name,
set to min_bucket*3 or min_bucket to min_split/3, as appropriate.</DD>
<DT>num_splits (optional)</DT>
- <DD>integer, default: 100. Continuous-valued features are binned into
+ <DD>integer, default: 20. Continuous-valued features are binned into
discrete quantiles to compute split boundaries. This global parameter
is used to compute the resolution of splits for continuous features.
Higher number of bins will lead to better prediction,
- but will also result in higher processing time.</DD>
+ but will also result in longer processing time and higher memory usage.</DD>
<DT>surrogate_params (optional)</DT>
<DD>text, Comma-separated string of key-value pairs controlling the behavior
@@ -358,10 +371,11 @@ forest_train(training_table_name,
is close to 0 may result in trees with only the root node.
This allows users to experiment with the function in a speedy fashion.</DD>
</DL>
- @note The main parameters that affect memory usage are: depth of tree, number
- of features, and number of values per feature (controlled by num_splits).
- If you are hitting VMEM limits,
- consider reducing one or more of these parameters.
+ @note The main parameters that affect memory usage are: depth of
+ tree (‘max_tree_depth’), number of features, number of values per
+ categorical feature, and number of bins for continuous features (‘num_splits’).
+ If you are hitting memory limits, consider reducing one or
+ more of these parameters.
@anchor predict
@par Prediction Function
@@ -858,7 +872,7 @@ File random_forest.sql_in documenting the training function
* @param num_random_features OPTIONAL (Default = sqrt(n) for classification,
* n/3 for regression) Number of features to randomly select at
* each split.
- * @param max_tree_depth OPTIONAL (Default = 10). Set the maximum depth
+ * @param max_tree_depth OPTIONAL (Default = 7). Set the maximum depth
* of any node of the final tree, with the root node counted
* as depth 0.
* @param min_split OPTIONAL (Default = 20). Minimum number of
@@ -869,12 +883,13 @@ File random_forest.sql_in documenting the training function
* one of minbucket or minsplit is specified, minsplit
* is set to minbucket*3 or minbucket to minsplit/3, as
* appropriate.
- * @param num_splits optional (default = 100) number of bins to use
- * during binning. continuous-valued features are binned
+ * @param num_splits optional (default = 20) number of bins to use
+ * during binning. Continuous-valued features are binned
* into discrete bins (per the quartile values) to compute
- * split bound- aries. this global parameter is used to
- * compute the resolution of the bins. higher number of
- * bins will lead to higher processing time.
+ * split boundaries. This global parameter is used to
+ * compute the resolution of the bins. Higher number of
+ * bins will lead to higher processing time and more
+ * memory usage.
* @param verbose optional (default = false) prints status
* information on the splits performed and any other
* information useful for debugging.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/206e1269/src/ports/postgres/modules/utilities/pivot.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/pivot.sql_in b/src/ports/postgres/modules/utilities/pivot.sql_in
index 7cdfbe0..4d239de 100644
--- a/src/ports/postgres/modules/utilities/pivot.sql_in
+++ b/src/ports/postgres/modules/utilities/pivot.sql_in
@@ -142,6 +142,8 @@ pivot(
If the total number of output columns exceeds this limit, then make this
parameter either 'array' (to combine the output columns into an array) or
'svec' (to cast the array output to <em>'madlib.svec'</em> type).
+ If you have an 'aggregate_func' that has an array return type,
+ it cannot be combined with 'output_type'='array' or 'svec'.
A dictionary will be created (<em>output_col_dictionary=TRUE</em>)
when 'output_type' is 'array' or 'svec' to define each index into the array.
@@ -364,7 +366,7 @@ val_avg_piv_30_piv2_300 |
-# Use multiple pivot columns (same as above) with an array output:
<pre class="example">
-DROP TABLE IF EXISTS pivout;
+DROP TABLE IF EXISTS pivout, pivout_dictionary;
SELECT madlib.pivot('pivset_ext', 'pivout', 'id', 'piv, piv2', 'val',
NULL, NULL, FALSE, FALSE, 'array');
\\x off
[11/34] incubator-madlib git commit: Bugfix: Install check for
elastic net fails on gpdb5
Posted by ok...@apache.org.
Bugfix: Install check for elastic net fails on gpdb5
MADLIB-1088
- Fixes concurrent delete issue with GPDB 5 on install check. This
also fixes the elastic net failure on cross validation, whose root
cause was drop and create table within the same query string.
- Fixes elastic net failure with IGD optimizer. Accessing warmup
lambdas was incorrect.
Closes #114
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/f3b906e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/f3b906e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/f3b906e9
Branch: refs/heads/latest_release
Commit: f3b906e9d11c8e4894b3a0a8aa8d29cfb481025e
Parents: bb209bb
Author: Nandish Jayaram <nj...@apache.org>
Authored: Tue Apr 11 15:45:33 2017 -0700
Committer: Nandish Jayaram <nj...@apache.org>
Committed: Thu Apr 13 09:32:38 2017 -0700
----------------------------------------------------------------------
src/ports/postgres/modules/elastic_net/elastic_net.py_in | 2 --
.../postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in | 4 ++--
2 files changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/f3b906e9/src/ports/postgres/modules/elastic_net/elastic_net.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/elastic_net.py_in b/src/ports/postgres/modules/elastic_net/elastic_net.py_in
index 762c473..fb46ba2 100644
--- a/src/ports/postgres/modules/elastic_net/elastic_net.py_in
+++ b/src/ports/postgres/modules/elastic_net/elastic_net.py_in
@@ -799,7 +799,6 @@ def elastic_net_predict_all(schema_madlib, tbl_model, tbl_new_source,
if grouping_col and grouping_col != 'NULL':
qstr = """
- DROP TABLE IF EXISTS {tbl_predict};
CREATE TABLE {tbl_predict} AS
SELECT
{elastic_net_predict_id},
@@ -819,7 +818,6 @@ def elastic_net_predict_all(schema_madlib, tbl_model, tbl_new_source,
""".format(**locals())
else:
qstr = """
- DROP TABLE IF EXISTS {tbl_predict};
CREATE TABLE {tbl_predict} AS
SELECT
{elastic_net_predict_id},
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/f3b906e9/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in b/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in
index 5685af2..091aefb 100644
--- a/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in
+++ b/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in
@@ -251,7 +251,7 @@ def _elastic_net_igd_train_compute(schema_madlib, func_step_aggregate,
'col_grp_state': unique_string(desp='col_grp_state'),
'col_grp_key': unique_string(desp='col_grp_key'),
'col_n_tuples': unique_string(desp='col_n_tuples'),
- 'lambda_count': 0,
+ 'lambda_count': 1,
'state_type': "double precision[]",
'rel_source': tbl_used,
'grouping_str': grouping_str,
@@ -282,7 +282,7 @@ def _elastic_net_igd_train_compute(schema_madlib, func_step_aggregate,
dimension=args["dimension"],
stepsize=args["stepsize"],
lambda_name=args["warmup_lambdas"],
- warmup_lambda_value=args.get('warmup_lambdas')[args["lambda_count"]],
+ warmup_lambda_value=args.get('warmup_lambdas')[args["lambda_count"]-1],
alpha=args["alpha"],
row_num=args["row_num"],
xmean_val=args["xmean_val"],
[34/34] incubator-madlib git commit: Packagemaker: Use .txt for cpack
license
Posted by ok...@apache.org.
Packagemaker: Use .txt for cpack license
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/8e2778a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/8e2778a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/8e2778a3
Branch: refs/heads/latest_release
Commit: 8e2778a3921aa99f009962756881ce4bea5eee16
Parents: 6c2f8e3
Author: Rahul Iyer <ri...@apache.org>
Authored: Thu May 4 16:13:21 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Thu May 4 16:13:21 2017 -0700
----------------------------------------------------------------------
deploy/PackageMaker/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8e2778a3/deploy/PackageMaker/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/deploy/PackageMaker/CMakeLists.txt b/deploy/PackageMaker/CMakeLists.txt
index 89617b9..81a6dcc 100644
--- a/deploy/PackageMaker/CMakeLists.txt
+++ b/deploy/PackageMaker/CMakeLists.txt
@@ -11,7 +11,7 @@
set(CPACK_RESOURCE_FILE_README
"${CPACK_PACKAGE_DESCRIPTION_FILE}" PARENT_SCOPE)
set(CPACK_RESOURCE_FILE_LICENSE
- "${CMAKE_SOURCE_DIR}/LICENSE" PARENT_SCOPE)
+ "${CMAKE_SOURCE_DIR}/licenses/MADlib.txt" PARENT_SCOPE)
set(CPACK_RESOURCE_FILE_WELCOME
"${CMAKE_CURRENT_SOURCE_DIR}/Welcome.html" PARENT_SCOPE)
set(CPACK_OSX_PACKAGE_VERSION "10.5" PARENT_SCOPE)
[06/34] incubator-madlib git commit: Multiple: Updates version number
and removes empty graph file.
Posted by ok...@apache.org.
Multiple: Updates version number and removes empty graph file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/aaf5f821
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/aaf5f821
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/aaf5f821
Branch: refs/heads/latest_release
Commit: aaf5f82149e7201ae70cd5b7ef0f69c40ef4d439
Parents: 63f59e2
Author: Orhan Kislal <ok...@pivotal.io>
Authored: Wed Mar 22 17:03:46 2017 -0700
Committer: Orhan Kislal <ok...@pivotal.io>
Committed: Wed Mar 22 17:03:46 2017 -0700
----------------------------------------------------------------------
src/config/Version.yml | 2 +-
src/ports/postgres/modules/graph/graph_utils.sql_in | 0
2 files changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/aaf5f821/src/config/Version.yml
----------------------------------------------------------------------
diff --git a/src/config/Version.yml b/src/config/Version.yml
index 6176098..097842c 100644
--- a/src/config/Version.yml
+++ b/src/config/Version.yml
@@ -1 +1 @@
-version: 1.10.0
+version: 1.11-dev
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/aaf5f821/src/ports/postgres/modules/graph/graph_utils.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/graph_utils.sql_in b/src/ports/postgres/modules/graph/graph_utils.sql_in
deleted file mode 100644
index e69de29..0000000
[10/34] incubator-madlib git commit: Pivot: Add support for array
output
Posted by ok...@apache.org.
Pivot: Add support for array output
JIRA: MADLIB-1066
When total pivoted columns exceed the Postgresql limit (250 - 1600
depending on the type of columns), an array output becomes
essential. This commit adds support to get each pivoted set of columns
(all columns related to a particular value-aggregate combination) as an
array. There is also support for getting the output as madlib.svec.
Closes #108
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/bb209bbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/bb209bbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/bb209bbb
Branch: refs/heads/latest_release
Commit: bb209bbb6e081a2838a3f698947529358792e47f
Parents: 6b466ea
Author: Rahul Iyer <ri...@apache.org>
Authored: Fri Mar 31 11:15:58 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Fri Mar 31 11:15:58 2017 -0700
----------------------------------------------------------------------
.../postgres/modules/utilities/pivot.py_in | 238 +++++++++-----
.../postgres/modules/utilities/pivot.sql_in | 327 ++++++++-----------
.../modules/utilities/test/pivot.sql_in | 74 +++--
.../postgres/modules/utilities/utilities.py_in | 13 +-
.../modules/utilities/validate_args.py_in | 4 +-
5 files changed, 354 insertions(+), 302 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/bb209bbb/src/ports/postgres/modules/utilities/pivot.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/pivot.py_in b/src/ports/postgres/modules/utilities/pivot.py_in
index 7e342f1..6d0ebae 100644
--- a/src/ports/postgres/modules/utilities/pivot.py_in
+++ b/src/ports/postgres/modules/utilities/pivot.py_in
@@ -42,6 +42,7 @@ from validate_args import columns_exist_in_table
from validate_args import table_is_empty
from validate_args import _get_table_schema_names
from validate_args import get_first_schema
+from validate_args import get_expr_type
m4_changequote(`<!', `!>')
@@ -49,14 +50,14 @@ m4_changequote(`<!', `!>')
def pivot(schema_madlib, source_table, out_table, index, pivot_cols,
pivot_values, aggregate_func=None, fill_value=None, keep_null=False,
- output_col_dictionary=False, **kwargs):
+ output_col_dictionary=False, output_type=None, **kwargs):
"""
Helper function that can be used to pivot tables
Args:
@param source_table The original data table
@param out_table The output table that contains the dummy
variable columns
- @param index The index columns to group by the records by
+ @param index The index columns to group the records by
@param pivot_cols The columns to pivot the table
@param pivot_values The value columns to be summarized in the
pivoted table
@@ -80,6 +81,16 @@ def pivot(schema_madlib, source_table, out_table, index, pivot_cols,
FROM pivset GROUP BY id ORDER BY id)
"""
+ def _fill_value_wrapper(sel_str):
+ """ Wrap a given SQL SELECT statement with COALESCE using a given fill value.
+
+ No-op if the fill value is not provided
+ """
+ if fill_value is not None:
+ return " COALESCE({0}, {1}) ".format(sel_str, fill_value)
+ else:
+ return sel_str
+
with MinWarning('warning'):
# If there are more than 1000 columns for the output table, we give a
@@ -93,6 +104,25 @@ def pivot(schema_madlib, source_table, out_table, index, pivot_cols,
indices = split_quoted_delimited_str(index)
pcols = split_quoted_delimited_str(pivot_cols)
pvals = split_quoted_delimited_str(pivot_values)
+
+ # output type for specific supported types
+ output_type = 'column' if not output_type else output_type.lower()
+ all_output_types = sorted(['array', 'column', 'svec'])
+ try:
+ # allow user to specify a prefix substring of
+ # supported output types. This works because the supported
+ # output types have unique prefixes.
+ output_type = next(s for s in all_output_types
+ if s.startswith(output_type))
+ except StopIteration:
+ # next() returns a StopIteration if no element found
+ plpy.error("Encoding categorical: Output type should be one of {0}".
+ format(','.join(all_output_types)))
+
+ is_array_output = output_type in ('array', 'svec')
+ # always build dictionary table if output is array
+ output_col_dictionary = True if is_array_output else output_col_dictionary
+
validate_pivot_coding(source_table, out_table, indices, pcols, pvals)
# Strip the end quotes for building output columns (this can only be
@@ -104,6 +134,8 @@ def pivot(schema_madlib, source_table, out_table, index, pivot_cols,
# value column.
agg_dict = parse_aggregates(pvals, aggregate_func)
+ validate_output_types(source_table, agg_dict, is_array_output)
+
# Find the distinct values of pivot_cols
array_agg_str = ', '.join("array_agg(DISTINCT {pcol}) AS {pcol}_values".
format(pcol=pcol) for pcol in pcols)
@@ -143,7 +175,7 @@ def pivot(schema_madlib, source_table, out_table, index, pivot_cols,
for pcol in pcols])))
# Check the max possible length of a output column name
- # If it is over 63 (psql upper limit) create table lookup
+ # If it is over 63 (postgresql upper limit) create dictionary lookup
for pval in pvals:
agg_func = agg_dict[pval]
# Length calculation: value column length + aggregate length +
@@ -159,122 +191,157 @@ def pivot(schema_madlib, source_table, out_table, index, pivot_cols,
format(**locals()))
output_col_dictionary = True
- # Create the output dictionary if needed
+ # Types of pivot columns are needed for building the right columns
+ # in the dictionary table and to decide if a pivot column value needs to
+ # be quoted during comparison (will be quoted if it's a text column)
+ types_str = ', '.join("pg_typeof(\"{pcol}\") as {pcol}".
+ format(pcol=p) for p in pcols)
+ pcol_types = plpy.execute("SELECT {0} FROM {1} LIMIT 1".
+ format(types_str, source_table))[0]
if output_col_dictionary:
out_dict = out_table + "_dictionary"
_assert(not table_exists(out_dict),
"Pivot: Output dictionary table already exists!")
-
- # Collect the types for pivot columns
- types_str = ','.join("pg_typeof(\"{pcol}\") as {pcol}_type".
- format(pcol=pcol) for pcol in pcols)
- pcol_types = plpy.execute("SELECT {0} FROM {1} LIMIT 1".
- format(types_str, source_table))
-
# Create the empty dictionary table
- dict_str = ', '.join(" {pcol} {pcol_type} ".
- format(pcol=pcol, pcol_type=pcol_types[0][pcol+"_type"])
- for pcol in pcols)
+ pcol_names_types = ', '.join(" {pcol} {pcol_type} ".
+ format(pcol=pcol,
+ pcol_type=pcol_types[pcol])
+ for pcol in pcols)
plpy.execute("""
CREATE TABLE {out_dict} (
- __pivot_cid__ VARCHAR, pval VARCHAR,
- agg VARCHAR, {dict_str}, col_name VARCHAR)
- """.format(**locals()))
-
- # The holder for rows to insert into output dictionary
- insert_str = []
+ __pivot_cid__ VARCHAR,
+ pval VARCHAR,
+ agg VARCHAR,
+ {pcol_names_types},
+ col_name VARCHAR)
+ """.format(out_dict=out_dict, pcol_names_types=pcol_names_types))
+
+ # List of rows to insert into output dictionary
+ dict_insert_str = []
# Counter for the new output column names
- dict_counter = 0
+ dict_counter = 1
- pivot_str_sel_list = []
- pivot_str_from_list = []
- # Prepare the wrapper for fill value
- if fill_value is not None:
- fill_str_begin = " COALESCE("
- fill_str_end = ", " + fill_value + " ) "
- else:
- fill_str_begin, fill_str_end = "", ""
+ pivot_sel_list = []
+ pivot_from_list = []
for pval in pvals:
agg_func = agg_dict[pval]
for agg in agg_func:
+
+ # is using array_output, create a new array for each pval-agg combo
+ if is_array_output:
+ # we store information in the dictionary table for each
+ # index in the array. 'index_counter' is the current index
+ # being updated (resets for each new array)
+ index_counter = 1
+
+ sub_pivot_sel_list = []
for comb in pivot_comb:
pivot_col_condition = []
- pivot_col_name = ["\"{pval}_{agg}".format(pval=pval, agg=agg)]
+ # note column name starts with double quotes
+ pivot_col_name = ['{pval}_{agg}'.format(pval=pval, agg=agg)]
if output_col_dictionary:
# Prepare the entry for the dictionary
- insert_str.append("(\'__p_{dict_counter}__\', \'{pval}\', "
- "\'{agg}\' ".format(dict_counter=dict_counter,
- pval=pval, agg=agg))
+ if not is_array_output:
+ index_name = ("__p_{dict_counter}__".
+ format(dict_counter=dict_counter))
+ else:
+ # for arrays, index_name is just the index into each array
+ index_name = str(index_counter)
+ index_counter += 1
+ dict_insert_str.append(
+ "(\'{index_name}\', \'{pval}\', \'{agg}\' ".
+ format(index_name=index_name, pval=pval, agg=agg))
# For every pivot column in a given combination
for counter, pcol in enumerate(pcols):
+ if comb[counter] is None:
+ quoted_pcol_value = "NULL"
+ elif pcol_types[pcol] in ("text", "varchar", "character varying"):
+ quoted_pcol_value = "'" + comb[counter] + "'"
+ else:
+ quoted_pcol_value = comb[counter]
+
# If we encounter a NULL value that means it is not filtered
# because of keep_null. Use "IS NULL" for comparison
if comb[counter] is None:
pivot_col_condition.append(" \"{0}\" IS NULL".format(pcol))
pivot_col_name.append("_{0}_null".format(pcol))
else:
- pivot_col_condition.append(" \"{0}\" = '{1}'".
- format(pcol, comb[counter]))
+ pivot_col_condition.append(" \"{0}\" = {1}".
+ format(pcol, quoted_pcol_value))
pivot_col_name.append("_{0}_{1}".format(pcol, comb[counter]))
- # Collect pcol values for the dict
if output_col_dictionary:
- insert_str.append("{0}".format(
- comb[counter] if comb[counter] is not None else "NULL"))
- pivot_col_name.append("\"")
+ dict_insert_str.append("{0}".format(quoted_pcol_value))
if output_col_dictionary:
- # Store the whole string in case some user wants it
- insert_str.append("\'{column_name}\')".
- format(column_name=''.join(pivot_col_name)))
- pivot_col_name = ["__p_"+str(dict_counter)+"__"]
+ # Store the whole string as additional info
+ dict_insert_str.append("'{0}')".format(''.join(pivot_col_name)))
+ pivot_col_name = ["__p_" + str(dict_counter) + "__"]
dict_counter += 1
+
# Collecting the whole sql query
# Please refer to the earlier comment for a sample output
-
# Build the pivot column with NULL values in tuples that don't
# satisfy that column's condition
- pivot_str_from = ("(CASE WHEN {condition} THEN {pval} END) "
- "AS {pivot_col_name}".
- format(pval=pval,
- condition=' AND '.join(pivot_col_condition),
- pivot_col_name=''.join(pivot_col_name)))
- pivot_str_from_list.append(pivot_str_from)
- # Aggregate over each pivot column, while filtering all NULL values
- # created by previous query.
- pivot_str_sel = ("{fill_str_begin}"
- " {agg} ({pivot_col_name}) "
- " FILTER (WHERE {pivot_col_name} IS NOT NULL) "
- "{fill_str_end} AS {pivot_col_name}".
- format(agg=agg, fill_str_begin=fill_str_begin,
- fill_str_end=fill_str_end,
- pivot_col_name=''.join(pivot_col_name)))
- pivot_str_sel_list.append(pivot_str_sel)
+ p_name = '"{0}"'.format(''.join(pivot_col_name))
+ pivot_str_from = (
+ "(CASE WHEN {condition} THEN {pval} END) AS {p_name}".
+ format(pval=pval,
+ condition=' AND '.join(pivot_col_condition),
+ p_name=p_name))
+ pivot_from_list.append(pivot_str_from)
+
+ # Aggregate over each pivot column, while filtering all NULL
+ # values created by previous query.
+ sub_pivot_str_sel = _fill_value_wrapper(
+ "{agg}({p_name}) "
+ " FILTER (WHERE {p_name} IS NOT NULL)".
+ format(agg=agg, p_name=p_name))
+ if not is_array_output:
+ # keep spaces around the 'AS'
+ sub_pivot_str_sel += " AS " + p_name
+ sub_pivot_sel_list.append(sub_pivot_str_sel)
+
+ if sub_pivot_sel_list:
+ if is_array_output:
+ if output_type is 'svec':
+ cast_str = '::FLOAT8[]::{0}.svec'.format(schema_madlib)
+ else:
+ cast_str = '::FLOAT8[]'
+ pivot_sel_list.append(
+ 'ARRAY[{all_pivot_sel}]{cast_str} AS "{pval}_{agg}"'.
+ format(all_pivot_sel=', '.join(sub_pivot_sel_list),
+ cast_str=cast_str,
+ pval=pval,
+ agg=agg))
+ else:
+ pivot_sel_list += sub_pivot_sel_list
try:
plpy.execute("""
CREATE TABLE {out_table} AS
SELECT {index},
- {pivot_str_sel_list}
+ {all_pivot_sel_str}
FROM (
SELECT {index},
- {pivot_str_from_list}
+ {all_pivot_from_str}
FROM {source_table}
) x
GROUP BY {index}
""".format(out_table=out_table,
index=index,
source_table=source_table,
- pivot_str_from_list=', '.join(pivot_str_from_list),
- pivot_str_sel_list=', '.join(pivot_str_sel_list)))
+ all_pivot_from_str=', '.join(pivot_from_list),
+ all_pivot_sel_str=', '.join(pivot_sel_list)
+ ))
if output_col_dictionary:
plpy.execute("INSERT INTO {out_dict} VALUES {insert_sql}".
format(out_dict=out_dict,
- insert_sql=', '.join(insert_str)))
+ insert_sql=', '.join(dict_insert_str)))
except plpy.SPIError:
# Warn user if the number of columns is over the limit
with MinWarning("warning"):
@@ -314,16 +381,16 @@ def parse_aggregates(pvals, aggregate_func):
5) A partial mapping (eg. 'val2=sum'): Use the default ('avg') for the
missing value columns
"""
- param_types = dict.fromkeys(pvals, list)
+ param_types = dict.fromkeys(pvals, tuple)
agg_dict = extract_keyvalue_params(aggregate_func, param_types)
if not agg_dict:
- agg_list = split_quoted_delimited_str(aggregate_func)
- agg_dict = dict.fromkeys(pvals, (agg_list if agg_list else ['avg']))
+ agg_list = tuple(split_quoted_delimited_str(aggregate_func))
+ agg_dict = dict.fromkeys(pvals, (agg_list if agg_list else ('avg', )))
else:
for pval in pvals:
if pval not in agg_dict:
- agg_dict[pval] = ['avg']
+ agg_dict[pval] = ('avg', )
return agg_dict
# ------------------------------------------------------------------------------
@@ -364,6 +431,26 @@ def validate_pivot_coding(source_table, out_table, indices, pivs, vals):
# ------------------------------------------------------------------------------
+def validate_output_types(source_table, agg_dict, is_array_output):
+ """
+ Args:
+ @param source_table: str, Name of table containing data
+ @param agg_dict: dict, Key-value pair containing aggregates applied for each val column
+ @param is_array_output: bool, Is the pivot output columnar (False) or array (True)
+
+ Returns:
+ None
+ """
+ for val, func_iterable in agg_dict.items():
+ for func in func_iterable:
+ func_call_str = '{0}({1})'.format(func, val)
+ _assert(not ('[]' in get_expr_type(func_call_str, source_table) and
+ is_array_output),
+ "Pivot: Aggregate {0} with an array return type cannot be "
+ "combined with output_type='array' or 'svec'".format(func))
+# ----------------------------------------------------------------------
+
+
def pivot_help(schema_madlib, message, **kwargs):
"""
Help function for pivot
@@ -401,14 +488,19 @@ For more details on function usage:
-- of the output pivot table
pivot_cols, -- Comma-separated columns that will form the
-- columns of the output pivot table
- pivot_values -- Comma-separated columns that contain the values
+ pivot_values, -- Comma-separated columns that contain the values
-- to be summarized in the output pivot table
- fill_value -- If specified, determines how to fill NULL values
+ fill_value, -- If specified, determines how to fill NULL values
-- resulting from pivot operation
- keep_null -- The flag for determining how to handle NULL
+ keep_null, -- The flag for determining how to handle NULL
-- values in pivot columns
- output_col_dictionary -- The flag for enabling the creation of the
+ output_col_dictionary, -- The flag for enabling the creation of the
-- output dictionary for shorter column names
+ output_type -- This parameter controls the output format
+ -- of the pivoted variables.
+ -- If 'column', a column is created for each pivot
+ -- If 'array', an array is created combining all pivots
+ -- If 'svec', the array is cast to madlib.svec
);
-----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/bb209bbb/src/ports/postgres/modules/utilities/pivot.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/pivot.sql_in b/src/ports/postgres/modules/utilities/pivot.sql_in
index cb2c223..7cdfbe0 100644
--- a/src/ports/postgres/modules/utilities/pivot.sql_in
+++ b/src/ports/postgres/modules/utilities/pivot.sql_in
@@ -59,7 +59,8 @@ pivot(
aggregate_func,
fill_value,
keep_null,
- output_col_dictionary
+ output_col_dictionary,
+ output_type
)
</pre>
\b Arguments
@@ -67,6 +68,7 @@ pivot(
<dt>source_table</dt>
<dd>VARCHAR. Name of the source table (or view) containing data to
pivot.</dd>
+
<dt>output_table</dt>
<dd>VARCHAR. Name of output table that contains the pivoted data.
The output table contains all the columns present in
@@ -81,18 +83,21 @@ pivot(
- aggregate function
- name of the pivot column <em>'pivot_cols'</em>
- values in the pivot column
-
</dd>
+
<dt>index </dt>
<dd>VARCHAR. Comma-separated columns that will form the index of the output
pivot table. By index we mean the values to group by; these are the rows
in the output pivot table.</dd>
+
<dt>pivot_cols </dt>
<dd>VARCHAR. Comma-separated columns that will form the columns of the
output pivot table.</dd>
+
<dt>pivot_values </dt>
<dd>VARCHAR. Comma-separated columns that contain the values to be
summarized in the output pivot table.</dd>
+
<dt>aggregate_func (optional)</dt>
<dd>VARCHAR. default: 'AVG'. A comma-separated list of aggregates to be
applied to values. These can be PostgreSQL built-in aggregates [1] or UDAs. It is
@@ -113,10 +118,12 @@ pivot(
values resulting from pivot operation. This is a global parameter (not
applied per aggregate) and is applied post-aggregation to the output
table.</dd>
+
<dt>keep_null (optional)</dt>
<dd>BOOLEAN. default: FALSE. If TRUE, then pivot columns are created
corresponding to NULL categories. If FALSE, then no pivot columns will be
created for NULL categories.</dd>
+
<dt>output_col_dictionary (optional)</dt>
<dd>BOOLEAN. default: FALSE. This parameter is used to handle
auto-generated column names that exceed the PostgreSQL limit of 63 bytes
@@ -127,6 +134,19 @@ pivot(
a dictionary output file will be created and a message given to the user.
</dd>
+ <dt>output_type (optional)</dt>
+ <dd>VARCHAR. default: 'column'. This parameter controls the output format
+ of the pivoted variables. If 'column', a column is created for each pivot
+ variable. PostgreSQL limits the number of columns in a table
+ (250 - 1600 depending on column types).
+ If the total number of output columns exceeds this limit, then make this
+ parameter either 'array' (to combine the output columns into an array) or
+ 'svec' (to cast the array output to <em>'madlib.svec'</em> type).
+
+ A dictionary will be created (<em>output_col_dictionary=TRUE</em>)
+ when 'output_type' is 'array' or 'svec' to define each index into the array.
+ </dd>
+
</dl>
@anchor notes
@@ -138,8 +158,8 @@ allowed so NULLs are ignored.
- It is not allowed to set the fill_value parameter without setting the
aggregate_func parameter due to possible ambiguity. Set
aggregate_func to NULL for the default behavior and use fill_value as desired.
-Please note that full_value must be of the same type as the output of the
-aggregate_func (or capable of being cast to the same type by PostgreSQL),
+Please note that full_value must be of the same type as the output of the
+aggregate_func (or capable of being cast to the same type by PostgreSQL),
or else an error will result.
- It is not allowed to set the output_col_dictionary parameter without setting
the keep_null parameter due to possible ambiguity. Set
@@ -303,13 +323,9 @@ SELECT * FROM pivout ORDER BY id,id2;
| 0 | 8 | |
</pre>
--# Turn on the extended view for readability:
+-# Use multiple pivot columns with columnar output:
<pre class="example">
\\x on
-</pre>
-
--# Use multiple pivot columns:
-<pre class="example">
DROP TABLE IF EXISTS pivout;
SELECT madlib.pivot('pivset_ext', 'pivout', 'id', 'piv, piv2', 'val');
SELECT * FROM pivout ORDER BY id;
@@ -346,10 +362,47 @@ val_avg_piv_30_piv2_300 |
...
</pre>
+-# Use multiple pivot columns (same as above) with an array output:
+<pre class="example">
+DROP TABLE IF EXISTS pivout;
+SELECT madlib.pivot('pivset_ext', 'pivout', 'id', 'piv, piv2', 'val',
+ NULL, NULL, FALSE, FALSE, 'array');
+\\x off
+SELECT * FROM pivout ORDER BY id;
+</pre>
+<pre class="result">
+ id | val_avg
+--------+------------------------------------------------------------
+ 0 | {1,2,NULL,NULL,NULL,3,NULL,NULL,NULL,NULL,NULL,NULL}
+ 1 | {NULL,NULL,7,NULL,NULL,4,NULL,NULL,NULL,NULL,5.5,NULL}
+ [NULL] | {NULL,NULL,NULL,8,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL}
+</pre>
+<pre class="example">
+-- Use the dictionary to understand what each index of an array corresponds to
+SELECT * FROM pivout_dictionary;
+</pre>
+<pre class="result">
+ __pivot_cid__ | pval | agg | piv | piv2 | col_name
+---------------+------+-----+-----+------+---------------------------
+ 1 | val | avg | 10 | 0 | "val_avg_piv_10_piv2_0"
+ 2 | val | avg | 10 | 100 | "val_avg_piv_10_piv2_100"
+ 3 | val | avg | 10 | 200 | "val_avg_piv_10_piv2_200"
+ 4 | val | avg | 10 | 300 | "val_avg_piv_10_piv2_300"
+ 5 | val | avg | 20 | 0 | "val_avg_piv_20_piv2_0"
+ 6 | val | avg | 20 | 100 | "val_avg_piv_20_piv2_100"
+ 7 | val | avg | 20 | 200 | "val_avg_piv_20_piv2_200"
+ 8 | val | avg | 20 | 300 | "val_avg_piv_20_piv2_300"
+ 9 | val | avg | 30 | 0 | "val_avg_piv_30_piv2_0"
+ 10 | val | avg | 30 | 100 | "val_avg_piv_30_piv2_100"
+ 11 | val | avg | 30 | 200 | "val_avg_piv_30_piv2_200"
+ 12 | val | avg | 30 | 300 | "val_avg_piv_30_piv2_300"
+</pre>
+
-# Use multiple value columns:
<pre class="example">
DROP TABLE IF EXISTS pivout;
SELECT madlib.pivot('pivset_ext', 'pivout', 'id', 'piv', 'val, val2');
+\\x on
SELECT * FROM pivout ORDER BY id;
</pre>
<pre class="result">
@@ -377,6 +430,7 @@ val2_avg_piv_30 | 15.5
<pre class="example">
DROP TABLE IF EXISTS pivout;
SELECT madlib.pivot('pivset_ext', 'pivout', 'id', 'piv', 'val', 'avg, sum');
+\\x on
SELECT * FROM pivout ORDER BY id;
</pre>
<pre class="result">
@@ -404,6 +458,7 @@ val_sum_piv_30 | 11
DROP TABLE IF EXISTS pivout;
SELECT madlib.pivot('pivset_ext', 'pivout', 'id', 'piv', 'val, val2',
'val=avg, val2=sum');
+\\x on
SELECT * FROM pivout ORDER BY id;
</pre>
<pre class="result">
@@ -431,6 +486,7 @@ val2_sum_piv_30 | 31
DROP TABLE IF EXISTS pivout;
SELECT madlib.pivot('pivset_ext', 'pivout', 'id', 'piv', 'val, val2',
'val=avg, val2=[avg,sum]');
+\\x on
SELECT * FROM pivout ORDER BY id;
</pre>
<pre class="result">
@@ -464,6 +520,7 @@ val2_sum_piv_30 | 31
DROP TABLE IF EXISTS pivout;
SELECT madlib.pivot('pivset_ext', 'pivout', 'id, id2', 'piv, piv2', 'val, val2',
'val=avg, val2=[avg,sum]', '111', True);
+\\x on
SELECT * FROM pivout ORDER BY id,id2;
</pre>
<pre class="result">
@@ -492,32 +549,7 @@ val2_avg_piv_null_piv2_200 | 111
val2_avg_piv_null_piv2_300 | 111
val2_avg_piv_10_piv2_0 | 11
val2_avg_piv_10_piv2_100 | 111
-val2_avg_piv_10_piv2_200 | 111
-val2_avg_piv_10_piv2_300 | 111
-val2_avg_piv_20_piv2_0 | 111
-val2_avg_piv_20_piv2_100 | 111
-val2_avg_piv_20_piv2_200 | 111
-val2_avg_piv_20_piv2_300 | 111
-val2_avg_piv_30_piv2_0 | 111
-val2_avg_piv_30_piv2_100 | 111
-val2_avg_piv_30_piv2_200 | 111
-val2_avg_piv_30_piv2_300 | 111
-val2_sum_piv_null_piv2_0 | 111
-val2_sum_piv_null_piv2_100 | 111
-val2_sum_piv_null_piv2_200 | 111
-val2_sum_piv_null_piv2_300 | 111
-val2_sum_piv_10_piv2_0 | 11
-val2_sum_piv_10_piv2_100 | 111
-val2_sum_piv_10_piv2_200 | 111
-val2_sum_piv_10_piv2_300 | 111
-val2_sum_piv_20_piv2_0 | 111
-val2_sum_piv_20_piv2_100 | 111
-val2_sum_piv_20_piv2_200 | 111
-val2_sum_piv_20_piv2_300 | 111
-val2_sum_piv_30_piv2_0 | 111
-val2_sum_piv_30_piv2_100 | 111
-val2_sum_piv_30_piv2_200 | 111
-val2_sum_piv_30_piv2_300 | 111
+...
-[ RECORD 2 ]--------------+-----
id | 0
id2 | 1
@@ -541,34 +573,6 @@ val2_avg_piv_null_piv2_0 | 111
val2_avg_piv_null_piv2_100 | 111
val2_avg_piv_null_piv2_200 | 111
val2_avg_piv_null_piv2_300 | 111
-val2_avg_piv_10_piv2_0 | 111
-val2_avg_piv_10_piv2_100 | 12
-val2_avg_piv_10_piv2_200 | 111
-val2_avg_piv_10_piv2_300 | 111
-val2_avg_piv_20_piv2_0 | 111
-val2_avg_piv_20_piv2_100 | 13
-val2_avg_piv_20_piv2_200 | 111
-val2_avg_piv_20_piv2_300 | 111
-val2_avg_piv_30_piv2_0 | 111
-val2_avg_piv_30_piv2_100 | 111
-val2_avg_piv_30_piv2_200 | 111
-val2_avg_piv_30_piv2_300 | 111
-val2_sum_piv_null_piv2_0 | 111
-val2_sum_piv_null_piv2_100 | 111
-val2_sum_piv_null_piv2_200 | 111
-val2_sum_piv_null_piv2_300 | 111
-val2_sum_piv_10_piv2_0 | 111
-val2_sum_piv_10_piv2_100 | 12
-val2_sum_piv_10_piv2_200 | 111
-val2_sum_piv_10_piv2_300 | 111
-val2_sum_piv_20_piv2_0 | 111
-val2_sum_piv_20_piv2_100 | 13
-val2_sum_piv_20_piv2_200 | 111
-val2_sum_piv_20_piv2_300 | 111
-val2_sum_piv_30_piv2_0 | 111
-val2_sum_piv_30_piv2_100 | 111
-val2_sum_piv_30_piv2_200 | 111
-val2_sum_piv_30_piv2_300 | 111
...
</pre>
@@ -577,74 +581,49 @@ val2_sum_piv_30_piv2_300 | 111
DROP TABLE IF EXISTS pivout, pivout_dictionary;
SELECT madlib.pivot('pivset_ext', 'pivout', 'id, id2', 'piv, piv2', 'val, val2',
'val=avg, val2=[avg,sum]', '111', True, True);
-SELECT * FROM pivout_dictionary;
+\\x off
+SELECT * FROM pivout_dictionary order by __pivot_cid__;
</pre>
<pre class="result">
- __pivot_cid__ | pval | agg | piv | piv2 | col_name
----------------+------+-----+-----+------+------------------------------
- __p_1__ | val | avg | | 100 | "val_avg_piv_null_piv2_100"
- __p_5__ | val | avg | 10 | 100 | "val_avg_piv_10_piv2_100"
- __p_9__ | val | avg | 20 | 100 | "val_avg_piv_20_piv2_100"
- __p_12__ | val | avg | 30 | 0 | "val_avg_piv_30_piv2_0"
- __p_16__ | val2 | avg | | 0 | "val2_avg_piv_null_piv2_0"
- __p_23__ | val2 | avg | 10 | 300 | "val2_avg_piv_10_piv2_300"
- __p_27__ | val2 | avg | 20 | 300 | "val2_avg_piv_20_piv2_300"
- __p_30__ | val2 | avg | 30 | 200 | "val2_avg_piv_30_piv2_200"
- __p_34__ | val2 | sum | | 200 | "val2_sum_piv_null_piv2_200"
- __p_38__ | val2 | sum | 10 | 200 | "val2_sum_piv_10_piv2_200"
- __p_41__ | val2 | sum | 20 | 100 | "val2_sum_piv_20_piv2_100"
- __p_45__ | val2 | sum | 30 | 100 | "val2_sum_piv_30_piv2_100"
- __p_2__ | val | avg | | 200 | "val_avg_piv_null_piv2_200"
- __p_6__ | val | avg | 10 | 200 | "val_avg_piv_10_piv2_200"
- __p_11__ | val | avg | 20 | 300 | "val_avg_piv_20_piv2_300"
- __p_15__ | val | avg | 30 | 300 | "val_avg_piv_30_piv2_300"
- __p_19__ | val2 | avg | | 300 | "val2_avg_piv_null_piv2_300"
- __p_20__ | val2 | avg | 10 | 0 | "val2_avg_piv_10_piv2_0"
- __p_24__ | val2 | avg | 20 | 0 | "val2_avg_piv_20_piv2_0"
- __p_28__ | val2 | avg | 30 | 0 | "val2_avg_piv_30_piv2_0"
- __p_33__ | val2 | sum | | 100 | "val2_sum_piv_null_piv2_100"
- __p_37__ | val2 | sum | 10 | 100 | "val2_sum_piv_10_piv2_100"
- __p_42__ | val2 | sum | 20 | 200 | "val2_sum_piv_20_piv2_200"
- __p_46__ | val2 | sum | 30 | 200 | "val2_sum_piv_30_piv2_200"
- __p_3__ | val | avg | | 300 | "val_avg_piv_null_piv2_300"
- __p_7__ | val | avg | 10 | 300 | "val_avg_piv_10_piv2_300"
- __p_10__ | val | avg | 20 | 200 | "val_avg_piv_20_piv2_200"
- __p_14__ | val | avg | 30 | 200 | "val_avg_piv_30_piv2_200"
- __p_18__ | val2 | avg | | 200 | "val2_avg_piv_null_piv2_200"
- __p_21__ | val2 | avg | 10 | 100 | "val2_avg_piv_10_piv2_100"
- __p_25__ | val2 | avg | 20 | 100 | "val2_avg_piv_20_piv2_100"
- __p_29__ | val2 | avg | 30 | 100 | "val2_avg_piv_30_piv2_100"
- __p_32__ | val2 | sum | | 0 | "val2_sum_piv_null_piv2_0"
- __p_36__ | val2 | sum | 10 | 0 | "val2_sum_piv_10_piv2_0"
- __p_43__ | val2 | sum | 20 | 300 | "val2_sum_piv_20_piv2_300"
- __p_47__ | val2 | sum | 30 | 300 | "val2_sum_piv_30_piv2_300"
- __p_0__ | val | avg | | 0 | "val_avg_piv_null_piv2_0"
- __p_4__ | val | avg | 10 | 0 | "val_avg_piv_10_piv2_0"
- __p_8__ | val | avg | 20 | 0 | "val_avg_piv_20_piv2_0"
- __p_13__ | val | avg | 30 | 100 | "val_avg_piv_30_piv2_100"
- __p_17__ | val2 | avg | | 100 | "val2_avg_piv_null_piv2_100"
- __p_22__ | val2 | avg | 10 | 200 | "val2_avg_piv_10_piv2_200"
- __p_26__ | val2 | avg | 20 | 200 | "val2_avg_piv_20_piv2_200"
- __p_31__ | val2 | avg | 30 | 300 | "val2_avg_piv_30_piv2_300"
- __p_35__ | val2 | sum | | 300 | "val2_sum_piv_null_piv2_300"
- __p_39__ | val2 | sum | 10 | 300 | "val2_sum_piv_10_piv2_300"
- __p_40__ | val2 | sum | 20 | 0 | "val2_sum_piv_20_piv2_0"
- __p_44__ | val2 | sum | 30 | 0 | "val2_sum_piv_30_piv2_0"
+__pivot_cid__ | pval | agg | piv | piv2 | col_name
+---------------+------+-----+--------+------+------------------------------
+ __p_1__ | val | avg | [NULL] | 0 | "val_avg_piv_null_piv2_0"
+ __p_2__ | val | avg | [NULL] | 100 | "val_avg_piv_null_piv2_100"
+ __p_3__ | val | avg | [NULL] | 200 | "val_avg_piv_null_piv2_200"
+ __p_4__ | val | avg | [NULL] | 300 | "val_avg_piv_null_piv2_300"
+ __p_5__ | val | avg | 10 | 0 | "val_avg_piv_10_piv2_0"
+ __p_6__ | val | avg | 10 | 100 | "val_avg_piv_10_piv2_100"
+ __p_7__ | val | avg | 10 | 200 | "val_avg_piv_10_piv2_200"
+ __p_8__ | val | avg | 10 | 300 | "val_avg_piv_10_piv2_300"
+ __p_9__ | val | avg | 20 | 0 | "val_avg_piv_20_piv2_0"
+ __p_10__ | val | avg | 20 | 100 | "val_avg_piv_20_piv2_100"
+ __p_11__ | val | avg | 20 | 200 | "val_avg_piv_20_piv2_200"
+ __p_12__ | val | avg | 20 | 300 | "val_avg_piv_20_piv2_300"
+ __p_13__ | val | avg | 30 | 0 | "val_avg_piv_30_piv2_0"
+ __p_14__ | val | avg | 30 | 100 | "val_avg_piv_30_piv2_100"
+ __p_15__ | val | avg | 30 | 200 | "val_avg_piv_30_piv2_200"
+ __p_16__ | val | avg | 30 | 300 | "val_avg_piv_30_piv2_300"
+ __p_17__ | val2 | avg | [NULL] | 0 | "val2_avg_piv_null_piv2_0"
+ __p_18__ | val2 | avg | [NULL] | 100 | "val2_avg_piv_null_piv2_100"
+ __p_19__ | val2 | avg | [NULL] | 200 | "val2_avg_piv_null_piv2_200"
+ __p_20__ | val2 | avg | [NULL] | 300 | "val2_avg_piv_null_piv2_300"
+ __p_21__ | val2 | avg | 10 | 0 | "val2_avg_piv_10_piv2_0"
+...
(48 rows)
</pre>
<pre class="example">
+\\x on
SELECT * FROM pivout ORDER BY id,id2;
</pre>
<pre class="result">
--[ RECORD 1 ]--
+-[ RECORD 1 ]----
id | 0
id2 | 0
-__p_0__ | 111
__p_1__ | 111
__p_2__ | 111
__p_3__ | 111
-__p_4__ | 1
-__p_5__ | 111
+__p_4__ | 111
+__p_5__ | 1
__p_6__ | 111
__p_7__ | 111
__p_8__ | 111
@@ -653,91 +632,40 @@ __p_10__ | 111
__p_11__ | 111
__p_12__ | 111
__p_13__ | 111
-__p_14__ | 111
-__p_15__ | 111
-__p_16__ | 111
-__p_17__ | 111
-__p_18__ | 111
-__p_19__ | 111
-__p_20__ | 11
-__p_21__ | 111
-__p_22__ | 111
-__p_23__ | 111
-__p_24__ | 111
-__p_25__ | 111
-__p_26__ | 111
-__p_27__ | 111
-__p_28__ | 111
-__p_29__ | 111
-__p_30__ | 111
-__p_31__ | 111
-__p_32__ | 111
-__p_33__ | 111
-__p_34__ | 111
-__p_35__ | 111
-__p_36__ | 11
-__p_37__ | 111
-__p_38__ | 111
-__p_39__ | 111
-__p_40__ | 111
-__p_41__ | 111
-__p_42__ | 111
-__p_43__ | 111
-__p_44__ | 111
-__p_45__ | 111
-__p_46__ | 111
-__p_47__ | 111
--[ RECORD 2 ]--
+...
+-[ RECORD 2 ]----
id | 0
id2 | 1
-__p_0__ | 111
__p_1__ | 111
__p_2__ | 111
__p_3__ | 111
__p_4__ | 111
-__p_5__ | 2
+__p_5__ | 111
+__p_6__ | 2
+__p_7__ | 111
+__p_8__ | 111
+__p_9__ | 111
+__p_10__ | 3
+__p_11__ | 111
+__p_12__ | 111
+__p_13__ | 111
+...
+-[ RECORD 3 ]----
+id | 1
+id2 | 0
+__p_1__ | 111
+__p_2__ | 111
+__p_3__ | 111
+__p_4__ | 111
+__p_5__ | 111
__p_6__ | 111
__p_7__ | 111
__p_8__ | 111
-__p_9__ | 3
+__p_9__ | 111
__p_10__ | 111
__p_11__ | 111
__p_12__ | 111
__p_13__ | 111
-__p_14__ | 111
-__p_15__ | 111
-__p_16__ | 111
-__p_17__ | 111
-__p_18__ | 111
-__p_19__ | 111
-__p_20__ | 111
-__p_21__ | 12
-__p_22__ | 111
-__p_23__ | 111
-__p_24__ | 111
-__p_25__ | 13
-__p_26__ | 111
-__p_27__ | 111
-__p_28__ | 111
-__p_29__ | 111
-__p_30__ | 111
-__p_31__ | 111
-__p_32__ | 111
-__p_33__ | 111
-__p_34__ | 111
-__p_35__ | 111
-__p_36__ | 111
-__p_37__ | 12
-__p_38__ | 111
-__p_39__ | 111
-__p_40__ | 111
-__p_41__ | 13
-__p_42__ | 111
-__p_43__ | 111
-__p_44__ | 111
-__p_45__ | 111
-__p_46__ | 111
-__p_47__ | 111
...
</pre>
@@ -786,7 +714,8 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pivot(
aggregate_func TEXT,
fill_value TEXT,
keep_null BOOLEAN,
- output_col_dictionary BOOLEAN
+ output_col_dictionary BOOLEAN,
+ output_type TEXT
) RETURNS VOID AS $$
PythonFunction(utilities, pivot, pivot)
@@ -794,6 +723,22 @@ $$ LANGUAGE plpythonu VOLATILE
m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `MODIFIES SQL DATA', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pivot(
+ source_table TEXT,
+ out_table TEXT,
+ index TEXT,
+ pivot_cols TEXT,
+ pivot_values TEXT,
+ aggregate_func TEXT,
+ fill_value TEXT,
+ keep_null BOOLEAN,
+ output_col_dictionary BOOLEAN
+
+) RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.pivot($1, $2, $3, $4, $5, $6, $7, $8, $9, NULL)
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pivot(
source_table TEXT,
out_table TEXT,
index TEXT,
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/bb209bbb/src/ports/postgres/modules/utilities/test/pivot.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/test/pivot.sql_in b/src/ports/postgres/modules/utilities/test/pivot.sql_in
index 79bcc57..3dafd18 100644
--- a/src/ports/postgres/modules/utilities/test/pivot.sql_in
+++ b/src/ports/postgres/modules/utilities/test/pivot.sql_in
@@ -28,16 +28,16 @@ CREATE TABLE pivset(
);
INSERT INTO pivset VALUES
- (0, 10, 1),
- (0, 10, 2),
- (0, 20, 3),
- (1, 20, 4),
- (1, 30, 5),
- (1, 30, 6),
- (1, 10, 7),
- (NULL, 10, 8),
- (0, NULL, 9),
- (0, 10, NULL);
+ (0, 10, 1),
+ (0, 10, 2),
+ (0, 20, 3),
+ (1, 20, 4),
+ (1, 30, 5),
+ (1, 30, 6),
+ (1, 10, 7),
+ (NULL, 10, 8),
+ (0, NULL, 9),
+ (0, 10, NULL);
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset', 'pivout', 'id', 'piv', 'val');
@@ -47,12 +47,12 @@ SELECT assert(val_avg_piv_20 = 3, 'Wrong output in pivoting') FROM pivout WHERE
DROP VIEW IF EXISTS pivset_ext;
CREATE VIEW pivset_ext AS
- SELECT *,
+ SELECT *,
COALESCE(id + (pivset.val / 3), 0) AS id2,
COALESCE(piv + (pivset.val / 3), 0) AS piv2,
COALESCE(val + 10, 0) AS val2
FROM pivset;
-SELECT id,id2,piv,piv2,val,val2 FROM pivset_ext
+SELECT id,id2,piv,piv2,val,val2 FROM pivset_ext
ORDER BY id,id2,piv,piv2,val,val2;
DROP TABLE IF EXISTS pivout;
@@ -60,87 +60,87 @@ SELECT pivot('pivset_ext', 'pivout', 'id,id2', 'piv', 'val');
SELECT * FROM pivout;
SELECT assert(val_avg_piv_10 = 1.5,
- 'Wrong output in pivoting: index columns') FROM pivout
- WHERE id = 0 AND id2 = 0;
+ 'Wrong output in pivoting: index columns') FROM pivout
+ WHERE id = 0 AND id2 = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv, piv2', 'val');
SELECT * FROM pivout;
SELECT assert(val_avg_piv_10_piv2_10 = 1.5,
- 'Wrong output in pivoting: pivot columns') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: pivot columns') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv', 'val, val2');
SELECT * FROM pivout;
SELECT assert(val2_avg_piv_20 = 13,
- 'Wrong output in pivoting: value columns') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: value columns') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv', 'val', 'sum');
SELECT * FROM pivout;
SELECT assert(val_sum_piv_10 = 3,
- 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv', 'val', 'sum', True);
SELECT * FROM pivout;
SELECT assert(val_sum_piv_null = 9,
- 'Wrong output in pivoting: keep null') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: keep null') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv', 'val', 'sum', '111');
SELECT * FROM pivout;
SELECT assert(val_sum_piv_30 = 111,
- 'Wrong output in pivoting: fill value') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: fill value') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv', 'val', 'sum', '111', True);
SELECT * FROM pivout;
SELECT assert(val_sum_piv_30 = 111 AND val_sum_piv_null = 9,
- 'Wrong output in pivoting: fill value') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: fill value') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv', 'val', 'avg, sum');
SELECT * FROM pivout;
SELECT assert(val_avg_piv_10 = 1.5 AND val_sum_piv_10 = 3,
- 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv, piv2', 'val', 'avg, sum');
SELECT * FROM pivout;
SELECT assert(val_avg_piv_10_piv2_10 = 1.5 AND val_sum_piv_10_piv2_10 = 3,
- 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv', 'val, val2', 'avg, sum');
SELECT * FROM pivout;
SELECT assert(val_sum_piv_10 = 3 AND val2_avg_piv_20 = 13,
- 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv', 'val, val2',
- 'val=avg, val2=sum');
+ 'val=avg, val2=sum');
SELECT * FROM pivout;
SELECT assert(val_avg_piv_10 = 1.5 AND val2_sum_piv_10 = 23,
- 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv', 'val, val2',
- 'val=avg, val2=[avg,sum]');
+ 'val=avg, val2=[avg,sum]');
SELECT * FROM pivout;
SELECT assert(val2_avg_piv_20 = 13 AND val2_sum_piv_10 = 23,
- 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
+ 'Wrong output in pivoting: aggregate functions') FROM pivout WHERE id = 0;
DROP TABLE IF EXISTS pivout;
DROP TABLE IF EXISTS pivout_dictionary;
@@ -148,9 +148,9 @@ SELECT pivot('pivset_ext', 'pivout', 'id, id2', 'piv, piv2', 'val, val2',
'val=avg, val2=[avg,sum]', '111', True, True);
SELECT * FROM pivout;
-SELECT assert(__p_7__ = 1.5,
- 'Wrong output in pivoting: Output dictionary') FROM pivout
- WHERE id = 0 AND id2 = 0;
+SELECT assert(__p_8__ = 1.5,
+ 'Wrong output in pivoting: Output dictionary') FROM pivout
+ WHERE id = 0 AND id2 = 0;
DROP FUNCTION IF EXISTS array_add1(ANYARRAY, ANYELEMENT);
DROP AGGREGATE IF EXISTS array_accum1 (anyelement);
@@ -167,4 +167,18 @@ DROP TABLE IF EXISTS pivout;
SELECT pivot('pivset_ext', 'pivout', 'id', 'piv', 'val', 'array_accum1');
SELECT * FROM pivout;
+DROP TABLE IF EXISTS pivout;
+DROP TABLE IF EXISTS pivout_dictionary;
+SELECT pivot('pivset_ext', 'pivout', 'id, id2', 'piv, piv2', 'val, val2',
+ 'val=avg, val2=[avg,sum]', '111', True, True, 'a');
+SELECT * FROM pivout;
+SELECT * FROM pivout_dictionary;
+
+DROP TABLE IF EXISTS pivout;
+DROP TABLE IF EXISTS pivout_dictionary;
+SELECT pivot('pivset_ext', 'pivout', 'id, id2', 'piv, piv2', 'val, val2',
+ 'val=avg, val2=[avg,sum]', '111', True, True, 's');
+SELECT * FROM pivout;
+SELECT * FROM pivout_dictionary;
+
DROP VIEW IF EXISTS pivset_ext;
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/bb209bbb/src/ports/postgres/modules/utilities/utilities.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/utilities.py_in b/src/ports/postgres/modules/utilities/utilities.py_in
index 0e47aea..126f4e6 100644
--- a/src/ports/postgres/modules/utilities/utilities.py_in
+++ b/src/ports/postgres/modules/utilities/utilities.py_in
@@ -1,4 +1,5 @@
+import collections
import re
import time
import random
@@ -583,16 +584,16 @@ def extract_keyvalue_params(input_params,
else:
continue
try:
- if param_type in (int, str, float):
- parameter_dict[param_name] = param_type(param_value)
- elif param_type == list:
- parameter_dict[param_name] = split_quoted_delimited_str(
- param_value.strip('[](){} '))
- elif param_type == bool:
+ if param_type == bool: # bool is not subclassable
# True values are y, yes, t, true, on and 1;
# False values are n, no, f, false, off and 0.
# Raises ValueError if anything else.
parameter_dict[param_name] = bool(strtobool(param_value))
+ elif param_type in (int, str, float):
+ parameter_dict[param_name] = param_type(param_value)
+ elif issubclass(param_type, collections.Iterable):
+ parameter_dict[param_name] = split_quoted_delimited_str(
+ param_value.strip('[](){} '))
else:
raise TypeError("Invalid input: {0} has unsupported type "
"{1}".format(param_name, usage_str))
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/bb209bbb/src/ports/postgres/modules/utilities/validate_args.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/validate_args.py_in b/src/ports/postgres/modules/utilities/validate_args.py_in
index 91f34b8..5832124 100644
--- a/src/ports/postgres/modules/utilities/validate_args.py_in
+++ b/src/ports/postgres/modules/utilities/validate_args.py_in
@@ -345,8 +345,9 @@ def get_cols_and_types(tbl):
def get_expr_type(expr, tbl):
- """ Temporary function to obtain the type of an expression
+ """ Return the type of an expression run on a given table
+ Note: this
Args:
@param expr
@@ -356,7 +357,6 @@ def get_expr_type(expr, tbl):
expr_type = plpy.execute("""
SELECT pg_typeof({0}) AS type
FROM {1}
- WHERE ({0}) IS NOT NULL
LIMIT 1
""".format(expr, tbl))[0]['type']
return expr_type.upper()
[29/34] incubator-madlib git commit: Release: Update the release
notes for v1.11.0.
Posted by ok...@apache.org.
Release: Update the release notes for v1.11.0.
Closes #128
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/4b0c3771
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/4b0c3771
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/4b0c3771
Branch: refs/heads/latest_release
Commit: 4b0c37714c17a4aae8759a4dd05e59f0c94ba598
Parents: 648b057
Author: Rashmi Raghu <rr...@pivotal.io>
Authored: Fri Apr 28 16:42:14 2017 -0700
Committer: Rashmi Raghu <rr...@pivotal.io>
Committed: Fri Apr 28 16:42:14 2017 -0700
----------------------------------------------------------------------
RELEASE_NOTES | 35 +++++++++++++++++++++++++++++++++++
1 file changed, 35 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/4b0c3771/RELEASE_NOTES
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 78f3d10..f425283 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -9,6 +9,41 @@ commit history located at https://github.com/apache/incubator-madlib/commits/mas
Current list of bugs and issues can be found at https://issues.apache.org/jira/browse/MADLIB.
—-------------------------------------------------------------------------
+MADlib v1.11.0:
+
+Release Date: 2017-May-05
+
+New features:
+* New module: Graph - PageRank
+ - Implements the original PageRank algorithm that assumes a random surfer model
+ (https://en.wikipedia.org/wiki/PageRank#Damping_factor) (MADLIB-1069)
+ - Grouping support is included for PageRank (MADLIB-1082)
+* Graph - Single Source Shortest Path (SSSP): Add grouping support (MADLIB-1081)
+* Pivot: Add support for array and svec output types (MADLIB-1066)
+* DT and RF:
+ - Change default values for 2 parameters (max_depth and num_splits)
+ - Reduce memory footprint: Assign memory only for reachable nodes (MADLIB-1057)
+ - Include rows with NULL features in training (MADLIB-1095)
+ - Update error message for invalid parameter specification (num_splits)
+* Array Operations: Add function to unnest 2-D arrays by one level into rows of 1-D arrays (MADLIB-1086)
+* Build process on Apache infrastructure (MADLIB-920, MADLIB-1080)
+* Updates for Apache Top Level Project readiness (MADLIB-1022, MADLIB-1076, MADLIB-1077, MADLIB 1090)
+* Support for GPDB 5.0
+
+Bug fixes:
+ - DT and RF:
+ - Fix accuracy issues related to integer categorical variables and tree depth
+ - Improve visualization of tree(s)
+ - Elastic Net:
+ - Fix install check on GPDB 5.0 and HAWQ 2.2 (MADLIB-1088)
+ - Fix inconsistent results with grouping (MADLIB-1092)
+ - PCA: Fix install check
+
+Other:
+ - PMML: Skip install check when run without the ‘-t’ option (MADLIB-1078)
+ - Multiple user documentation improvements
+
+—-------------------------------------------------------------------------
MADlib v1.10.0
Release Date: 2017-February-17
[18/34] incubator-madlib git commit: Update README.md
Posted by ok...@apache.org.
Update README.md
Updated 3rd party components list + a few minor edits
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/658ecdef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/658ecdef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/658ecdef
Branch: refs/heads/latest_release
Commit: 658ecdeff4051d4594a9ce95e5f9a9b3f4597498
Parents: 206e126
Author: Frank McQuillan <fm...@pivotal.io>
Authored: Wed Apr 19 15:04:03 2017 -0700
Committer: GitHub <no...@github.com>
Committed: Wed Apr 19 15:04:03 2017 -0700
----------------------------------------------------------------------
README.md | 23 +++++++++++++++--------
1 file changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/658ecdef/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 1d507a3..a54ba5e 100644
--- a/README.md
+++ b/README.md
@@ -9,9 +9,10 @@ machine learning methods for structured and unstructured data.
Installation and Contribution
==============================
-See the project webpage [`MADlib Home`](http://madlib.incubator.apache.org/) for links to the
+See the project website [`MADlib Home`](http://madlib.incubator.apache.org/) for links to the
latest binary and source packages. For installation and contribution guides,
-please see [`MADlib Wiki`](https://cwiki.apache.org/confluence/display/MADLIB/)
+and other useful information
+please refer to the [`MADlib Wiki`](https://cwiki.apache.org/confluence/display/MADLIB/)
Development with Docker
=======================
@@ -79,13 +80,19 @@ architecture.
Third Party Components
======================
-MADlib incorporates material from the following third-party components
+MADlib incorporates software from the following third-party components. Bundled with source code:
-1. [`argparse 1.2.1`](http://code.google.com/p/argparse/) "provides an easy, declarative interface for creating command line tools"
-2. [`Boost 1.47.0 (or newer)`](http://www.boost.org/) "provides peer-reviewed portable C++ source libraries"
-3. [`Eigen 3.2.2`](http://eigen.tuxfamily.org/index.php?title=Main_Page) "is a C++ template library for linear algebra"
-4. [`PyYAML 3.10`](http://pyyaml.org/wiki/PyYAML) "is a YAML parser and emitter for Python"
-5. [`PyXB 1.2.4`](http://pyxb.sourceforge.net/) "is a Python library for XML Schema Bindings"
+1. [`libstemmer`](http://snowballstem.org/) "small string processing language"
+2. [`m_widen_init`](https://github.com/apache/incubator-madlib/blob/master/licenses/third_party/_M_widen_init.txt) "allows compilation with recent versions of gcc with runtime dependencies from earlier versions of libstdc++"
+3. [`argparse 1.2.1`](http://code.google.com/p/argparse/) "provides an easy, declarative interface for creating command line tools"
+4. [`PyYAML 3.10`](http://pyyaml.org/wiki/PyYAML) "YAML parser and emitter for Python"
+5. [`UseLATEX.cmake`](https://github.com/kmorel/UseLATEX/blob/master/UseLATEX.cmake) "CMAKE commands to use the LaTeX compiler"
+
+Downloaded at build time:
+
+6. [`Boost 1.61.0 (or newer)`](http://www.boost.org/) "provides peer-reviewed portable C++ source libraries"
+7. [`PyXB 1.2.4`](http://pyxb.sourceforge.net/) "Python library for XML Schema Bindings"
+8. [`Eigen 3.2.2`](http://eigen.tuxfamily.org/index.php?title=Main_Page) "C++ template library for linear algebra"
Licensing
==========
[09/34] incubator-madlib git commit: Feature: PageRank
Posted by ok...@apache.org.
Feature: PageRank
JIRA: MADLIB-1069
- Introduces a new module that computes the PageRank of all nodes
in a directed graph.
- Implements the original PageRank algorithm that assumes a random
surfer model (https://en.wikipedia.org/wiki/PageRank#Damping_factor)
- This version does not perform convergence test yet, so the PageRank
computation runs through all iterations. Exiting on convergence
will be handled as part of MADLIB-1082. The threshold parameter
specified will be ignored.
Closes #109
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/6b466ea6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/6b466ea6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/6b466ea6
Branch: refs/heads/latest_release
Commit: 6b466ea6d19731e8500cc89058d1a77bf778121a
Parents: d344f1f
Author: Nandish Jayaram <nj...@apache.org>
Authored: Thu Mar 16 12:02:40 2017 -0700
Committer: Nandish Jayaram <nj...@apache.org>
Committed: Thu Mar 30 17:06:17 2017 -0700
----------------------------------------------------------------------
doc/mainpage.dox.in | 3 +
.../postgres/modules/graph/graph_utils.py_in | 8 +-
src/ports/postgres/modules/graph/pagerank.py_in | 288 +++++++++++++++++++
.../postgres/modules/graph/pagerank.sql_in | 271 +++++++++++++++++
.../postgres/modules/graph/test/pagerank.sql_in | 62 ++++
5 files changed, 628 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/6b466ea6/doc/mainpage.dox.in
----------------------------------------------------------------------
diff --git a/doc/mainpage.dox.in b/doc/mainpage.dox.in
index 9131c10..94950e7 100644
--- a/doc/mainpage.dox.in
+++ b/doc/mainpage.dox.in
@@ -122,6 +122,9 @@ complete matrix stored as a distributed table.
@ingroup grp_datatrans
@defgroup grp_graph Graph
@{Contains graph algorithms. @}
+ @defgroup grp_pagerank PageRank
+ @ingroup grp_graph
+
@defgroup grp_sssp Single Source Shortest Path
@ingroup grp_graph
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/6b466ea6/src/ports/postgres/modules/graph/graph_utils.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/graph_utils.py_in b/src/ports/postgres/modules/graph/graph_utils.py_in
index fb43491..2d83301 100644
--- a/src/ports/postgres/modules/graph/graph_utils.py_in
+++ b/src/ports/postgres/modules/graph/graph_utils.py_in
@@ -69,11 +69,11 @@ def validate_graph_coding(vertex_table, vertex_id, edge_table, edge_params,
existing_cols = set(unquote_ident(i) for i in get_cols(vertex_table))
_assert(vertex_id in existing_cols,
- """Graph {func_name}: The vertex column {vertex_id} is not present in
- vertex table ({vertex_table}) """.format(**locals()))
+ """Graph {func_name}: The vertex column {vertex_id} is not present in vertex table ({vertex_table}) """.
+ format(**locals()))
_assert(columns_exist_in_table(edge_table, edge_params.values()),
- """Graph {func_name}: Not all columns from {cols} present in edge
- table ({edge_table})""".format(cols=edge_params.values(), **locals()))
+ """Graph {func_name}: Not all columns from {cols} present in edge table ({edge_table})""".
+ format(cols=edge_params.values(), **locals()))
return None
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/6b466ea6/src/ports/postgres/modules/graph/pagerank.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/pagerank.py_in b/src/ports/postgres/modules/graph/pagerank.py_in
new file mode 100644
index 0000000..13cdcc5
--- /dev/null
+++ b/src/ports/postgres/modules/graph/pagerank.py_in
@@ -0,0 +1,288 @@
+# coding=utf-8
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# PageRank
+
+# Please refer to the pagerank.sql_in file for the documentation
+
+"""
+@file pagerank.py_in
+
+@namespace graph
+"""
+
+import plpy
+from utilities.control import MinWarning
+from utilities.utilities import _assert
+from utilities.utilities import extract_keyvalue_params
+from utilities.utilities import unique_string
+from utilities.control import IterationController2S
+from graph_utils import *
+
+import time
+
+m4_changequote(`<!', `!>')
+
+def validate_pagerank_args(vertex_table, vertex_id, edge_table, edge_params,
+ out_table, damping_factor, max_iter, threshold, module_name):
+ """
+ Function to validate input parameters for PageRank
+ """
+ validate_graph_coding(vertex_table, vertex_id, edge_table, edge_params,
+ out_table, module_name)
+ _assert(damping_factor >= 0.0 and damping_factor <= 1.0,
+ """PageRank: Invalid damping factor value ({0}), must be between 0 and 1."""
+ .format(damping_factor))
+ _assert(threshold >= 0.0 and threshold <= 1.0,
+ """PageRank: Invalid threshold value ({0}), must be between 0 and 1."""
+ .format(threshold))
+ _assert(max_iter > 0,
+ """PageRank: Invalid max_iter value ({0}), must be a positive integer. """
+ .format(max_iter))
+
+def pagerank(schema_madlib, vertex_table, vertex_id, edge_table, edge_args,
+ out_table, damping_factor, max_iter, threshold, **kwargs):
+ """
+ Function that computes the PageRank
+
+ Args:
+ @param vertex_table
+ @param vertex_id
+ @param edge_table
+ @param source_vertex
+ @param dest_vertex
+ @param out_table
+ @param damping_factor
+ @param max_iter
+ @param threshold
+ """
+ old_msg_level = plpy.execute("""
+ SELECT setting
+ FROM pg_settings
+ WHERE name='client_min_messages'
+ """)[0]['setting']
+ plpy.execute('SET client_min_messages TO warning')
+ params_types = {'src': str, 'dest': str}
+ default_args = {'src': 'src', 'dest': 'dest'}
+ edge_params = extract_keyvalue_params(edge_args, params_types, default_args)
+
+ # populate default values for optional params if null
+ if damping_factor is None:
+ damping_factor = 0.85
+ if max_iter is None:
+ max_iter = 100
+ if threshold is None:
+ threshold = 0.00001
+ if vertex_id is None:
+ vertex_id = "id"
+ validate_pagerank_args(vertex_table, vertex_id, edge_table, edge_params,
+ out_table, damping_factor, max_iter, threshold, 'PageRank')
+ src = edge_params["src"]
+ dest = edge_params["dest"]
+
+ edge_temp_table = unique_string(desp='temp_edge')
+ distribution = m4_ifdef(<!__POSTGRESQL__!>, <!''!>,
+ <!"DISTRIBUTED BY ({0})".format(dest)!>)
+ plpy.execute("""
+ DROP TABLE IF EXISTS {edge_temp_table};
+ CREATE TEMP TABLE {edge_temp_table} AS
+ SELECT * FROM {edge_table}
+ {distribution}
+ """.format(**locals()))
+ # GPDB and HAWQ have distributed by clauses to help them with indexing.
+ # For Postgres we add the indices manually.
+ sql_index = m4_ifdef(<!__POSTGRESQL__!>,
+ <!"""CREATE INDEX ON {edge_temp_table} ({src});
+ """.format(**locals())!>,
+ <!''!>)
+ plpy.execute(sql_index)
+
+ nvertices = plpy.execute("""
+ SELECT COUNT({0}) AS cnt
+ FROM {1}
+ """.format(vertex_id, vertex_table))[0]["cnt"]
+ init_value = 1.0/nvertices
+ random_prob = (1.0-damping_factor)/nvertices
+ cur = unique_string(desp='cur')
+ message = unique_string(desp='message')
+ plpy.execute("""
+ CREATE TEMP TABLE {cur} AS
+ SELECT {vertex_id}, {init_value}::DOUBLE PRECISION AS pagerank
+ FROM {vertex_table}
+ """.format(**locals()))
+ v1 = unique_string(desp='v1')
+
+ out_cnts = unique_string(desp='out_cnts')
+ out_cnts_cnt = unique_string(desp='cnt')
+ # Compute the out-degree of every node in the graph.
+ cnts_distribution = m4_ifdef(<!__POSTGRESQL__!>, <!''!>,
+ <!"DISTRIBUTED BY ({0})".format(vertex_id)!>)
+
+ plpy.execute("""
+ DROP TABLE IF EXISTS {out_cnts};
+ CREATE TEMP TABLE {out_cnts} AS
+ SELECT {src} AS {vertex_id}, COUNT({dest}) AS {out_cnts_cnt}
+ FROM {edge_table}
+ GROUP BY {src}
+ {cnts_distribution}
+ """.format(**locals()))
+
+ for i in range(max_iter):
+ #####################################################################
+ # PageRank for node 'A' at any given iteration 'i' is given by:
+ # PR_i(A) = damping_factor(PR_i-1(B)/degree(B) + PR_i-1(C)/degree(C) + ...) + (1-damping_factor)/N
+ # where 'N' is the number of vertices in the graph,
+ # B, C are nodes that have edges to node A, and
+ # degree(node) represents the number of outgoing edges from 'node'
+ #####################################################################
+ # Essentially, the pagerank for a node is based on an aggregate of a
+ # fraction of the pagerank values of all the nodes that have incoming
+ # edges to it, along with a small random probability.
+ # More information can be found at:
+ # https://en.wikipedia.org/wiki/PageRank#Damping_factor
+
+ # The query below computes the PageRank of each node using the above formula.
+ plpy.execute("""
+ CREATE TABLE {message} AS
+ SELECT {edge_temp_table}.{dest} AS {vertex_id},
+ SUM({v1}.pagerank/{out_cnts}.{out_cnts_cnt})*{damping_factor}+{random_prob} AS pagerank
+ FROM {edge_temp_table}
+ INNER JOIN {cur} ON {edge_temp_table}.{dest}={cur}.{vertex_id}
+ INNER JOIN {out_cnts} ON {out_cnts}.{vertex_id}={edge_temp_table}.{src}
+ INNER JOIN {cur} AS {v1} ON {v1}.{vertex_id}={edge_temp_table}.{src}
+ GROUP BY {edge_temp_table}.{dest}
+ """.format(**locals()))
+ # If there are nodes that have no incoming edges, they are not captured in the message table.
+ # Insert entries for such nodes, with random_prob.
+ plpy.execute("""
+ INSERT INTO {message}
+ SELECT {vertex_id}, {random_prob}::DOUBLE PRECISION AS pagerank
+ FROM {cur}
+ WHERE {vertex_id} NOT IN (
+ SELECT {vertex_id}
+ FROM {message}
+ )
+ """.format(**locals()))
+ # Check for convergence will be done as part of grouping support for pagerank:
+ # https://issues.apache.org/jira/browse/MADLIB-1082. So, the threshold parameter
+ # is a dummy variable at the moment, the PageRank computation happens for
+ # {max_iter} number of times.
+ plpy.execute("""
+ DROP TABLE IF EXISTS {cur};
+ ALTER TABLE {message} RENAME TO {cur}
+ """.format(**locals()))
+
+ plpy.execute("ALTER TABLE {cur} RENAME TO {out_table}".format(**locals()))
+
+ ## Step 4: Cleanup
+ plpy.execute("""
+ DROP TABLE IF EXISTS {0},{1},{2},{3};
+ """.format(out_cnts, edge_temp_table, cur, message))
+ plpy.execute("SET client_min_messages TO %s" % old_msg_level)
+
+def pagerank_help(schema_madlib, message, **kwargs):
+ """
+ Help function for pagerank
+
+ Args:
+ @param schema_madlib
+ @param message: string, Help message string
+ @param kwargs
+
+ Returns:
+ String. Help/usage information
+ """
+ if message is not None and \
+ message.lower() in ("usage", "help", "?"):
+ help_string = "Get from method below"
+ help_string = get_graph_usage(schema_madlib, 'PageRank',
+ """out_table TEXT, -- Name of the output table for PageRank
+ damping_factor, DOUBLE PRECISION, -- Damping factor in random surfer model
+ -- (DEFAULT = 0.85)
+ max_iter, INTEGER, -- Maximum iteration number (DEFAULT = 100)
+ threshold DOUBLE PRECISION -- Stopping criteria (DEFAULT = 1e-5)
+""")
+ else:
+ if message is not None and \
+ message.lower() in ("example", "examples"):
+ help_string = """
+----------------------------------------------------------------------------
+ EXAMPLES
+----------------------------------------------------------------------------
+-- Create a graph, represented as vertex and edge tables.
+DROP TABLE IF EXISTS vertex, edge;
+CREATE TABLE vertex(
+ id INTEGER
+ );
+CREATE TABLE edge(
+ src INTEGER,
+ dest INTEGER
+ );
+INSERT INTO vertex VALUES
+(0),
+(1),
+(2),
+(3),
+(4),
+(5),
+(6);
+INSERT INTO edge VALUES
+(0, 1),
+(0, 2),
+(0, 4),
+(1, 2),
+(1, 3),
+(2, 3),
+(2, 5),
+(2, 6),
+(3, 0),
+(4, 0),
+(5, 6),
+(6, 3);
+
+-- Compute the PageRank:
+DROP TABLE IF EXISTS pagerank_out;
+SELECT madlib.pagerank(
+ 'vertex', -- Vertex table
+ 'id', -- Vertix id column
+ 'edge', -- Edge table
+ 'src=src, dest=dest', -- Comma delimted string of edge arguments
+ 'pagerank_out') -- Output table of PageRank
+
+-- View the PageRank of all vertices, sorted by their scores.
+SELECT * FROM pagerank_out ORDER BY pagerank desc;
+"""
+ else:
+ help_string = """
+----------------------------------------------------------------------------
+ SUMMARY
+----------------------------------------------------------------------------
+Given a directed graph, pagerank algorithm finds the PageRank score of all
+the vertices in the graph.
+--
+For an overview on usage, run:
+SELECT {schema_madlib}.pagerank('usage');
+
+For some examples, run:
+SELECT {schema_madlib}.pagerank('example')
+--
+"""
+
+ return help_string.format(schema_madlib=schema_madlib)
+# ---------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/6b466ea6/src/ports/postgres/modules/graph/pagerank.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/pagerank.sql_in b/src/ports/postgres/modules/graph/pagerank.sql_in
new file mode 100644
index 0000000..712d146
--- /dev/null
+++ b/src/ports/postgres/modules/graph/pagerank.sql_in
@@ -0,0 +1,271 @@
+/* ----------------------------------------------------------------------- *//**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ *
+ * @file graph.sql_in
+ *
+ * @brief SQL functions for graph analytics
+ * @date Nov 2016
+ *
+ * @sa Provides various graph algorithms.
+ *
+ *//* ----------------------------------------------------------------------- */
+m4_include(`SQLCommon.m4')
+
+
+/**
+@addtogroup grp_pagerank
+
+<div class="toc"><b>Contents</b>
+<ul>
+<li><a href="#pagerank">PageRank</a></li>
+<li><a href="#notes">Notes</a></li>
+<li><a href="#examples">Examples</a></li>
+<li><a href="#literature">Literature</a></li>
+</ul>
+</div>
+
+@brief Find the PageRank of all vertices in a directed graph.
+
+Given a graph, the PageRank algorithm outputs a probability distribution representing the
+likelihood that a person randomly traversing the graph will arrive at any particular vertex.
+This algorithm was originally used by Google to rank websites where the World Wide Web was
+modeled as a directed graph with the vertices representing the websites.
+
+@anchor pagerank
+@par PageRank
+<pre class="syntax">
+pagerank( vertex_table,
+ vertex_id,
+ edge_table,
+ edge_args,
+ out_table,
+ damping_factor,
+ max_iter,
+ threshold
+ )
+</pre>
+
+\b Arguments
+<dl class="arglist">
+<dt>vertex_table</dt>
+<dd>TEXT. Name of the table containing the vertex data for the graph. Must contain the
+column specified in the 'vertex_id' parameter below.</dd>
+
+<dt>vertex_id</dt>
+<dd>TEXT, default = 'id'. Name of the column in 'vertex_table' containing
+vertex ids. The vertex ids are of type INTEGER with no duplicates.
+They do not need to be contiguous.</dd>
+
+<dt>edge_table</dt>
+<dd>TEXT. Name of the table containing the edge data. The edge table must
+contain columns for source vertex and destination vertex.</dd>
+
+<dt>edge_args</dt>
+<dd>TEXT. A comma-delimited string containing multiple named arguments of
+the form "name=value". The following parameters are supported for
+this string argument:
+ - src (INTEGER): Name of the column containing the source vertex ids in the edge table.
+ Default column name is 'src'.
+ - dest (INTEGER): Name of the column containing the destination vertex ids in the edge table.
+ Default column name is 'dest'.</dd>
+
+<dt>out_table</dt>
+<dd>TEXT. Name of the table to store the result of PageRank.
+It will contain a row for every vertex from 'vertex_table' with
+the following columns:
+ - vertex_id : The id of a vertex. Will use the input parameter 'vertex_id' for column naming.
+ - pagerank : The vertex's PageRank.</dd>
+
+<dt>damping_factor</dt>
+<dd>FLOAT8, default 0.85. The probability, at any step, that a user will continue following the links in a random surfer model.</dd>
+
+<dt>max_iter</dt>
+<dd>INTEGER, default: 100. The maximum number of iterations allowed.</dd>
+
+<dt>threshold</dt>
+<dd>FLOAT8, default: 1e-5. If the difference between the PageRank of every vertex of two consecutive
+iterations is smaller than 'threshold', or the iteration number is larger than 'max_iter', the
+computation stops. If you set the threshold to zero, then you will force the algorithm to run for the full number of iterations specified in 'max_iter'.</dd>
+
+</dl>
+
+@anchor notes
+@par Notes
+
+The PageRank algorithm proposed by Larry Page and Sergey Brin is used [1].
+
+@anchor examples
+@examp
+
+-# Create vertex and edge tables to represent the graph:
+<pre class="syntax">
+DROP TABLE IF EXISTS vertex, edge;
+CREATE TABLE vertex(
+ id INTEGER
+ );
+CREATE TABLE edge(
+ src INTEGER,
+ dest INTEGER
+ );
+INSERT INTO vertex VALUES
+(0),
+(1),
+(2),
+(3),
+(4),
+(5),
+(6);
+INSERT INTO edge VALUES
+(0, 1),
+(0, 2),
+(0, 4),
+(1, 2),
+(1, 3),
+(2, 3),
+(2, 5),
+(2, 6),
+(3, 0),
+(4, 0),
+(5, 6),
+(6, 3);
+</pre>
+
+-# Compute the PageRank:
+<pre class="syntax">
+DROP TABLE IF EXISTS pagerank_out;
+SELECT madlib.pagerank(
+ 'vertex', -- Vertex table
+ 'id', -- Vertix id column
+ 'edge', -- Edge table
+ 'src=src, dest=dest', -- Comma delimted string of edge arguments
+ 'pagerank_out'); -- Output table of PageRank
+SELECT * FROM pagerank_out ORDER BY pagerank desc;
+</pre>
+<pre class="result">
+ id | pagerank
+----+--------------------
+ 0 | 0.278256122055856
+ 3 | 0.201882680839737
+ 2 | 0.142878491945534
+ 6 | 0.114538731993905
+ 1 | 0.100266150276761
+ 4 | 0.100266150276761
+ 5 | 0.061911672611445
+(7 rows)
+</pre>
+
+-# Run PageRank with a damping factor of 0.5 results in different final values:
+<pre class="syntax">
+DROP TABLE IF EXISTS pagerank_out;
+SELECT madlib.pagerank(
+ 'vertex', -- Vertex table
+ 'id', -- Vertix id column
+ 'edge', -- Edge table
+ 'src=src, dest=dest', -- Comma delimted string of edge arguments
+ 'pagerank_out', -- Output table of PageRank
+ 0.5); -- Damping factor
+SELECT * FROM pagerank_out ORDER BY pagerank desc;
+</pre>
+<pre class="result">
+ id | pagerank
+----+-------------------
+ 0 | 0.221378135793372
+ 3 | 0.191574922960784
+ 6 | 0.140994575864846
+ 2 | 0.135406336658892
+ 4 | 0.108324751971412
+ 1 | 0.108324751971412
+ 5 | 0.093996524779681
+(7 rows)
+</pre>
+
+@anchor literature
+@par Literature
+
+[1] PageRank algorithm. https://en.wikipedia.org/wiki/PageRank
+*/
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank(
+ vertex_table TEXT,
+ vertex_id TEXT,
+ edge_table TEXT,
+ edge_args TEXT,
+ out_table TEXT,
+ damping_factor FLOAT8,
+ max_iter INTEGER,
+ threshold FLOAT8
+) RETURNS VOID AS $$
+ PythonFunction(graph, pagerank, pagerank)
+$$ LANGUAGE plpythonu VOLATILE
+m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `MODIFIES SQL DATA', `');
+-------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank(
+ vertex_table TEXT,
+ vertex_id TEXT,
+ edge_table TEXT,
+ edge_args TEXT,
+ out_table TEXT,
+ damping_factor FLOAT8,
+ max_iter INTEGER
+) RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.pagerank($1, $2, $3, $4, $5, $6, $7, 0.00001)
+$$ LANGUAGE SQL
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
+-------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank(
+ vertex_table TEXT,
+ vertex_id TEXT,
+ edge_table TEXT,
+ edge_args TEXT,
+ out_table TEXT,
+ damping_factor FLOAT8
+) RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.pagerank($1, $2, $3, $4, $5, $6, 100, 0.00001)
+$$ LANGUAGE SQL
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
+-------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank(
+ vertex_table TEXT,
+ vertex_id TEXT,
+ edge_table TEXT,
+ edge_args TEXT,
+ out_table TEXT
+) RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.pagerank($1, $2, $3, $4, $5, 0.85, 100, 0.00001)
+$$ LANGUAGE SQL
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
+-------------------------------------------------------------------------
+
+-- Online help
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank(
+ message VARCHAR
+) RETURNS VARCHAR AS $$
+ PythonFunction(graph, pagerank, pagerank_help)
+$$ LANGUAGE plpythonu IMMUTABLE
+m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `');
+
+--------------------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.pagerank()
+RETURNS VARCHAR AS $$
+ SELECT MADLIB_SCHEMA.pagerank('');
+$$ LANGUAGE sql IMMUTABLE
+m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `');
+--------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/6b466ea6/src/ports/postgres/modules/graph/test/pagerank.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/test/pagerank.sql_in b/src/ports/postgres/modules/graph/test/pagerank.sql_in
new file mode 100644
index 0000000..1d695e2
--- /dev/null
+++ b/src/ports/postgres/modules/graph/test/pagerank.sql_in
@@ -0,0 +1,62 @@
+/* ----------------------------------------------------------------------- *//**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ *//* ----------------------------------------------------------------------- */
+
+DROP TABLE IF EXISTS vertex, edge, pagerank_out;
+CREATE TABLE vertex(
+ id INTEGER
+ );
+CREATE TABLE edge(
+ src INTEGER,
+ dest INTEGER
+ );
+INSERT INTO vertex VALUES
+(0),
+(1),
+(2),
+(3),
+(4),
+(5),
+(6);
+INSERT INTO edge VALUES
+(0, 1),
+(0, 2),
+(0, 4),
+(1, 2),
+(1, 3),
+(2, 3),
+(2, 5),
+(2, 6),
+(3, 0),
+(4, 0),
+(5, 6),
+(6, 3);
+
+SELECT madlib.pagerank(
+ 'vertex', -- Vertex table
+ 'id', -- Vertix id column
+ 'edge', -- Edge table
+ 'src=src, dest=dest', -- Edge args
+ 'pagerank_out'); -- Output table of PageRank
+
+-- View the PageRank of all vertices, sorted by their scores.
+SELECT assert(relative_error(SUM(pagerank), 1) < 0.00001,
+ 'PageRank: Scores do not sum up to 1.'
+ ) FROM pagerank_out;
[21/34] incubator-madlib git commit: MADLIB-1076. Review LICENSE file
and README.md
Posted by ok...@apache.org.
MADLIB-1076. Review LICENSE file and README.md
Closes #123
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/0d815f2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/0d815f2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/0d815f2b
Branch: refs/heads/latest_release
Commit: 0d815f2ba3b8421c32a9bfbd7b334285d83fa347
Parents: c8bfbf8
Author: Roman Shaposhnik <rv...@apache.org>
Authored: Thu Apr 20 11:02:43 2017 -0700
Committer: Roman Shaposhnik <rv...@apache.org>
Committed: Thu Apr 20 15:42:06 2017 -0700
----------------------------------------------------------------------
HAWQ_Install.txt | 2 +-
LICENSE | 444 +++++++++++++++++++++++++++++++++++++
README.md | 20 +-
RELEASE_NOTES | 2 +-
ReadMe_Build.txt | 17 +-
deploy/RPM/CMakeLists.txt | 2 +-
deploy/description.txt | 2 +-
doc/etc/developer.doxyfile.in | 2 +-
licenses/MADlib.txt | 11 +-
src/CMakeLists.txt | 8 +-
src/patch/PyXB.sh | 3 +-
11 files changed, 484 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/HAWQ_Install.txt
----------------------------------------------------------------------
diff --git a/HAWQ_Install.txt b/HAWQ_Install.txt
index e933624..8cef22e 100644
--- a/HAWQ_Install.txt
+++ b/HAWQ_Install.txt
@@ -5,7 +5,7 @@ MADlib is a library of statistics and machine learning functions that can be
installed in HAWQ. MADlib is installed separately from the main HAWQ
installation. For a description of the general MADlib installation process,
refer to the MADlib installation guide for PostgreSQL and GPDB:
-https://github.com/madlib/madlib/wiki/Installation-Guide
+https://cwiki.apache.org/confluence/display/MADLIB/Installation+Guide
An installation script, hawq_install.sh, installs the MADlib RPM distribution on
the HAWQ master and segment nodes. It installs the MADlib files but does not
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 2fd044f..ffb7ae3 100644
--- a/LICENSE
+++ b/LICENSE
@@ -341,3 +341,447 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***********************************************************************
+
+The binary distribution of MADlib statically links and otherwise ships
+code available under the following licenses (either directly compatible
+with Apache License version 2, or explicitly approved by Apache Software
+Foundation to be compatible with inclusion in a binary form within an
+Apache product if the inclusion is appropriately labeled):
+
+----------------------------------------------------------------------------
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+----------------------------------------------------------------------------
+Eigen
+
+From the Eigen Licensing page
+(http://eigen.tuxfamily.org/index.php?title=Main_Page)
+
+Eigen is Free Software. Starting from the 3.1.1 version, it is licensed
+under the MPL2, which is a simple weak copyleft license. Common
+questions about the MPL2 are answered in the official MPL2 FAQ
+(http://www.mozilla.org/MPL/2.0/FAQ.html).
+
+Note that currently, a few features rely on third-party code licensed
+under the LGPL: SimplicialCholesky, AMD ordering, and constrained_cg.
+Such features can be explicitly disabled by compiling with the
+EIGEN_MPL2_ONLY preprocessor symbol defined.
+
+Virtually any software may use Eigen. For example, closed-source
+software may use Eigen without having to disclose its own source code.
+Many proprietary and closed-source software projects are using Eigen
+right now, as well as many BSD-licensed projects.
+
+------
+
+Mozilla Public License
+Version 2.0
+
+1. Definitions
+
+1.1. “Contributor”
+means each individual or legal entity that creates, contributes to the
+creation of, or owns Covered Software.
+
+1.2. “Contributor Version”
+means the combination of the Contributions of others (if any) used by
+a Contributor and that particular Contributor’s Contribution.
+
+1.3. “Contribution”
+means Covered Software of a particular Contributor.
+
+1.4. “Covered Software”
+means Source Code Form to which the initial Contributor has attached
+the notice in Exhibit A, the Executable Form of such Source Code Form,
+and Modifications of such Source Code Form, in each case including
+portions thereof.
+
+1.5. “Incompatible With Secondary Licenses”
+means
+
+• that the initial Contributor has attached the notice described in
+Exhibit B to the Covered Software; or
+
+• that the Covered Software was made available under the terms of
+version 1.1 or earlier of the License, but not also under the terms of
+a Secondary License.
+
+1.6. “Executable Form”
+means any form of the work other than Source Code Form.
+
+1.7. “Larger Work”
+means a work that combines Covered Software with other material, in a
+separate file or files, that is not Covered Software.
+
+1.8. “License”
+means this document.
+
+1.9. “Licensable”
+means having the right to grant, to the maximum extent possible,
+whether at the time of the initial grant or subsequently, any and all
+of the rights conveyed by this License.
+
+1.10. “Modifications”
+means any of the following:
+
+• any file in Source Code Form that results from an addition to,
+deletion from, or modification of the contents of Covered Software; or
+
+• any new file in Source Code Form that contains any Covered Software.
+
+1.11. “Patent Claims” of a Contributor
+means any patent claim(s), including without limitation, method,
+process, and apparatus claims, in any patent Licensable by such
+Contributor that would be infringed, but for the grant of the License,
+by the making, using, selling, offering for sale, having made, import,
+or transfer of either its Contributions or its Contributor Version.
+
+1.12. “Secondary License”
+means either the GNU General Public License, Version 2.0, the GNU
+Lesser General Public License, Version 2.1, the GNU Affero General
+Public License, Version 3.0, or any later versions of those licenses.
+
+1.13. “Source Code Form”
+means the form of the work preferred for making modifications.
+
+1.14. “You” (or “Your”)
+means an individual or a legal entity exercising rights under this
+License. For legal entities, “You” includes any entity that
+controls, is controlled by, or is under common control with You. For
+purposes of this definition, “control” means (a) the power, direct or
+indirect, to cause the direction or management of such entity, whether
+by contract or otherwise, or (b) ownership of more than fifty percent
+(50%) of the outstanding shares or beneficial ownership of such entity.
+
+2. License Grants and Conditions
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+• under intellectual property rights (other than patent or trademark)
+Licensable by such Contributor to use, reproduce, make available,
+modify, display, perform, distribute, and otherwise exploit its
+Contributions, either on an unmodified basis, with Modifications, or as
+part of a Larger Work; and
+
+• under Patent Claims of such Contributor to make, use, sell, offer
+for sale, have made, import, and otherwise transfer either its
+Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor
+first distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted
+under this License. No additional rights or licenses will be implied
+from the distribution or licensing of Covered Software under this
+License. Notwithstanding Section 2.1(b) above, no patent license is
+granted by a Contributor:
+
+• for any code that a Contributor has removed from Covered Software;
+or
+
+• for infringements caused by: (i) Your and any other third party’s
+modifications of Covered Software, or (ii) the combination of its
+Contributions with other software (except as part of its Contributor
+Version); or
+
+• under Patent Claims infringed by Covered Software in the absence of
+its Contributions.
+
+This License does not grant any rights in the trademarks, service
+marks, or logos of any Contributor (except as may be necessary to
+comply with the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License
+(if permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients’ rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+• such Covered Software must also be made available in Source Code
+Form, as described in Section 3.1, and You must inform recipients of
+the Executable Form how they can obtain a copy of such Source Code Form
+by reasonable means in a timely manner, at a charge no more than the
+cost of distribution to the recipient; and
+
+• You may distribute such Executable Form under the terms of this
+License, or sublicense it under different terms, provided that the
+license for the Executable Form does not attempt to limit or alter the
+recipients’ rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and
+the Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered
+by You alone, and You hereby agree to indemnify every Contributor for
+any liability incurred by such Contributor as a result of warranty,
+support, indemnity or liability terms You offer. You may include
+additional disclaimers of warranty and limitations of liability
+specific to any jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description
+must be placed in a text file included with all distributions of the
+Covered Software under this License. Except to the extent prohibited
+by statute or regulation, such description must be sufficiently
+detailed for a recipient of ordinary skill to be able to understand it.
+
+5. Termination
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a
+particular Contributor are reinstated (a) provisionally, unless and
+until such Contributor explicitly and finally terminates Your grants,
+and (b) on an ongoing basis, if such Contributor fails to notify You
+of the non-compliance by some reasonable means prior to 60 days after
+You have come back into compliance. Moreover, Your grants from a
+particular Contributor are reinstated on an ongoing basis if such
+Contributor notifies You of the non-compliance by some reasonable
+means, this is the first time You have received notice of
+non-compliance with this License from such Contributor, and You become
+compliant prior to 30 days after Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a
+patent infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers)
+which have been validly granted by You or Your distributors under this
+License prior to termination shall survive termination.
+
+6. Disclaimer of Warranty
+
+Covered Software is provided under this License on an “as is”
+basis, without warranty of any kind, either expressed, implied, or
+statutory, including, without limitation, warranties that the Covered
+Software is free of defects, merchantable, fit for a particular purpose
+or non-infringing. The entire risk as to the quality and performance of
+the Covered Software is with You. Should any Covered Software prove
+defective in any respect, You (not any Contributor) assume the cost of
+any necessary servicing, repair, or correction. This disclaimer of
+warranty constitutes an essential part of this License. No use of any
+Covered Software is authorized under this License except under this
+disclaimer.
+
+7. Limitation of Liability
+
+Under no circumstances and under no legal theory, whether tort
+(including negligence), contract, or otherwise, shall any Contributor,
+or anyone who distributes Covered Software as permitted above, be
+liable to You for any direct, indirect, special, incidental, or
+consequential damages of any character including, without limitation,
+damages for lost profits, loss of goodwill, work stoppage, computer
+failure or malfunction, or any and all other commercial damages or
+losses, even if such party shall have been informed of the possibility
+of such damages. This limitation of liability shall not apply to
+liability for death or personal injury resulting from such party’s
+negligence to the extent applicable law prohibits such limitation. Some
+jurisdictions do not allow the exclusion or limitation of incidental or
+consequential damages, so this exclusion and limitation may not apply
+to You.
+
+8. Litigation
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party’s ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in
+Section 10.3, no one other than the license steward has the right to
+modify or publish new versions of this License. Each version will be
+given a distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered
+Software, or under the terms of any subsequent version published by the
+license steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+
+This Source Code Form is subject to the terms of the Mozilla Public
+License, v. 2.0. If a copy of the MPL was not distributed with this
+file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - “Incompatible With Secondary Licenses” Notice
+
+This Source Code Form is “Incompatible With Secondary Licenses”, as
+defined by the Mozilla Public License, v. 2.0.
+
+MADlib builds Eigen using the EIGEN_MPL2_ONLY flag enabled.
+----------------------------------------------------------------------------
+std::ctype<char>::_M_widen_init() is a function authored by Jerry Quinn
+<jl...@optonline.net>, which was added to libstdc++ with revision 74662 on
+Dec 16, 2003 [1].
+
+With permission from Jerry (thankfully received on Oct 9, 2012), we include a
+copy of this function in the MADlib repository. The sole intention is to allow
+compiling MADlib with recent versions of gcc while still keeping the runtime
+dependencies limited to earlier versions of libstdc++. Technical details are
+given in src/utils/libstdcxx-compatibility.cpp.
+
+Revision 74662 of the libstdc++-v3 file include/bits/locale_facets.h, where
+std::ctype<char>::_M_widen_init() has been copied from, also included the
+following notice in the file header [2]:
+
+// As a special exception, you may use this file as part of a free software
+// library without restriction. [...]
+
+Links:
+[1] http://gcc.gnu.org/viewcvs?diff_format=h&view=revision&revision=74662
+[2] http://gcc.gnu.org/viewcvs/trunk/libstdc%2B%2B-v3/include/bits/locale_facets.h?diff_format=h&view=markup&pathrev=74662
+----------------------------------------------------------------------------
+argparse is (c) 2006-2009 Steven J. Bethard <st...@gmail.com>.
+
+The argparse module was contributed to Python as of Python 2.7 and thus
+was licensed under the Python license. Same license applies to all files in
+the argparse package project.
+
+For details about the Python License, please see Python_License_v2.7.1.txt.
+
+History
+-------
+
+Before (and including) argparse 1.1, the argparse package was licensed under
+Apache License v2.0.
+
+After argparse 1.1, all project files from the argparse project were deleted
+due to license compatibility issues between Apache License 2.0 and GNU GPL v2.
+
+The project repository then had a clean start with some files taken from
+Python 2.7.1, so definitely all files are under Python License now.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index a54ba5e..ecbb345 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-  
+  
=================================================
**MADlib<sup>®</sup>** is an open-source library for scalable in-database analytics.
It provides data-parallel implementations of mathematical, statistical and
@@ -63,6 +63,8 @@ docker kill madlib
docker rm madlib
```
+Detailed build instructions are available in [`ReadMe_Build.txt`](ReadMe_Build.txt)
+
User and Developer Documentation
==================================
The latest documentation of MADlib modules can be found at [`MADlib
@@ -75,7 +77,7 @@ The following block-diagram gives a high-level overview of MADlib's
architecture.
-
+
Third Party Components
@@ -83,12 +85,12 @@ Third Party Components
MADlib incorporates software from the following third-party components. Bundled with source code:
1. [`libstemmer`](http://snowballstem.org/) "small string processing language"
-2. [`m_widen_init`](https://github.com/apache/incubator-madlib/blob/master/licenses/third_party/_M_widen_init.txt) "allows compilation with recent versions of gcc with runtime dependencies from earlier versions of libstdc++"
+2. [`m_widen_init`](licenses/third_party/_M_widen_init.txt) "allows compilation with recent versions of gcc with runtime dependencies from earlier versions of libstdc++"
3. [`argparse 1.2.1`](http://code.google.com/p/argparse/) "provides an easy, declarative interface for creating command line tools"
4. [`PyYAML 3.10`](http://pyyaml.org/wiki/PyYAML) "YAML parser and emitter for Python"
5. [`UseLATEX.cmake`](https://github.com/kmorel/UseLATEX/blob/master/UseLATEX.cmake) "CMAKE commands to use the LaTeX compiler"
-Downloaded at build time:
+Downloaded at build time (or supplied as build dependencies):
6. [`Boost 1.61.0 (or newer)`](http://www.boost.org/) "provides peer-reviewed portable C++ source libraries"
7. [`PyXB 1.2.4`](http://pyxb.sourceforge.net/) "Python library for XML Schema Bindings"
@@ -96,13 +98,17 @@ Downloaded at build time:
Licensing
==========
-License information regarding MADlib and included third-party libraries can be
-found inside the [`license`](https://github.com/apache/incubator-madlib/blob/master/licenses) directory.
+Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the [`NOTICE`](NOTICE) file distributed with this work for additional information regarding copyright ownership. The ASF licenses this project to You under the Apache License, Version 2.0 (the "License"); you may not use this project except in compliance with the License. You may obtain a copy of the License at [`LICENSE`](LICENSE).
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+
+As specified in [`LICENSE`](LICENSE) additional license information regarding included third-party libraries can be
+found inside the [`licenses`](licenses) directory.
Release Notes
=============
Changes between MADlib versions are described in the
-[`ReleaseNotes.txt`](https://github.com/apache/incubator-madlib/blob/master/RELEASE_NOTES) file.
+[`ReleaseNotes.txt`](RELEASE_NOTES) file.
Papers and Talks
=================
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/RELEASE_NOTES
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 29d850c..78f3d10 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -5,7 +5,7 @@ These release notes contain the significant changes in each MADlib release,
with most recent versions listed at the top.
A complete list of changes for each release can be obtained by viewing the git
-commit history located at https://github.com/madlib/madlib/commits/master.
+commit history located at https://github.com/apache/incubator-madlib/commits/master.
Current list of bugs and issues can be found at https://issues.apache.org/jira/browse/MADLIB.
—-------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/ReadMe_Build.txt
----------------------------------------------------------------------
diff --git a/ReadMe_Build.txt b/ReadMe_Build.txt
index 4e21c82..ccb6207 100644
--- a/ReadMe_Build.txt
+++ b/ReadMe_Build.txt
@@ -17,7 +17,8 @@ Building and Installing from Source
- CMake >= 2.8.4
- Internet connection to automatically download MADlib's dependencies if needed
- (Boost, Eigen). See configuration options below.
+ (Boost, Eigen). You can avoid this and build MADlib in a networkless mode
+ by providing tarballs of 3 external dependencies. See configuration options below.
Optional:
@@ -40,6 +41,12 @@ Optional:
+ Greenplum 4.2, 4.3
+ All requirements for generating user-level documentation (see above)
+** Build-time Debian package dependencies (optional read):
+-------------------------------------------
+
+On Debian based platform you can install the required dependencies (aside from
+Boost, Eigen and PyXB) by running the following command:
+ apt-get install cmake g++ m4 python flex bison doxygen graphviz postgresql-server-dev-all texlive-full poppler-utils
** Build instructions (required read):
--------------------------------------
@@ -158,10 +165,16 @@ root directory) for more options, after having run `cmake` the first time.
- `EIGEN_TAR_SOURCE` (default: *empty*)
- Eigen is downloaded automatically, unless the you call `./configure`
+ Eigen is downloaded automatically, unless you call `./configure`
with `-DEIGEN_TAR_SOURCE=/path/to/eigen_x.tar.gz`, in which case
this tarball is used.
+- `PYXB_TAR_SOURCE` (default: *empty*)
+
+ PyXB is downloaded automatically, unless you call `./configure`
+ with `-DPYXB_TAR_SOURCE=/path/to/pyxb_x.tar.gz`, in which case
+ this tarball is used.
+
Debugging
=========
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/deploy/RPM/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/deploy/RPM/CMakeLists.txt b/deploy/RPM/CMakeLists.txt
index 8bf1675..da31da2 100644
--- a/deploy/RPM/CMakeLists.txt
+++ b/deploy/RPM/CMakeLists.txt
@@ -12,7 +12,7 @@ rh_version(RH_VERSION)
# -- Set RPM-specific variables ------------------------------------------------
set(CPACK_RPM_PACKAGE_ARCHITECTURE x86_64 PARENT_SCOPE)
-set(CPACK_RPM_PACKAGE_LICENSE "New BSD License" PARENT_SCOPE)
+set(CPACK_RPM_PACKAGE_LICENSE "ASL 2.0" PARENT_SCOPE)
set(CPACK_RPM_PACKAGE_GROUP "Development/Libraries" PARENT_SCOPE)
set(CPACK_PACKAGING_INSTALL_PREFIX "/usr/local/madlib/Versions/${MADLIB_VERSION_STRING}" PARENT_SCOPE)
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/deploy/description.txt
----------------------------------------------------------------------
diff --git a/deploy/description.txt b/deploy/description.txt
index c3b16cd..77175ac 100644
--- a/deploy/description.txt
+++ b/deploy/description.txt
@@ -7,4 +7,4 @@ analytic skills, by harnessing efforts from commercial practice,
academic research, and open-source development.
To more information, please see the MADlib wiki at
-https://github.com/madlib/madlib/wiki
+https://cwiki.apache.org/confluence/display/MADLIB
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/doc/etc/developer.doxyfile.in
----------------------------------------------------------------------
diff --git a/doc/etc/developer.doxyfile.in b/doc/etc/developer.doxyfile.in
index 02558c9..0f6ab3b 100644
--- a/doc/etc/developer.doxyfile.in
+++ b/doc/etc/developer.doxyfile.in
@@ -840,7 +840,7 @@ FILTER_SOURCE_PATTERNS =
# (index.html). This can be useful if you have a project on for instance GitHub
# and want reuse the introduction page also for the doxygen output.
-USE_MDFILE_AS_MAINPAGE = "https://github.com/madlib/madlib/blob/master/README.md"
+USE_MDFILE_AS_MAINPAGE = "https://github.com/apache/incubator-madlib/blob/master/README.md"
#---------------------------------------------------------------------------
# configuration options related to source browsing
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/licenses/MADlib.txt
----------------------------------------------------------------------
diff --git a/licenses/MADlib.txt b/licenses/MADlib.txt
deleted file mode 100644
index 9027809..0000000
--- a/licenses/MADlib.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-Portions of this software Copyright (c) 2010-2013 by EMC Corporation. All rights reserved.
-Portions of this software Copyright (c) 2010-2013 by Regents of the University of California. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
-- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/licenses/MADlib.txt b/licenses/MADlib.txt
new file mode 120000
index 0000000..ea5b606
--- /dev/null
+++ b/licenses/MADlib.txt
@@ -0,0 +1 @@
+../LICENSE
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ca7961b..c8e0e2e 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -18,10 +18,10 @@ set(BITBUCKET_BASE_URL
"${MADLIB_REDIRECT_PREFIX}https://bitbucket.org"
CACHE STRING
"Base URL for Bitbucket projects. May be overridden for testing purposes.")
-set(GITHUB_MADLIB_BASE_URL
- "${MADLIB_REDIRECT_PREFIX}https://github.com/madlib"
+set(EIGEN_BASE_URL
+ "${MADLIB_REDIRECT_PREFIX}https://github.com/madlib/eigen/archive"
CACHE STRING
- "Base URL for MADlib Github projects. May be overridden for testing purposes.")
+ "Base URL for Eigen projects. May be overridden for testing purposes.")
# Boost might not be present on the system (or simply too old). In this case, we
# download the following version (unless it is already present in
@@ -52,7 +52,7 @@ endif (NOT BOOST_TAR_SOURCE)
# -DEIGEN_TAR_SOURCE=/path/to/eigen-x.x.x.tar.gz
set(EIGEN_VERSION "branches/3.2")
-set(EIGEN_URL "${GITHUB_MADLIB_BASE_URL}/eigen/archive/${EIGEN_VERSION}.tar.gz")
+set(EIGEN_URL "${EIGEN_BASE_URL}/${EIGEN_VERSION}.tar.gz")
set(EIGEN_TAR_MD5 13bc1043270d8f4397339c0cb0b62938)
set(EIGEN_MPL2_ONLY TRUE)
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0d815f2b/src/patch/PyXB.sh
----------------------------------------------------------------------
diff --git a/src/patch/PyXB.sh b/src/patch/PyXB.sh
index 408ba92..ed2aacb 100755
--- a/src/patch/PyXB.sh
+++ b/src/patch/PyXB.sh
@@ -223,5 +223,6 @@ patch -N -p1 <<'EOF'
EOF
echo "PyXB: Removing GPL component from code base"
+echo "PyXB: see https://github.com/pabigot/pyxb/issues/77 for details"
rm -f doc/extapi.py
-rm -f doc/extapi.pyc
\ No newline at end of file
+rm -f doc/extapi.pyc
[16/34] incubator-madlib git commit: DTree: Update defaults for
max_depth, num_splits
Posted by ok...@apache.org.
DTree: Update defaults for max_depth, num_splits
Reduce the defaults for max_depth to 7 and num_splits to 20 to decrease
the chances of running out of memory when initializing tree for problems
with many features or with features with many categorical values.
Closes #117
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/3eec0a82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/3eec0a82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/3eec0a82
Branch: refs/heads/latest_release
Commit: 3eec0a82ee522101264c6557457602f9e0dbee52
Parents: 8faf622
Author: Rahul Iyer <ri...@apache.org>
Authored: Tue Apr 18 11:53:36 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Tue Apr 18 17:19:54 2017 -0700
----------------------------------------------------------------------
.../recursive_partitioning/decision_tree.py_in | 23 ++++----
.../recursive_partitioning/decision_tree.sql_in | 59 +++++++++++---------
.../test/decision_tree.sql_in | 2 +-
3 files changed, 46 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/3eec0a82/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
index fb18278..f7c4bd8 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
@@ -223,7 +223,7 @@ SELECT {schema_madlib}.tree_train(
is NULL
'weights', -- A Column name containing weights for
each observation. Default is NULL
- max_depth, -- Maximum depth of any node, default is 10
+ max_depth, -- Maximum depth of any node, default is 7
min_split, -- Minimum number of observations that must
exist in a node for a split to be
attemped, default is 20
@@ -231,7 +231,7 @@ SELECT {schema_madlib}.tree_train(
terminal node, default is min_split/3
n_bins, -- Number of bins to find possible node
split threshold values for continuous
- variables, default is 100 (Must be greater than 1)
+ variables, default is 20 (Must be greater than 1)
pruning_params, -- A comma-separated text containing
key=value pairs of parameters for pruning.
Parameters accepted:
@@ -341,7 +341,6 @@ def _extract_pruning_params(pruning_params_str):
@param pruning_param: str, Parameters used for pruning the tree
cp = Cost-complexity for pruning
n_folds = Number of folds for cross-validation
-
Returns:
dict. A dictionary containing the pruning parameters
"""
@@ -567,17 +566,21 @@ def tree_train(schema_madlib, training_table_name, output_table_name,
"""
msg_level = "notice" if verbose_mode else "warning"
- # Set default values for optional arguments
- min_split = 20 if (min_split is None and min_bucket is None) else min_split
- min_bucket = min_split // 3 if min_bucket is None else min_bucket
- min_split = min_bucket * 3 if min_split is None else min_split
- n_bins = 100 if n_bins is None else n_bins
+ # Set default values for all arguments
split_criterion = 'gini' if not split_criterion else split_criterion
- plpy.notice("split_criterion:" + split_criterion)
+ max_depth = 7 if max_depth is None else max_depth
+ if min_split is None and min_bucket is None:
+ min_split = 20
+ min_bucket = 6
+ else:
+ min_bucket = min_split // 3 if min_bucket is None else min_bucket
+ min_split = min_bucket * 3 if min_split is None else min_split
+ n_bins = 20 if n_bins is None else n_bins
+
+ # defaults for cp and n_folds set within _extract_pruning_params
pruning_param_dict = _extract_pruning_params(pruning_params)
cp = pruning_param_dict['cp']
n_folds = pruning_param_dict['n_folds']
-
surrogate_param_dict = extract_keyvalue_params(surrogate_params,
dict(max_surrogates=int),
dict(max_surrogates=0))
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/3eec0a82/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
index 97e8471..ef671fc 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
@@ -258,7 +258,7 @@ tree_train(
<DD>TEXT. Column name containing weights for each observation.</DD>
<DT>max_depth (optional)</DT>
- <DD>INTEGER, default: 10. Maximum depth of any node of the final tree,
+ <DD>INTEGER, default: 7. Maximum depth of any node of the final tree,
with the root node counted as depth 0.</DD>
<DT>min_split (optional)</DT>
@@ -272,7 +272,7 @@ tree_train(
set to min_bucket*3 or min_bucket to min_split/3, as appropriate.</DD>
<DT>num_splits (optional)</DT>
- <DD>INTEGER, default: 100. Continuous-valued features are binned into
+ <DD>INTEGER, default: 20. Continuous-valued features are binned into
discrete quantiles to compute split boundaries. This global parameter
is used to compute the resolution of splits for continuous features.
Higher number of bins will lead to better prediction,
@@ -920,7 +920,7 @@ File decision_tree.sql_in documenting the training function
* multiple decision trees, one for each group.
* @param weights OPTIONAL. Column name containing weights for
* each observation.
- * @param max_depth OPTIONAL (Default = 10). Set the maximum depth
+ * @param max_depth OPTIONAL (Default = 7). Set the maximum depth
* of any node of the final tree, with the root node counted
* as depth 0.
* @param min_split OPTIONAL (Default = 20). Minimum number of
@@ -931,13 +931,13 @@ File decision_tree.sql_in documenting the training function
* one of minbucket or minsplit is specified, minsplit
* is set to minbucket*3 or minbucket to minsplit/3, as
* appropriate.
- * @param n_bins optional (default = 100) number of bins to use
+ * @param n_bins optional (default = 20) number of bins to use
* during binning. continuous-valued features are binned
* into discrete bins (per the quartile values) to compute
* split bound- aries. this global parameter is used to
* compute the resolution of the bins. higher number of
* bins will lead to higher processing time.
- * @param pruning_params (default = 'cp=0.01') pruning parameter string
+ * @param pruning_params (default: cp=0) pruning parameter string
* containing key-value pairs.
* the keys can be:
* cp (default = 0.01) a complexity parameter
@@ -1574,8 +1574,10 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
pruning_params TEXT,
surrogate_params TEXT
) RETURNS VOID AS $$
+ -- verbose = false
SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
$11, $12, $13, $14, $15, FALSE);
+
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1596,7 +1598,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
pruning_params TEXT
) RETURNS VOID AS $$
SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
- $11, $12, $13, $14, 'max_surrogates=0', FALSE);
+ $11, $12, $13, $14, NULL::text, FALSE);
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1616,8 +1618,8 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
n_bins INTEGER
) RETURNS VOID AS $$
SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
- $11, $12, $13, 'cp=0.01'::TEXT,
- 'max_surrogates=0', FALSE::BOOLEAN);
+ $11, $12, $13, NULL::TEXT,
+ NULL::TEXT, FALSE::BOOLEAN);
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1635,8 +1637,9 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
min_split INTEGER,
min_bucket INTEGER
) RETURNS VOID AS $$
- SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
- 100::INTEGER, 'cp=0.01'::TEXT, 'max_surrogates=0', FALSE::BOOLEAN);
+ SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
+ $11, $12, NULL::INTEGER, NULL::TEXT,
+ NULL::TEXT, FALSE::BOOLEAN);
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1654,8 +1657,8 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
min_split INTEGER
) RETURNS VOID AS $$
SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11,
- ($11/3)::INTEGER, 100::INTEGER, 'cp=0.01'::TEXT, 'max_surrogates=0',
- FALSE::BOOLEAN);
+ NULL::INTEGER, NULL::INTEGER, NULL::TEXT,
+ NULL::TEXT, FALSE::BOOLEAN);
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1672,8 +1675,8 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
max_depth INTEGER
) RETURNS VOID AS $$
SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
- 20::INTEGER, 6::INTEGER, 100::INTEGER, 'cp=0.01'::TEXT,
- 'max_surrogates=0', FALSE::BOOLEAN);
+ NULL::INTEGER, NULL::INTEGER, NULL::INTEGER,
+ NULL::TEXT, NULL::TEXT, FALSE::BOOLEAN);
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1689,8 +1692,9 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
weights TEXT
) RETURNS VOID AS $$
SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6, $7, $8, $9,
- 10::INTEGER, 20::INTEGER, 6::INTEGER, 100::INTEGER,
- 'cp=0.01'::TEXT, 'max_surrogates=0', FALSE::BOOLEAN);
+ NULL::INTEGER, NULL::INTEGER, NULL::INTEGER,
+ NULL::INTEGER, NULL::TEXT, NULL::TEXT,
+ FALSE::BOOLEAN);
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1705,8 +1709,9 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
grouping_cols TEXT
) RETURNS VOID AS $$
SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6, $7, $8,
- NULL::TEXT, 10::INTEGER, 20::INTEGER, 6::INTEGER, 100::INTEGER,
- 'cp=0.01'::TEXT, 'max_surrogates=0', FALSE::BOOLEAN);
+ NULL::TEXT, NULL::INTEGER, NULL::INTEGER,
+ NULL::INTEGER, NULL::INTEGER, NULL::TEXT, NULL::TEXT,
+ FALSE::BOOLEAN);
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1720,9 +1725,9 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
split_criterion TEXT
) RETURNS VOID AS $$
SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6, $7,
- NULL::TEXT, NULL::TEXT, 10::INTEGER, 20::INTEGER,
- 6::INTEGER, 100::INTEGER, 'cp=0.01'::TEXT,
- 'max_surrogates=0', FALSE::BOOLEAN);
+ NULL::TEXT, NULL::TEXT, NULL::INTEGER, NULL::INTEGER,
+ NULL::INTEGER, NULL::INTEGER, NULL::TEXT,
+ NULL::TEXT, FALSE::BOOLEAN);
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1735,9 +1740,9 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
list_of_features_to_exclude TEXT
) RETURNS VOID AS $$
SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5, $6,
- 'gini'::TEXT, NULL::TEXT, NULL::TEXT, 10::INTEGER,
- 20::INTEGER, 6::INTEGER, 100::INTEGER, 'cp=0.01'::TEXT,
- 'max_surrogates=0', FALSE::BOOLEAN);
+ NULL::TEXT, NULL::TEXT, NULL::TEXT, NULL::INTEGER,
+ NULL::INTEGER, NULL::INTEGER, NULL::INTEGER, NULL::TEXT,
+ NULL::TEXT, FALSE::BOOLEAN);
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
@@ -1749,9 +1754,9 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.tree_train(
list_of_features TEXT
) RETURNS VOID AS $$
SELECT MADLIB_SCHEMA.tree_train($1, $2, $3, $4, $5,
- NULL::TEXT, 'gini'::TEXT, NULL::TEXT, NULL::TEXT,
- 10::INTEGER, 20::INTEGER, 6::INTEGER, 100::INTEGER,
- 'cp=0.01'::TEXT, 'max_surrogates=0', FALSE::BOOLEAN);
+ NULL::TEXT, NULL::TEXT, NULL::TEXT, NULL::TEXT,
+ NULL::INTEGER, NULL::INTEGER, NULL::INTEGER, NULL::INTEGER,
+ NULL::TEXT, NULL::text, FALSE::BOOLEAN);
$$ LANGUAGE SQL VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-- -------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/3eec0a82/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
index 1863b64..28a4647 100644
--- a/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
@@ -325,7 +325,7 @@ SELECT tree_train('dt_golf'::text, -- source table
'mse'::text, -- split criterion
NULL::text, -- no grouping
NULL::text, -- no weights
- 10::integer, -- max depth
+ NULL::integer, -- max depth
6::integer, -- min split
2::integer, -- min bucket
8::integer, -- number of bins per continuous variable
[28/34] incubator-madlib git commit: Release 1.11: Upgrade related
changes
Posted by ok...@apache.org.
Release 1.11: Upgrade related changes
Additional Author: Orhan Kislal <ok...@pivotal.io>
Updates the changelists and other related files for upgrade.
Note that upgrade is not supported from versions prior to 1.9.
Closes #121
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/648b0579
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/648b0579
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/648b0579
Branch: refs/heads/latest_release
Commit: 648b05798826956e9621027447af501c194392b8
Parents: c4fd91e
Author: Nandish Jayaram <nj...@apache.org>
Authored: Fri Apr 28 14:55:08 2017 -0700
Committer: Orhan Kislal <ok...@pivotal.io>
Committed: Fri Apr 28 14:55:28 2017 -0700
----------------------------------------------------------------------
deploy/gppkg/CMakeLists.txt | 2 +-
doc/mainpage.dox.in | 1 +
pom.xml | 2 +-
src/config/Version.yml | 2 +-
src/madpack/changelist.yaml | 95 +--
src/madpack/changelist_1.8_1.10.yaml | 857 ----------------------------
src/madpack/changelist_1.9.1_1.11.yaml | 137 +++++
src/madpack/changelist_1.9_1.10.yaml | 175 ------
src/madpack/changelist_1.9_1.11.yaml | 175 ++++++
src/madpack/diff_udf.sql | 2 +-
src/madpack/diff_udt.sql | 2 +-
src/madpack/madpack.py | 4 +-
src/madpack/upgrade_util.py | 12 +-
13 files changed, 337 insertions(+), 1129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/deploy/gppkg/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/deploy/gppkg/CMakeLists.txt b/deploy/gppkg/CMakeLists.txt
index 268d926..fb2ea14 100644
--- a/deploy/gppkg/CMakeLists.txt
+++ b/deploy/gppkg/CMakeLists.txt
@@ -2,7 +2,7 @@
# Packaging for Greenplum's gppkg
# ------------------------------------------------------------------------------
-set(MADLIB_GPPKG_VERSION "1.9.7")
+set(MADLIB_GPPKG_VERSION "1.9.8")
set(MADLIB_GPPKG_RELEASE_NUMBER 1)
set(MADLIB_GPPKG_RPM_SOURCE_DIR
"${CMAKE_BINARY_DIR}/_CPack_Packages/Linux/RPM/${CPACK_PACKAGE_FILE_NAME}"
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/doc/mainpage.dox.in
----------------------------------------------------------------------
diff --git a/doc/mainpage.dox.in b/doc/mainpage.dox.in
index 510ab1b..c94260b 100644
--- a/doc/mainpage.dox.in
+++ b/doc/mainpage.dox.in
@@ -17,6 +17,7 @@ Useful links:
<li><a href="https://mail-archives.apache.org/mod_mbox/incubator-madlib-user/">User mailing list</a></li>
<li><a href="https://mail-archives.apache.org/mod_mbox/incubator-madlib-dev/">Dev mailing list</a></li>
<li>User documentation for earlier releases:
+ <a href="../v1.10.0/index.html">v1.10.0</a>,
<a href="../v1.9.1/index.html">v1.9.1</a>,
<a href="../v1.9/index.html">v1.9</a>,
<a href="../v1.8/index.html">v1.8</a>,
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f033334..adffa8c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,7 @@
<groupId>org.apache.madlib</groupId>
<artifactId>madlib</artifactId>
- <version>1.11-dev</version>
+ <version>1.11</version>
<packaging>pom</packaging>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/src/config/Version.yml
----------------------------------------------------------------------
diff --git a/src/config/Version.yml b/src/config/Version.yml
index 097842c..3e4a7a8 100644
--- a/src/config/Version.yml
+++ b/src/config/Version.yml
@@ -1 +1 @@
-version: 1.11-dev
+version: 1.11
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/src/madpack/changelist.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist.yaml b/src/madpack/changelist.yaml
index 16e4144..6747ee0 100644
--- a/src/madpack/changelist.yaml
+++ b/src/madpack/changelist.yaml
@@ -1,4 +1,4 @@
-# Changelist for MADlib version 1.9 to 1.9.1
+# Changelist for MADlib version 1.10.0 to 1.11
# This file contains all changes that were introduced in a new version of
# MADlib. This changelist is used by the upgrade script to detect what objects
@@ -9,17 +9,11 @@
# file installed on the upgrade version. All other files (that don't have
# updates), are cleaned up to remove object replacements
new module:
- # ----------------- Changes from 1.9.1 to 1.0 ----------
- sssp:
- encode_categorical:
- knn:
+ # ----------------- Changes from 1.10.0 to 1.11 --------
+ pagerank:
# Changes in the types (UDT) including removal and modification
udt:
- # ----------------- Changes from 1.9.1 to 1.0 ----------
- _tree_result_type:
- _prune_result_type:
- kmeans_result:
- kmeans_state:
+
# List of the UDF changes that affect the user externally. This includes change
# in function name, return type, argument order or types, or removal of
@@ -28,80 +22,13 @@ udt:
# are user views dependent on this function, since the original function will
# not be present in the upgraded version.
udf:
- # ----------------- Changes from 1.9.1 to 1.0 ----------
- - _dt_apply:
- rettype: schema_madlib._tree_result_type
- argument: schema_madlib.bytea8, schema_madlib.bytea8, schema_madlib.bytea8, smallint, smallint, smallint, boolean, integer
- - _prune_and_cplist:
- rettype: schema_madlib._prune_result_type
- argument: schema_madlib.bytea8, double precision, boolean
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying, integer, double precision
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying, integer
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[]
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying, integer, double precision
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying, integer
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision, double precision
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer
- - internal_execute_using_kmeans_args:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, integer, double precision
-
+ # ----------------- Changes from 1.10.0 to 1.11 ----------
+ - __build_tree:
+ rettype: void
+ argument: boolean, text, text, text, text, text, boolean, character varying[], character varying[], character varying[], text, text, integer, integer, integer, integer, text, smallint, text, integer
+ - graph_sssp_get_path:
+ rettype: integer[]
+ argument: text, integer
# Changes to aggregates (UDA) including removal and modification
# Overloaded functions should be mentioned separately
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/src/madpack/changelist_1.8_1.10.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.8_1.10.yaml b/src/madpack/changelist_1.8_1.10.yaml
deleted file mode 100644
index d85877b..0000000
--- a/src/madpack/changelist_1.8_1.10.yaml
+++ /dev/null
@@ -1,857 +0,0 @@
-# ------------------------------------------------------------------------------
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# ------------------------------------------------------------------------------
-
-# Changelist for MADlib version 1.8 to 1.10
-
-# This file contains all changes that were introduced in a new version of
-# MADlib. This changelist is used by the upgrade script to detect what objects
-# should be upgraded (while retaining all other objects from the previous version)
-
-# New modules (actually .sql_in files) added in upgrade version
-# For these files the sql_in code is retained as is with the functions in the
-# file installed on the upgrade version. All other files (that don't have
-# updates), are cleaned up to remove object replacements
-new module:
- # ----------------- Changes from 1.9.1 to 1.0 ----------
- sssp:
- encode_categorical:
- knn:
-
-# Changes in the types (UDT) including removal and modification
-udt:
-
- # ----------------- Changes from 1.8 to 1.9 ----------
- __enc_tbl_result:
- __gen_acc_time:
- __rep_type:
- __train_result:
- c45_classify_result:
- c45_train_result:
- correlation_result:
- lsvm_sgd_model_rec:
- lsvm_sgd_result:
- rf_classify_result:
- rf_train_result:
- svm_cls_result:
- svm_model_pr:
- svm_model_rec:
- svm_nd_result:
- svm_reg_result:
- svm_support_vector:
- _prune_result_type:
- _tree_result_type:
- linear_svm_result:
- # ----------------- Changes from 1.9 to 1.9.1 ----------
- profile_result:
- # ----------------- Changes from 1.9.1 to 1.0 ----------
- _tree_result_type:
- _prune_result_type:
- kmeans_result:
- kmeans_state:
-
-# List of the UDF changes that affect the user externally. This includes change
-# in function name, return type, argument order or types, or removal of
-# the function. In each case, the original function is as good as removed and a
-# new function is created. In such cases, we should abort the upgrade if there
-# are user views dependent on this function, since the original function will
-# not be present in the upgraded version.
-udf:
-
- # ----------------- Changes from 1.8 to 1.9 ----------
- - _dt_apply:
- rettype: schema_madlib._tree_result_type
- argument: schema_madlib.bytea8,schema_madlib.bytea8,schema_madlib.bytea8,smallint,smallint,smallint,boolean,integer
-
- - internal_linear_svm_igd_result:
- rettype: schema_madlib.linear_svm_result
- argument: double precision[]
-
- - _prune_and_cplist:
- rettype: schema_madlib._prune_result_type
- argument: schema_madlib.bytea8,double precision,boolean
-
- - __array_elem_in:
- rettype: boolean[]
- argument: anyarray, anyarray
-
- - __array_indexed_agg_ffunc:
- rettype: double precision[]
- argument: double precision[]
-
- - __array_indexed_agg_prefunc:
- rettype: double precision[]
- argument: double precision[], double precision[]
-
- - __array_indexed_agg_sfunc:
- rettype: double precision[]
- argument: double precision[], double precision, bigint, bigint
-
- - __array_search:
- rettype: boolean
- argument: anyelement, anyarray
-
- - __array_sort:
- rettype: anyarray
- argument: anyarray
-
- - __assert:
- rettype: void
- argument: boolean, text
-
- - __assert_table:
- rettype: void
- argument: text, boolean
-
- - __best_scv_prefunc:
- rettype: double precision[]
- argument: double precision[], double precision[]
-
- - __best_scv_sfunc:
- rettype: double precision[]
- argument: double precision[], double precision[], integer, double precision
-
- - __bigint_array_add:
- rettype: bigint[]
- argument: bigint[], bigint[]
-
- - __breakup_table:
- rettype: void
- argument: text, text, text, text, text, text[], boolean[], integer, integer
-
- - __check_dt_common_params:
- rettype: void
- argument: text, text, text, text, text, text, text, text, integer, double precision, double precision, integer, text
-
- - __check_training_table:
- rettype: void
- argument: text, text[], text[], text, text, integer
-
- - __column_exists:
- rettype: boolean
- argument: text, text
-
- - __columns_in_table:
- rettype: boolean
- argument: text[], text
-
- - __create_metatable:
- rettype: void
- argument: text
-
- - __create_tree_tables:
- rettype: void
- argument: text
-
- - __csvstr_to_array:
- rettype: text[]
- argument: text
-
- - __display_node_sfunc:
- rettype: text
- argument: text, integer, boolean, text, text, double precision, double precision, text, integer
-
- - __display_tree_no_ordered_aggr:
- rettype: text
- argument: text, integer, integer, integer, boolean, double precision, text, integer, integer
-
- - __distinct_feature_value:
- rettype: integer
- argument: text, integer
-
- - __drop_metatable:
- rettype: void
- argument: text
-
- - __dt_acc_count_sfunc:
- rettype: bigint[]
- argument: bigint[], integer, bigint, integer
-
- - __dt_get_node_split_fids:
- rettype: integer[]
- argument: integer, integer, integer, integer[]
-
- - __ebp_calc_errors:
- rettype: double precision
- argument: double precision, double precision, double precision
-
- - __ebp_prune_tree:
- rettype: void
- argument: text
-
- - __encode_and_train:
- rettype: record
- argument: text, text, integer, integer, text, text, text, text, text, text, text, double precision, text, integer, double precision, boolean, double precision, double precision, text, integer
-
- - __encode_columns:
- rettype: void
- argument: text, text, integer, integer
-
- - __encode_table:
- rettype: void
- argument: text, text, text, integer, integer
-
- - __encode_table:
- rettype: void
- argument: text, text, text[], text, text[], text, text, integer, integer
-
- - __find_best_split:
- rettype: void
- argument: text, double precision, text, integer, integer, text, integer, integer
-
- - __format:
- rettype: text
- argument: text, text
-
- - __format:
- rettype: text
- argument: text, text, text
-
- - __format:
- rettype: text
- argument: text, text, text, text
-
- - __format:
- rettype: text
- argument: text, text, text, text, text
-
- - __format:
- rettype: text
- argument: text, text[]
-
- - __gen_acc:
- rettype: __gen_acc_time
- argument: text, text, text, text, text, integer, integer, boolean, integer
-
- - __gen_enc_meta_names:
- rettype: text[]
- argument: text, text
-
- - __gen_horizontal_encoded_table:
- rettype: void
- argument: text, text, integer, integer
-
- - __gen_vertical_encoded_table:
- rettype: void
- argument: text, text, text, boolean, integer
-
- - __generate_final_tree:
- rettype: void
- argument: text
-
- - __get_class_column_name:
- rettype: text
- argument: text
-
- - __get_class_value:
- rettype: text
- argument: integer, text
-
- - __get_classtable_name:
- rettype: text
- argument: text
-
- - __get_column_value:
- rettype: text
- argument: integer, integer, character, text
-
- - __get_feature_name:
- rettype: text
- argument: integer, text
-
- - __get_feature_value:
- rettype: text
- argument: integer, integer, text
-
- - __get_features_of_nodes:
- rettype: text
- argument: text, text, integer, integer, integer
-
- - __get_id_column_name:
- rettype: text
- argument: text
-
- - __get_schema_name:
- rettype: text
- argument: text
-
- - __get_table_name:
- rettype: text
- argument: text
-
- - __insert_into_metatable:
- rettype: void
- argument: text, integer, text, character, boolean, text, integer
-
- - __is_valid_enc_table:
- rettype: boolean
- argument: text
-
- - __num_of_class:
- rettype: integer
- argument: text
-
- - __num_of_columns:
- rettype: integer
- argument: text
-
- - __num_of_feature:
- rettype: integer
- argument: text
-
- - __regclass_to_text:
- rettype: text
- argument: regclass
-
- - __rename_table:
- rettype: void
- argument: text, text
-
- - __rep_aggr_class_count_ffunc:
- rettype: bigint[]
- argument: bigint[]
-
- - __rep_aggr_class_count_sfunc:
- rettype: bigint[]
- argument: bigint[], integer, integer, integer
-
- - __rep_prune_tree:
- rettype: void
- argument: text, text, integer
-
- - __sample_with_replacement:
- rettype: void
- argument: integer, bigint, text, text
-
- - __sample_within_range:
- rettype: SETOF bigint
- argument: bigint, bigint, bigint
-
- - __scv_aggr_ffunc:
- rettype: double precision[]
- argument: double precision[]
-
- - __scv_aggr_prefunc:
- rettype: double precision[]
- argument: double precision[], double precision[]
-
- - __scv_aggr_sfunc:
- rettype: double precision[]
- argument: double precision[], integer, boolean, integer, double precision[], double precision[], bigint
-
- - __strip_schema_name:
- rettype: text
- argument: text
-
- - __svm_random_ind2:
- rettype: double precision[]
- argument: integer
-
- - __svm_random_ind:
- rettype: double precision[]
- argument: integer
-
- - __svm_target_cl_func:
- rettype: double precision
- argument: double precision[]
-
- - __svm_target_reg_func:
- rettype: double precision
- argument: double precision[]
-
- - __table_exists:
- rettype: boolean
- argument: text
-
- - __train_tree:
- rettype: __train_result
- argument: text, integer, integer, text, text, text, text, text, text, double precision, integer, double precision, double precision, double precision, boolean, integer, integer
-
- - __treemodel_classify_internal:
- rettype: text[]
- argument: text, text, integer
-
- - __treemodel_classify_internal_serial:
- rettype: text[]
- argument: text, text, integer
-
- - __treemodel_display_no_ordered_aggr:
- rettype: SETOF text
- argument: text, integer[], integer
-
- - __treemodel_display_with_ordered_aggr:
- rettype: SETOF text
- argument: text, integer[], integer
-
- - __treemodel_get_vote_result:
- rettype: void
- argument: text, text
-
- - __treemodel_score:
- rettype: double precision
- argument: text, text, integer
-
- - __validate_input_table:
- rettype: void
- argument: text, text[], text, text
-
- - __validate_metatable:
- rettype: void
- argument: text
-
- - c45_classify:
- rettype: c45_classify_result
- argument: text, text, text
-
- - c45_classify:
- rettype: c45_classify_result
- argument: text, text, text, integer
-
- - c45_clean:
- rettype: boolean
- argument: text
-
- - c45_display:
- rettype: SETOF text
- argument: text
-
- - c45_display:
- rettype: SETOF text
- argument: text, integer
-
- - c45_genrule:
- rettype: SETOF text
- argument: text
-
- - c45_genrule:
- rettype: SETOF text
- argument: text, integer
-
- - c45_score:
- rettype: double precision
- argument: text, text
-
- - c45_score:
- rettype: double precision
- argument: text, text, integer
-
- - c45_train:
- rettype: c45_train_result
- argument: text, text, text
-
- - c45_train:
- rettype: c45_train_result
- argument: text, text, text, text, text, text, text, text, double precision, text
-
- - c45_train:
- rettype: c45_train_result
- argument: text, text, text, text, text, text, text, text, double precision, text, integer, double precision, double precision, integer
-
- - correlation:
- rettype: correlation_result
- argument: character varying, character varying
-
- - correlation:
- rettype: correlation_result
- argument: character varying, character varying, character varying
-
- - correlation:
- rettype: correlation_result
- argument: character varying, character varying, character varying, boolean
-
- - linear_svm_igd_transition:
- rettype: double precision[]
- argument: double precision[], double precision[], boolean, double precision[], integer, double precision, double precision
-
- - lsvm_classification:
- rettype: SETOF lsvm_sgd_result
- argument: text, text
-
- - lsvm_classification:
- rettype: SETOF lsvm_sgd_result
- argument: text, text, boolean
-
- - lsvm_classification:
- rettype: SETOF lsvm_sgd_result
- argument: text, text, boolean, boolean, double precision, double precision
-
- - lsvm_classification:
- rettype: SETOF lsvm_sgd_result
- argument: text, text, boolean, boolean, double precision, double precision, integer
-
- - lsvm_predict:
- rettype: double precision
- argument: double precision[], double precision[]
-
- - lsvm_predict_batch:
- rettype: text
- argument: text, text, text, text, text
-
- - lsvm_predict_batch:
- rettype: text
- argument: text, text, text, text, text, boolean
-
- - matrix_block_trans:
- rettype: matrix_result
- argument: text, text, text, text, boolean
-
- - matrix_densify:
- rettype: matrix_result
- argument: text, text, text, text, boolean
-
- - matrix_sparsify:
- rettype: matrix_result
- argument: text, text, text, text, boolean
-
- - matrix_trans:
- rettype: matrix_result
- argument: text, text, text, text, boolean
-
- - rf_classify:
- rettype: rf_classify_result
- argument: text, text, text
-
- - rf_classify:
- rettype: rf_classify_result
- argument: text, text, text, boolean, integer
-
- - rf_classify:
- rettype: rf_classify_result
- argument: text, text, text, integer
-
- - rf_clean:
- rettype: boolean
- argument: text
-
- - rf_display:
- rettype: SETOF text
- argument: text
-
- - rf_display:
- rettype: SETOF text
- argument: text, integer[]
-
- - rf_display:
- rettype: SETOF text
- argument: text, integer[], integer
-
- - rf_score:
- rettype: double precision
- argument: text, text
-
- - rf_score:
- rettype: double precision
- argument: text, text, integer
-
- - rf_train:
- rettype: rf_train_result
- argument: text, text, text
-
- - rf_train:
- rettype: rf_train_result
- argument: text, text, text, integer, integer, double precision, text, text, text, text, text, integer, double precision, double precision, integer
-
- - svdmf_run:
- rettype: text
- argument: text, text, text, text, integer
-
- - svdmf_run:
- rettype: text
- argument: text, text, text, text, integer, integer, double precision
-
- - svm_classification:
- rettype: SETOF svm_cls_result
- argument: text, text, boolean, text
-
- - svm_classification:
- rettype: SETOF svm_cls_result
- argument: text, text, boolean, text, boolean, double precision, double precision
-
- - svm_classification:
- rettype: SETOF svm_cls_result
- argument: text, text, boolean, text, boolean, double precision, double precision, double precision
-
- - svm_classification:
- rettype: SETOF svm_cls_result
- argument: text, text, boolean, text, double precision
-
- - svm_cls_update:
- rettype: schema_madlib.svm_model_rec
- argument: schema_madlib.svm_model_rec, double precision[], double precision, text, double precision, double precision, double precision
-
- - svm_data_normalization:
- rettype: void
- argument: text
-
- - svm_dot:
- rettype: double precision
- argument: double precision[], double precision[]
-
- - svm_dot:
- rettype: double precision
- argument: double precision[], double precision[], double precision
-
- - svm_drop_model:
- rettype: void
- argument: text
-
- - svm_gaussian:
- rettype: double precision
- argument: double precision[], double precision[], double precision
-
- - svm_generate_cls_data:
- rettype: void
- argument: text, integer, integer
-
- - svm_generate_nd_data:
- rettype: void
- argument: text, integer, integer
-
- - svm_generate_reg_data:
- rettype: void
- argument: text, integer, integer
-
- - svm_nd_update:
- rettype: schema_madlib.svm_model_rec
- argument: schema_madlib.svm_model_rec, double precision[], text, double precision, double precision, double precision
-
- - svm_novelty_detection:
- rettype: SETOF schema_madlib.svm_nd_result
- argument: text, text, boolean, text
-
- - svm_novelty_detection:
- rettype: SETOF schema_madlib.svm_nd_result
- argument: text, text, boolean, text, boolean, double precision, double precision
-
- - svm_novelty_detection:
- rettype: SETOF schema_madlib.svm_nd_result
- argument: text, text, boolean, text, boolean, double precision, double precision, double precision
-
- - svm_polynomial:
- rettype: double precision
- argument: double precision[], double precision[], double precision
-
- - svm_predict:
- rettype: double precision
- argument: schema_madlib.svm_model_rec, double precision[], text, double precision
-
- - svm_predict_batch:
- rettype: text
- argument: text, text, text, text, text, boolean
-
- - svm_predict_sub:
- rettype: double precision
- argument: integer, integer, double precision[], double precision[], double precision[], text, double precision
-
- - svm_reg_update:
- rettype: schema_madlib.svm_model_rec
- argument: schema_madlib.svm_model_rec, double precision[], double precision, text, double precision, double precision, double precision, double precision
-
- - svm_regression:
- rettype: SETOF svm_reg_result
- argument: text, text, boolean, text
-
- - svm_regression:
- rettype: SETOF svm_reg_result
- argument: text, text, boolean, text, boolean, double precision, double precision, double precision
-
- - svm_regression:
- rettype: SETOF svm_reg_result
- argument: text, text, boolean, text, boolean, double precision, double precision, double precision, double precision
-
- - svm_store_model:
- rettype: void
- argument: text, text, text
-
- # ----------------- Changes from 1.9 to 1.9.1 ----------
- - array_collapse:
- rettype: anyarray
- argument: anyarray
- - linear_svm_igd_transition:
- rettype: double precision[]
- argument: double precision[], double precision[], double precision, double precision[], integer, double precision, double precision, boolean, integer, double precision, boolean
- - profile:
- rettype: SETOF schema_madlib.profile_result
- argument: text
- - profile_full:
- rettype: SETOF schema_madlib.profile_result
- argument: text, integer
- - profile:
- rettype: schema_madlib.profile_result
- argument: text
- - profile_full:
- rettype: schema_madlib.profile_result
- argument: text, integer
- - quantile:
- rettype: double precision
- argument: text, text, double precision
- - quantile_big:
- rettype: double precision
- argument: text, text, double precision
-
- # ----------------- Changes from 1.9.1 to 1.0 ----------
- - _dt_apply:
- rettype: schema_madlib._tree_result_type
- argument: schema_madlib.bytea8, schema_madlib.bytea8, schema_madlib.bytea8, smallint, smallint, smallint, boolean, integer
- - _prune_and_cplist:
- rettype: schema_madlib._prune_result_type
- argument: schema_madlib.bytea8, double precision, boolean
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying, integer, double precision
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying, integer
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[]
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying, integer, double precision
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying, integer
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision, double precision
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer
- - internal_execute_using_kmeans_args:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, integer, double precision
-
-
-# Changes to aggregates (UDA) including removal and modification
-# Overloaded functions should be mentioned separately
-uda:
-
- # ----------------- Changes from 1.8 to 1.9 ----------
- - __array_indexed_agg:
- rettype: double precision[]
- argument: double precision, bigint, bigint
-
- - __best_scv_aggr:
- rettype: double precision[]
- argument: double precision[], integer, double precision
-
- - __bigint_array_sum:
- rettype: bigint[]
- argument: bigint[]
-
- - __display_tree_aggr:
- rettype: text
- argument: integer, boolean, text, text, double precision, double precision, text, integer
-
- - __dt_acc_count_aggr:
- rettype: bigint[]
- argument: integer, bigint, integer
-
- - __rep_aggr_class_count:
- rettype: bigint[]
- argument: integer, integer, integer
-
- - __scv_aggr:
- rettype: double precision[]
- argument: integer, boolean, integer, double precision[], double precision[], bigint
-
- - linear_svm_igd_step:
- rettype: double precision[]
- argument: double precision[], boolean, double precision[], integer, double precision, double precision
-
- - linear_svm_igd_step_serial:
- rettype: double precision[]
- argument: double precision[], boolean, double precision[], integer, double precision, double precision
-
- - svm_cls_agg:
- rettype: schema_madlib.svm_model_rec
- argument: double precision[], double precision, text, double precision, double precision, double precision
-
- - svm_nd_agg:
- rettype: schema_madlib.svm_model_rec
- argument: double precision[], text, double precision, double precision, double precision
-
- - svm_reg_agg:
- rettype: schema_madlib.svm_model_rec
- argument: double precision[], double precision, text, double precision, double precision, double precision, double precision
-
- - __svm_random_ind2:
- rettype: double precision[]
- argument: integer
-
- # ----------------- Changes from 1.9 to 1.9.1 ----------
- - array_agg:
- rettype: anyarray
- argument: anyelement
- - linear_svm_igd_step:
- rettype: double precision[]
- argument: double precision[], double precision, double precision[], integer, double precision, double precision, boolean, integer, double precision, boolean
-
-# Casts (UDC) updated/removed
-udc:
- # ----------------- Changes from 1.8 to 1.9 ----------
-
-# Operators (UDO) removed/updated
-udo:
- # ----------------- Changes from 1.8 to 1.9 ----------
-
-# Operator Classes (UDOC) removed/updated
-udoc:
- # ----------------- Changes from 1.8 to 1.9 ----------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/src/madpack/changelist_1.9.1_1.11.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.9.1_1.11.yaml b/src/madpack/changelist_1.9.1_1.11.yaml
new file mode 100644
index 0000000..6e8a15c
--- /dev/null
+++ b/src/madpack/changelist_1.9.1_1.11.yaml
@@ -0,0 +1,137 @@
+# ------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------------------------
+
+# Changelist for MADlib version 1.9.1 to 1.11
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+ # ----------------- Changes from 1.9.1 to 1.10.0 ----------
+ sssp:
+ encode_categorical:
+ knn:
+ # ----------------- Changes from 1.10.0 to 1.11 --------
+ pagerank:
+# Changes in the types (UDT) including removal and modification
+udt:
+ # ----------------- Changes from 1.9.1 to 1.10.0 ----------
+ kmeans_result:
+ kmeans_state:
+
+# List of the UDF changes that affect the user externally. This includes change
+# in function name, return type, argument order or types, or removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+ # ----------------- Changes from 1.9.1 to 1.10 ----------
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying, character varying, integer, double precision
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying, character varying, integer
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying, character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[]
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, character varying, integer, double precision
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, character varying, integer
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer, double precision, double precision
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer, double precision
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer, double precision
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer
+ - internal_execute_using_kmeans_args:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, integer, double precision
+ # ----------------- Changes from 1.10.0 to 1.11 ----------
+ - __build_tree:
+ rettype: void
+ argument: boolean, text, text, text, text, text, boolean, character varying[], character varying[], character varying[], text, text, integer, integer, integer, integer, text, smallint, text, integer
+ - graph_sssp_get_path:
+ rettype: integer[]
+ argument: text, integer
+
+
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+
+# Casts (UDC) updated/removed
+udc:
+
+# Operators (UDO) removed/updated
+udo:
+
+# Operator Classes (UDOC) removed/updated
+udoc:
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/src/madpack/changelist_1.9_1.10.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.9_1.10.yaml b/src/madpack/changelist_1.9_1.10.yaml
deleted file mode 100644
index 8d1a773..0000000
--- a/src/madpack/changelist_1.9_1.10.yaml
+++ /dev/null
@@ -1,175 +0,0 @@
-# ------------------------------------------------------------------------------
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# ------------------------------------------------------------------------------
-
-# Changelist for MADlib version 1.9 to 1.10
-
-# This file contains all changes that were introduced in a new version of
-# MADlib. This changelist is used by the upgrade script to detect what objects
-# should be upgraded (while retaining all other objects from the previous version)
-
-# New modules (actually .sql_in files) added in upgrade version
-# For these files the sql_in code is retained as is with the functions in the
-# file installed on the upgrade version. All other files (that don't have
-# updates), are cleaned up to remove object replacements
-new module:
- # ----------------- Changes from 1.9.1 to 1.0 ----------
- sssp:
- encode_categorical:
- knn:
-
-# Changes in the types (UDT) including removal and modification
-udt:
- # ----------------- Changes from 1.9 to 1.9.1 ----------
- profile_result:
- # ----------------- Changes from 1.9.1 to 1.0 ----------
- _tree_result_type:
- _prune_result_type:
- kmeans_result:
- kmeans_state:
-
-# List of the UDF changes that affect the user externally. This includes change
-# in function name, return type, argument order or types, or removal of
-# the function. In each case, the original function is as good as removed and a
-# new function is created. In such cases, we should abort the upgrade if there
-# are user views dependent on this function, since the original function will
-# not be present in the upgraded version.
-udf:
- # ----------------- Changes from 1.9 to 1.9.1 ----------
- - array_collapse:
- rettype: anyarray
- argument: anyarray
- - linear_svm_igd_transition:
- rettype: double precision[]
- argument: double precision[], double precision[], double precision, double precision[], integer, double precision, double precision, boolean, integer, double precision, boolean
- - profile:
- rettype: SETOF schema_madlib.profile_result
- argument: text
- - profile_full:
- rettype: SETOF schema_madlib.profile_result
- argument: text, integer
- - profile:
- rettype: schema_madlib.profile_result
- argument: text
- - profile_full:
- rettype: schema_madlib.profile_result
- argument: text, integer
- - quantile:
- rettype: double precision
- argument: text, text, double precision
- - quantile_big:
- rettype: double precision
- argument: text, text, double precision
- # ----------------- Changes from 1.9.1 to 1.0 ----------
- - _dt_apply:
- rettype: schema_madlib._tree_result_type
- argument: schema_madlib.bytea8, schema_madlib.bytea8, schema_madlib.bytea8, smallint, smallint, smallint, boolean, integer
- - _prune_and_cplist:
- rettype: schema_madlib._prune_result_type
- argument: schema_madlib.bytea8, double precision, boolean
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying, integer, double precision
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying, integer
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[]
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying, integer, double precision
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying, integer
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision, double precision
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer
- - internal_execute_using_kmeans_args:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, integer, double precision
-
-
-# Changes to aggregates (UDA) including removal and modification
-# Overloaded functions should be mentioned separately
-uda:
- # ----------------- Changes from 1.9 to 1.9.1 ----------
- - array_agg:
- rettype: anyarray
- argument: anyelement
- - linear_svm_igd_step:
- rettype: double precision[]
- argument: double precision[], double precision, double precision[], integer, double precision, double precision, boolean, integer, double precision, boolean
-
-
-# Casts (UDC) updated/removed
-udc:
- # ----------------- Changes from 1.9 to 1.9.1 ----------
-
-# Operators (UDO) removed/updated
-udo:
- # ----------------- Changes from 1.9 to 1.9.1 ----------
-
-# Operator Classes (UDOC) removed/updated
-udoc:
- # ----------------- Changes from 1.9 to 1.9.1 ----------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/src/madpack/changelist_1.9_1.11.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.9_1.11.yaml b/src/madpack/changelist_1.9_1.11.yaml
new file mode 100644
index 0000000..2c9647f
--- /dev/null
+++ b/src/madpack/changelist_1.9_1.11.yaml
@@ -0,0 +1,175 @@
+# ------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------------------------
+
+# Changelist for MADlib version 1.9 to 1.11
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+ # ----------------- Changes from 1.9.1 to 1.10.0 ----------
+ sssp:
+ encode_categorical:
+ knn:
+ # ----------------- Changes from 1.10.0 to 1.11 --------
+ pagerank:
+# Changes in the types (UDT) including removal and modification
+udt:
+ # ----------------- Changes from 1.9 to 1.9.1 ----------
+ profile_result:
+ # ----------------- Changes from 1.9.1 to 1.10.0 ----------
+ kmeans_result:
+ kmeans_state:
+
+# List of the UDF changes that affect the user externally. This includes change
+# in function name, return type, argument order or types, or removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+ # ----------------- Changes from 1.9 to 1.9.1 ----------
+ - array_collapse:
+ rettype: anyarray
+ argument: anyarray
+ - linear_svm_igd_transition:
+ rettype: double precision[]
+ argument: double precision[], double precision[], double precision, double precision[], integer, double precision, double precision, boolean, integer, double precision, boolean
+ - profile:
+ rettype: SETOF schema_madlib.profile_result
+ argument: text
+ - profile_full:
+ rettype: SETOF schema_madlib.profile_result
+ argument: text, integer
+ - profile:
+ rettype: schema_madlib.profile_result
+ argument: text
+ - profile_full:
+ rettype: schema_madlib.profile_result
+ argument: text, integer
+ - quantile:
+ rettype: double precision
+ argument: text, text, double precision
+ - quantile_big:
+ rettype: double precision
+ argument: text, text, double precision
+ # ----------------- Changes from 1.9.1 to 1.10.0 ----------
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying, character varying, integer, double precision
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying, character varying, integer
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying, character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[]
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, character varying, integer, double precision
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, character varying, integer
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer, double precision, double precision
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer, double precision
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer, double precision
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer
+ - internal_execute_using_kmeans_args:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, integer, double precision
+ # ----------------- Changes from 1.10.0 to 1.11 ----------
+ - __build_tree:
+ rettype: void
+ argument: boolean, text, text, text, text, text, boolean, character varying[], character varying[], character varying[], text, text, integer, integer, integer, integer, text, smallint, text, integer
+ - graph_sssp_get_path:
+ rettype: integer[]
+ argument: text, integer
+
+
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+ # ----------------- Changes from 1.9 to 1.9.1 ----------
+ - array_agg:
+ rettype: anyarray
+ argument: anyelement
+ - linear_svm_igd_step:
+ rettype: double precision[]
+ argument: double precision[], double precision, double precision[], integer, double precision, double precision, boolean, integer, double precision, boolean
+
+
+# Casts (UDC) updated/removed
+udc:
+ # ----------------- Changes from 1.9 to 1.9.1 ----------
+
+# Operators (UDO) removed/updated
+udo:
+ # ----------------- Changes from 1.9 to 1.9.1 ----------
+
+# Operator Classes (UDOC) removed/updated
+udoc:
+ # ----------------- Changes from 1.9 to 1.9.1 ----------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/src/madpack/diff_udf.sql
----------------------------------------------------------------------
diff --git a/src/madpack/diff_udf.sql b/src/madpack/diff_udf.sql
index da345b5..4e0b9aa 100644
--- a/src/madpack/diff_udf.sql
+++ b/src/madpack/diff_udf.sql
@@ -2,7 +2,7 @@
-- name but the content changes (e.g. add a field in composite type)
SET client_min_messages to ERROR;
-
+\x on
CREATE OR REPLACE FUNCTION filter_schema(argstr text, schema_name text)
RETURNS text AS $$
if argstr is None:
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/src/madpack/diff_udt.sql
----------------------------------------------------------------------
diff --git a/src/madpack/diff_udt.sql b/src/madpack/diff_udt.sql
index e521a09..dbb0bfb 100644
--- a/src/madpack/diff_udt.sql
+++ b/src/madpack/diff_udt.sql
@@ -1,5 +1,5 @@
SET client_min_messages to ERROR;
-
+\x on
CREATE OR REPLACE FUNCTION filter_schema(argstr text, schema_name text)
RETURNS text AS $$
if argstr is None:
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/src/madpack/madpack.py
----------------------------------------------------------------------
diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py
index c5dd1f9..fe0671d 100755
--- a/src/madpack/madpack.py
+++ b/src/madpack/madpack.py
@@ -657,9 +657,9 @@ def _db_upgrade(schema, dbrev):
_info("Current MADlib version already up to date.", True)
return
- if _is_rev_gte([1,7,1],_get_rev_num(dbrev)):
+ if _is_rev_gte([1,8],_get_rev_num(dbrev)):
_error("""
- MADlib versions prior to v1.8 are not supported for upgrade.
+ MADlib versions prior to v1.9 are not supported for upgrade.
Please try upgrading to v1.9.1 and then upgrade to this version.
""", True)
return
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/648b0579/src/madpack/upgrade_util.py
----------------------------------------------------------------------
diff --git a/src/madpack/upgrade_util.py b/src/madpack/upgrade_util.py
index 21ddd55..0ecf86d 100644
--- a/src/madpack/upgrade_util.py
+++ b/src/madpack/upgrade_util.py
@@ -141,15 +141,15 @@ class ChangeHandler(UpgradeBase):
@brief Load the configuration file
"""
- # _mad_dbrev = 1.8
- if self._mad_dbrev.split('.') < '1.9'.split('.'):
- filename = os.path.join(self._maddir, 'madpack',
- 'changelist_1.8_1.10.yaml')
# _mad_dbrev = 1.9
- elif self._mad_dbrev.split('.') < '1.9.1'.split('.'):
+ if self._mad_dbrev.split('.') < '1.9.1'.split('.'):
filename = os.path.join(self._maddir, 'madpack',
- 'changelist_1.9_1.10.yaml')
+ 'changelist_1.9_1.11.yaml')
# _mad_dbrev = 1.9.1
+ elif self._mad_dbrev.split('.') < '1.10.0'.split('.'):
+ filename = os.path.join(self._maddir, 'madpack',
+ 'changelist_1.9.1_1.11.yaml')
+ # _mad_dbrev = 1.10.0
else:
filename = os.path.join(self._maddir, 'madpack',
'changelist.yaml')
[33/34] incubator-madlib git commit: Build: Update multiple files for
correct verbage
Posted by ok...@apache.org.
Build: Update multiple files for correct verbage
JIRA: MADLIB-1098
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/6c2f8e39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/6c2f8e39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/6c2f8e39
Branch: refs/heads/latest_release
Commit: 6c2f8e39fc4f48783eac34ddc2ee9ffe0b56f971
Parents: ef4101e
Author: Rahul Iyer <ri...@apache.org>
Authored: Thu May 4 14:37:33 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Thu May 4 14:37:33 2017 -0700
----------------------------------------------------------------------
HAWQ_Install.txt | 32 ++++++++++++++++----------------
deploy/PackageMaker/CMakeLists.txt | 2 +-
deploy/PackageMaker/Welcome.html | 12 +++++++-----
deploy/gppkg/madlib.spec.in | 4 ++--
4 files changed, 26 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/6c2f8e39/HAWQ_Install.txt
----------------------------------------------------------------------
diff --git a/HAWQ_Install.txt b/HAWQ_Install.txt
index 8cef22e..52fd6b1 100644
--- a/HAWQ_Install.txt
+++ b/HAWQ_Install.txt
@@ -1,11 +1,11 @@
-Installing MADlib on Pivotal HAWQ
-=================================
+Installing Apache MADlib (incubating) on Apache HAWQ (incubating)
+=================================================================
-MADlib is a library of statistics and machine learning functions that can be
-installed in HAWQ. MADlib is installed separately from the main HAWQ
-installation. For a description of the general MADlib installation process,
-refer to the MADlib installation guide for PostgreSQL and GPDB:
-https://cwiki.apache.org/confluence/display/MADLIB/Installation+Guide
+Apache MADlib (incubating) is a library of statistics and machine learning
+functions that can be installed in Apache HAWQ. MADlib is installed separately
+from the main HAWQ installation. For a description of the general MADlib
+installation process, refer to the MADlib installation guide for PostgreSQL and
+GPDB: https://cwiki.apache.org/confluence/display/MADLIB/Installation+Guide
An installation script, hawq_install.sh, installs the MADlib RPM distribution on
the HAWQ master and segment nodes. It installs the MADlib files but does not
@@ -17,16 +17,16 @@ After adding new segment nodes to HAWQ, MADlib must be installed on the new
segment nodes. This should be done after the HAWQ binaries are properly
installed and preferably before running gpexpand.
-Upgrading HAWQ from 1.1 to 1.2
-------------------------------
+Apache MADlib is an effort undergoing incubation at the Apache Software
+Foundation (ASF), sponsored by the Apache Incubator PMC.
-In HAWQ 1.1 a portion of MADlib v0.5 came preinstalled. These functions in their
-original form are incompatible with HAWQ 1.2 and will be removed as part of the
-HAWQ 1.2 upgrade. Dependencies on MADlib 0.5 should be removed from the
-installation before performing the HAWQ 1.2 upgrade. When the HAWQ upgrade is
-complete, install MADlib 1.5 or higher and then reinstall the MADlib database
-objects using the madpack utility.
+Incubation is required of all newly accepted projects until a further review
+indicates that the infrastructure, communications, and decision making process
+have stabilized in a manner consistent with other successful ASF projects.
+While incubation status is not necessarily a reflection of the completeness or
+stability of the code, it does indicate that the project has yet to be fully
+endorsed by the ASF.
Requirements
------------
@@ -76,5 +76,5 @@ Optional Settings
Example
-------
- hawq_install.sh -r /home/gpadmin/madlib/madlib-1.5-Linux.rpm -f /usr/local/greenplum-db/hostfile
+ hawq_install.sh -r /home/gpadmin/madlib/madlib-1.11-Linux.rpm -f /usr/local/greenplum-db/hostfile
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/6c2f8e39/deploy/PackageMaker/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/deploy/PackageMaker/CMakeLists.txt b/deploy/PackageMaker/CMakeLists.txt
index 81a6dcc..89617b9 100644
--- a/deploy/PackageMaker/CMakeLists.txt
+++ b/deploy/PackageMaker/CMakeLists.txt
@@ -11,7 +11,7 @@
set(CPACK_RESOURCE_FILE_README
"${CPACK_PACKAGE_DESCRIPTION_FILE}" PARENT_SCOPE)
set(CPACK_RESOURCE_FILE_LICENSE
- "${CMAKE_SOURCE_DIR}/licenses/MADlib.txt" PARENT_SCOPE)
+ "${CMAKE_SOURCE_DIR}/LICENSE" PARENT_SCOPE)
set(CPACK_RESOURCE_FILE_WELCOME
"${CMAKE_CURRENT_SOURCE_DIR}/Welcome.html" PARENT_SCOPE)
set(CPACK_OSX_PACKAGE_VERSION "10.5" PARENT_SCOPE)
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/6c2f8e39/deploy/PackageMaker/Welcome.html
----------------------------------------------------------------------
diff --git a/deploy/PackageMaker/Welcome.html b/deploy/PackageMaker/Welcome.html
index 725cec4..d18338a 100644
--- a/deploy/PackageMaker/Welcome.html
+++ b/deploy/PackageMaker/Welcome.html
@@ -3,13 +3,15 @@
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-<title>Welcome to MADlib</title>
+<title>Welcome to Apache MADlib (incubating)</title>
<body>
-<h2>Welcome to Apache MADlib (incubating)!</h2>
-<p>This installer will guide you through the process of installing MADlib onto
-your computer.</p>
+<h2>Welcome to Apache MADlib (incubating)</h2>
<p>
-Apache MADlib is an effort undergoing incubation at the Apache Software
+ This installer will guide you through the process of installing MADlib onto
+your computer.
+</p>
+<p>
+ Apache MADlib is an effort undergoing incubation at the Apache Software
Foundation (ASF), sponsored by the Apache Incubator PMC.
Incubation is required of all newly accepted projects until a further
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/6c2f8e39/deploy/gppkg/madlib.spec.in
----------------------------------------------------------------------
diff --git a/deploy/gppkg/madlib.spec.in b/deploy/gppkg/madlib.spec.in
index 123eb1c..414bf5c 100644
--- a/deploy/gppkg/madlib.spec.in
+++ b/deploy/gppkg/madlib.spec.in
@@ -5,9 +5,9 @@
%define _madlib_version @MADLIB_VERSION_STRING@
BuildRoot: @MADLIB_GPPKG_RPM_SOURCE_DIR@
-Summary: MADlib for @GPDB_VARIANT@ Database
+Summary: Apache MADlib (incubating) for @GPDB_VARIANT@ Database
License: @CPACK_RPM_PACKAGE_LICENSE@
-Name: madlib
+Name: Apache MADlib (incubating)
Version: @MADLIB_VERSION_STRING_NO_HYPHEN@
Release: @MADLIB_GPPKG_RELEASE_NUMBER@
Group: @CPACK_RPM_PACKAGE_GROUP@
[13/34] incubator-madlib git commit: Task: Skip install-check for pmml
Posted by ok...@apache.org.
Task: Skip install-check for pmml
JIRA: MADLIB-1078
Skip install-check for pmml when run without the '-t' option. We
can still run install-check for pmml if the '-t' option is
specified.
Closes #115
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/975d34e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/975d34e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/975d34e4
Branch: refs/heads/latest_release
Commit: 975d34e43a8d416e5ae5b2ba668fac80dbbc15ea
Parents: c694893
Author: Nandish Jayaram <nj...@apache.org>
Authored: Fri Apr 14 11:45:35 2017 -0700
Committer: Nandish Jayaram <nj...@apache.org>
Committed: Fri Apr 14 14:30:38 2017 -0700
----------------------------------------------------------------------
src/madpack/madpack.py | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/975d34e4/src/madpack/madpack.py
----------------------------------------------------------------------
diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py
index 3b66b17..049adf5 100755
--- a/src/madpack/madpack.py
+++ b/src/madpack/madpack.py
@@ -1424,6 +1424,11 @@ def main(argv):
# Skip if doesn't meet specified modules
if modset is not None and len(modset) > 0 and module not in modset:
continue
+ # JIRA: MADLIB-1078 fix
+ # Skip pmml during install-check (when run without the -t option).
+ # We can still run install-check on pmml with '-t' option.
+ if not modset and module in ['pmml']:
+ continue
_info("> - %s" % module, verbose)
# Make a temp dir for this module (if doesn't exist)
[19/34] incubator-madlib git commit: Build: Update and move ReadMe.txt
Posted by ok...@apache.org.
Build: Update and move ReadMe.txt
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/9362ba80
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/9362ba80
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/9362ba80
Branch: refs/heads/latest_release
Commit: 9362ba803c62229a764d946f650c58685f8965d4
Parents: 658ecde
Author: Rahul Iyer <ri...@apache.org>
Authored: Wed Apr 19 16:09:27 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Wed Apr 19 16:09:27 2017 -0700
----------------------------------------------------------------------
CMakeLists.txt | 1 -
ReadMe.txt | 52 ----------------------------------
deploy/PGXN/CMakeLists.txt | 1 +
deploy/PGXN/META.json.in | 2 +-
deploy/PGXN/ReadMe.txt | 62 +++++++++++++++++++++++++++++++++++++++++
doc/CMakeLists.txt | 2 +-
doc/mainpage.dox.in | 10 ++++---
7 files changed, 71 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/9362ba80/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b2e6cf9..b2172ef 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -182,7 +182,6 @@ install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/licenses"
)
install(
FILES
- "${CMAKE_CURRENT_SOURCE_DIR}/ReadMe.txt"
"${CMAKE_CURRENT_SOURCE_DIR}/README.md"
"${CMAKE_CURRENT_SOURCE_DIR}/RELEASE_NOTES"
DESTINATION doc
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/9362ba80/ReadMe.txt
----------------------------------------------------------------------
diff --git a/ReadMe.txt b/ReadMe.txt
deleted file mode 100644
index 2116d7f..0000000
--- a/ReadMe.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-MADlib Read Me
---------------
-
-MADlib is an open-source library for scalable in-database analytics.
-It provides data-parallel implementations of mathematical, statistical
-and machine learning methods for structured and unstructured data.
-
-See the project web site located at http://madlib.incubator.apache.org/ for links to the latest
-binary and source packages.
-
-For installation and contribution guides, please see the MADlib wiki at
-https://github.com/madlib/madlib/wiki.
-
-The latest documentation of MADlib modules can be found at http://madlib.incubator.apache.org/docs
-or can be accessed directly from the MADlib installation directory by opening
-doc/user/html/index.html.
-
-Changes between MADlib versions are described in the ReleaseNotes.txt file.
-
-MADlib incorporates material from the following third-party components:
-- argparse 1.2.1 "provides an easy, declarative interface for creating command
- line tools"
- http://code.google.com/p/argparse/
-- Boost 1.47.0 (or newer) "provides peer-reviewed portable C++ source
- libraries"
- http://www.boost.org/
-- doxypy 0.4.2 "is an input filter for Doxygen"
- http://code.foosel.org/doxypy
-- Eigen 3.2.2 "is a C++ template library for linear algebra"
- http://eigen.tuxfamily.org/index.php?title=Main_Page
-- PyYAML 3.10 "is a YAML parser and emitter for Python"
- http://pyyaml.org/wiki/PyYAML
-
-License information regarding MADlib and included third-party libraries can be
-found inside the 'licenses' directory.
-
--------------------------------------------------------------------------
-
-The following list of functions have been deprecated and will be removed on
-upgrading to the next major version:
- - All overloaded functions 'cox_prop_hazards' and 'cox_prop_hazards_regr'.
- - All overloaded functions 'mlogregr'.
- - Overloaded forms of function 'robust_variance_mlogregr' that accept
- individual optimizer parameters (max_iter, optimizer, tolerance). These
- parameters have been replaced with a single optimizer parameter.
- - Overloaded forms of function 'clusterd_variance_mlogregr' that accept
- individual optimizer parameters (max_iter, optimizer, tolerance). These
- parameters have been replaced with a single optimizer parameter.
- - Overloaded forms of function 'margins_mlogregr' that accept
- individual optimizer parameters (max_iter, optimizer, tolerance). These
- parameters have been replaced with a single optimizer parameter.
- - All overloaded functions 'margins_logregr'.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/9362ba80/deploy/PGXN/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/deploy/PGXN/CMakeLists.txt b/deploy/PGXN/CMakeLists.txt
index 39c4c82..22195b9 100644
--- a/deploy/PGXN/CMakeLists.txt
+++ b/deploy/PGXN/CMakeLists.txt
@@ -12,6 +12,7 @@ set(MADLIB_PGXN_NAME "madlib-pgxn-${MADLIB_PGXN_VERSION_STR}")
configure_file(META.json.in META.json)
configure_file(generate_package.sh.in generate_package.sh @ONLY)
configure_file(zipignore.in zipignore)
+configure_file(ReadMe.txt ReadMe.txt COPYONLY)
add_custom_command(
OUTPUT madlib.zip
COMMAND "${CMAKE_COMMAND}" -E create_symlink
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/9362ba80/deploy/PGXN/META.json.in
----------------------------------------------------------------------
diff --git a/deploy/PGXN/META.json.in b/deploy/PGXN/META.json.in
index 914454d..e1196c0 100644
--- a/deploy/PGXN/META.json.in
+++ b/deploy/PGXN/META.json.in
@@ -8,7 +8,7 @@
"provides": {
"madlib": {
"file": "madlib--@MADLIB_VERSION_MAJOR@.@MADLIB_VERSION_MINOR@.@MADLIB_VERSION_PATCH@.sql",
- "docfile": "ReadMe.txt",
+ "docfile": "deploy/PGXN/ReadMe.txt",
"version": "@MADLIB_VERSION_MAJOR@.@MADLIB_VERSION_MINOR@.@MADLIB_VERSION_PATCH@"
}
},
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/9362ba80/deploy/PGXN/ReadMe.txt
----------------------------------------------------------------------
diff --git a/deploy/PGXN/ReadMe.txt b/deploy/PGXN/ReadMe.txt
new file mode 100644
index 0000000..2982477
--- /dev/null
+++ b/deploy/PGXN/ReadMe.txt
@@ -0,0 +1,62 @@
+MADlib Read Me
+--------------
+
+MADlib is an open-source library for scalable in-database analytics.
+It provides data-parallel implementations of mathematical, statistical
+and machine learning methods for structured and unstructured data.
+
+See the project web site located at http://madlib.incubator.apache.org/ for
+links to the latest binary and source packages.
+
+For installation and contribution guides, please see the MADlib wiki at
+https://cwiki.apache.org/confluence/display/MADLIB.
+
+The latest documentation of MADlib modules can be found at http://madlib.incubator.apache.org/docs
+or can be accessed directly from the MADlib installation directory by opening
+doc/user/html/index.html.
+
+Changes between MADlib versions are described in the ReleaseNotes.txt file.
+
+MADlib incorporates material from the following third-party components:
+
+Bundled with source code:
+- libstemmer "small string processing language"
+ http://snowballstem.org/
+- m_widen_init "allows compilation with recent versions of gcc with runtime
+ dependencies from earlier versions of libstdc++"
+ https://github.com/apache/incubator-madlib/blob/master/licenses/third_party/_M_widen_init.txt
+- PyYAML 3.10 "is a YAML parser and emitter for Python"
+ http://pyyaml.org/wiki/PyYAML
+- argparse 1.2.1 "provides an easy, declarative interface for creating command
+ line tools"
+ http://code.google.com/p/argparse/
+- UseLATEX.cmake "CMAKE commands to use the LaTeX compiler"
+ https://github.com/kmorel/UseLATEX/blob/master/UseLATEX.cmake
+
+Downloaded at build time:
+- Boost 1.61.0 (or newer) "provides peer-reviewed portable C++ source
+ libraries"
+ http://www.boost.org/
+- Eigen 3.2 "is a C++ template library for linear algebra"
+ http://eigen.tuxfamily.org/index.php?title=Main_Page
+- PyXB 1.2.4 "Python library for XML Schema Bindings"
+
+License information regarding MADlib and included third-party libraries can be
+found inside the 'licenses' directory.
+
+-------------------------------------------------------------------------
+
+The following list of functions have been deprecated and will be removed on
+upgrading to the next major version:
+ - All overloaded functions 'cox_prop_hazards' and 'cox_prop_hazards_regr'.
+ - All overloaded functions 'mlogregr'.
+ - Overloaded forms of function 'robust_variance_mlogregr' that accept
+ individual optimizer parameters (max_iter, optimizer, tolerance). These
+ parameters have been replaced with a single optimizer parameter.
+ - Overloaded forms of function 'clusterd_variance_mlogregr' that accept
+ individual optimizer parameters (max_iter, optimizer, tolerance). These
+ parameters have been replaced with a single optimizer parameter.
+ - Overloaded forms of function 'margins_mlogregr' that accept
+ individual optimizer parameters (max_iter, optimizer, tolerance). These
+ parameters have been replaced with a single optimizer parameter.
+ - All overloaded functions 'margins_logregr'.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/9362ba80/doc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt
index aa969dc..d85272e 100644
--- a/doc/CMakeLists.txt
+++ b/doc/CMakeLists.txt
@@ -2,7 +2,7 @@
# MADlib Documentation
# ------------------------------------------------------------------------------
-set(DOXYGEN_README_FILE "../ReadMe.txt" CACHE STRING
+set(DOXYGEN_README_FILE "../README.md" CACHE STRING
"Path to ReadMe file relative to the doc directory after installation")
set(DOXYGEN_LICENSE_DIR "../../licenses" CACHE STRING
"Path to license directory relative to the doc directory after installation")
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/9362ba80/doc/mainpage.dox.in
----------------------------------------------------------------------
diff --git a/doc/mainpage.dox.in b/doc/mainpage.dox.in
index 94950e7..510ab1b 100644
--- a/doc/mainpage.dox.in
+++ b/doc/mainpage.dox.in
@@ -30,10 +30,12 @@ Useful links:
</li>
</ul>
-Please refer to the <a href="https://github.com/apache/incubator-madlib/blob/master/ReadMe.txt">Read-Me</a> file for information
-about incorporated third-party material. License information regarding MADlib
-and included third-party libraries can be found inside the
-<a href="https://github.com/apache/incubator-madlib/blob/master/LICENSE">License</a> directory.
+Please refer to the
+<a href="https://github.com/apache/incubator-madlib/blob/master/README.md">ReadMe</a>
+file for information about incorporated third-party material. License information
+regarding MADlib and included third-party libraries can be found inside the
+<a href="https://github.com/apache/incubator-madlib/blob/master/LICENSE">
+License</a> directory.
@defgroup grp_datatrans Data Types and Transformations
@{Data types and transformation operations @}
[24/34] incubator-madlib git commit: Array Operations: Unnest 2-D
arrays by one level.
Posted by ok...@apache.org.
Array Operations: Unnest 2-D arrays by one level.
JIRA: MADLIB-1086
Unnest 2-D arrays by one level (i.e. into rows of 1-D arrays).
Example usage in k-Means shows how to unnest the 2-D centroid array
to get one centroid per row for follow on operations.
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/3af18a93
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/3af18a93
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/3af18a93
Branch: refs/heads/latest_release
Commit: 3af18a9388d144920d5bca3e5cde27edee6e0eac
Parents: 20b1158
Author: Rashmi Raghu <rr...@pivotal.io>
Authored: Tue Apr 25 14:41:09 2017 -0700
Committer: Rashmi Raghu <rr...@pivotal.io>
Committed: Wed Apr 26 11:35:23 2017 -0700
----------------------------------------------------------------------
methods/array_ops/src/pg_gp/array_ops.sql_in | 102 ++++++++-
.../array_ops/src/pg_gp/test/array_ops.sql_in | 218 +++++++++++++++++++
src/ports/postgres/modules/kmeans/kmeans.sql_in | 89 +++++---
3 files changed, 375 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/3af18a93/methods/array_ops/src/pg_gp/array_ops.sql_in
----------------------------------------------------------------------
diff --git a/methods/array_ops/src/pg_gp/array_ops.sql_in b/methods/array_ops/src/pg_gp/array_ops.sql_in
index c83a947..08ba377 100644
--- a/methods/array_ops/src/pg_gp/array_ops.sql_in
+++ b/methods/array_ops/src/pg_gp/array_ops.sql_in
@@ -24,7 +24,7 @@ m4_include(`SQLCommon.m4')
@brief Provides fast array operations supporting other MADlib modules.
-This module provides a set of basic array operations implemented in C.
+This module provides a set of basic array operations implemented in C and SQL.
It is a support module for several machine learning algorithms that
require fast array operations.
@@ -42,6 +42,8 @@ These functions support several numeric types:
- DOUBLE PRECISION (FLOAT8)
- NUMERIC (internally casted into FLOAT8, loss of precisions can happen)
+Additionally, array_unnest_2d_to_1d() supports other data types such as TEXT or VARCHAR.
+
Several of the function require NO NULL VALUES, while others omit NULLs and return results. See details in description of individual functions.
@anchor list
@@ -126,6 +128,11 @@ Several of the function require NO NULL VALUES, while others omit NULLs and retu
<tr><th>normalize()</th><td> This function normalizes an array as sum of squares to be 1. It requires that the array is 1-D and all the values are NON-NULL.
</td></tr>
+
+<tr><th>array_unnest_2d_to_1d()</th><td> This function takes a 2-D array as the input and unnests it by one level. It returns a set of 1-D arrays that correspond to rows of
+ the input array as well as an ID column with values corresponding to row positions occupied by those 1-D arrays within the 2-D array.
+</td></tr>
+
</table>
@anchor examples
@@ -220,6 +227,30 @@ Result:
{1.3,1.3,1.3,1.3,1.3,1.3,1.3,1.3,1.3}
(1 row)
</pre>
+-# Unnest a column of 2-D arrays into sets of 1-D arrays.
+<pre class="example">
+SELECT id, (madlib.array_unnest_2d_to_1d(val)).*
+FROM (
+ SELECT 1::INT AS id, ARRAY[[1.3,2.0,3.2],[10.3,20.0,32.2]]::FLOAT8[][] AS val
+ UNION ALL
+ SELECT 2, ARRAY[[pi(),pi()/2],[2*pi(),pi()],[pi()/4,4*pi()]]::FLOAT8[][]
+) t
+ORDER BY 1,2;
+</pre>
+Result:
+<pre class="result">
+ id | unnest_row_id | unnest_result
+----+---------------+--------------------------------------
+ 1 | 1 | {1.3,2,3.2}
+ 1 | 2 | {10.3,20,32.2}
+ 2 | 1 | {3.14159265358979,1.5707963267949}
+ 2 | 2 | {6.28318530717959,3.14159265358979}
+ 2 | 3 | {0.785398163397448,12.5663706143592}
+(5 rows)
+</pre>
+If the function is called without the .* notation then it will return a
+composite record type with two attributes: the row ID and corresponding
+unnested array result.
@anchor related
@par Related Topics
@@ -636,3 +667,72 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_cum_prod(x anyarray) RETURNS anya
AS 'MODULE_PATHNAME', 'array_cum_prod'
LANGUAGE C IMMUTABLE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+
+/**
+ * @brief This function takes a 2-D array as the input and unnests it
+ * by one level.
+ * It returns a set of 1-D arrays that correspond to rows of the
+ * input array as well as an ID column containing row positions occupied by
+ * those 1-D arrays within the 2-D array (the ID column values start with
+ * 1 and not 0)
+ *
+ * @param x Array x
+ * @returns Set of 1-D arrays that corrspond to rows of x and an ID column.
+ *
+ */
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_unnest_2d_to_1d(
+ x ANYARRAY,
+ OUT unnest_row_id INT,
+ OUT unnest_result ANYARRAY
+)
+RETURNS SETOF RECORD
+AS
+$BODY$
+ SELECT t2.r::int, array_agg($1[t2.r][t2.c] order by t2.c) FROM
+ (
+ SELECT generate_series(array_lower($1,2),array_upper($1,2)) as c, t1.r
+ FROM
+ (
+ SELECT generate_series(array_lower($1,1),array_upper($1,1)) as r
+ ) t1
+ ) t2
+GROUP BY t2.r
+$BODY$ LANGUAGE SQL IMMUTABLE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.array_unnest_2d_to_1d()
+RETURNS TEXT AS $$
+ return """
+------------------------------------------------------------------
+ SUMMARY
+------------------------------------------------------------------
+This function takes a 2-D array as the input and unnests it by
+one level.
+It returns a set of 1-D arrays that correspond to rows of the
+input array as well as an ID column containing row positions occupied by
+those 1-D arrays within the 2-D array (the ID column values start with
+1 and not 0).
+
+------------------------------------------------------------------
+ USAGE
+------------------------------------------------------------------
+
+ SELECT ({schema_madlib}.array_unnest_2d_to_1d(input_array)).* from input_table;
+
+If the function is called without the .* notation then it will return a
+composite record type with two attributes: the row ID and corresponding
+unnested array result.
+
+------------------------------------------------------------------
+ EXAMPLE
+------------------------------------------------------------------
+SELECT id, (madlib.array_unnest_2d_to_1d(val)).*
+FROM (
+ SELECT 1::INT AS id, ARRAY[[1.3,2.0,3.2],[10.3,20.0,32.2]]::FLOAT8[][] AS val
+ UNION ALL
+ SELECT 2, ARRAY[[pi(),pi()/2],[2*pi(),pi()],[pi()/4,4*pi()]]::FLOAT8[][]
+) t
+ORDER BY 1,2;
+ """.format(schema_madlib='MADLIB_SCHEMA')
+$$ LANGUAGE PLPYTHONU IMMUTABLE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/3af18a93/methods/array_ops/src/pg_gp/test/array_ops.sql_in
----------------------------------------------------------------------
diff --git a/methods/array_ops/src/pg_gp/test/array_ops.sql_in b/methods/array_ops/src/pg_gp/test/array_ops.sql_in
index 473e32e..b05d0b7 100644
--- a/methods/array_ops/src/pg_gp/test/array_ops.sql_in
+++ b/methods/array_ops/src/pg_gp/test/array_ops.sql_in
@@ -89,3 +89,221 @@ SELECT array_scalar_mult(
(1.0/MADLIB_SCHEMA.array_sum(ARRAY[1.,2,3,4]))
);
+--------------------------------------------------------------
+-- TESTING array_unnest_2d_to_1d FUNCTION
+--------------------------------------------------------------
+-- 2-element float8 arrays
+DROP TABLE IF EXISTS unnest_2d_tbl01;
+CREATE TABLE unnest_2d_tbl01 (id INT, val DOUBLE PRECISION[][]);
+INSERT INTO unnest_2d_tbl01 VALUES
+ (1, ARRAY[[1::float8,2],[3::float8,4],[5::float8,6]]),
+ (2, ARRAY[[101::float8,202],[303::float8,404],[505::float8,606]])
+;
+
+DROP TABLE IF EXISTS unnest_2d_tbl01_groundtruth;
+CREATE TABLE unnest_2d_tbl01_groundtruth (
+ id INT,
+ unnest_row_id INT,
+ val DOUBLE PRECISION[]
+);
+INSERT INTO unnest_2d_tbl01_groundtruth VALUES
+ (1, 1, ARRAY[1::float8,2]),
+ (1, 2, ARRAY[3::float8,4]),
+ (1, 3, ARRAY[5::float8,6]),
+ (2, 1, ARRAY[101::float8,202]),
+ (2, 2, ARRAY[303::float8,404]),
+ (2, 3, ARRAY[505::float8,606])
+;
+
+DROP TABLE IF EXISTS unnest_2d_tbl01_out;
+CREATE TABLE unnest_2d_tbl01_out AS
+ SELECT id, (array_unnest_2d_to_1d(val)).* FROM unnest_2d_tbl01;
+
+SELECT assert(
+ unnest_result = val,
+ 'array_unnest_2d_to_1d: Wrong results for test table "unnest_2d_tbl01"'
+)
+FROM (
+ SELECT * FROM
+ unnest_2d_tbl01_out t1
+ JOIN
+ unnest_2d_tbl01_groundtruth t2
+ USING (id,unnest_row_id)
+) t3;
+
+-- 3-element float8 arrays
+DROP TABLE IF EXISTS unnest_2d_tbl02;
+CREATE TABLE unnest_2d_tbl02 (id INT, val DOUBLE PRECISION[][]);
+INSERT INTO unnest_2d_tbl02 VALUES
+ (1, ARRAY[[1.57::float8,2,3],[4::float8,5,6]]),
+ (2, ARRAY[[101::float8,202,303],[PI(),505,606]]),
+ (3, ARRAY[[1011::float8,2022,3033],[4044,5055,60.66]])
+;
+
+DROP TABLE IF EXISTS unnest_2d_tbl02_groundtruth;
+CREATE TABLE unnest_2d_tbl02_groundtruth (
+ id INT,
+ unnest_row_id INT,
+ val DOUBLE PRECISION[]
+);
+INSERT INTO unnest_2d_tbl02_groundtruth VALUES
+ (1, 1, array[1.57::float8,2,3]),
+ (1, 2, array[4::float8,5,6]),
+ (2, 1, array[101::float8,202,303]),
+ (2, 2, array[pi(),505,606]),
+ (3, 1, array[1011::float8,2022,3033]),
+ (3, 2, array[4044,5055,60.66])
+;
+
+DROP TABLE IF EXISTS unnest_2d_tbl02_out;
+CREATE TABLE unnest_2d_tbl02_out AS
+ SELECT id, (array_unnest_2d_to_1d(val)).* FROM unnest_2d_tbl02;
+
+SELECT assert(
+ unnest_result = val,
+ 'array_unnest_2d_to_1d: Wrong results for test table "unnest_2d_tbl02"'
+)
+FROM (
+ SELECT * FROM
+ unnest_2d_tbl02_out t1
+ JOIN
+ unnest_2d_tbl02_groundtruth t2
+ USING (id,unnest_row_id)
+) t3;
+
+-- 2-element text arrays
+DROP TABLE IF EXISTS unnest_2d_tbl03;
+CREATE TABLE unnest_2d_tbl03 (id INT, val TEXT[][]);
+INSERT INTO unnest_2d_tbl03 VALUES
+ (1, ARRAY[['a','b'],['c','d'],['e','f']]),
+ (2, ARRAY[['apple','banana'],['cherries','kiwi'],['lemon','mango']])
+;
+
+DROP TABLE IF EXISTS unnest_2d_tbl03_groundtruth;
+CREATE TABLE unnest_2d_tbl03_groundtruth (
+ id INT,
+ unnest_row_id INT,
+ val TEXT[]
+);
+INSERT INTO unnest_2d_tbl03_groundtruth VALUES
+ (1, 1, ARRAY['a','b']),
+ (1, 2, ARRAY['c','d']),
+ (1, 3, ARRAY['e','f']),
+ (2, 1, ARRAY['apple','banana']),
+ (2, 2, ARRAY['cherries','kiwi']),
+ (2, 3, ARRAY['lemon','mango'])
+;
+
+DROP TABLE IF EXISTS unnest_2d_tbl03_out;
+CREATE TABLE unnest_2d_tbl03_out AS
+ SELECT id, (array_unnest_2d_to_1d(val)).* FROM unnest_2d_tbl03;
+
+SELECT assert(
+ unnest_result = val,
+ 'array_unnest_2d_to_1d: Wrong results for test table "unnest_2d_tbl03"'
+)
+FROM (
+ SELECT * FROM
+ unnest_2d_tbl03_out t1
+ JOIN
+ unnest_2d_tbl03_groundtruth t2
+ USING (id,unnest_row_id)
+) t3;
+
+-- 3-element float8 arrays with some NULLs
+DROP TABLE IF EXISTS unnest_2d_tbl04;
+CREATE TABLE unnest_2d_tbl04 (id INT, val DOUBLE PRECISION[][]);
+INSERT INTO unnest_2d_tbl04 VALUES
+ (1, ARRAY[[1::float8,NULL,3],[4.0,5,NULL]]),
+ (2, ARRAY[[101::float8,NULL,303],
+ [NULL::float8,NULL,NULL]]::double precision[][]),
+ (3, ARRAY[[NULL,2022::float8],[4044::float8,NULL]])
+;
+
+DROP TABLE IF EXISTS unnest_2d_tbl04_groundtruth;
+CREATE TABLE unnest_2d_tbl04_groundtruth (
+ id INT,
+ unnest_row_id INT,
+ val DOUBLE PRECISION[]
+);
+INSERT INTO unnest_2d_tbl04_groundtruth VALUES
+ (1, 1, ARRAY[1::float8,NULL,3]),
+ (1, 2, ARRAY[4.0::float8,5,NULL]),
+ (2, 1, ARRAY[101::float8,NULL,303]),
+ (2, 2, ARRAY[NULL::float8,NULL,NULL]),
+ (3, 1, ARRAY[NULL,2022::float8]),
+ (3, 2, ARRAY[4044::float8,NULL])
+;
+
+DROP TABLE IF EXISTS unnest_2d_tbl04_out;
+CREATE TABLE unnest_2d_tbl04_out AS
+ SELECT id, (array_unnest_2d_to_1d(val)).* FROM unnest_2d_tbl04;
+
+SELECT assert(
+ unnest_result = val,
+ 'array_unnest_2d_to_1d: Wrong results for test table "unnest_2d_tbl04"'
+)
+FROM (
+ SELECT * FROM
+ unnest_2d_tbl04_out t1
+ JOIN
+ unnest_2d_tbl04_groundtruth t2
+ USING (id,unnest_row_id)
+) t3;
+
+-- 3-element timestamp arrays with NULLs
+DROP TABLE IF EXISTS unnest_2d_tbl05;
+CREATE TABLE unnest_2d_tbl05 (id INT, val TIMESTAMP WITHOUT TIME ZONE[][]);
+INSERT INTO unnest_2d_tbl05 VALUES
+ (1, array[['2017-01-01 11:00:02'::TIMESTAMP WITHOUT TIME ZONE,
+ '2017-01-01 13:00:05',
+ '2017-01-02 11:55:00'],
+ ['2016-10-12 12:00:02'::TIMESTAMP WITHOUT TIME ZONE,
+ '2016-10-12 13:15:22',
+ NULL]]),
+ (2, NULL),
+ (3, array[['2014-02-01 11:00:02'::TIMESTAMP WITHOUT TIME ZONE,
+ '2014-02-01 13:00:05',
+ '2014-02-02 11:55:00'],
+ ['2013-07-12 12:00:02'::TIMESTAMP WITHOUT TIME ZONE,
+ NULL,
+ '2013-07-12 13:15:22']])
+;
+
+DROP TABLE IF EXISTS unnest_2d_tbl05_groundtruth;
+CREATE TABLE unnest_2d_tbl05_groundtruth (
+ id INT,
+ unnest_row_id INT,
+ val TIMESTAMP WITHOUT TIME ZONE[]
+);
+INSERT INTO unnest_2d_tbl05_groundtruth VALUES
+ (1, 1, ARRAY['2017-01-01 11:00:02'::TIMESTAMP WITHOUT TIME ZONE,
+ '2017-01-01 13:00:05',
+ '2017-01-02 11:55:00']),
+ (1, 2, ARRAY['2016-10-12 12:00:02'::TIMESTAMP WITHOUT TIME ZONE,
+ '2016-10-12 13:15:22',
+ NULL]),
+ (2, NULL, NULL),
+ (3, 1, ARRAY['2014-02-01 11:00:02'::TIMESTAMP WITHOUT TIME ZONE,
+ '2014-02-01 13:00:05',
+ '2014-02-02 11:55:00']),
+ (3, 2, ARRAY['2013-07-12 12:00:02'::TIMESTAMP WITHOUT TIME ZONE,
+ NULL,
+ '2013-07-12 13:15:22'])
+;
+
+DROP TABLE IF EXISTS unnest_2d_tbl05_out;
+CREATE TABLE unnest_2d_tbl05_out AS
+ SELECT id, (array_unnest_2d_to_1d(val)).* FROM unnest_2d_tbl05;
+
+SELECT assert(
+ unnest_result = val,
+ 'array_unnest_2d_to_1d: Wrong results for test table "unnest_2d_tbl05"'
+)
+FROM (
+ SELECT * FROM
+ unnest_2d_tbl05_out t1
+ JOIN
+ unnest_2d_tbl05_groundtruth t2
+ USING (id,unnest_row_id)
+) t3;
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/3af18a93/src/ports/postgres/modules/kmeans/kmeans.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/kmeans/kmeans.sql_in b/src/ports/postgres/modules/kmeans/kmeans.sql_in
index f689dd6..b3cdd55 100644
--- a/src/ports/postgres/modules/kmeans/kmeans.sql_in
+++ b/src/ports/postgres/modules/kmeans/kmeans.sql_in
@@ -239,75 +239,98 @@ INSERT INTO km_sample VALUES
</pre>
-# Run k-means clustering using kmeans++ for centroid seeding:
<pre class="example">
+DROP TABLE IF EXISTS km_result;
+-- Run kmeans algorithm
+CREATE TABLE km_result AS
+SELECT * FROM madlib.kmeanspp('km_sample', 'points', 2,
+ 'madlib.squared_dist_norm2',
+ 'madlib.avg', 20, 0.001);
\\x on;
-SELECT * FROM madlib.kmeanspp( 'km_sample', -- Table of source data
- 'points', -- Column containing point co-ordinates
- 2, -- Number of centroids to calculate
- 'madlib.squared_dist_norm2', -- Distance function
- 'madlib.avg', -- Aggregate function
- 20, -- Number of iterations
- 0.001 -- Fraction of centroids reassigned to keep iterating
- );
+SELECT * FROM km_result;
</pre>
Result:
<pre class="result">
-centroids | {{13.7533333333333,1.905,2.425,16.0666666666667,90.3333333333333,2.805,2.98,0.29,2.005,5.40663333333333,1.04166666666667, 3.31833333333333,1020.83333333333},
- {14.255,1.9325,2.5025,16.05,110.5,3.055,2.9775,0.2975,1.845,6.2125,0.9975,3.365,1378.75}}
-cluster_variance | {122999.110416013,30561.74805}
-objective_fn | 153560.858466013
+centroids | {{14.036,2.018,2.536,16.56,108.6,3.004,3.03,0.298,2.038,6.10598,1.004,3.326,1340},{13.872,1.814,2.376,15.56,88.2,2.806,2.928,0.288,1.844,5.35198,1.044,3.348,988}}
+cluster_variance | {60672.638245208,90512.324426408}
+objective_fn | 151184.962671616
frac_reassigned | 0
-num_iterations | 3
+num_iterations | 2
</pre>
-# Calculate the simplified silhouette coefficient:
<pre class="example">
SELECT * FROM madlib.simple_silhouette( 'km_sample',
'points',
- (SELECT centroids FROM
- madlib.kmeanspp('km_sample',
- 'points',
- 2,
- 'madlib.squared_dist_norm2',
- 'madlib.avg',
- 20,
- 0.001)),
+ (SELECT centroids FROM km_result),
'madlib.dist_norm2'
);
</pre>
Result:
<pre class="result">
-simple_silhouette | 0.686314347664694
+simple_silhouette | 0.68978804882941
</pre>
-# Find the cluster assignment for each point:
<pre class="example">
\\x off;
-DROP TABLE IF EXISTS km_result;
--- Run kmeans algorithm
-CREATE TABLE km_result AS
-SELECT * FROM madlib.kmeanspp('km_sample', 'points', 2,
- 'madlib.squared_dist_norm2',
- 'madlib.avg', 20, 0.001);
-- Get point assignment
SELECT data.*, (madlib.closest_column(centroids, points)).column_id as cluster_id
FROM km_sample as data, km_result
ORDER BY data.pid;
</pre>
+Result:
<pre class="result">
- pid | points | cluster_id
+ pid | points | cluster_id
-----+--------------------------------------------------------------------+------------
- 1 | {14.23,1.71,2.43,15.6,127,2.8,3.06,0.28,2.29,5.64,1.04,3.92,1065} | 0
- 2 | {13.2,1.78,2.14,11.2,1,2.65,2.76,0.26,1.28,4.38,1.05,3.49,1050} | 0
+ 1 | {14.23,1.71,2.43,15.6,127,2.8,3.06,0.28,2.29,5.64,1.04,3.92,1065} | 1
+ 2 | {13.2,1.78,2.14,11.2,1,2.65,2.76,0.26,1.28,4.38,1.05,3.49,1050} | 1
3 | {13.16,2.36,2.67,18.6,101,2.8,3.24,0.3,2.81,5.6799,1.03,3.17,1185} | 0
4 | {14.37,1.95,2.5,16.8,113,3.85,3.49,0.24,2.18,7.8,0.86,3.45,1480} | 0
5 | {13.24,2.59,2.87,21,118,2.8,2.69,0.39,1.82,4.32,1.04,2.93,735} | 1
6 | {14.2,1.76,2.45,15.2,112,3.27,3.39,0.34,1.97,6.75,1.05,2.85,1450} | 0
7 | {14.39,1.87,2.45,14.6,96,2.5,2.52,0.3,1.98,5.25,1.02,3.58,1290} | 0
8 | {14.06,2.15,2.61,17.6,121,2.6,2.51,0.31,1.25,5.05,1.06,3.58,1295} | 0
- 9 | {14.83,1.64,2.17,14,97,2.8,2.98,0.29,1.98,5.2,1.08,2.85,1045} | 0
- 10 | {13.86,1.35,2.27,16,98,2.98,3.15,0.22,1.85,7.2199,1.01,3.55,1045} | 0
+ 9 | {14.83,1.64,2.17,14,97,2.8,2.98,0.29,1.98,5.2,1.08,2.85,1045} | 1
+ 10 | {13.86,1.35,2.27,16,98,2.98,3.15,0.22,1.85,7.2199,1.01,3.55,1045} | 1
(10 rows)
</pre>
+-# Unnest the cluster centroids 2-D array to get a set of 1-D centroid arrays:
+<pre class="example">
+DROP TABLE IF EXISTS km_centroids_unnest;
+-- Run unnest function
+CREATE TABLE km_centroids_unnest AS
+SELECT (madlib.array_unnest_2d_to_1d(centroids)).*
+FROM km_result;
+SELECT * FROM km_centroids_unnest ORDER BY 1;
+</pre>
+Result:
+<pre class="result">
+ unnest_row_id | unnest_result
+---------------+----------------------------------------------------------------------------------
+ 1 | {14.036,2.018,2.536,16.56,108.6,3.004,3.03,0.298,2.038,6.10598,1.004,3.326,1340}
+ 2 | {13.872,1.814,2.376,15.56,88.2,2.806,2.928,0.288,1.844,5.35198,1.044,3.348,988}
+(2 rows)
+</pre>
+Note that the ID column returned by array_unnest_2d_to_1d()
+is not guaranteed to be the same as the cluster ID assigned by k-means.
+See below to create the correct cluster IDs.
+
+-# Create cluster IDs for 1-D centroid arrays so that cluster ID for any centroid
+can be matched to the cluster assignment for the data points:
+<pre class="example">
+SELECT cent.*, (madlib.closest_column(centroids, unnest_result)).column_id as cluster_id
+FROM km_centroids_unnest as cent, km_result
+ORDER BY cent.unnest_row_id;
+</pre>
+Result:
+<pre class="result">
+ unnest_row_id | unnest_result | cluster_id
+---------------+----------------------------------------------------------------------------------+------------
+ 1 | {14.036,2.018,2.536,16.56,108.6,3.004,3.03,0.298,2.038,6.10598,1.004,3.326,1340} | 0
+ 2 | {13.872,1.814,2.376,15.56,88.2,2.806,2.928,0.288,1.844,5.35198,1.044,3.348,988} | 1
+(2 rows)
+</pre>
+
-# Run the same example as above, but using array input. Create the input table:
<pre class="example">
DROP TABLE IF EXISTS km_arrayin CASCADE;
[30/34] incubator-madlib git commit: Bugfix: Elastic net gives
inconsistent result
Posted by ok...@apache.org.
Bugfix: Elastic net gives inconsistent result
JIRA: MADLIB-1092
- Elastic net used to consider the number of rows as the total number
of rows in the table even when grouping was used. This fix changes
that to consider the number of rows in a group while computing IGD.
- Elastic net used to consider mean and standard deviation for both
independent and dependent variables based on the entire table even
when grouping was used. This is now computed based on a group,
which is used to computed the scaled data when standardize=TRUE
for Gaussian IGD.
- One approximation still remains. During gradient computation (C++),
every value in the independent variable (for each dimension) is
subtracted with the mean computed based on the entire table and
not groups. This approximiation was adopted since it is messy to
pass group specific mean values for every row in the table to the
C++ layer.
Closes #126
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/0ff829a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/0ff829a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/0ff829a7
Branch: refs/heads/latest_release
Commit: 0ff829a7060d08f284e8468ebf35c31b6e231d58
Parents: 4b0c377
Author: Nandish Jayaram <nj...@apache.org>
Authored: Mon Apr 24 09:46:03 2017 -0700
Committer: Nandish Jayaram <nj...@apache.org>
Committed: Fri Apr 28 17:47:20 2017 -0700
----------------------------------------------------------------------
.../modules/convex/utils_regularization.py_in | 157 ++++++++++--
.../elastic_net_generate_result.py_in | 89 ++++---
.../elastic_net_optimizer_fista.py_in | 20 +-
.../elastic_net/elastic_net_optimizer_igd.py_in | 106 ++++----
.../modules/elastic_net/elastic_net_utils.py_in | 242 +++++++------------
5 files changed, 341 insertions(+), 273 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0ff829a7/src/ports/postgres/modules/convex/utils_regularization.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/convex/utils_regularization.py_in b/src/ports/postgres/modules/convex/utils_regularization.py_in
index 64204aa..879c0d6 100644
--- a/src/ports/postgres/modules/convex/utils_regularization.py_in
+++ b/src/ports/postgres/modules/convex/utils_regularization.py_in
@@ -6,16 +6,17 @@ from validation.cv_utils import __cv_split_data_using_id_col_compute
from validation.cv_utils import __cv_split_data_using_id_tbl_compute
from validation.cv_utils import __cv_generate_random_id
from utilities.utilities import __mad_version
+from utilities.utilities import split_quoted_delimited_str
version_wrapper = __mad_version()
mad_vec = version_wrapper.select_vecfunc()
# ========================================================================
-
-def __utils_ind_var_scales(**kwargs):
+def __utils_ind_var_scales(tbl_data, col_ind_var, dimension, schema_madlib):
"""
- The mean and standard deviation for each dimension of an array stored in a column.
+ The mean and standard deviation for each dimension of an array stored
+ in a column.
Returns:
Dictionary with keys 'mean' and 'std' each with a value of an array of
@@ -32,19 +33,41 @@ def __utils_ind_var_scales(**kwargs):
FROM
{tbl_data}
) q2
- """.format(**kwargs))[0]
+ """.format(**locals()))[0]
x_scales["mean"] = mad_vec(x_scales["mean"], text=False)
x_scales["std"] = mad_vec(x_scales["std"], text=False)
return x_scales
# ========================================================================
-def __utils_dep_var_scale(**kwargs):
+def __utils_ind_var_scales_grouping(tbl_data, col_ind_var, dimension,
+ schema_madlib, grouping_col, x_mean_table):
+ """
+ The mean and standard deviation for each dimension of an array stored in
+ a column. Creates a table containing the mean (array) and std of each
+ dimension of the independent variable, for each group.
+ """
+ group_col = _cast_if_null(grouping_col, unique_string('grp_col'))
+ x_scales = plpy.execute(
+ """
+ CREATE TEMP TABLE {x_mean_table} AS
+ SELECT (f).*, {group_col}
+ FROM (
+ SELECT {group_col},
+ {schema_madlib}.__utils_var_scales_result(
+ {schema_madlib}.utils_var_scales({col_ind_var}, {dimension})) as f
+ FROM
+ {tbl_data}
+ GROUP BY {group_col}
+ ) q2
+ """.format(**locals()))
+# ========================================================================
+
+def __utils_dep_var_scale(schema_madlib, tbl_data, col_ind_var,
+ col_dep_var):
"""
The mean and standard deviation for each element of the dependent variable,
which is a scalar in ridge and lasso.
- The output will be stored in a temp table: a mean array and a std array
-
This function is also used in lasso.
Parameters:
@@ -53,18 +76,109 @@ def __utils_dep_var_scale(**kwargs):
col_ind_var -- independent variables column
col_dep_var -- dependent variable column
"""
-
y_scale = plpy.execute(
"""
- select
- avg(case when not {schema_madlib}.array_contains_null({col_ind_var}) then {col_dep_var} end) as mean,
- 1 as std
- from {tbl_data}
- """.format(**kwargs))[0]
-
+ SELECT
+ avg(CASE WHEN NOT {schema_madlib}.array_contains_null({col_ind_var}) THEN {col_dep_var} END) AS mean,
+ 1 AS std
+ FROM {tbl_data}
+ """.format(**locals()))[0]
return y_scale
# ========================================================================
+def __utils_dep_var_scale_grouping(y_mean_table, tbl_data, grouping_col,
+ family, schema_madlib=None, col_ind_var=None, col_dep_var=None):
+ """
+ The mean and standard deviation for each element of the dependent variable,
+ w.r.t a group, which is a scalar in ridge and lasso.
+
+ The output will be stored in a temp table: a mean array and a std array,
+ for each group.
+ If the family is Binomial, mean and std for each group is set to 0 and 1
+ respectively.
+
+ This function is also used in lasso.
+
+ Parameters:
+ y_mean_table -- name of the output table to write into
+ tbl_data -- input table
+ grouping_col -- the columns to group the data on
+ family -- if family is Gaussian, ALL following parameters must be defined
+ schema_madlib -- madlib schema
+ col_ind_var -- independent variables column
+ col_dep_var -- dependent variable column
+ """
+ group_col = _cast_if_null(grouping_col, unique_string('grp_col'))
+ if family == 'binomial':
+ mean_str = '0'
+ else:
+ # If the family is Gaussian, schema_madlib, col_ind_var and
+ # col_dep_var must be passed along.
+ if schema_madlib is None or col_ind_var is None or col_dep_var is None:
+ plpy.error("Schema name, indpendent column and dependent column names required.")
+ mean_str = ' avg(CASE WHEN NOT {0}.array_contains_null({1}) THEN {2} END) '.format(
+ schema_madlib, col_ind_var, col_dep_var)
+ plpy.execute(
+ """
+ CREATE TEMP TABLE {y_mean_table} AS
+ SELECT {group_col},
+ {mean_str} AS mean,
+ 1 AS std
+ FROM {tbl_data}
+ GROUP BY {group_col}
+ """.format(**locals()))
+# ========================================================================
+
+def __utils_normalize_data_grouping(y_decenter=True, **kwargs):
+ """
+ Normalize the independent and dependent variables using the calculated
+ mean's and std's in __utils_ind_var_scales and __utils_dep_var_scale.
+
+ Compute the scaled variables by: scaled_value = (origin_value - mean) / std,
+ and special care is needed if std is zero.
+
+ The output is a table with scaled independent and dependent variables,
+ based on mean and std for each group. This function is also used in lasso.
+
+ Parameters:
+ tbl_data -- original data
+ col_ind_var -- independent variables column
+ dimension -- length of independent variable array
+ col_dep_var -- dependent variable column
+ tbl_ind_scales -- independent variables scales array
+ tbl_dep_scale -- dependent variable scale
+ tbl_data_scaled -- scaled data result
+ col_ind_var_norm_new -- create a new name for the scaled array
+ to be compatible with array[...] expressions
+ x_mean_table -- name of the table containing mean of 'x' for each group
+ y_mean_table -- name of the table containing mean of 'y' for each group
+ grouping_col -- columns to group the data on
+ """
+ group_col = kwargs.get('grouping_col')
+ group_col_list = split_quoted_delimited_str(group_col)
+ group_where_x = ' AND '.join(['{tbl_data}.{grp}=__x__.{grp}'.format(grp=grp,
+ **kwargs) for grp in group_col_list])
+ group_where_y = ' AND '.join(['{tbl_data}.{grp}=__y__.{grp}'.format(grp=grp,
+ **kwargs) for grp in group_col_list])
+ ydecenter_str = "- __y__.mean".format(**kwargs) if y_decenter else ""
+ plpy.execute(
+ """
+ CREATE TEMP TABLE {tbl_data_scaled} AS
+ SELECT
+ ({schema_madlib}.utils_normalize_data({col_ind_var},
+ __x__.mean::double precision[],
+ __x__.std::double precision[]))
+ AS {col_ind_var_norm_new},
+ ({col_dep_var} {ydecenter_str}) AS {col_dep_var_norm_new},
+ {tbl_data}.{group_col}
+ FROM {tbl_data}
+ INNER JOIN {x_mean_table} AS __x__ ON {group_where_x}
+ INNER JOIN {y_mean_table} AS __y__ ON {group_where_y}
+ """.format(ydecenter_str=ydecenter_str, group_col=group_col,
+ group_where_x=group_where_x, group_where_y=group_where_y, **kwargs))
+ return None
+# ========================================================================
+
def __utils_normalize_data(y_decenter=True, **kwargs):
"""
Normalize the independent and dependent variables using the calculated mean's and std's
@@ -88,25 +202,22 @@ def __utils_normalize_data(y_decenter=True, **kwargs):
col_ind_var_norm_new -- create a new name for the scaled array
to be compatible with array[...] expressions
"""
- group_col = _cast_if_null(kwargs.get('grouping_col', None), unique_string('grp_col'))
ydecenter_str = "- {y_mean}".format(**kwargs) if y_decenter else ""
plpy.execute(
"""
- create temp table {tbl_data_scaled} as
- select
+ CREATE TEMP TABLE {tbl_data_scaled} AS
+ SELECT
({schema_madlib}.utils_normalize_data({col_ind_var},
'{x_mean_str}'::double precision[],
'{x_std_str}'::double precision[]))
- as {col_ind_var_norm_new},
- ({col_dep_var} {ydecenter_str}) as {col_dep_var_norm_new},
- {group_col}
- from {tbl_data}
- """.format(ydecenter_str=ydecenter_str, group_col=group_col, **kwargs))
+ AS {col_ind_var_norm_new},
+ ({col_dep_var} {ydecenter_str}) AS {col_dep_var_norm_new}
+ FROM {tbl_data}
+ """.format(ydecenter_str=ydecenter_str, **kwargs))
return None
# ========================================================================
-
def __utils_cv_preprocess(kwargs):
"""
Some common processes used in both ridge and lasso cross validation functions:
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0ff829a7/src/ports/postgres/modules/elastic_net/elastic_net_generate_result.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/elastic_net_generate_result.py_in b/src/ports/postgres/modules/elastic_net/elastic_net_generate_result.py_in
index 6246ed9..df5489f 100644
--- a/src/ports/postgres/modules/elastic_net/elastic_net_generate_result.py_in
+++ b/src/ports/postgres/modules/elastic_net/elastic_net_generate_result.py_in
@@ -2,7 +2,7 @@ import plpy
from elastic_net_utils import _process_results
from elastic_net_utils import _compute_log_likelihood
from utilities.validate_args import get_cols_and_types
-
+from utilities.utilities import split_quoted_delimited_str
def _elastic_net_generate_result(optimizer, iteration_run, **args):
"""
@@ -10,6 +10,10 @@ def _elastic_net_generate_result(optimizer, iteration_run, **args):
"""
standardize_flag = "True" if args["normalization"] else "False"
source_table = args["rel_source"]
+ data_scaled = False
+ if args["normalization"] or optimizer == "igd":
+ # x_mean_table and y_mean_table are created only in these conditions.
+ data_scaled = True
if optimizer == "fista":
result_func = "__gaussian_fista_result({0})".format(args["col_grp_state"])
elif optimizer == "igd":
@@ -30,38 +34,54 @@ def _elastic_net_generate_result(optimizer, iteration_run, **args):
grouping_str = args['grouping_str']
cols_types = dict(get_cols_and_types(args["tbl_source"]))
grouping_str1 = grouping_column + ","
- select_grouping_info = ','.join([grp_col.strip() + "\t" + cols_types[grp_col.strip()]
- for grp_col in grouping_column.split(',')]) + ","
+
+ select_mean_and_std = ''
+ inner_join_x = ''
+ inner_join_y = ''
+ if data_scaled:
+ grouping_cols_list = split_quoted_delimited_str(grouping_column)
+ select_grouping_info = ','.join([
+ grp_col.strip()+"\t"+cols_types[grp_col.strip()]
+ for grp_col in grouping_column.split(',')]) + ","
+ select_grp = ','.join(['n_tuples_including_nulls_subq.'+str(grp)
+ for grp in grouping_cols_list]) + ','
+ x_grp_cols = ' AND '.join([
+ 'n_tuples_including_nulls_subq.{0}={1}.{2}'.format(grp,
+ args["x_mean_table"], grp) for grp in grouping_cols_list])
+ y_grp_cols = ' AND '.join([
+ 'n_tuples_including_nulls_subq.{0}={1}.{2}'.format(grp,
+ args["y_mean_table"], grp) for grp in grouping_cols_list])
+ select_mean_and_std = ' {0}.mean AS x_mean, '.format(args["x_mean_table"]) +\
+ ' {0}.mean AS y_mean, '.format(args["y_mean_table"]) +\
+ ' {0}.std AS x_std, '.format(args["x_mean_table"])
+ inner_join_x = ' INNER JOIN {0} ON {1} '.format(
+ args["x_mean_table"], x_grp_cols)
+ inner_join_y = ' INNER JOIN {0} ON {1} '.format(
+ args["y_mean_table"], y_grp_cols)
out_table_qstr = """
SELECT
- {grouping_str1}
+ {select_grp}
+ {select_mean_and_std}
(result).coefficients AS coef,
(result).intercept AS intercept
FROM
(
- SELECT {schema_madlib}.{result_func} AS result, {col_grp_key}
- FROM {tbl_state}
- WHERE {col_grp_iteration} = {iteration_run}
- ) t
- JOIN
- (
SELECT
{grouping_str1}
array_to_string(ARRAY[{grouping_str}], ',') AS {col_grp_key}
FROM {source_table}
- GROUP BY {grouping_col}, {col_grp_key}
+ GROUP BY {grouping_column}, {col_grp_key}
) n_tuples_including_nulls_subq
- USING ({col_grp_key})
- """.format(result_func=result_func,
- tbl_state=tbl_state,
- grouping_col=grouping_column,
- col_grp_iteration=args["col_grp_iteration"],
- iteration_run=iteration_run,
- grouping_str1=grouping_str1,
- grouping_str=grouping_str,
- col_grp_key=col_grp_key,
- source_table=source_table,
- schema_madlib=args["schema_madlib"])
+ INNER JOIN
+ (
+ SELECT {schema_madlib}.{result_func} AS result, {col_grp_key}
+ FROM {tbl_state}
+ WHERE {col_grp_iteration} = {iteration_run}
+ ) t USING ({col_grp_key})
+ {inner_join_x}
+ {inner_join_y}
+ """.format(schema_madlib=args["schema_madlib"],
+ col_grp_iteration=args["col_grp_iteration"], **locals())
else:
# It's a much simpler query when there is no grouping.
grouping_str1 = ""
@@ -139,7 +159,12 @@ def build_output_table(res, grouping_column, grouping_str1,
r_coef = res["coef"]
if r_coef:
if args["normalization"]:
- (coef, intercept) = _restore_scale(r_coef, res["intercept"], args)
+ if grouping_column:
+ (coef, intercept) = _restore_scale(r_coef, res["intercept"],
+ args, res["x_mean"], res["x_std"], res["y_mean"])
+ else:
+ (coef, intercept) = _restore_scale(r_coef,
+ res["intercept"], args)
else:
coef = r_coef
intercept = res["intercept"]
@@ -167,20 +192,22 @@ def build_output_table(res, grouping_column, grouping_str1,
**args)
plpy.execute(fquery)
# ------------------------------------------------------------------------
-
-
-def _restore_scale(coef, intercept, args):
+def _restore_scale(coef, intercept, args,
+ x_mean=None, x_std=None, y_mean=None):
"""
Restore the original scales
"""
+ if x_mean is None and x_std is None and y_mean is None:
+ x_mean = args["x_scales"]["mean"]
+ y_mean = args["y_scale"]["mean"]
+ x_std = args["x_scales"]["std"]
rcoef = [0] * len(coef)
if args["family"] == "gaussian":
- rintercept = float(args["y_scale"]["mean"])
+ rintercept = float(y_mean)
elif args["family"] == "binomial":
rintercept = float(intercept)
for i in range(len(coef)):
- if args["x_scales"]["std"][i] != 0:
- rcoef[i] = coef[i] / args["x_scales"]["std"][i]
- rintercept -= (coef[i] * args["x_scales"]["mean"][i] /
- args["x_scales"]["std"][i])
+ if x_std[i] != 0:
+ rcoef[i] = coef[i] / x_std[i]
+ rintercept -= (coef[i] * x_mean[i] / x_std[i])
return (rcoef, rintercept)
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0ff829a7/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_fista.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_fista.py_in b/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_fista.py_in
index f50a214..a6ef699 100644
--- a/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_fista.py_in
+++ b/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_fista.py_in
@@ -136,6 +136,8 @@ def _fista_cleanup_temp_tbls(**kwargs):
drop table if exists {tbl_data_scaled};
drop table if exists {tbl_fista_args};
drop table if exists pg_temp.{tbl_fista_state};
+ drop table if exists {x_mean_table};
+ drop table if exists {y_mean_table};
""".format(**kwargs))
return None
@@ -209,7 +211,8 @@ def _elastic_net_fista_train_compute(schema_madlib, func_step_aggregate,
lambda_value,
tolerance,
schema_madlib))
-
+ args.update({'x_mean_table':unique_string(desp='x_mean_table')})
+ args.update({'y_mean_table':unique_string(desp='y_mean_table')})
args.update({'grouping_col': grouping_col})
# use normalized data or not
if normalization:
@@ -226,19 +229,15 @@ def _elastic_net_fista_train_compute(schema_madlib, func_step_aggregate,
if args["warmup_lambdas"] is not None:
args["warm_no"] = len(args["warmup_lambdas"])
- args["warmup_lambdas"] = args["warmup_lambdas"]
if args["warmup"] and args["warmup_lambdas"] is None:
# average squares of each feature
# used to estimate the largest lambda value
args["sq"] = _compute_average_sq(**args)
args["warmup_lambdas"] = \
- _generate_warmup_lambda_sequence(
- tbl_used, args["col_ind_var_new"], args["col_dep_var_new"],
- dimension, row_num, lambda_value, alpha,
- args["warmup_lambda_no"], args["sq"])
+ _generate_warmup_lambda_sequence(lambda_value,
+ args["warmup_lambda_no"])
args["warm_no"] = len(args["warmup_lambdas"])
- args["warmup_lambdas"] = args["warmup_lambdas"]
elif args["warmup"] is False:
args["warm_no"] = 1
args["warmup_lambdas"] = [lambda_value] # only one value
@@ -340,6 +339,11 @@ def _compute_fista(schema_madlib, func_step_aggregate, func_state_diff,
if (it.kwargs["lambda_count"] > len(args.get('lambda_name'))):
break
it.kwargs["warmup_lambda_value"] = args.get('lambda_name')[it.kwargs["lambda_count"] - 1]
+ # Fix for JIRA MADLIB-1092
+ # 'col_n_tuples' is supposed to refer to the number of rows in the
+ # table, or the number of rows in a group. col_n_tuples gets
+ # the right value in in_mem_group_control, so using this instead
+ # of row_num (which was used hitherto).
it.update("""
{schema_madlib}.{func_step_aggregate}(
({col_ind_var})::double precision[],
@@ -348,7 +352,7 @@ def _compute_fista(schema_madlib, func_step_aggregate, func_state_diff,
({warmup_lambda_value})::double precision,
({alpha})::double precision,
({dimension})::integer,
- ({row_num})::integer,
+ ({col_n_tuples})::integer,
({max_stepsize})::double precision,
({eta})::double precision,
({use_active_set})::integer,
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0ff829a7/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in b/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in
index 091aefb..d73a754 100644
--- a/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in
+++ b/src/ports/postgres/modules/elastic_net/elastic_net_optimizer_igd.py_in
@@ -144,6 +144,8 @@ def _igd_cleanup_temp_tbls(**args):
drop table if exists {tbl_data_scaled};
drop table if exists {tbl_igd_args};
drop table if exists pg_temp.{tbl_igd_state};
+ drop table if exists {x_mean_table};
+ drop table if exists {y_mean_table};
""".format(**args))
return None
# ------------------------------------------------------------------------
@@ -193,17 +195,19 @@ def _elastic_net_igd_train_compute(schema_madlib, func_step_aggregate,
is '{arg = value, ...}'::varchar[]
"""
with MinWarning('error'):
- (dimension, row_num) = _tbl_dimension_rownum(schema_madlib, tbl_source, col_ind_var)
+ (dimension, row_num) = _tbl_dimension_rownum(schema_madlib,
+ tbl_source, col_ind_var)
# generate a full dict to ease the following string format
# including several temporary table names
- args = _igd_construct_dict(schema_madlib, family, tbl_source, col_ind_var,
- col_dep_var, tbl_result,
- dimension, row_num, lambda_value, alpha, normalization,
- max_iter, tolerance, outstr_array,
- _igd_params_parser(optimizer_params, lambda_value,
- tolerance, schema_madlib))
-
+ args = _igd_construct_dict(schema_madlib, family, tbl_source,
+ col_ind_var, col_dep_var, tbl_result, dimension, row_num,
+ lambda_value, alpha, normalization, max_iter, tolerance,
+ outstr_array, _igd_params_parser(optimizer_params, lambda_value,
+ tolerance, schema_madlib))
+
+ args.update({'x_mean_table':unique_string(desp='x_mean_table')})
+ args.update({'y_mean_table':unique_string(desp='y_mean_table')})
args.update({'grouping_col': grouping_col})
# use normalized data or not
if normalization:
@@ -216,9 +220,10 @@ def _elastic_net_igd_train_compute(schema_madlib, func_step_aggregate,
tbl_used = tbl_source
args["col_ind_var_new"] = col_ind_var
args["col_dep_var_new"] = col_dep_var
-
args["tbl_used"] = tbl_used
+ # parameter values required by the IGD optimizer
+ (xmean, ymean) = _compute_means(args)
# average squares of each feature
# used to estimate the largest lambda value
# also used to screen out tiny values, so order is needed
@@ -227,23 +232,16 @@ def _elastic_net_igd_train_compute(schema_madlib, func_step_aggregate,
if args["warmup_lambdas"] is not None:
args["warm_no"] = len(args["warmup_lambdas"])
- args["warmup_lambdas"] = args["warmup_lambdas"]
if args["warmup"] and args["warmup_lambdas"] is None:
args["warmup_lambdas"] = \
- _generate_warmup_lambda_sequence(
- args["tbl_used"], args["col_ind_var_new"], args["col_dep_var_new"],
- dimension, row_num, lambda_value, alpha,
- args["warmup_lambda_no"], args["sq"])
+ _generate_warmup_lambda_sequence(lambda_value,
+ args["warmup_lambda_no"])
args["warm_no"] = len(args["warmup_lambdas"])
- args["warmup_lambdas"] = args["warmup_lambdas"]
elif args["warmup"] is False:
args["warm_no"] = 1
args["warmup_lambdas"] = [lambda_value] # only one value
- # parameter values required by the IGD optimizer
- (xmean, ymean) = _compute_means(**args)
-
args.update({
'rel_args': args["tbl_igd_args"],
'rel_state': args["tbl_igd_state"],
@@ -263,38 +261,39 @@ def _elastic_net_igd_train_compute(schema_madlib, func_step_aggregate,
if not args.get('parallel'):
func_step_aggregate += "_single_seg"
# perform the actual calculation
- iteration_run = _compute_igd(schema_madlib,
- func_step_aggregate,
- func_state_diff,
- args["tbl_igd_args"],
- args["tbl_igd_state"],
- tbl_used,
- args["col_ind_var_new"],
- args["col_dep_var_new"],
- grouping_str,
- grouping_col,
- start_iter=0,
- max_iter=args["max_iter"],
- tolerance=args["tolerance"],
- warmup_tolerance=args["warmup_tolerance"],
- warm_no=args["warm_no"],
- step_decay=args["step_decay"],
- dimension=args["dimension"],
- stepsize=args["stepsize"],
- lambda_name=args["warmup_lambdas"],
- warmup_lambda_value=args.get('warmup_lambdas')[args["lambda_count"]-1],
- alpha=args["alpha"],
- row_num=args["row_num"],
- xmean_val=args["xmean_val"],
- ymean_val=args["ymean_val"],
- lambda_count=args["lambda_count"],
- rel_state=args["tbl_igd_state"],
- col_grp_iteration=args["col_grp_iteration"],
- col_grp_state=args["col_grp_state"],
- col_grp_key=args["col_grp_key"],
- col_n_tuples=args["col_n_tuples"],
- rel_source=args["rel_source"],
- state_type=args["state_type"],)
+ iteration_run = _compute_igd(
+ schema_madlib,
+ func_step_aggregate,
+ func_state_diff,
+ args["tbl_igd_args"],
+ args["tbl_igd_state"],
+ tbl_used,
+ args["col_ind_var_new"],
+ args["col_dep_var_new"],
+ grouping_str,
+ grouping_col,
+ start_iter=0,
+ max_iter=args["max_iter"],
+ tolerance=args["tolerance"],
+ warmup_tolerance=args["warmup_tolerance"],
+ warm_no=args["warm_no"],
+ step_decay=args["step_decay"],
+ dimension=args["dimension"],
+ stepsize=args["stepsize"],
+ lambda_name=args["warmup_lambdas"],
+ warmup_lambda_value=args.get('warmup_lambdas')[args["lambda_count"]-1],
+ alpha=args["alpha"],
+ row_num=args["row_num"],
+ xmean_val=args["xmean_val"],
+ ymean_val=args["ymean_val"],
+ lambda_count=args["lambda_count"],
+ rel_state=args["tbl_igd_state"],
+ col_grp_iteration=args["col_grp_iteration"],
+ col_grp_state=args["col_grp_state"],
+ col_grp_key=args["col_grp_key"],
+ col_n_tuples=args["col_n_tuples"],
+ rel_source=args["rel_source"],
+ state_type=args["state_type"])
_elastic_net_generate_result("igd", iteration_run, **args)
@@ -341,6 +340,11 @@ def _compute_igd(schema_madlib, func_step_aggregate, func_state_diff,
if (it.kwargs["lambda_count"] > len(args.get('lambda_name'))):
break
it.kwargs["warmup_lambda_value"] = args.get('lambda_name')[it.kwargs["lambda_count"] - 1]
+ # Fix for JIRA MADLIB-1092
+ # 'col_n_tuples' is supposed to refer to the number of rows in the
+ # table, or the number of rows in a group. col_n_tuples gets
+ # the right value in in_mem_group_control, so using this instead
+ # of row_num (which was used hitherto).
it.update("""
{schema_madlib}.{func_step_aggregate}(
({col_ind_var})::double precision[],
@@ -350,7 +354,7 @@ def _compute_igd(schema_madlib, func_step_aggregate, func_state_diff,
({alpha})::double precision,
({dimension})::integer,
({stepsize})::double precision,
- ({row_num})::integer,
+ ({col_n_tuples})::integer,
('{xmean_val}')::double precision[],
({ymean_val})::double precision,
({step_decay})::double precision
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/0ff829a7/src/ports/postgres/modules/elastic_net/elastic_net_utils.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/elastic_net_utils.py_in b/src/ports/postgres/modules/elastic_net/elastic_net_utils.py_in
index ce6b280..b2f2505 100644
--- a/src/ports/postgres/modules/elastic_net/elastic_net_utils.py_in
+++ b/src/ports/postgres/modules/elastic_net/elastic_net_utils.py_in
@@ -6,8 +6,10 @@ from utilities.utilities import _array_to_string
from convex.utils_regularization import __utils_ind_var_scales
from convex.utils_regularization import __utils_dep_var_scale
from convex.utils_regularization import __utils_normalize_data
+from convex.utils_regularization import __utils_ind_var_scales_grouping
+from convex.utils_regularization import __utils_dep_var_scale_grouping
+from convex.utils_regularization import __utils_normalize_data_grouping
from utilities.validate_args import table_exists
-from utilities.control import IterationController2S
from collections import namedtuple
@@ -108,7 +110,6 @@ def _generate_warmup_lambda_sequence(lambda_value, n_steps):
return seq
# ------------------------------------------------------------------------
-
def _compute_average_sq(**args):
"""
Compute the average squares of all features, used to estimtae the largest lambda
@@ -192,16 +193,27 @@ def _elastic_net_validate_args(tbl_source, col_ind_var, col_dep_var,
return None
# ------------------------------------------------------------------------
+def _compute_data_scales_grouping(args):
+ __utils_ind_var_scales_grouping(args["tbl_source"], args["col_ind_var"],
+ args["dimension"], args["schema_madlib"], args["grouping_col"],
+ args["x_mean_table"])
+ if args["family"] == "binomial":
+ # set mean and std to 0 and 1 respectively, for each group.
+ __utils_dep_var_scale_grouping(args["y_mean_table"],
+ args["tbl_source"], args["grouping_col"], args["family"])
+ else:
+ __utils_dep_var_scale_grouping(args["y_mean_table"],
+ args["tbl_source"], args["grouping_col"], args["family"],
+ args["schema_madlib"], args["col_ind_var"], args["col_dep_var"])
def _compute_data_scales(args):
- args["x_scales"] = __utils_ind_var_scales(tbl_data=args["tbl_source"], col_ind_var=args["col_ind_var"],
- dimension=args["dimension"], schema_madlib=args["schema_madlib"])
-
+ args["x_scales"] = __utils_ind_var_scales(args["tbl_source"],
+ args["col_ind_var"], args["dimension"], args["schema_madlib"])
if args["family"] == "binomial":
args["y_scale"] = dict(mean=0, std=1)
else:
- args["y_scale"] = __utils_dep_var_scale(schema_madlib=args["schema_madlib"], tbl_data=args["tbl_source"],
- col_ind_var=args["col_ind_var"], col_dep_var=args["col_dep_var"])
+ args["y_scale"] = __utils_dep_var_scale(args["schema_madlib"],
+ args["tbl_source"], args["col_ind_var"], args["col_dep_var"])
args["xmean_str"] = _array_to_string(args["x_scales"]["mean"])
# ------------------------------------------------------------------------
@@ -214,23 +226,45 @@ def _normalize_data(args):
The output is stored in tbl_data_scaled
"""
- _compute_data_scales(args)
-
y_decenter = True if args["family"] == "gaussian" else False
-
- __utils_normalize_data(y_decenter=y_decenter,
- tbl_data=args["tbl_source"],
- col_ind_var=args["col_ind_var"],
- col_dep_var=args["col_dep_var"],
- tbl_data_scaled=args["tbl_data_scaled"],
- col_ind_var_norm_new=args["col_ind_var_norm_new"],
- col_dep_var_norm_new=args["col_dep_var_norm_new"],
- schema_madlib=args["schema_madlib"],
- x_mean_str=args["xmean_str"],
- x_std_str=_array_to_string(args["x_scales"]["std"]),
- y_mean=args["y_scale"]["mean"],
- y_std=args["y_scale"]["std"],
- grouping_col=args["grouping_col"])
+ if args["grouping_col"]:
+ # When grouping_col is defined, we must find an array containing
+ # the mean of every dimension in the independent variable (x), the
+ # mean of dependent variable (y) and the standard deviation for them
+ # specific to groups. Store these results in temp tables x_mean_table
+ # and y_mean_table.
+ _compute_data_scales_grouping(args)
+ # __utils_normalize_data_grouping reads the various means and stds
+ # from the tables.
+ __utils_normalize_data_grouping(y_decenter=y_decenter,
+ tbl_data=args["tbl_source"],
+ col_ind_var=args["col_ind_var"],
+ col_dep_var=args["col_dep_var"],
+ tbl_data_scaled=args["tbl_data_scaled"],
+ col_ind_var_norm_new=args["col_ind_var_norm_new"],
+ col_dep_var_norm_new=args["col_dep_var_norm_new"],
+ schema_madlib=args["schema_madlib"],
+ x_mean_table=args["x_mean_table"],
+ y_mean_table=args["y_mean_table"],
+ grouping_col=args["grouping_col"])
+ else:
+ # When no grouping_col is defined, the mean and std for both 'x' and
+ # 'y' can be defined using strings, stored in x_mean_str, x_std_str
+ # etc. We don't need a table like how we needed for grouping.
+ _compute_data_scales(args)
+ __utils_normalize_data(y_decenter=y_decenter,
+ tbl_data=args["tbl_source"],
+ col_ind_var=args["col_ind_var"],
+ col_dep_var=args["col_dep_var"],
+ tbl_data_scaled=args["tbl_data_scaled"],
+ col_ind_var_norm_new=args["col_ind_var_norm_new"],
+ col_dep_var_norm_new=args["col_dep_var_norm_new"],
+ schema_madlib=args["schema_madlib"],
+ x_mean_str=args["xmean_str"],
+ x_std_str=_array_to_string(args["x_scales"]["std"]),
+ y_mean=args["y_scale"]["mean"],
+ y_std=args["y_scale"]["std"],
+ grouping_col=args["grouping_col"])
return None
# ------------------------------------------------------------------------
@@ -242,27 +276,27 @@ def _tbl_dimension_rownum(schema_madlib, tbl_source, col_ind_var):
"""
# independent variable array length
dimension = plpy.execute("""
- select array_upper({col_ind_var},1) as dimension
- from {tbl_source} limit 1
- """.format(tbl_source=tbl_source,
- col_ind_var=col_ind_var))[0]["dimension"]
+ SELECT array_upper({col_ind_var},1) AS dimension
+ FROM {tbl_source} LIMIT 1
+ """.format(tbl_source=tbl_source,
+ col_ind_var=col_ind_var))[0]["dimension"]
# total row number of data source table
- # The WHERE clause here ignores rows in the table that contain one or more NULLs in the
- # independent variable (x). There is no NULL check made for the dependent variable (y),
- # since one of the hard requirements/assumptions of the input data to elastic_net is that the
- # dependent variable cannot be NULL.
+ # The WHERE clause here ignores rows in the table that contain one or more
+ # NULLs in the independent variable (x). There is no NULL check made for
+ # the dependent variable (y), since one of the hard assumptions of the
+ # input data to elastic_net is that the dependent variable cannot be NULL.
row_num = plpy.execute("""
- select count(*) from {tbl_source}
- WHERE not {schema_madlib}.array_contains_null({col_ind_var})
- """.format(tbl_source=tbl_source,
- schema_madlib=schema_madlib,
- col_ind_var=col_ind_var))[0]["count"]
+ SELECT COUNT(*) FROM {tbl_source}
+ WHERE NOT {schema_madlib}.array_contains_null({col_ind_var})
+ """.format(tbl_source=tbl_source,
+ schema_madlib=schema_madlib,
+ col_ind_var=col_ind_var))[0]["count"]
return (dimension, row_num)
# ------------------------------------------------------------------------
-def _compute_means(**args):
+def _compute_means(args):
"""
Compute the averages of dependent (y) and independent (x) variables
"""
@@ -270,127 +304,15 @@ def _compute_means(**args):
xmean_str = _array_to_string([0] * args["dimension"])
ymean = 0
return (xmean_str, ymean)
- else:
- return (args["xmean_str"], args["y_scale"]["mean"])
+ if args["grouping_col"]:
+ # We can use the mean of the entire table instead of groups here.
+ # The absolute correct thing to do is to use group specific
+ # mean, but we will need to add a new column and change the input
+ # table contents to do that (it has to be accessed by the group
+ # iteration controller, C++ code). That is a lot more messier,
+ # so living with this approximation for now.
+ _compute_data_scales(args)
+ # If there is no grouping_col, note that _compute_data_scales() was
+ # already called, so we don't have to call it again.
+ return (args["xmean_str"], args["y_scale"]["mean"])
# ------------------------------------------------------------------------
-
-
-class IterationControllerNoTableDrop (IterationController2S):
-
- """
- IterationController but without table dropping
-
- Useful if one wants to use it in cross validation
- where dropping tables in a loop would use up all the locks
- and get "out of memory" error
- """
- # ------------------------------------------------------------------------
-
- def __init__(self, rel_args, rel_state, stateType,
- temporaryTables=True,
- truncAfterIteration=False,
- schema_madlib="MADLIB_SCHEMA_MISSING",
- verbose=False,
- **kwargs):
- # Need to call super class's init method to initialize
- # member fields
- super(IterationControllerNoTableDrop, self).__init__(
- self, rel_args, rel_state, stateType, temporaryTables,
- truncAfterIteration, schema_madlib, verbose, **kwargs)
- # self.kwargs["rel_state"] = "pg_temp" + rel_state, but for testing
- # the existence of a table, schema name should be used together
- self.state_exists = plpy.execute(
- "select count(*) from information_schema.tables "
- "where table_name = '{0}' and table_schema = 'pg_temp'".
- format(rel_state))[0]['count'] == 1
- # The current total row number of rel_state table
- if self.state_exists:
- self.state_row_num = plpy.execute("select count(*) from {rel_state}".
- format(**self.kwargs))[0]["count"]
-
- # ------------------------------------------------------------------------
-
- def update(self, newState):
- """
- Update state of calculation.
- """
- newState = newState.format(iteration=self.iteration, **self.kwargs)
- self.iteration += 1
- if self.state_exists and self.iteration <= self.state_row_num:
- # If the rel_state table already exists, and
- # iteration number is smaller than total row number,
- # use UPDATE instead of append. UPDATE does not use
- # extra locks.
- self.runSQL("""
- update {rel_state} set _state = ({newState})
- where _iteration = {iteration}
- """.format(iteration=self.iteration,
- newState=newState,
- **self.kwargs))
- else:
- # rel_state table is newly created, and
- # append data to this table
- self.runSQL("""
- INSERT INTO {rel_state}
- SELECT
- {iteration},
- ({newState})
- """.format(iteration=self.iteration,
- newState=newState,
- **self.kwargs))
- # ------------------------------------------------------------------------
-
- def __enter__(self):
- """
- __enter__ and __exit__ methods are special. They are automatically called
- when using "with" block.
- """
- if self.state_exists is False:
- # create rel_state table when it does not already exist
- super(IterationControllerNoTableDrop, self).__enter__(self)
- self.inWith = True
- return self
-# ------------------------------------------------------------------------
-
-
-class IterationControllerTableAppend (IterationControllerNoTableDrop):
-
- def __init__(self, rel_args, rel_state, stateType,
- temporaryTables=True,
- truncAfterIteration=False,
- schema_madlib="MADLIB_SCHEMA_MISSING",
- verbose=False,
- **kwargs):
- self.kwargs = kwargs
- self.kwargs.update(
- rel_args=rel_args,
- rel_state=rel_state,
- stateType=stateType.format(schema_madlib=schema_madlib),
- schema_madlib=schema_madlib)
- self.temporaryTables = temporaryTables
- self.truncAfterIteration = truncAfterIteration
- self.verbose = verbose
- self.inWith = False
- self.iteration = -1
-
- self.state_exists = plpy.execute("""
- select count(*)
- from information_schema.tables
- where table_name = '{rel_state}'
- """.format(**self.kwargs))[0]['count'] == 1
- # ------------------------------------------------------------------------
-
- def update(self, newState):
- """
- Update state of calculation.
- """
- newState = newState.format(iteration=self.iteration, **self.kwargs)
- self.iteration += 1
- self.runSQL("""
- INSERT INTO {rel_state}
- SELECT
- {iteration},
- ({newState})
- """.format(iteration=self.iteration,
- newState=newState,
- **self.kwargs))
[27/34] incubator-madlib git commit: DT: Update error message for
invalid num_splits
Posted by ok...@apache.org.
DT: Update error message for invalid num_splits
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/c4fd91e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/c4fd91e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/c4fd91e1
Branch: refs/heads/latest_release
Commit: c4fd91e16827a5f8be4051eb3ea0d311d3e957f2
Parents: a3d54be
Author: Rahul Iyer <ri...@apache.org>
Authored: Thu Apr 27 12:12:48 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Thu Apr 27 12:12:48 2017 -0700
----------------------------------------------------------------------
src/modules/recursive_partitioning/feature_encoding.cpp | 8 ++++++--
.../recursive_partitioning/test/decision_tree.sql_in | 12 +++++++++---
2 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c4fd91e1/src/modules/recursive_partitioning/feature_encoding.cpp
----------------------------------------------------------------------
diff --git a/src/modules/recursive_partitioning/feature_encoding.cpp b/src/modules/recursive_partitioning/feature_encoding.cpp
index 20856e2..3b0a452 100644
--- a/src/modules/recursive_partitioning/feature_encoding.cpp
+++ b/src/modules/recursive_partitioning/feature_encoding.cpp
@@ -39,7 +39,7 @@ dst_compute_con_splits_transition::run(AnyType &args){
if (!state.empty() && state.num_rows >= state.buff_size) {
return args[0];
}
- // NULL-handling is done in python to make sure consistency b/w
+ // NULLs are handled by caller to ensure consistency between
// feature encoding and tree training
MappedColumnVector con_features = args[1].getAs<MappedColumnVector>();
@@ -71,8 +71,12 @@ dst_compute_con_splits_final::run(AnyType &args){
if (state.num_rows <= state.num_splits) {
std::stringstream error_msg;
+ // In message below, add 1 to state.num_splits since the meaning of
+ // "splits" for the caller is the number of quantiles, where as
+ // "splits" in this function is the number of values dividing the data
+ // into quantiles.
error_msg << "Decision tree error: Number of splits ("
- << state.num_splits
+ << state.num_splits + 1
<< ") is larger than the number of records ("
<< state.num_rows << ")";
throw std::runtime_error(error_msg.str());
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c4fd91e1/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
index 28a4647..dd861a0 100644
--- a/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
@@ -287,7 +287,7 @@ SELECT tree_train('dt_golf'::text, -- source table
'train_output'::text, -- output model table
'id'::text, -- id column
'temperature::double precision'::text, -- response
- 'humidity, windy'::text, -- features
+ '"OUTLOOK", humidity, windy'::text, -- features
NULL::text, -- exclude columns
'gini'::text, -- split criterion
'class'::text, -- grouping
@@ -301,13 +301,19 @@ SELECT tree_train('dt_golf'::text, -- source table
SELECT _print_decision_tree(tree) from train_output;
SELECT tree_display('train_output', False);
-SELECT tree_predict('train_output', 'dt_golf', 'predict_output');
+
+CREATE TABLE dt_golf2 as
+SELECT * FROM dt_golf
+UNION
+SELECT 15 as id, 'humid' as "OUTLOOK", 71 as temperature, 80 as humidity,
+ true as windy, 'Don''t Play' as class;
+SELECT tree_predict('train_output', 'dt_golf2', 'predict_output');
\x off
SELECT *
FROM
predict_output
JOIN
- dt_golf
+ dt_golf2
USING (id);
\x on
select * from train_output;
[31/34] incubator-madlib git commit: Build: Fix the version number in
postflight.py
Posted by ok...@apache.org.
Build: Fix the version number in postflight.py
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/d54be2b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/d54be2b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/d54be2b8
Branch: refs/heads/latest_release
Commit: d54be2b8574c5bf0ace96b94ba81f3e5cbf70a35
Parents: 0ff829a
Author: Orhan Kislal <ok...@pivotal.io>
Authored: Tue May 2 11:34:34 2017 -0700
Committer: Orhan Kislal <ok...@pivotal.io>
Committed: Tue May 2 11:34:34 2017 -0700
----------------------------------------------------------------------
deploy/postflight.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/d54be2b8/deploy/postflight.sh
----------------------------------------------------------------------
diff --git a/deploy/postflight.sh b/deploy/postflight.sh
index ddc1f81..ec36535 100755
--- a/deploy/postflight.sh
+++ b/deploy/postflight.sh
@@ -2,7 +2,7 @@
# $0 - Script Path, $1 - Package Path, $2 - Target Location, and $3 - Target Volumn
-MADLIB_VERSION=1.10.0
+MADLIB_VERSION=1.11
find $2/usr/local/madlib/bin -type d -exec cp -RPf {} $2/usr/local/madlib/old_bin \; 2>/dev/null
find $2/usr/local/madlib/bin -depth -type d -exec rm -r {} \; 2>/dev/null
[20/34] incubator-madlib git commit: Multiple: Minor changes for
GPDB5 and HAWQ2.2 support
Posted by ok...@apache.org.
Multiple: Minor changes for GPDB5 and HAWQ2.2 support
- Separate multi-command plpy.execute commands
- Disable some install check tests temporarily
- Add libstemmer_porter2 license
Closes #119
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/c8bfbf81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/c8bfbf81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/c8bfbf81
Branch: refs/heads/latest_release
Commit: c8bfbf81fa3de96ae4cff4e93ded49bd1ce88123
Parents: 9362ba8
Author: Orhan Kislal <ok...@pivotal.io>
Authored: Thu Apr 20 09:52:06 2017 -0700
Committer: Orhan Kislal <ok...@pivotal.io>
Committed: Thu Apr 20 09:52:06 2017 -0700
----------------------------------------------------------------------
licenses/third_party/libstemmer_porter2.txt | 20 +++++
.../test/elastic_net_install_check.sql_in | 48 +++++-----
src/ports/postgres/modules/graph/sssp.py_in | 31 ++++---
.../postgres/modules/graph/test/pagerank.sql_in | 24 ++---
src/ports/postgres/modules/pca/test/pca.sql_in | 64 ++++++-------
.../validation/test/cross_validation.sql_in | 94 ++++++++++----------
6 files changed, 156 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c8bfbf81/licenses/third_party/libstemmer_porter2.txt
----------------------------------------------------------------------
diff --git a/licenses/third_party/libstemmer_porter2.txt b/licenses/third_party/libstemmer_porter2.txt
new file mode 100644
index 0000000..6bd6e82
--- /dev/null
+++ b/licenses/third_party/libstemmer_porter2.txt
@@ -0,0 +1,20 @@
+License details from
+http://snowballstem.org/license.html
+
+Except where explicitly noted, all the software given out on this Snowball site is covered by the 3-clause BSD License:
+
+Copyright (c) 2001, Dr Martin Porter,
+Copyright (c) 2002, Richard Boulton.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Essentially, all this means is that you can do what you like with the code, except claim another Copyright for it, or claim that it is issued under a different license. The software is also issued without warranties, which means that if anyone suffers through its use, they cannot come back and sue you. You also have to alert anyone to whom you give the Snowball software to the fact that it is covered by the BSD license.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c8bfbf81/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in b/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
index 5146b93..cda7549 100644
--- a/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
+++ b/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
@@ -840,27 +840,27 @@ SELECT elastic_net_train(
SELECT * FROM house_en;
SELECT * FROM house_en_summary;
-DROP TABLE if exists house_en, house_en_summary, house_en_cv;
-SELECT elastic_net_train(
- 'lin_housing_wi',
- 'house_en',
- 'y',
- 'x',
- 'gaussian',
- 0.1,
- 0.2,
- True,
- NULL,
- 'fista',
- $$ eta = 2, max_stepsize = 0.5, use_active_set = f,
- n_folds = 3, validation_result=house_en_cv,
- n_lambdas = 3, alpha = {0, 0.1, 1},
- warmup = True, warmup_lambdas = {10, 1, 0.1}
- $$,
- NULL,
- 100,
- 1e-6
-);
-SELECT * FROM house_en;
-SELECT * FROM house_en_summary;
-SELECT * FROM house_en_cv;
+-- DROP TABLE if exists house_en, house_en_summary, house_en_cv;
+-- SELECT elastic_net_train(
+-- 'lin_housing_wi',
+-- 'house_en',
+-- 'y',
+-- 'x',
+-- 'gaussian',
+-- 0.1,
+-- 0.2,
+-- True,
+-- NULL,
+-- 'fista',
+-- $$ eta = 2, max_stepsize = 0.5, use_active_set = f,
+-- n_folds = 3, validation_result=house_en_cv,
+-- n_lambdas = 3, alpha = {0, 0.1, 1},
+-- warmup = True, warmup_lambdas = {10, 1, 0.1}
+-- $$,
+-- NULL,
+-- 100,
+-- 1e-6
+-- );
+-- SELECT * FROM house_en;
+-- SELECT * FROM house_en_summary;
+-- SELECT * FROM house_en_cv;
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c8bfbf81/src/ports/postgres/modules/graph/sssp.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/sssp.py_in b/src/ports/postgres/modules/graph/sssp.py_in
index 2520830..4dbd1b1 100644
--- a/src/ports/postgres/modules/graph/sssp.py_in
+++ b/src/ports/postgres/modules/graph/sssp.py_in
@@ -314,9 +314,13 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
{checkg_oo})
UNION
SELECT {grp_comma} id, {weight}, parent FROM {oldupdate};
- DROP TABLE {out_table};
- ALTER TABLE {temp_table} RENAME TO {out_table};
- CREATE TABLE {temp_table} AS (
+ """
+ plpy.execute(sql.format(**locals()))
+ sql = "DROP TABLE {out_table}"
+ plpy.execute(sql.format(**locals()))
+ sql = "ALTER TABLE {temp_table} RENAME TO {out_table}"
+ plpy.execute(sql.format(**locals()))
+ sql = """ CREATE TABLE {temp_table} AS (
SELECT * FROM {out_table} LIMIT 0)
{distribution};"""
plpy.execute(sql.format(**locals()))
@@ -409,7 +413,7 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
# It is possible that not all groups has negative cycles.
else:
- # gsql is the string created by collating grouping columns.
+ # grp is the string created by collating grouping columns.
# By looking at the oldupdate table we can see which groups
# are in a negative cycle.
@@ -419,9 +423,6 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
""".format(**locals()))[0]['grp']
# Delete the groups with negative cycles from the output table.
- sql_del = """ DELETE FROM {out_table}
- USING {oldupdate} AS oldupdate
- WHERE {checkg_oo_sub}"""
if is_hawq:
sql_del = """
TRUNCATE TABLE {temp_table};
@@ -432,11 +433,17 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
SELECT 1
FROM {oldupdate} as oldupdate
WHERE {checkg_oo_sub}
- );
- DROP TABLE {out_table};
- ALTER TABLE {temp_table} RENAME TO {out_table};"""
-
- plpy.execute(sql_del.format(**locals()))
+ );"""
+ plpy.execute(sql_del.format(**locals()))
+ sql_del = "DROP TABLE {out_table}"
+ plpy.execute(sql_del.format(**locals()))
+ sql_del = "ALTER TABLE {temp_table} RENAME TO {out_table};"
+ plpy.execute(sql_del.format(**locals()))
+ else:
+ sql_del = """ DELETE FROM {out_table}
+ USING {oldupdate} AS oldupdate
+ WHERE {checkg_oo_sub}"""
+ plpy.execute(sql_del.format(**locals()))
# If every group has a negative cycle,
# drop the output table as well.
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c8bfbf81/src/ports/postgres/modules/graph/test/pagerank.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/test/pagerank.sql_in b/src/ports/postgres/modules/graph/test/pagerank.sql_in
index 2e84f35..4c02df3 100644
--- a/src/ports/postgres/modules/graph/test/pagerank.sql_in
+++ b/src/ports/postgres/modules/graph/test/pagerank.sql_in
@@ -73,25 +73,29 @@ SELECT assert(relative_error(SUM(pagerank), 1) < 0.00001,
'PageRank: Scores do not sum up to 1.'
) FROM pagerank_out;
-DROP TABLE IF EXISTS pagerank_out, pagerank_out_summary;
+DROP TABLE IF EXISTS pagerank_gr_out;
+DROP TABLE IF EXISTS pagerank_gr_out_summary;
SELECT madlib.pagerank(
'vertex', -- Vertex table
'id', -- Vertix id column
'edge', -- Edge table
'src=src, dest=dest', -- Edge args
- 'pagerank_out', -- Output table of PageRank
+ 'pagerank_gr_out', -- Output table of PageRank
+ NULL,
NULL,
NULL,
- 0.00000001,
'user_id');
-- View the PageRank of all vertices, sorted by their scores.
SELECT assert(relative_error(SUM(pagerank), 1) < 0.00001,
'PageRank: Scores do not sum up to 1 for group 1.'
- ) FROM pagerank_out WHERE user_id=1;
-SELECT assert(relative_error(__iterations__, 27) = 0,
- 'PageRank: Incorrect iterations for group 1.'
- ) FROM pagerank_out_summary WHERE user_id=1;
-SELECT assert(relative_error(__iterations__, 31) = 0,
- 'PageRank: Incorrect iterations for group 2.'
- ) FROM pagerank_out_summary WHERE user_id=2;
+ ) FROM pagerank_gr_out WHERE user_id=1;
+SELECT assert(relative_error(SUM(pagerank), 1) < 0.00001,
+ 'PageRank: Scores do not sum up to 1 for group 2.'
+ ) FROM pagerank_gr_out WHERE user_id=2;
+-- SELECT assert(relative_error(__iterations__, 27) = 0,
+-- 'PageRank: Incorrect iterations for group 1.'
+-- ) FROM pagerank_gr_out_summary WHERE user_id=1;
+-- SELECT assert(relative_error(__iterations__, 31) = 0,
+-- 'PageRank: Incorrect iterations for group 2.'
+-- ) FROM pagerank_gr_out_summary WHERE user_id=2;
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c8bfbf81/src/ports/postgres/modules/pca/test/pca.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/pca/test/pca.sql_in b/src/ports/postgres/modules/pca/test/pca.sql_in
index 12d8ab1..fe397fc 100644
--- a/src/ports/postgres/modules/pca/test/pca.sql_in
+++ b/src/ports/postgres/modules/pca/test/pca.sql_in
@@ -119,40 +119,40 @@ select * from result_table_214712398172490837;
select * from result_table_214712398172490838;
-- Test dense data with grouping
-DROP TABLE IF EXISTS mat;
-CREATE TABLE mat (
- id integer,
- row_vec double precision[],
- grp integer
-);
-
-COPY mat (id, row_vec, grp) FROM stdin delimiter '|';
-1|{396,840,353,446,318,886,15,584,159,383}|1
-2|{691,58,899,163,159,533,604,582,269,390}|1
-3|{293,742,298,75,404,857,941,662,846,2}|1
-4|{462,532,787,265,982,306,600,608,212,885}|1
-5|{304,151,337,387,643,753,603,531,459,652}|1
-6|{327,946,368,943,7,516,272,24,591,204}|1
-7|{877,59,260,302,891,498,710,286,864,675}|1
-8|{458,959,774,376,228,354,300,669,718,565}|2
-9|{824,390,818,844,180,943,424,520,65,913}|2
-10|{882,761,398,688,761,405,125,484,222,873}|2
-11|{528,1,860,18,814,242,314,965,935,809}|2
-12|{492,220,576,289,321,261,173,1,44,241}|2
-13|{415,701,221,503,67,393,479,218,219,916}|2
-14|{350,192,211,633,53,783,30,444,176,932}|2
-15|{909,472,871,695,930,455,398,893,693,838}|2
-16|{739,651,678,577,273,935,661,47,373,618}|2
-\.
+-- DROP TABLE IF EXISTS mat;
+-- CREATE TABLE mat (
+-- id integer,
+-- row_vec double precision[],
+-- grp integer
+-- );
+
+-- COPY mat (id, row_vec, grp) FROM stdin delimiter '|';
+-- 1|{396,840,353,446,318,886,15,584,159,383}|1
+-- 2|{691,58,899,163,159,533,604,582,269,390}|1
+-- 3|{293,742,298,75,404,857,941,662,846,2}|1
+-- 4|{462,532,787,265,982,306,600,608,212,885}|1
+-- 5|{304,151,337,387,643,753,603,531,459,652}|1
+-- 6|{327,946,368,943,7,516,272,24,591,204}|1
+-- 7|{877,59,260,302,891,498,710,286,864,675}|1
+-- 8|{458,959,774,376,228,354,300,669,718,565}|2
+-- 9|{824,390,818,844,180,943,424,520,65,913}|2
+-- 10|{882,761,398,688,761,405,125,484,222,873}|2
+-- 11|{528,1,860,18,814,242,314,965,935,809}|2
+-- 12|{492,220,576,289,321,261,173,1,44,241}|2
+-- 13|{415,701,221,503,67,393,479,218,219,916}|2
+-- 14|{350,192,211,633,53,783,30,444,176,932}|2
+-- 15|{909,472,871,695,930,455,398,893,693,838}|2
+-- 16|{739,651,678,577,273,935,661,47,373,618}|2
+-- \.
-- Learn individaul PCA models based on grouping column (grp)
-drop table if exists result_table_214712398172490837;
-drop table if exists result_table_214712398172490837_mean;
-drop table if exists result_table_214712398172490838;
-select pca_train('mat', 'result_table_214712398172490837', 'id', 0.8,
-'grp', 5, FALSE, 'result_table_214712398172490838');
-select * from result_table_214712398172490837;
-select * from result_table_214712398172490838;
+-- drop table if exists result_table_214712398172490837;
+-- drop table if exists result_table_214712398172490837_mean;
+-- drop table if exists result_table_214712398172490838;
+-- select pca_train('mat', 'result_table_214712398172490837', 'id', 0.8,
+-- 'grp', 5, FALSE, 'result_table_214712398172490838');
+-- select * from result_table_214712398172490837;
+-- select * from result_table_214712398172490838;
-- Matrix in the column format
DROP TABLE IF EXISTS cmat;
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/c8bfbf81/src/ports/postgres/modules/validation/test/cross_validation.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/validation/test/cross_validation.sql_in b/src/ports/postgres/modules/validation/test/cross_validation.sql_in
index 258be29..3548178 100644
--- a/src/ports/postgres/modules/validation/test/cross_validation.sql_in
+++ b/src/ports/postgres/modules/validation/test/cross_validation.sql_in
@@ -1365,53 +1365,53 @@ select check_cv0();
-- select check_cv_ridge();
-m4_ifdef(<!__HAWQ__!>, <!!>, <!
-CREATE TABLE houses (
- id SERIAL NOT NULL,
- tax INTEGER,
- bedroom REAL,
- bath REAL,
- price INTEGER,
- size INTEGER,
- lot INTEGER
-);
+-- m4_ifdef(<!__HAWQ__!>, <!!>, <!
+-- CREATE TABLE houses (
+-- id SERIAL NOT NULL,
+-- tax INTEGER,
+-- bedroom REAL,
+-- bath REAL,
+-- price INTEGER,
+-- size INTEGER,
+-- lot INTEGER
+-- );
-INSERT INTO houses(tax, bedroom, bath, price, size, lot) VALUES
-( 590, 2, 1, 50000, 770, 22100),
-(1050, 3, 2, 85000, 1410, 12000),
-( 20, 3, 1, 22500, 1060, 3500 ),
-( 870, 2, 2, 90000, 1300, 17500),
-(1320, 3, 2, 133000, 1500, 30000),
-(1350, 2, 1, 90500, 820, 25700),
-(2790, 3, 2.5, 260000, 2130, 25000),
-( 680, 2, 1, 142500, 1170, 22000),
-(1840, 3, 2, 160000, 1500, 19000),
-(3680, 4, 2, 240000, 2790, 20000),
-(1660, 3, 1, 87000, 1030, 17500),
-(1620, 3, 2, 118600, 1250, 20000),
-(3100, 3, 2, 140000, 1760, 38000),
-(2070, 2, 3, 148000, 1550, 14000),
-( 650, 3, 1.5, 65000, 1450, 12000);
+-- INSERT INTO houses(tax, bedroom, bath, price, size, lot) VALUES
+-- ( 590, 2, 1, 50000, 770, 22100),
+-- (1050, 3, 2, 85000, 1410, 12000),
+-- ( 20, 3, 1, 22500, 1060, 3500 ),
+-- ( 870, 2, 2, 90000, 1300, 17500),
+-- (1320, 3, 2, 133000, 1500, 30000),
+-- (1350, 2, 1, 90500, 820, 25700),
+-- (2790, 3, 2.5, 260000, 2130, 25000),
+-- ( 680, 2, 1, 142500, 1170, 22000),
+-- (1840, 3, 2, 160000, 1500, 19000),
+-- (3680, 4, 2, 240000, 2790, 20000),
+-- (1660, 3, 1, 87000, 1030, 17500),
+-- (1620, 3, 2, 118600, 1250, 20000),
+-- (3100, 3, 2, 140000, 1760, 38000),
+-- (2070, 2, 3, 148000, 1550, 14000),
+-- ( 650, 3, 1.5, 65000, 1450, 12000);
-SELECT cross_validation_general(
- 'MADLIB_SCHEMA.elastic_net_train', -- modelling_func
- '{%data%, %model%, (price>100000), "array[tax, bath, size]", binomial, 1, lambda, TRUE, NULL, fista, "{eta = 2, max_stepsize = 2, use_active_set = t}", NULL, 2000, 1e-6}'::varchar[], -- modeling_params
- '{varchar, varchar, varchar, varchar, varchar, double precision, double precision, boolean, varchar, varchar, varchar, varchar, integer, double precision}'::varchar[], -- modelling_params_type
- 'lambda', -- param_explored
- '{0.04, 0.08, 0.12, 0.16, 0.20, 0.24, 0.28, 0.32, 0.36}'::varchar[], -- explore_values
- 'MADLIB_SCHEMA.elastic_net_predict', -- predict_func
- '{%model%, %data%, %id%, %prediction%}'::varchar[], -- predict_params
- '{text, text, text, text}'::varchar[], -- predict_params_type
- 'MADLIB_SCHEMA.misclassification_avg', -- metric_func
- '{%prediction%, %data%, %id%, (price>100000), %error%}'::varchar[], -- metric_params
- '{varchar, varchar, varchar, varchar, varchar}'::varchar[], -- metric_params_type
- 'houses', -- data_tbl
- 'id', -- data_id
- TRUE, -- id_is_random
- 'valid_rst_houses', -- validation_result
- '{tax,bath,size, price}'::varchar[], -- data_cols
- 3 -- fold_num
-);
+-- SELECT cross_validation_general(
+-- 'MADLIB_SCHEMA.elastic_net_train', -- modelling_func
+-- '{%data%, %model%, (price>100000), "array[tax, bath, size]", binomial, 1, lambda, TRUE, NULL, fista, "{eta = 2, max_stepsize = 2, use_active_set = t}", NULL, 2000, 1e-6}'::varchar[], -- modeling_params
+-- '{varchar, varchar, varchar, varchar, varchar, double precision, double precision, boolean, varchar, varchar, varchar, varchar, integer, double precision}'::varchar[], -- modelling_params_type
+-- 'lambda', -- param_explored
+-- '{0.04, 0.08, 0.12, 0.16, 0.20, 0.24, 0.28, 0.32, 0.36}'::varchar[], -- explore_values
+-- 'MADLIB_SCHEMA.elastic_net_predict', -- predict_func
+-- '{%model%, %data%, %id%, %prediction%}'::varchar[], -- predict_params
+-- '{text, text, text, text}'::varchar[], -- predict_params_type
+-- 'MADLIB_SCHEMA.misclassification_avg', -- metric_func
+-- '{%prediction%, %data%, %id%, (price>100000), %error%}'::varchar[], -- metric_params
+-- '{varchar, varchar, varchar, varchar, varchar}'::varchar[], -- metric_params_type
+-- 'houses', -- data_tbl
+-- 'id', -- data_id
+-- TRUE, -- id_is_random
+-- 'valid_rst_houses', -- validation_result
+-- '{tax,bath,size, price}'::varchar[], -- data_cols
+-- 3 -- fold_num
+-- );
-select * from valid_rst_houses;
-!>)
+-- select * from valid_rst_houses;
+-- !>)
[02/34] incubator-madlib git commit: Build: Add docker image for
MADlib
Posted by ok...@apache.org.
Build: Add docker image for MADlib
JIRA: MADLIB-920
- Add docker files that would help developers download a docker image
with Postgres-9.6 and MADlib depedencies installed. A developer's
local source code changes can be built on this image's container
to quickly build and run install-checks. Requires docker installed
on the developer's environment.
- Add a bash script (jenkins_build.sh) that would be a starting
point towards getting a Jenkins build for MADlib master branch.
- Add help in README on how to build MADlib source code, after
pulling down the base MADlib image from docker hub.
Closes #103
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/8679cbdf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/8679cbdf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/8679cbdf
Branch: refs/heads/latest_release
Commit: 8679cbdf94600b1d4b49a0c20c82e2529ba3ff9c
Parents: 01586c0
Author: Nandish Jayaram <nj...@apache.org>
Authored: Tue Mar 14 10:04:58 2017 -0700
Committer: Nandish Jayaram <nj...@apache.org>
Committed: Tue Mar 14 10:04:58 2017 -0700
----------------------------------------------------------------------
README.md | 44 ++++++++++++
tool/docker/base/Dockerfile_gpdb_4_3_10 | 71 ++++++++++++++++++++
tool/docker/base/Dockerfile_postgres_9_6 | 57 ++++++++++++++++
.../docker/base/Dockerfile_postgres_9_6_Jenkins | 42 ++++++++++++
tool/jenkins/jenkins_build.sh | 43 ++++++++++++
5 files changed, 257 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8679cbdf/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 12678e0..eaff324 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,50 @@ See the project webpage [`MADlib Home`](http://madlib.incubator.apache.org/) fo
latest binary and source packages. For installation and contribution guides,
please see [`MADlib Wiki`](https://cwiki.apache.org/confluence/display/MADLIB/)
+Development with Docker
+=======================
+We provide a Docker image with necessary dependencies required to compile and test MADlib on PostgreSQL 9.6. You can view the dependency Docker file at ./tool/docker/base/Dockerfile_postgres_9_6. The image is hosted on Docker Hub at madlib/postgres_9.6:latest. Later we will provide a similar Docker image for Greenplum Database.
+
+Some useful commands to use the docker file:
+```
+## 1) Pull down the `madlib/postgres_9.6:latest` image from docker hub:
+docker pull madlib/postgres_9.6:latest
+
+## 2) Launch a container corresponding to the MADlib image, mounting the source code folder to the container:
+docker run -d -it --name madlib -v (path to incubator-madlib directory):/incubator-madlib/ madlib/postgres_9.6
+# where incubator-madlib is the directory where the MADlib source code resides.
+
+############################################## * WARNING * ##################################################
+# Please be aware that when mounting a volume as shown above, any changes you make in the "incubator-madlib"
+# folder inside the Docker container will be reflected on your local disk (and vice versa). This means that
+# deleting data in the mounted volume from a Docker container will delete the data from your local disk also.
+#############################################################################################################
+
+## 3) When the container is up, connect to it and build MADlib:
+docker exec -it madlib bash
+mkdir /incubator-madlib/build-docker
+cd /incubator-madlib/build-docker
+cmake ..
+make
+make doc
+make install
+
+## 4) Install MADlib:
+src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install
+
+## 5) Several other commands, apart from the ones above can now be run, such as:
+# Run install check, on all modules:
+src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check
+# Run install check, on a specific module, say svm:
+src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check -t svm
+# Reinstall MADlib:
+src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres reinstall
+
+## 6) Kill and remove containers (after exiting the container):
+docker kill madlib
+docker rm madlib
+```
+
User and Developer Documentation
==================================
The latest documentation of MADlib modules can be found at [`MADlib
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8679cbdf/tool/docker/base/Dockerfile_gpdb_4_3_10
----------------------------------------------------------------------
diff --git a/tool/docker/base/Dockerfile_gpdb_4_3_10 b/tool/docker/base/Dockerfile_gpdb_4_3_10
new file mode 100644
index 0000000..620b379
--- /dev/null
+++ b/tool/docker/base/Dockerfile_gpdb_4_3_10
@@ -0,0 +1,71 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+### This is work in progress, does not work at the moment.
+#FROM pivotaldata/gpdb4310:latest
+#
+#### Get postgres specific add-ons
+#RUN yum -y update \
+# && yum -y groupinstall "Development tools" \
+# && yum -y install epel-release \
+# && yum -y install cmake \
+# openssl-devel \
+# openssl-libs \
+# openssh-server \
+# python-devel
+#
+#
+#### Build MADlib
+#ADD ./ /incubator-madlib
+##RUN cd incubator-madlib && \
+## mkdir build && \
+# cd build && \
+# cmake .. && \
+# make && \
+# make install
+#
+###################################################################################################
+################## PLACEHOLDER COMMANDS ##################
+#### WARNING: This is under construction, for future reference####################
+### Build the image from this docker file:
+## docker build -t gpdb -f tool/gpdb/Dockerfile_4_3_10 .
+#
+#### Steps to use the image for installing MADlib, building changed source code:
+### Run the container, mounting the source code's folder to the container. For example:
+## 1) docker run -d -it --name gpdb -v (path-to-incubator-madlib)/src:/incubator-madlib/src gpdb bash
+#
+### When the container is up, connect to it and execute (Install MADlib):
+## 2) docker exec -it gpdb /incubator-madlib/build/src/bin/madpack -p greenplum -c gpadmin@127.0.0.1:5432/gpadmin install
+#
+### Go into the container to build and run commands like install-check for modules:
+## 3) docker exec -it gpdb sh
+#
+### The above command gives us terminal access to the container, run commands such as:
+## - cd /incubator-madlib/build
+## - make (This can be run after changing code in the incubator-madlib source code)
+## - src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check -t svm
+### Install or reinstall MADlib if required:
+## - src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install
+## - src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres reinstall
+#
+#
+#### Common docker commands:
+### Kill and remove containers:
+## - docker kill gpdb
+## - docker rm gpdb
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8679cbdf/tool/docker/base/Dockerfile_postgres_9_6
----------------------------------------------------------------------
diff --git a/tool/docker/base/Dockerfile_postgres_9_6 b/tool/docker/base/Dockerfile_postgres_9_6
new file mode 100644
index 0000000..4dc5a4c
--- /dev/null
+++ b/tool/docker/base/Dockerfile_postgres_9_6
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM postgres:9.6
+
+### Get postgres specific add-ons
+RUN apt-get update && apt-get install -y wget \
+ build-essential \
+ postgresql-server-dev-9.6 \
+ postgresql-plpython-9.6 \
+ openssl \
+ libssl-dev \
+ libboost-all-dev \
+ m4 \
+ wget \
+ vim \
+ pgxnclient \
+ flex \
+ bison \
+ graphviz
+
+### Build custom CMake with SSQL support
+RUN wget https://cmake.org/files/v3.6/cmake-3.6.1.tar.gz && \
+ tar -zxvf cmake-3.6.1.tar.gz && \
+ cd cmake-3.6.1 && \
+ sed -i 's/-DCMAKE_BOOTSTRAP=1/-DCMAKE_BOOTSTRAP=1 -DCMAKE_USE_OPENSSL=ON/g' bootstrap && \
+ ./configure && \
+ make -j2 && \
+ make install
+
+### Install doxygen-1.8.13:
+RUN wget http://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.13.src.tar.gz && \
+ tar xf doxygen-1.8.13.src.tar.gz && \
+ cd doxygen-1.8.13 && \
+ mkdir build && \
+ cd build && \
+ cmake -G "Unix Makefiles" .. && \
+ make && \
+ make install
+
+## To build an image from this docker file, from incubator-madlib folder, run:
+# docker build -t madlib/postgres_9.6:latest -f tool/docker/base/Dockerfile_postgres_9_6 .
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8679cbdf/tool/docker/base/Dockerfile_postgres_9_6_Jenkins
----------------------------------------------------------------------
diff --git a/tool/docker/base/Dockerfile_postgres_9_6_Jenkins b/tool/docker/base/Dockerfile_postgres_9_6_Jenkins
new file mode 100644
index 0000000..137842e
--- /dev/null
+++ b/tool/docker/base/Dockerfile_postgres_9_6_Jenkins
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM postgres:9.6
+
+### Get postgres specific add-ons
+RUN apt-get update && apt-get install -y wget \
+ build-essential \
+ postgresql-server-dev-9.6 \
+ postgresql-plpython-9.6 \
+ openssl \
+ libssl-dev \
+ libboost-all-dev \
+ m4 \
+ wget
+
+### Build custom CMake with SSQL support
+RUN wget https://cmake.org/files/v3.6/cmake-3.6.1.tar.gz && \
+ tar -zxvf cmake-3.6.1.tar.gz && \
+ cd cmake-3.6.1 && \
+ sed -i 's/-DCMAKE_BOOTSTRAP=1/-DCMAKE_BOOTSTRAP=1 -DCMAKE_USE_OPENSSL=ON/g' bootstrap && \
+ ./configure && \
+ make -j2 && \
+ make install
+
+## To build an image from this docker file, from incubator-madlib folder, run:
+# docker build -t madlib/postgres_9.6:jenkins -f tool/docker/base/Dockerfile_postgres_9_6_Jenkins .
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8679cbdf/tool/jenkins/jenkins_build.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_build.sh b/tool/jenkins/jenkins_build.sh
new file mode 100644
index 0000000..72ada55
--- /dev/null
+++ b/tool/jenkins/jenkins_build.sh
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#!/bin/sh
+
+#####################################################################################
+### If this bash script is executed as a stand-alone file, assuming this
+### is not part of the MADlib source code, then the following two commands
+### may have to be used:
+# git clone https://github.com/apache/incubator-madlib.git
+# pushd incubator-madlib
+#####################################################################################
+
+# Pull down the base docker images
+docker pull madlib/postgres_9_6:jenkins
+# Assuming git clone of incubator-madlib has been done, launch a container with the volume mounted
+docker run -d --name madlib -v incubator-madlib:/incubator-madlib madlib/postgres_9.6:jenkins
+## This sleep is required since it takes a couple of seconds for the docker
+## container to come up, which is required by the docker exec command that follows.
+sleep 5
+# cmake, make and make install MADlib
+docker exec madlib bash -c 'mkdir /incubator-madlib/build ; cd /incubator-madlib/build ; cmake .. ; make ; make install'
+# Install MADlib and run install check
+docker exec -it madlib /incubator-madlib/build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install
+docker exec -it madlib /incubator-madlib/build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check
+
+docker kill madlib
+docker rm madlib
[25/34] incubator-madlib git commit: Jenkins: Get error message from
install-check FAIL
Posted by ok...@apache.org.
Jenkins: Get error message from install-check FAIL
Closes #124
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/8bd4947f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/8bd4947f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/8bd4947f
Branch: refs/heads/latest_release
Commit: 8bd4947fefb29a977f1239905472fda41fa76ae0
Parents: 3af18a9
Author: Rahul Iyer <ri...@apache.org>
Authored: Thu Apr 20 18:01:05 2017 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Wed Apr 26 14:17:06 2017 -0700
----------------------------------------------------------------------
src/madpack/madpack.py | 9 +++++----
tool/jenkins/jenkins_build.sh | 4 +++-
tool/jenkins/junit_export.py | 37 ++++++++++++++++++++++++++++++-------
3 files changed, 38 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8bd4947f/src/madpack/madpack.py
----------------------------------------------------------------------
diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py
index 049adf5..c5dd1f9 100755
--- a/src/madpack/madpack.py
+++ b/src/madpack/madpack.py
@@ -932,7 +932,7 @@ def _db_create_objects(schema, old_schema, upgrade=False, sc=None, testcase="",
sc)
# Check the exit status
if retval != 0:
- _error("Failed executing %s" % tmpfile, False)
+ _error("TEST CASE RESULTed executing %s" % tmpfile, False)
_error("Check the log at %s" % logfile, False)
raise Exception
# ------------------------------------------------------------------------------
@@ -1489,8 +1489,6 @@ def main(argv):
# Check the exit status
if retval != 0:
- _error("Failed executing %s" % tmpfile, False)
- _error("Check the log at %s" % logfile, False)
result = 'FAIL'
keeplogs = True
# Since every single statement in the test file gets logged,
@@ -1501,11 +1499,14 @@ def main(argv):
else:
result = 'ERROR'
- # Spit the line
+ # Output result
print "TEST CASE RESULT|Module: " + module + \
"|" + os.path.basename(sqlfile) + "|" + result + \
"|Time: %d milliseconds" % (milliseconds)
+ if result == 'FAIL':
+ _error("Failed executing %s" % tmpfile, False)
+ _error("Check the log at %s" % logfile, False)
# Cleanup test schema for the module
_internal_run_query("DROP SCHEMA IF EXISTS %s CASCADE;" % (test_schema), True)
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8bd4947f/tool/jenkins/jenkins_build.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_build.sh b/tool/jenkins/jenkins_build.sh
index f03bc78..d0f5510 100644
--- a/tool/jenkins/jenkins_build.sh
+++ b/tool/jenkins/jenkins_build.sh
@@ -64,7 +64,9 @@ echo "---------- Installing and running install-check --------------------"
# Install MADlib and run install check
cat <<EOF
docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install | tee $workdir/logs/madlib_install.log
-docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check | tee $workdir/logs/madlib_install_check.log
+
+mkdir -p $workdir/tmp
+docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres -d $workdir/tmp install-check | tee $workdir/logs/madlib_install_check.log
EOF
docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install | tee $workdir/logs/madlib_install.log
docker exec madlib /build/src/bin/madpack -p postgres -c postgres/postgres@localhost:5432/postgres install-check | tee $workdir/logs/madlib_install_check.log
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8bd4947f/tool/jenkins/junit_export.py
----------------------------------------------------------------------
diff --git a/tool/jenkins/junit_export.py b/tool/jenkins/junit_export.py
index ce30320..1836ea4 100644
--- a/tool/jenkins/junit_export.py
+++ b/tool/jenkins/junit_export.py
@@ -19,6 +19,7 @@
import re
import sys
+import subprocess
from collections import namedtuple
""" Convert install-check results into a standardized JUnit XML format
@@ -36,7 +37,7 @@ Example of JUnit output:
"""
-TestResult = namedtuple("TestResult", 'name suite status duration')
+TestResult = namedtuple("TestResult", 'name suite status duration message')
def _test_result_factory(install_check_log):
@@ -48,11 +49,29 @@ def _test_result_factory(install_check_log):
Next result of type test_result
"""
with open(install_check_log, 'r') as ic_log:
- for line in ic_log:
+ line = ic_log.readline()
+ while line:
m = re.match(r"^TEST CASE RESULT\|Module: (.*)\|(.*)\|(.*)\|Time: ([0-9]+)(.*)", line)
if m:
- yield TestResult(name=m.group(2), suite=m.group(1),
- status=m.group(3), duration=m.group(4))
+ suite, name, status, duration = [m.group(i) for i in range(1, 5)]
+ message = ""
+ if status == 'FAIL':
+ # get the tmp file and log file containing error
+ # these two lines are output after each failure
+ tmp_file_line = ic_log.readline()
+ log_file_line = ic_log.readline()
+ failure_m = re.match(r".* Check the log at (.*)", log_file_line)
+ if failure_m:
+ log_file = failure_m.group(1)
+ try:
+ message = subprocess.check_output(['tail', '-n 100', log_file],
+ stderr=subprocess.STDOUT)
+ except subprocess.CalledProcessError as e:
+ message = e.output
+ yield TestResult(name=name, suite=suite,
+ status=status, duration=duration,
+ message=message)
+ line = ic_log.readline()
# ----------------------------------------------------------------------
@@ -68,15 +87,17 @@ def _add_footer(out_log):
def _add_test_case(out_log, test_results):
- for res in test_results:
+ for t in test_results:
try:
# convert duration from milliseconds to seconds
- duration = float(res.duration)/1000
+ duration = float(t.duration)/1000
except TypeError:
duration = 0.0
output = ['<testcase classname="{t.suite}" name="{t.name}" '
'status="{t.status}" time="{d}">'.
- format(t=res, d=duration)]
+ format(t=t, d=duration)]
+ if t.status == "FAIL":
+ output.append('<failure>{0}</failure>'.format(t.message))
output.append('</testcase>')
out_log.write('\n'.join(output))
@@ -93,4 +114,6 @@ def main(install_check_log, test_output_log):
if __name__ == "__main__":
+ # argv[1] = install check log
+ # argv[2] = output file to store xml
main(sys.argv[1], sys.argv[2])
[15/34] incubator-madlib git commit: Mulltiple: Add grouping support
for SSSP and support GPDB5
Posted by ok...@apache.org.
Mulltiple: Add grouping support for SSSP and support GPDB5
JIRA: MADLIB-1081
- This commit adds grouping support for SSSP as well as its path function.
- Update chi2 test for GPDB5 alpha compatibility.
- Decouple DROP and CREATE statements for various modules.
Closes #113
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/8faf6226
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/8faf6226
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/8faf6226
Branch: refs/heads/latest_release
Commit: 8faf62263f6c5aa4281e2d3dc33e389d41784c0e
Parents: c82b9d0
Author: Orhan Kislal <ok...@pivotal.io>
Authored: Mon Apr 17 11:17:41 2017 -0700
Committer: Orhan Kislal <ok...@pivotal.io>
Committed: Mon Apr 17 11:17:41 2017 -0700
----------------------------------------------------------------------
.../elastic_net_generate_result.py_in | 2 +-
.../postgres/modules/graph/graph_utils.py_in | 2 +-
src/ports/postgres/modules/graph/sssp.py_in | 716 ++++++++++++++-----
src/ports/postgres/modules/graph/sssp.sql_in | 132 +++-
.../postgres/modules/graph/test/sssp.sql_in | 75 +-
src/ports/postgres/modules/pca/pca.py_in | 6 +-
.../modules/stats/test/chi2_test.sql_in | 2 +-
.../validation/internal/cross_validation.py_in | 6 +-
8 files changed, 739 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8faf6226/src/ports/postgres/modules/elastic_net/elastic_net_generate_result.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/elastic_net_generate_result.py_in b/src/ports/postgres/modules/elastic_net/elastic_net_generate_result.py_in
index c48beca..6246ed9 100644
--- a/src/ports/postgres/modules/elastic_net/elastic_net_generate_result.py_in
+++ b/src/ports/postgres/modules/elastic_net/elastic_net_generate_result.py_in
@@ -81,8 +81,8 @@ def _elastic_net_generate_result(optimizer, iteration_run, **args):
schema_madlib=args["schema_madlib"])
# Create the output table
+ plpy.execute("DROP TABLE IF EXISTS {tbl_result}".format(**args))
plpy.execute("""
- DROP TABLE IF EXISTS {tbl_result};
CREATE TABLE {tbl_result} (
{select_grouping_info}
family text,
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8faf6226/src/ports/postgres/modules/graph/graph_utils.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/graph_utils.py_in b/src/ports/postgres/modules/graph/graph_utils.py_in
index 2d83301..25f70a5 100644
--- a/src/ports/postgres/modules/graph/graph_utils.py_in
+++ b/src/ports/postgres/modules/graph/graph_utils.py_in
@@ -72,7 +72,7 @@ def validate_graph_coding(vertex_table, vertex_id, edge_table, edge_params,
"""Graph {func_name}: The vertex column {vertex_id} is not present in vertex table ({vertex_table}) """.
format(**locals()))
_assert(columns_exist_in_table(edge_table, edge_params.values()),
- """Graph {func_name}: Not all columns from {cols} present in edge table ({edge_table})""".
+ """Graph {func_name}: Not all columns from {cols} are present in edge table ({edge_table})""".
format(cols=edge_params.values(), **locals()))
return None
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8faf6226/src/ports/postgres/modules/graph/sssp.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/sssp.py_in b/src/ports/postgres/modules/graph/sssp.py_in
index 4d27761..2520830 100644
--- a/src/ports/postgres/modules/graph/sssp.py_in
+++ b/src/ports/postgres/modules/graph/sssp.py_in
@@ -33,27 +33,56 @@ from utilities.control import MinWarning
from utilities.utilities import _assert
from utilities.utilities import extract_keyvalue_params
from utilities.utilities import unique_string
-from utilities.validate_args import get_cols
-from utilities.validate_args import unquote_ident
+from utilities.utilities import _string_to_array
+from utilities.utilities import split_quoted_delimited_str
from utilities.validate_args import table_exists
from utilities.validate_args import columns_exist_in_table
from utilities.validate_args import table_is_empty
+from utilities.validate_args import get_cols_and_types
+from utilities.validate_args import get_expr_type
m4_changequote(`<!', `!>')
+
+def _check_groups(tbl1, tbl2, grp_list):
+
+ """
+ Helper function for joining tables with groups.
+ Args:
+ @param tbl1 Name of the first table
+ @param tbl2 Name of the second table
+ @param grp_list The list of grouping columns
+ """
+
+ return ' AND '.join([" {tbl1}.{i} = {tbl2}.{i} ".format(**locals())
+ for i in grp_list])
+
+def _grp_from_table(tbl, grp_list):
+
+ """
+ Helper function for selecting grouping columns of a table
+ Args:
+ @param tbl Name of the table
+ @param grp_list The list of grouping columns
+ """
+ return ' , '.join([" {tbl}.{i} ".format(**locals())
+ for i in grp_list])
+
def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
- edge_args, source_vertex, out_table, **kwargs):
+ edge_args, source_vertex, out_table, grouping_cols, **kwargs):
+
"""
Single source shortest path function for graphs using the Bellman-Ford
algorhtm [1].
Args:
- @param vertex_table Name of the table that contains the vertex data.
- @param vertex_id Name of the column containing the vertex ids.
- @param edge_table Name of the table that contains the edge data.
- @param edge_args A comma-delimited string containing multiple
- named arguments of the form "name=value".
- @param source_vertex The source vertex id for the algorithm to start.
- @param out_table Name of the table to store the result of SSSP.
+ @param vertex_table Name of the table that contains the vertex data.
+ @param vertex_id Name of the column containing the vertex ids.
+ @param edge_table Name of the table that contains the edge data.
+ @param edge_args A comma-delimited string containing multiple
+ named arguments of the form "name=value".
+ @param source_vertex The source vertex id for the algorithm to start.
+ @param out_table Name of the table to store the result of SSSP.
+ @param grouping_cols The list of grouping columns.
[1] https://en.wikipedia.org/wiki/Bellman-Ford_algorithm
"""
@@ -61,6 +90,7 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
with MinWarning("warning"):
INT_MAX = 2147483647
+ INFINITY = "'Infinity'"
EPSILON = 0.000001
message = unique_string(desp='message')
@@ -73,8 +103,23 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
edge_params = extract_keyvalue_params(edge_args,
params_types,
default_args)
+
+ # Prepare the input for recording in the summary table
if vertex_id is None:
+ v_st= "NULL"
vertex_id = "id"
+ else:
+ v_st = vertex_id
+ if edge_args is None:
+ e_st = "NULL"
+ else:
+ e_st = edge_args
+ if grouping_cols is None:
+ g_st = "NULL"
+ glist = None
+ else:
+ g_st = grouping_cols
+ glist = split_quoted_delimited_str(grouping_cols)
src = edge_params["src"]
dest = edge_params["dest"]
@@ -85,47 +130,91 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
local_distribution = m4_ifdef(<!__POSTGRESQL__!>, <!''!>,
<!"DISTRIBUTED BY (id)"!>)
- validate_sssp(vertex_table, vertex_id, edge_table,
- edge_params, source_vertex, out_table)
+ is_hawq = m4_ifdef(<!__HAWQ__!>, <!True!>, <!False!>)
+ _validate_sssp(vertex_table, vertex_id, edge_table,
+ edge_params, source_vertex, out_table, glist)
plpy.execute(" DROP TABLE IF EXISTS {0},{1},{2}".format(
message,oldupdate,newupdate))
+ # Initialize grouping related variables
+ comma_grp = ""
+ comma_grp_e = ""
+ comma_grp_m = ""
+ grp_comma = ""
+ checkg_oo = ""
+ checkg_eo = ""
+ checkg_ex = ""
+ checkg_om = ""
+ group_by = ""
+
+ if grouping_cols is not None:
+ comma_grp = " , " + grouping_cols
+ group_by = " , " + _grp_from_table(edge_table,glist)
+ comma_grp_e = " , " + _grp_from_table(edge_table,glist)
+ comma_grp_m = " , " + _grp_from_table("message",glist)
+ grp_comma = grouping_cols + " , "
+
+ checkg_oo_sub = _check_groups(out_table,"oldupdate",glist)
+ checkg_oo = " AND " + checkg_oo_sub
+ checkg_eo = " AND " + _check_groups(edge_table,"oldupdate",glist)
+ checkg_ex = " AND " + _check_groups(edge_table,"x",glist)
+ checkg_om = " AND " + _check_groups("out_table","message",glist)
+
+ w_type = get_expr_type(weight,edge_table).lower()
+ init_w = INT_MAX
+ if w_type in ['double precision','float8']:
+ init_w = INFINITY
+
# We keep a table of every vertex, the minimum cost to that destination
# seen so far and the parent to this vertex in the associated shortest
- # path. This table will be updated throughtout the execution.
+ # path. This table will be updated throughout the execution.
plpy.execute(
- """ CREATE TABLE {out_table} AS
- SELECT {vertex_id} AS {vertex_id},
- CAST('Infinity' AS DOUBLE PRECISION) AS {weight},
- NULL::INT AS parent
- FROM {vertex_table}
- WHERE {vertex_id} IS NOT NULL
+ """ CREATE TABLE {out_table} AS ( SELECT
+ {grp_comma} {src} AS {vertex_id}, {weight},
+ {src} AS parent FROM {edge_table} LIMIT 0)
{distribution} """.format(**locals()))
+ # We keep a summary table to keep track of the parameters used for this
+ # SSSP run. This table is used in the path finding function to eliminate
+ # the need for repetition.
+ plpy.execute( """ CREATE TABLE {out_table}_summary (
+ vertex_table TEXT,
+ vertex_id TEXT,
+ edge_table TEXT,
+ edge_args TEXT,
+ source_vertex INTEGER,
+ out_table TEXT,
+ grouping_cols TEXT)
+ """.format(**locals()))
+ plpy.execute( """ INSERT INTO {out_table}_summary VALUES
+ ('{vertex_table}', '{v_st}', '{edge_table}', '{e_st}',
+ {source_vertex}, '{out_table}', '{g_st}')
+ """.format(**locals()))
+
# We keep 2 update tables and alternate them during the execution.
# This is necessary since we need to know which vertices are updated in
# the previous iteration to calculate the next set of updates.
plpy.execute(
- """ CREATE TEMP TABLE {oldupdate}(
- id INT, val DOUBLE PRECISION, parent INT)
+ """ CREATE TEMP TABLE {oldupdate} AS ( SELECT
+ {src} AS id, {weight},
+ {src} AS parent {comma_grp} FROM {edge_table} LIMIT 0)
{local_distribution}
""".format(**locals()))
plpy.execute(
- """ CREATE TEMP TABLE {newupdate}(
- id INT, val DOUBLE PRECISION, parent INT)
+ """ CREATE TEMP TABLE {newupdate} AS ( SELECT
+ {src} AS id, {weight},
+ {src} AS parent {comma_grp} FROM {edge_table} LIMIT 0)
{local_distribution}
""".format(**locals()))
# Since HAWQ does not allow us to update, we create a new table and
- # rename at every iteration
- temp_table = unique_string(desp='temp')
- sql = m4_ifdef(<!__HAWQ__!>,
- """ CREATE TABLE {temp_table} (
- {vertex_id} INT, {weight} DOUBLE PRECISION, parent INT)
- {distribution};
- """, <!''!>)
- plpy.execute(sql.format(**locals()))
+ # rename at every iteration.
+ if is_hawq:
+ temp_table = unique_string(desp='temp')
+ sql =""" CREATE TABLE {temp_table} AS ( SELECT * FROM {out_table} )
+ {distribution} """
+ plpy.execute(sql.format(**locals()))
# GPDB and HAWQ have distributed by clauses to help them with indexing.
# For Postgres we add the indices manually.
@@ -137,45 +226,117 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
<!''!>)
plpy.execute(sql_index)
- # The source can be reached with 0 cost and it has itself as the parent.
- plpy.execute(
- """ INSERT INTO {oldupdate}
- VALUES({source_vertex},0,{source_vertex})
- """.format(**locals()))
+ # The initialization step is quite different when grouping is involved
+ # since not every group (subgraph) will have the same set of vertices.
+
+ # Example:
+ # Assume there are two grouping columns g1 and g2
+ # g1 values are 0 and 1. g2 values are 5 and 6
+ if grouping_cols is not None:
+
+ distinct_grp_table = unique_string(desp='grp')
+ plpy.execute(""" DROP TABLE IF EXISTS {distinct_grp_table} """.
+ format(**locals()))
+ plpy.execute( """ CREATE TEMP TABLE {distinct_grp_table} AS
+ SELECT DISTINCT {grouping_cols} FROM {edge_table} """.
+ format(**locals()))
+ subq = unique_string(desp='subquery')
+
+ checkg_ds_sub = _check_groups(distinct_grp_table,subq,glist)
+ grp_d_comma = _grp_from_table(distinct_grp_table,glist) +","
+
+ plpy.execute(
+ """ INSERT INTO {out_table}
+ SELECT {grp_d_comma} {vertex_id} AS {vertex_id},
+ {init_w} AS {weight}, NULL::INT AS parent
+ FROM {distinct_grp_table} INNER JOIN
+ (
+ SELECT {src} AS {vertex_id} {comma_grp}
+ FROM {edge_table}
+ UNION
+ SELECT {dest} AS {vertex_id} {comma_grp}
+ FROM {edge_table}
+ ) {subq} ON ({checkg_ds_sub})
+ WHERE {vertex_id} IS NOT NULL
+ """.format(**locals()))
+
+ plpy.execute(
+ """ INSERT INTO {oldupdate}
+ SELECT {source_vertex}, 0, {source_vertex},
+ {grouping_cols}
+ FROM {distinct_grp_table}
+ """.format(**locals()))
+
+ # The maximum number of vertices for any group.
+ # Used for determining negative cycles.
+ v_cnt = plpy.execute(
+ """ SELECT max(count) as max FROM (
+ SELECT count({vertex_id}) AS count
+ FROM {out_table}
+ GROUP BY {grouping_cols}) x
+ """.format(**locals()))[0]['max']
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(distinct_grp_table))
+ else:
+ plpy.execute(
+ """ INSERT INTO {out_table}
+ SELECT {vertex_id} AS {vertex_id},
+ {init_w} AS {weight},
+ NULL AS parent
+ FROM {vertex_table}
+ WHERE {vertex_id} IS NOT NULL
+ """.format(**locals()))
+
+ # The source can be reached with 0 cost and it has itself as the
+ # parent.
+ plpy.execute(
+ """ INSERT INTO {oldupdate}
+ VALUES({source_vertex},0,{source_vertex})
+ """.format(**locals()))
+
+ v_cnt = plpy.execute(
+ """ SELECT count(*) FROM {vertex_table}
+ WHERE {vertex_id} IS NOT NULL
+ """.format(**locals()))[0]['count']
- v_cnt = plpy.execute(
- """SELECT count(*) FROM {vertex_table}
- WHERE {vertex_id} IS NOT NULL""".format(**locals()))[0]['count']
for i in range(0,v_cnt+1):
- # Apply the updates calculated in the last iteration
- sql = m4_ifdef(<!__HAWQ__!>,
- <!"""
+ # Apply the updates calculated in the last iteration.
+ if is_hawq:
+ sql = """
TRUNCATE TABLE {temp_table};
INSERT INTO {temp_table}
SELECT *
FROM {out_table}
- WHERE {out_table}.{vertex_id} NOT IN (
- SELECT {oldupdate}.id FROM {oldupdate})
+ WHERE NOT EXISTS (
+ SELECT 1
+ FROM {oldupdate} as oldupdate
+ WHERE {out_table}.{vertex_id} = oldupdate.id
+ {checkg_oo})
UNION
- SELECT * FROM {oldupdate};
+ SELECT {grp_comma} id, {weight}, parent FROM {oldupdate};
DROP TABLE {out_table};
ALTER TABLE {temp_table} RENAME TO {out_table};
- CREATE TABLE {temp_table} (
- {vertex_id} INT, {weight} DOUBLE PRECISION, parent INT)
- {distribution};
- """!>,
- <!"""
+ CREATE TABLE {temp_table} AS (
+ SELECT * FROM {out_table} LIMIT 0)
+ {distribution};"""
+ plpy.execute(sql.format(**locals()))
+ ret = plpy.execute("SELECT id FROM {0} LIMIT 1".
+ format(oldupdate))
+ else:
+ sql = """
UPDATE {out_table} SET
- {weight}=oldupdate.val,
+ {weight}=oldupdate.{weight},
parent=oldupdate.parent
FROM
{oldupdate} AS oldupdate
WHERE
- {out_table}.{vertex_id}=oldupdate.id
- """!>)
- plpy.execute(sql.format(**locals()))
+ {out_table}.{vertex_id}=oldupdate.id AND
+ {out_table}.{weight}>oldupdate.{weight} {checkg_oo}
+ """
+ ret = plpy.execute(sql.format(**locals()))
+ if ret.nrows() == 0:
+ break
plpy.execute("TRUNCATE TABLE {0}".format(newupdate))
@@ -194,105 +355,237 @@ def graph_sssp(schema_madlib, vertex_table, vertex_id, edge_table,
# for comparison.
# Once we have a list of edges and values (stores as 'message'),
- # we check if these values are lower than the existing shortest path
- # values.
+ # we check if these values are lower than the existing shortest
+ # path values.
sql = (""" INSERT INTO {newupdate}
- SELECT DISTINCT ON (message.id) message.id AS id,
- message.val AS val,
- message.parent AS parent
+ SELECT DISTINCT ON (message.id {comma_grp})
+ message.id AS id,
+ message.{weight} AS {weight},
+ message.parent AS parent {comma_grp_m}
FROM {out_table} AS out_table INNER JOIN
(
- SELECT edge_table.{dest} AS id, x.val AS val,
- oldupdate.id AS parent
+ SELECT {edge_table}.{dest} AS id, x.{weight} AS {weight},
+ oldupdate.id AS parent {comma_grp_e}
+ FROM {oldupdate} AS oldupdate INNER JOIN
+ {edge_table} ON
+ ({edge_table}.{src} = oldupdate.id {checkg_eo})
+ INNER JOIN
+ (
+ SELECT {edge_table}.{dest} AS id,
+ min(oldupdate.{weight} +
+ {edge_table}.{weight}) AS {weight} {comma_grp_e}
FROM {oldupdate} AS oldupdate INNER JOIN
- {edge_table} AS edge_table ON
- (edge_table.{src} = oldupdate.id) INNER JOIN
- (
- SELECT edge_table.{dest} AS id,
- min(oldupdate.val + edge_table.{weight})
- AS val
- FROM {oldupdate} AS oldupdate INNER JOIN
- {edge_table} AS edge_table ON
- (edge_table.{src}=oldupdate.id)
- GROUP BY edge_table.{dest}
- ) x ON (edge_table.{dest} = x.id)
- WHERE ABS(oldupdate.val + edge_table.{weight} - x.val)
- < {EPSILON}
- ) AS message ON (message.id = out_table.{vertex_id})
- WHERE message.val<out_table.{weight}
+ {edge_table} ON
+ ({edge_table}.{src}=oldupdate.id {checkg_eo})
+ GROUP BY {edge_table}.{dest} {comma_grp_e}
+ ) x
+ ON ({edge_table}.{dest} = x.id {checkg_ex} )
+ WHERE ABS(oldupdate.{weight} + {edge_table}.{weight}
+ - x.{weight}) < {EPSILON}
+ ) message
+ ON (message.id = out_table.{vertex_id} {checkg_om})
+ WHERE message.{weight}<out_table.{weight}
""".format(**locals()))
- # If there are no updates, SSSP is finalized
- ret = plpy.execute(sql)
- if ret.nrows() == 0:
- break
+ plpy.execute(sql)
- # Swap the update tables for the next iteration
+ # Swap the update tables for the next iteration.
tmp = oldupdate
oldupdate = newupdate
newupdate = tmp
- # Bellman-Ford should converge in |V|-1 iterations.
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(newupdate))
+ # The algorithm should converge in less than |V| iterations.
+ # Otherwise there is a negative cycle in the graph.
if i == v_cnt:
- plpy.execute("DROP TABLE IF EXISTS {out_table}".format(**locals()))
- plpy.error("Graph SSSP: Detected a negative cycle in the graph.")
-
- m4_ifdef(<!__HAWQ__!>,
- plpy.execute("DROP TABLE {temp_table} ".format(**locals())), <!''!>)
+ if grouping_cols is None:
+ plpy.execute("DROP TABLE IF EXISTS {0},{1},{2}".
+ format(out_table, out_table+"_summary", oldupdate))
+ if is_hawq:
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(temp_table))
+ plpy.error("Graph SSSP: Detected a negative cycle in the graph.")
+
+ # It is possible that not all groups has negative cycles.
+ else:
+
+ # gsql is the string created by collating grouping columns.
+ # By looking at the oldupdate table we can see which groups
+ # are in a negative cycle.
+
+ negs = plpy.execute(
+ """ SELECT array_agg(DISTINCT ({grouping_cols})) AS grp
+ FROM {oldupdate}
+ """.format(**locals()))[0]['grp']
+
+ # Delete the groups with negative cycles from the output table.
+ sql_del = """ DELETE FROM {out_table}
+ USING {oldupdate} AS oldupdate
+ WHERE {checkg_oo_sub}"""
+ if is_hawq:
+ sql_del = """
+ TRUNCATE TABLE {temp_table};
+ INSERT INTO {temp_table}
+ SELECT *
+ FROM {out_table}
+ WHERE NOT EXISTS(
+ SELECT 1
+ FROM {oldupdate} as oldupdate
+ WHERE {checkg_oo_sub}
+ );
+ DROP TABLE {out_table};
+ ALTER TABLE {temp_table} RENAME TO {out_table};"""
+
+ plpy.execute(sql_del.format(**locals()))
+
+ # If every group has a negative cycle,
+ # drop the output table as well.
+ if table_is_empty(out_table):
+ plpy.execute("DROP TABLE IF EXISTS {0},{1}".
+ format(out_table,out_table+"_summary"))
+
+ plpy.warning(
+ """Graph SSSP: Detected a negative cycle in the """ +
+ """sub-graphs of following groups: {0}.""".
+ format(str(negs)[1:-1]))
+
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(oldupdate))
+ if is_hawq:
+ plpy.execute("DROP TABLE IF EXISTS {temp_table} ".
+ format(**locals()))
return None
-def graph_sssp_get_path(schema_madlib, sssp_table, dest_vertex, **kwargs):
+def graph_sssp_get_path(schema_madlib, sssp_table, dest_vertex, path_table,
+ **kwargs):
"""
- Helper function that can be used to get the shortest path for a vertex
+ Helper function that can be used to get the shortest path for a vertex
Args:
- @param source_table Name of the table that contains the SSSP output.
- @param out_table The vertex that will be the destination of the
- desired path.
+ @param sssp_table Name of the table that contains the SSSP output.
+ @param dest_vertex The vertex that will be the destination of the
+ desired path.
+ @param path_table Name of the output table that contains the path.
"""
+ with MinWarning("warning"):
+ _validate_get_path(sssp_table, dest_vertex, path_table)
- validate_get_path(sssp_table, dest_vertex)
- cur = dest_vertex
- cols = get_cols(sssp_table)
- id = cols[0]
- ret = [dest_vertex]
- plan_name = unique_string(desp='plan')
-
- # Follow the 'parent' chain until you reach the source.
- # We don't need to know what the source is since it is the only vertex with
- # itself as its parent
- plpy.execute(""" PREPARE {plan_name} (int) AS
- SELECT parent FROM {sssp_table} WHERE {id} = $1 LIMIT 1
- """.format(**locals()))
- sql = "EXECUTE {plan_name} ({cur})"
- parent = plpy.execute(sql.format(**locals()))
+ temp1_name = unique_string(desp='temp1')
+ temp2_name = unique_string(desp='temp2')
- if parent.nrows() == 0:
- plpy.error(
- "Graph SSSP: Vertex {0} is not present in the sssp table {1}".
- format(dest_vertex,sssp_table))
-
- while 1:
- parent = parent[0]['parent']
- if parent == cur:
- ret.reverse()
- return ret
- else:
- ret.append(parent)
- cur = parent
- parent = plpy.execute(sql.format(**locals()))
+ select_grps = ""
+ check_grps_t1 = ""
+ check_grps_t2 = ""
+ check_grps_pt1 = ""
+ check_grps_pt2 = ""
+ checkg_po = ""
+ grp_comma = ""
+ tmp = ""
+
+ summary = plpy.execute("SELECT * FROM {0}_summary".format(sssp_table))
+ vertex_id = summary[0]['vertex_id']
+ source_vertex = summary[0]['source_vertex']
+
+ if vertex_id == "NULL":
+ vertex_id = "id"
+
+ grouping_cols = summary[0]['grouping_cols']
+ if grouping_cols == "NULL":
+ grouping_cols = None
+
+ if grouping_cols is not None:
+ glist = split_quoted_delimited_str(grouping_cols)
+ select_grps = _grp_from_table(sssp_table,glist) + " , "
+ check_grps_t1 = " AND " + _check_groups(
+ sssp_table,temp1_name,glist)
+ check_grps_t2 = " AND " + _check_groups(
+ sssp_table,temp2_name,glist)
+
+ checkg_po = " WHERE " + _check_groups(
+ path_table,"oldupdate",glist)
+ grp_comma = grouping_cols + " , "
+
+ if source_vertex == dest_vertex:
+ plpy.execute("""
+ CREATE TABLE {path_table} AS
+ SELECT {grp_comma} '{{{dest_vertex}}}'::INT[] AS path
+ FROM {sssp_table} WHERE {vertex_id} = {dest_vertex}
+ """.format(**locals()))
+ return
+
+ plpy.execute( "DROP TABLE IF EXISTS {0},{1}".
+ format(temp1_name,temp2_name));
+ out = plpy.execute(""" CREATE TEMP TABLE {temp1_name} AS
+ SELECT {grp_comma} {sssp_table}.parent AS {vertex_id},
+ ARRAY[{dest_vertex}] AS path
+ FROM {sssp_table}
+ WHERE {vertex_id} = {dest_vertex}
+ AND {sssp_table}.parent IS NOT NULL
+ """.format(**locals()))
+
+ plpy.execute("""
+ CREATE TEMP TABLE {temp2_name} AS
+ SELECT * FROM {temp1_name} LIMIT 0
+ """.format(**locals()))
+
+ # Follow the 'parent' chain until you reach the source.
+ while out.nrows() > 0:
+
+ plpy.execute("TRUNCATE TABLE {temp2_name}".format(**locals()))
+ # If the vertex id is not the source vertex,
+ # Add it to the path and move to its parent
+ out = plpy.execute(
+ """ INSERT INTO {temp2_name}
+ SELECT {select_grps} {sssp_table}.parent AS {vertex_id},
+ {sssp_table}.{vertex_id} || {temp1_name}.path AS path
+ FROM {sssp_table} INNER JOIN {temp1_name} ON
+ ({sssp_table}.{vertex_id} = {temp1_name}.{vertex_id}
+ {check_grps_t1})
+ WHERE {source_vertex} <> {sssp_table}.{vertex_id}
+ """.format(**locals()))
+
+ tmp = temp2_name
+ temp2_name = temp1_name
+ temp1_name = tmp
+
+ tmp = check_grps_t1
+ check_grps_t1 = check_grps_t2
+ check_grps_t2 = tmp
+
+ # Add the source vertex to the beginning of every path and
+ # add the empty arrays for the groups that don't have a path to reach
+ # the destination vertex
+ plpy.execute("""
+ CREATE TABLE {path_table} AS
+ SELECT {grp_comma} {source_vertex} || path AS path
+ FROM {temp2_name}
+ UNION
+ SELECT {grp_comma} '{{}}'::INT[] AS path
+ FROM {sssp_table}
+ WHERE {vertex_id} = {dest_vertex}
+ AND {sssp_table}.parent IS NULL
+ """.format(**locals()))
+
+ out = plpy.execute("SELECT 1 FROM {0} LIMIT 1".format(path_table))
+
+ if out.nrows() == 0:
+ plpy.error(
+ "Graph SSSP: Vertex {0} is not present in the SSSP table {1}".
+ format(dest_vertex,sssp_table))
+
+ plpy.execute("DROP TABLE IF EXISTS {temp1_name}, {temp1_name}".
+ format(**locals()))
return None
-def validate_sssp(vertex_table, vertex_id, edge_table, edge_params,
- source_vertex, out_table, **kwargs):
+
+def _validate_sssp(vertex_table, vertex_id, edge_table, edge_params,
+ source_vertex, out_table, glist, **kwargs):
validate_graph_coding(vertex_table, vertex_id, edge_table, edge_params,
out_table,'SSSP')
_assert(isinstance(source_vertex,int),
- """Graph SSSP: Source vertex {source_vertex} has to be an integer """.
+ """Graph SSSP: Source vertex {source_vertex} has to be an integer.""".
format(**locals()))
src_exists = plpy.execute("""
SELECT * FROM {vertex_table} WHERE {vertex_id}={source_vertex}
@@ -300,8 +593,8 @@ def validate_sssp(vertex_table, vertex_id, edge_table, edge_params,
if src_exists.nrows() == 0:
plpy.error(
- """Graph SSSP: Source vertex {source_vertex} is not present in the
- vertex table {vertex_table} """.format(**locals()))
+ """Graph SSSP: Source vertex {source_vertex} is not present in the vertex table {vertex_table}.""".
+ format(**locals()))
vt_error = plpy.execute(
""" SELECT {vertex_id}
@@ -312,12 +605,20 @@ def validate_sssp(vertex_table, vertex_id, edge_table, edge_params,
if vt_error.nrows() != 0:
plpy.error(
- """Graph SSSP: Source vertex table {vertex_table}
- contains duplicate vertex id's """.format(**locals()))
+ """Graph SSSP: Source vertex table {vertex_table} contains duplicate vertex id's.""".
+ format(**locals()))
+
+ _assert(not table_exists(out_table+"_summary"),
+ "Graph SSSP: Output summary table already exists!")
+
+ if glist is not None:
+ _assert(columns_exist_in_table(edge_table, glist),
+ """Graph SSSP: Not all columns from {glist} are present in edge table ({edge_table}).""".
+ format(**locals()))
return None
-def validate_get_path(sssp_table, dest_vertex, **kwargs):
+def _validate_get_path(sssp_table, dest_vertex, path_table, **kwargs):
_assert(sssp_table and sssp_table.strip().lower() not in ('null', ''),
"Graph SSSP: Invalid SSSP table name!")
@@ -326,21 +627,31 @@ def validate_get_path(sssp_table, dest_vertex, **kwargs):
_assert(not table_is_empty(sssp_table),
"Graph SSSP: SSSP table ({0}) is empty!".format(sssp_table))
+ summary = sssp_table+"_summary"
+ _assert(table_exists(summary),
+ "Graph SSSP: SSSP summary table ({0}) is missing!".format(summary))
+ _assert(not table_is_empty(summary),
+ "Graph SSSP: SSSP summary table ({0}) is empty!".format(summary))
+
+ _assert(not table_exists(path_table),
+ "Graph SSSP: Output path table already exists!")
+
+ return None
def graph_sssp_help(schema_madlib, message, **kwargs):
- """
- Help function for graph_sssp and graph_sssp_get_path
+ """
+ Help function for graph_sssp and graph_sssp_get_path
- Args:
- @param schema_madlib
- @param message: string, Help message string
- @param kwargs
+ Args:
+ @param schema_madlib
+ @param message: string, Help message string
+ @param kwargs
- Returns:
- String. Help/usage information
- """
- if not message:
- help_string = """
+ Returns:
+ String. Help/usage information
+ """
+ if not message:
+ help_string = """
-----------------------------------------------------------------------
SUMMARY
-----------------------------------------------------------------------
@@ -352,41 +663,120 @@ weights of its constituent edges is minimized.
For more details on function usage:
SELECT {schema_madlib}.graph_sssp('usage')
"""
- elif message in ['usage', 'help', '?']:
- help_string = """
+ elif message.lower() in ['usage', 'help', '?']:
+ help_string = """
{graph_usage}
To retrieve the path for a specific vertex:
SELECT {schema_madlib}.graph_sssp_get_path(
sssp_table TEXT, -- Name of the table that contains the SSSP output.
- dest_vertex INT -- The vertex that will be the destination of the
- -- desired path.
+ dest_vertex INT, -- The vertex that will be the destination of the
+ -- desired path.
+ path_table TEXT -- Name of the output table that contains the path.
);
----------------------------------------------------------------------------
OUTPUT
----------------------------------------------------------------------------
-The output table ('out_table' above) will contain a row for every vertex from
-vertex_table and have the following columns:
-
-vertex_id : The id for the destination. Will use the input parameter
- (vertex_id) for column naming.
-weight : The total weight of the shortest path from the source vertex
- to this particular vertex. Will use the input parameter (weight)
- for column naming.
-parent : The parent of this vertex in the shortest path from source.
- Will use "parent" for column naming.
-
-The graph_sssp_get_path function will return an INT array that contains the
-shortest path from the initial source vertex to the desired destination vertex.
+The output of SSSP ('out_table' above) contains a row for every vertex of
+every group and have the following columns (in addition to the grouping
+columns):
+ - vertex_id : The id for the destination. Will use the input parameter
+ 'vertex_id' for column naming.
+ - weight : The total weight of the shortest path from the source vertex
+ to this particular vertex.
+ Will use the input parameter 'weight' for column naming.
+ - parent : The parent of this vertex in the shortest path from source.
+ Will use 'parent' for column naming.
+
+The output of graph_sssp_get_path ('path_table' above) contains a row for
+every group and has the following columns:
+ - grouping_cols : The grouping columns given in the creation of the SSSP
+ table. If there are no grouping columns, these columns
+ will not exist and the table will have a single row.
+ - path (ARRAY) : The shortest path from the source vertex (as specified
+ in the SSSP execution) to the destination vertex.
+"""
+ elif message.lower() in ("example", "examples"):
+ help_string = """
+----------------------------------------------------------------------------
+ EXAMPLES
+----------------------------------------------------------------------------
+-- Create a graph, represented as vertex and edge tables.
+DROP TABLE IF EXISTS vertex,edge,out,out_summary,out_path;
+CREATE TABLE vertex(
+ id INTEGER
+ );
+CREATE TABLE edge(
+ src INTEGER,
+ dest INTEGER,
+ weight DOUBLE PRECISION
+);
+
+INSERT INTO vertex VALUES
+(0),
+(1),
+(2),
+(3),
+(4),
+(5),
+(6),
+(7)
+;
+INSERT INTO edge VALUES
+(0, 1, 1),
+(0, 2, 1),
+(0, 4, 10),
+(1, 2, 2),
+(1, 3, 10),
+(2, 3, 1),
+(2, 5, 1),
+(2, 6, 3),
+(3, 0, 1),
+(4, 0, -2),
+(5, 6, 1),
+(6, 7, 1)
+;
+
+-- Compute the SSSP:
+DROP TABLE IF EXISTS pagerank_out;
+SELECT madlib.graph_sssp(
+ 'vertex', -- Vertex table
+ 'id', -- Vertix id column
+ 'edge', -- Edge table
+ 'src=src, dest=dest, weight=weight', -- Comma delimted string of edge arguments
+ 0, -- The source vertex
+ 'out' -- Output table of SSSP
+);
+-- View the SSSP costs for every vertex:
+SELECT * FROM out ORDER BY id;
+
+-- View the actual shortest path for a vertex:
+SELECT graph_sssp_get_path('out',5,'out_path');
+SELECT * FROM out_path;
+
+-- Create a graph with 2 groups:
+DROP TABLE IF EXISTS edge_gr;
+CREATE TABLE edge_gr AS
+(
+ SELECT *, 0 AS grp FROM edge
+ UNION
+ SELECT *, 1 AS grp FROM edge WHERE src < 6 AND dest < 6
+);
+INSERT INTO edge_gr VALUES
+(4,5,-20,1);
+
+-- Find SSSP for all groups:
+DROP TABLE IF EXISTS out_gr, out_gr_summary;
+SELECT graph_sssp('vertex',NULL,'edge_gr',NULL,0,'out_gr','grp');
"""
- else:
- help_string = "No such option. Use {schema_madlib}.graph_sssp()"
+ else:
+ help_string = "No such option. Use {schema_madlib}.graph_sssp()"
- return help_string.format(schema_madlib=schema_madlib,
- graph_usage=get_graph_usage(schema_madlib, 'graph_sssp',
+ return help_string.format(schema_madlib=schema_madlib,
+ graph_usage=get_graph_usage(schema_madlib, 'graph_sssp',
"""source_vertex INT, -- The source vertex id for the algorithm to start.
- out_table TEXT -- Name of the table to store the result of SSSP."""))
+ out_table TEXT, -- Name of the table to store the result of SSSP.
+ grouping_cols TEXT -- The list of grouping columns."""))
# ---------------------------------------------------------------------
-
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8faf6226/src/ports/postgres/modules/graph/sssp.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/sssp.sql_in b/src/ports/postgres/modules/graph/sssp.sql_in
index 7f89823..be433dc 100644
--- a/src/ports/postgres/modules/graph/sssp.sql_in
+++ b/src/ports/postgres/modules/graph/sssp.sql_in
@@ -55,7 +55,8 @@ graph_sssp( vertex_table,
edge_table,
edge_args,
source_vertex,
- out_table
+ out_table,
+ grouping_cols
)
</pre>
@@ -89,12 +90,18 @@ exist in the 'vertex_id' column of 'vertex_table'.</dd>
<dt>out_table</dt>
<dd>TEXT. Name of the table to store the result of SSSP.
-It will contain a row for every vertex from 'vertex_table' and have
-the following columns:
+It contains a row for every vertex of every group and have
+the following columns (in addition to the grouping columns):
- vertex_id : The id for the destination. Will use the input parameter 'vertex_id' for column naming.
- weight : The total weight of the shortest path from the source vertex to this particular vertex.
- Will use the input parameter (weight) for column naming.
- - parent : The parent of this vertex in the shortest path from source. Will use 'parent' for column naming.</dd>
+ Will use the input parameter 'weight' for column naming.
+ - parent : The parent of this vertex in the shortest path from source. Will use 'parent' for column naming.
+
+A summary table named <out_table>_summary is also created. This is an internal table that keeps a record of the input parameters and is used by the path function described below.
+</dd>
+
+<dt>grouping_cols</dt>
+<dd>TEXT, default = NULL. List of columns used to group the input into discrete subgraphs. These columns must exist in the edge table. When this value is null, no grouping is used and a single SSSP result is generated. </dd>
</dl>
@par Path Retrieval
@@ -103,9 +110,10 @@ The path retrieval function returns the shortest path from the
source vertex to a specified desination vertex.
<pre class="syntax">
-graph_sssp( sssp_table,
- dest_vertex
- )
+graph_sssp_get_path( sssp_table,
+ dest_vertex,
+ path_table
+ )
</pre>
\b Arguments
@@ -115,6 +123,14 @@ graph_sssp( sssp_table,
<dt>dest_vertex</dt>
<dd>INTEGER. The vertex that will be the destination of the desired path.</dd>
+
+<dt>path_table</dt>
+<dd>TEXT. Name of the output table that contains the path.
+It contains a row for every group and has the following columns:
+ - grouping_cols : The grouping columns given in the creation of the SSSP table. If there are no grouping columns, these columns will not exist and the table will have a single row.
+ - path (ARRAY) : The shortest path from the source vertex (as specified in the SSSP execution) to the destination vertex.
+</dd>
+
</dl>
@anchor notes
@@ -167,7 +183,7 @@ INSERT INTO edge VALUES
-# Calculate the shortest paths from vertex 0:
<pre class="syntax">
-DROP TABLE IF EXISTS out;
+DROP TABLE IF EXISTS out, out_summary;
SELECT madlib.graph_sssp(
'vertex', -- Vertex table
NULL, -- Vertix id column (NULL means use default naming)
@@ -191,14 +207,16 @@ SELECT * FROM out ORDER BY id;
(8 rows)
</pre>
--# Get the shortest path to vertex 6:
+-# Get the shortest path to vertex 5:
<pre class="syntax">
-SELECT madlib.graph_sssp_get_path('out',6) AS spath;
+DROP TABLE IF EXISTS out_path;
+SELECT madlib.graph_sssp_get_path('out',5,'out_path');
+SELECT * FROM out_path;
</pre>
<pre class="result">
- spath
-\-----------
- {0,2,5,6}
+ path
+\---------
+ {0,2,5}
</pre>
-# Now let's do a similar example except using
@@ -212,10 +230,10 @@ CREATE TABLE edge_alt AS SELECT src AS e_src, dest, weight AS e_weight FROM edge
-# Get the shortest path from vertex 1:
<pre class="syntax">
-DROP TABLE IF EXISTS out_alt;
+DROP TABLE IF EXISTS out_alt, out_alt_summary;
SELECT madlib.graph_sssp(
'vertex_alt', -- Vertex table
- 'v_id', -- Vertix id column (NULL means use default naming)
+ 'v_id', -- Vertex id column (NULL means use default naming)
'edge_alt', -- Edge table
'src=e_src, weight=e_weight', -- Edge arguments (NULL means use default naming)
1, -- Source vertex for path calculation
@@ -236,6 +254,65 @@ SELECT * FROM out_alt ORDER BY v_id;
(8 rows)
</pre>
+-# Create a graph with 2 groups:
+<pre class="syntax">
+DROP TABLE IF EXISTS edge_gr;
+CREATE TABLE edge_gr AS
+(
+ SELECT *, 0 AS grp FROM edge
+ UNION
+ SELECT *, 1 AS grp FROM edge WHERE src < 6 AND dest < 6
+);
+INSERT INTO edge_gr VALUES
+(4,5,-20,1);
+</pre>
+
+-# Find SSSP for all groups
+<pre class="syntax">
+DROP TABLE IF EXISTS out_gr, out_gr_summary;
+SELECT madlib.graph_sssp(
+ 'vertex', -- Vertex table
+ NULL, -- Vertex id column (NULL means use default naming)
+ 'edge_gr', -- Edge table
+ NULL, -- Edge arguments (NULL means use default naming)
+ 0, -- Source vertex for path calculation
+ 'out_gr', -- Output table of shortest paths
+ 'grp' -- Grouping columns
+);
+SELECT * FROM out_gr ORDER BY grp,id;
+</pre>
+<pre class="result">
+ grp | id | weight | parent
+-----+----+--------+--------
+ 0 | 0 | 0 | 0
+ 0 | 1 | 1 | 0
+ 0 | 2 | 1 | 0
+ 0 | 3 | 2 | 2
+ 0 | 4 | 10 | 0
+ 0 | 5 | 2 | 2
+ 0 | 6 | 3 | 5
+ 0 | 7 | 4 | 6
+ 1 | 0 | 0 | 0
+ 1 | 1 | 1 | 0
+ 1 | 2 | 1 | 0
+ 1 | 3 | 2 | 2
+ 1 | 4 | 10 | 0
+ 1 | 5 | -10 | 4
+</pre>
+
+-# Find the path to vertex 5 in every group
+<pre class="syntax">
+DROP TABLE IF EXISTS out_gr_path;
+SELECT madlib.graph_sssp_get_path('out_gr',5,'out_gr_path');
+SELECT * FROM out_gr_path ORDER BY grp;
+</pre>
+<pre class="result">
+ grp | path
+-----+---------
+ 0 | {0,2,5}
+ 1 | {0,4,5}
+</pre>
+
@anchor literature
@par Literature
@@ -253,21 +330,36 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.graph_sssp(
edge_table TEXT,
edge_args TEXT,
source_vertex INT,
- out_table TEXT
+ out_table TEXT,
+ grouping_cols TEXT
) RETURNS VOID AS $$
PythonFunction(graph, sssp, graph_sssp)
$$ LANGUAGE plpythonu VOLATILE
m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `MODIFIES SQL DATA', `');
-------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.graph_sssp(
+ vertex_table TEXT,
+ vertex_id TEXT,
+ edge_table TEXT,
+ edge_args TEXT,
+ source_vertex INT,
+ out_table TEXT
+
+) RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.graph_sssp($1, $2, $3, $4, $5, $6, NULL);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `MODIFIES SQL DATA', `');
+-------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.graph_sssp_get_path(
sssp_table TEXT,
- dest_vertex INT
+ dest_vertex INT,
+ path_table TEXT
-) RETURNS INT[] AS $$
+) RETURNS VOID AS $$
PythonFunction(graph, sssp, graph_sssp_get_path)
$$ LANGUAGE plpythonu VOLATILE
-m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `CONTAINS SQL', `');
+m4_ifdef(`\_\_HAS_FUNCTION_PROPERTIES\_\_', `MODIFIES SQL DATA', `');
-------------------------------------------------------------------------
-- Online help
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8faf6226/src/ports/postgres/modules/graph/test/sssp.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/graph/test/sssp.sql_in b/src/ports/postgres/modules/graph/test/sssp.sql_in
index e2342c5..c3545c2 100644
--- a/src/ports/postgres/modules/graph/test/sssp.sql_in
+++ b/src/ports/postgres/modules/graph/test/sssp.sql_in
@@ -20,7 +20,10 @@
*//* ----------------------------------------------------------------------- */
-DROP TABLE IF EXISTS vertex,edge,out,vertex_alt,edge_alt,out_alt;
+DROP TABLE IF EXISTS vertex,edge,out,out_summary,out_path,
+ vertex_alt,edge_alt,out_alt,out_alot_summary,
+ edge_gr,out_gr,out_gr_summary,out_gr_path,
+ edge_gr2, out_gr2, out_gr2_summary;
CREATE TABLE vertex(
@@ -30,7 +33,7 @@ CREATE TABLE vertex(
CREATE TABLE edge(
src INTEGER,
dest INTEGER,
- weight INTEGER
+ weight DOUBLE PRECISION
);
INSERT INTO vertex VALUES
@@ -62,17 +65,69 @@ SELECT graph_sssp('vertex',NULL,'edge',NULL,0,'out');
SELECT * FROM out;
-SELECT assert(weight = 3, 'Wrong output in graph (SSSP)') FROM out WHERE id = 6;
-SELECT assert(parent = 5, 'Wrong parent in graph (SSSP)') FROM out WHERE id = 6;
+SELECT assert(weight = 3, 'Wrong output in graph (SSSP)')
+ FROM out WHERE id = 6;
+SELECT assert(parent = 5, 'Wrong parent in graph (SSSP)')
+ FROM out WHERE id = 6;
-SELECT graph_sssp_get_path('out',6);
+SELECT graph_sssp_get_path('out',6,'out_path');
-CREATE TABLE vertex_alt AS SELECT id AS v_id FROM vertex;
-CREATE TABLE edge_alt AS SELECT src AS e_src, dest, weight AS e_weight FROM edge;
+CREATE TABLE vertex_alt AS SELECT id AS v_id
+ FROM vertex;
+CREATE TABLE edge_alt AS SELECT src AS e_src, dest, weight AS e_weight
+ FROM edge;
-SELECT graph_sssp('vertex_alt','v_id','edge_alt','src=e_src, weight=e_weight',1,'out_alt');
+SELECT graph_sssp('vertex_alt','v_id','edge_alt','src=e_src, weight=e_weight'
+ ,1,'out_alt');
SELECT * FROM out_alt;
-SELECT assert(e_weight = 4, 'Wrong output in graph (SSSP)') FROM out_alt WHERE v_id = 6;
-SELECT assert(parent = 5, 'Wrong parent in graph (SSSP)') FROM out_alt WHERE v_id = 6;
+SELECT assert(e_weight = 4, 'Wrong output in graph (SSSP)')
+ FROM out_alt WHERE v_id = 6;
+SELECT assert(parent = 5, 'Wrong parent in graph (SSSP)')
+ FROM out_alt WHERE v_id = 6;
+
+CREATE TABLE edge_gr AS
+( SELECT *, 0 AS grp FROM edge
+ UNION
+ SELECT *, 1 AS grp FROM edge WHERE src < 6 AND dest < 6
+ UNION
+ SELECT *, 2 AS grp FROM edge WHERE src < 6 AND dest < 6
+);
+
+INSERT INTO edge_gr VALUES
+(7,NULL,NULL,1),
+(4,0,-20,2);
+
+SELECT graph_sssp('vertex',NULL,'edge_gr',NULL,0,'out_gr','grp');
+
+SELECT assert(weight = 3, 'Wrong output in graph (SSSP)')
+ FROM out_gr WHERE id = 6 AND grp = 0;
+SELECT assert(parent = 5, 'Wrong parent in graph (SSSP)')
+ FROM out_gr WHERE id = 6 AND grp = 0;
+
+SELECT assert(weight = 2, 'Wrong output in graph (SSSP)')
+ FROM out_gr WHERE id = 5 AND grp = 1;
+SELECT assert(parent = 2, 'Wrong parent in graph (SSSP)')
+ FROM out_gr WHERE id = 5 AND grp = 1;
+
+SELECT assert(weight = 'Infinity', 'Wrong output in graph (SSSP)')
+ FROM out_gr WHERE id = 7 AND grp = 1;
+
+SELECT graph_sssp_get_path('out_gr',5,'out_gr_path');
+
+CREATE TABLE edge_gr2 AS
+( SELECT *, 0 AS grp1, 0 AS grp2 FROM edge
+ UNION
+ SELECT *, 1 AS grp1, 0 AS grp2 FROM edge WHERE src < 6 AND dest < 6
+ UNION
+ SELECT *, 1 AS grp1, 1 AS grp2 FROM edge WHERE src < 6 AND dest < 6
+);
+
+SELECT graph_sssp('vertex',NULL,'edge_gr2',NULL,0,'out_gr2','grp1,grp2');
+
+
+SELECT assert(weight = 3, 'Wrong output in graph (SSSP)')
+ FROM out_gr2 WHERE id = 6 AND grp1 = 0 AND grp2 = 0;
+SELECT assert(parent = 5, 'Wrong parent in graph (SSSP)')
+ FROM out_gr2 WHERE id = 6 AND grp1 = 0 AND grp2 = 0;
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8faf6226/src/ports/postgres/modules/pca/pca.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/pca/pca.py_in b/src/ports/postgres/modules/pca/pca.py_in
index 196c558..680e9f6 100644
--- a/src/ports/postgres/modules/pca/pca.py_in
+++ b/src/ports/postgres/modules/pca/pca.py_in
@@ -144,16 +144,16 @@ def pca_wrap(schema_madlib, source_table, pc_table, row_id,
)
""".format(pc_table=pc_table, grouping_cols_clause=grouping_cols_clause))
pc_table_mean = add_postfix(pc_table, "_mean")
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(pc_table_mean))
plpy.execute("""
- DROP TABLE IF EXISTS {pc_table_mean};
CREATE TABLE {pc_table_mean} (
column_mean double precision[]
{grouping_cols_clause}
)
""".format(pc_table_mean=pc_table_mean, grouping_cols_clause=grouping_cols_clause))
if result_summary_table:
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(result_summary_table))
plpy.execute("""
- DROP TABLE IF EXISTS {0};
CREATE TABLE {0} (
rows_used INTEGER,
"exec_time (ms)" numeric,
@@ -947,7 +947,7 @@ SELECT {schema_madlib}.pca_train( 'mat',
'id',
3
);
-
+
SELECT * FROM result_table ORDER BY row_id;
DROP TABLE IF EXISTS mat_group;
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8faf6226/src/ports/postgres/modules/stats/test/chi2_test.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/stats/test/chi2_test.sql_in b/src/ports/postgres/modules/stats/test/chi2_test.sql_in
index c49d996..62648a0 100644
--- a/src/ports/postgres/modules/stats/test/chi2_test.sql_in
+++ b/src/ports/postgres/modules/stats/test/chi2_test.sql_in
@@ -58,7 +58,7 @@ CREATE TABLE chi2_independence_est_1 AS
SELECT (chi2_gof_test(observed, expected, deg_freedom)).*
FROM (
SELECT
- observed,
+ id_x,id_y,observed,
sum(observed) OVER (PARTITION BY id_x)::DOUBLE PRECISION
* sum(observed) OVER (PARTITION BY id_y) AS expected
FROM chi2_test_friendly_unpivoted
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/8faf6226/src/ports/postgres/modules/validation/internal/cross_validation.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/validation/internal/cross_validation.py_in b/src/ports/postgres/modules/validation/internal/cross_validation.py_in
index c1b2561..11cde2f 100644
--- a/src/ports/postgres/modules/validation/internal/cross_validation.py_in
+++ b/src/ports/postgres/modules/validation/internal/cross_validation.py_in
@@ -200,9 +200,9 @@ def _cv_copy_data(rel_origin, dependent_varname,
"""
"""
target_col, features_col = 'y', 'x'
+ plpy.execute("drop table if exists {0}".format(rel_copied))
plpy.execute("""
select setseed(0.5);
- drop table if exists {rel_copied};
create temp table {rel_copied} as
select
row_number() over (order by random()) as {random_id},
@@ -233,15 +233,15 @@ def _cv_split_data(rel_source, col_data, col_id, row_num,
# which corresponds to rows outside of [start_row, end_row).
# Extract the validation part of data,
# which corresponds to rows inside of [start_row, end_row).
+ plpy.execute("drop view if exists {rel_train}".format(**kwargs))
plpy.execute("""
- drop view if exists {rel_train};
create temp view {rel_train} as
select {col_id}, {col_string} from {rel_source}
where {col_id} < {start_row}
or {col_id} >= {end_row}
""".format(**kwargs))
+ plpy.execute("drop view if exists {rel_valid}".format(**kwargs))
plpy.execute("""
- drop view if exists {rel_valid};
create temp view {rel_valid} as
select {col_id}, {col_string} from {rel_source}
where {col_id} >= {start_row}