You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ok...@apache.org on 2016/12/22 17:26:58 UTC
incubator-madlib git commit: Adds cluster variance to kmeans for
PivotalR support.
Repository: incubator-madlib
Updated Branches:
refs/heads/master 6939fd63b -> b045f7e2e
Adds cluster variance to kmeans for PivotalR support.
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/b045f7e2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/b045f7e2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/b045f7e2
Branch: refs/heads/master
Commit: b045f7e2e6c8f1a1f9cc6ce0c22b75764c81c1f1
Parents: 6939fd6
Author: Orhan Kislal <ok...@pivotal.io>
Authored: Thu Dec 22 09:25:44 2016 -0800
Committer: Orhan Kislal <ok...@pivotal.io>
Committed: Thu Dec 22 09:25:44 2016 -0800
----------------------------------------------------------------------
src/ports/postgres/modules/kmeans/kmeans.py_in | 4 +++-
src/ports/postgres/modules/kmeans/kmeans.sql_in | 18 ++++++++++++------
2 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/b045f7e2/src/ports/postgres/modules/kmeans/kmeans.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/kmeans/kmeans.py_in b/src/ports/postgres/modules/kmeans/kmeans.py_in
index ffac9b2..d99ddd8 100644
--- a/src/ports/postgres/modules/kmeans/kmeans.py_in
+++ b/src/ports/postgres/modules/kmeans/kmeans.py_in
@@ -280,7 +280,7 @@ def compute_kmeans(schema_madlib, rel_args, rel_state, rel_source,
it.update("""
SELECT
- CAST((_args.initial_centroids, NULL, 'Inf', 1.0) AS
+ CAST((_args.initial_centroids, NULL, NULL, 'Inf', 1.0) AS
{schema_madlib}.kmeans_state)
FROM {rel_args} AS _args
""")
@@ -295,6 +295,7 @@ def compute_kmeans(schema_madlib, rel_args, rel_state, rel_source,
_centroid::FLOAT8[]
ORDER BY _new_centroid_id),
array_agg(_new_centroid_id ORDER BY _new_centroid_id),
+ array_agg(_objective_fn ORDER BY _new_centroid_id),
sum(_objective_fn),
CAST(sum(_num_reassigned) AS DOUBLE PRECISION)
/ sum(_num_points)
@@ -350,6 +351,7 @@ def compute_kmeans(schema_madlib, rel_args, rel_state, rel_source,
(SELECT fn_dist_name FROM {rel_args})
),
(%(old_centroid)s),
+ NULL,
'Inf',
1.0
) AS {schema_madlib}.kmeans_state)
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/b045f7e2/src/ports/postgres/modules/kmeans/kmeans.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/kmeans/kmeans.sql_in b/src/ports/postgres/modules/kmeans/kmeans.sql_in
index 0b44286..2352bc1 100644
--- a/src/ports/postgres/modules/kmeans/kmeans.sql_in
+++ b/src/ports/postgres/modules/kmeans/kmeans.sql_in
@@ -178,6 +178,10 @@ The output of the k-means module is a composite type with the following columns:
<td>DOUBLE PRECISION[][]. The final centroid positions.</td>
</tr>
<tr>
+ <th>cluster_variance</th>
+ <td>DOUBLE PRECISION[]. The value of the objective function per cluster.</td>
+ </tr>
+ <tr>
<th>objective_fn</th>
<td>DOUBLE PRECISION. The value of the objective function.</td>
</tr>
@@ -254,11 +258,12 @@ SELECT * FROM madlib.kmeanspp( 'km_sample',
</pre>
Result:
<pre class="result">
-centroids | {{13.872,1.814,2.376,15.56,88.2,2.806,2.928,0.288,1.844,5.35198,1.044,3.348,988},
+centroids | {{13.872,1.814,2.376,15.56,88.2,2.806,2.928,0.288,1.844,5.35198,1.044,3.348,988},
{14.036,2.018,2.536,16.56,108.6,3.004,3.03,0.298,2.038,6.10598,1.004,3.326,1340}}
-objective_fn | 151184.962672
-frac_reassigned | 0
-num_iterations | 2
+cluster_variance | {90512.324426408,60672.638245208}
+objective_fn | 151184.962672
+frac_reassigned | 0
+num_iterations | 2
</pre>
-# Calculate the simplified silhouette coefficient:
<pre class="example">
@@ -435,6 +440,7 @@ File kmeans.sql_in documenting the k-Means SQL functions
DROP TYPE IF EXISTS MADLIB_SCHEMA.kmeans_result CASCADE;
CREATE TYPE MADLIB_SCHEMA.kmeans_result AS (
centroids DOUBLE PRECISION[][],
+ cluster_variance DOUBLE PRECISION[],
objective_fn DOUBLE PRECISION,
frac_reassigned DOUBLE PRECISION,
num_iterations INTEGER
@@ -456,6 +462,7 @@ DROP TYPE IF EXISTS MADLIB_SCHEMA.kmeans_state CASCADE;
CREATE TYPE MADLIB_SCHEMA.kmeans_state AS (
centroids DOUBLE PRECISION[][],
old_centroid_ids INTEGER[],
+ cluster_variance DOUBLE PRECISION[],
objective_fn DOUBLE PRECISION,
frac_reassigned DOUBLE PRECISION
);
@@ -732,7 +739,7 @@ BEGIN
-- Retrieve result from state table and return it
EXECUTE
$sql$
- SELECT (_state).centroids, (_state).objective_fn,
+ SELECT (_state).centroids, (_state).cluster_variance, (_state).objective_fn,
(_state).frac_reassigned, NULL
FROM _madlib_kmeans_state
WHERE _iteration = $sql$ || theIteration || $sql$
@@ -1616,4 +1623,3 @@ AS $$
'MADLIB_SCHEMA.dist_norm2')
$$
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `READS SQL DATA', `');
-