You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2018/06/15 15:28:59 UTC
[1/2] madlib git commit: Upgrade: Fix multiple bugs
Repository: madlib
Updated Branches:
refs/heads/master 8e34f68d7 -> b88f60464
Upgrade: Fix multiple bugs
1. Appended schema_madlib to the mlp_igd_final return type. The missing
schema name caused the upgrade to fail from 1.12 to 1.x if there was a
dependency on mlp_igd_final.
2. A new changelist was created for changes from v1.14 to 1.15-dev. We
will rename this at the 1.15 release from 1.14_1.15-dev.yaml to
1.14_1.15.yaml.
3. Commit 8e34f68 added a new function called `_write_to_file` that
takes 2 arguments. Some of the calls to this function were not passing
the first file handle argument.
Closes #278
Co-authored-by : Orhan Kislal <ok...@pivotal.io>
Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/89bcdb78
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/89bcdb78
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/89bcdb78
Branch: refs/heads/master
Commit: 89bcdb785816716f2ef6c5e4599edbf95584595d
Parents: 8e34f68
Author: Nikhil Kak <nk...@pivotal.io>
Authored: Fri Jun 15 08:18:34 2018 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Fri Jun 15 08:22:34 2018 -0700
----------------------------------------------------------------------
src/madpack/changelist_1.12_1.13.yaml | 2 +-
src/madpack/changelist_1.14_1.15-dev.yaml | 58 ++++++++++++++++++++++++++
src/madpack/upgrade_util.py | 14 +++----
3 files changed, 66 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/madlib/blob/89bcdb78/src/madpack/changelist_1.12_1.13.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.12_1.13.yaml b/src/madpack/changelist_1.12_1.13.yaml
index 0e6c3df..49169c3 100644
--- a/src/madpack/changelist_1.12_1.13.yaml
+++ b/src/madpack/changelist_1.12_1.13.yaml
@@ -49,7 +49,7 @@ udf:
rettype: void
argument: character varying, character varying, character varying, character varying
- mlp_igd_final:
- rettype: mlp_step_result
+ rettype: schema_madlib.mlp_step_result
argument: double precision[]
- mlp_igd_transition:
rettype: double precision[]
http://git-wip-us.apache.org/repos/asf/madlib/blob/89bcdb78/src/madpack/changelist_1.14_1.15-dev.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.14_1.15-dev.yaml b/src/madpack/changelist_1.14_1.15-dev.yaml
new file mode 100644
index 0000000..88bb886
--- /dev/null
+++ b/src/madpack/changelist_1.14_1.15-dev.yaml
@@ -0,0 +1,58 @@
+# ------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------------------------
+
+# Changelist for MADlib version 1.14 to 1.15
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+ # ----------------- Changes from 1.14 to 1.15 --------
+
+
+# Changes in the types (UDT) including removal and modification
+udt:
+
+# List of the UDF changes that affect the user externally. This includes change
+# in function name, return type, argument order or types, or removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+ # ----------------- Changes from 1.14 to 1.15 ----------
+
+
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+
+# Casts (UDC) updated/removed
+udc:
+
+# Operators (UDO) removed/updated
+udo:
+
+# Operator Classes (UDOC) removed/updated
+udoc:
http://git-wip-us.apache.org/repos/asf/madlib/blob/89bcdb78/src/madpack/upgrade_util.py
----------------------------------------------------------------------
diff --git a/src/madpack/upgrade_util.py b/src/madpack/upgrade_util.py
index c77b893..53ff748 100644
--- a/src/madpack/upgrade_util.py
+++ b/src/madpack/upgrade_util.py
@@ -406,7 +406,7 @@ class ChangeHandler(UpgradeBase):
cascade_str = 'CASCADE' if udt in ('svec', 'bytea8') else ''
# CASCADE DROP for svec and bytea8 because the recv/send
# functions and the type depend on each other
- _write_to_file("DROP TYPE IF EXISTS {0}.{1} {2};".
+ _write_to_file(self.output_filehandle, "DROP TYPE IF EXISTS {0}.{1} {2};".
format(self._schema, udt, cascade_str))
def drop_changed_udf(self):
@@ -420,7 +420,7 @@ class ChangeHandler(UpgradeBase):
# so dropping that function needs this extra check.
udf_arglist = item['argument'] if 'argument' in item else ''
- _write_to_file("DROP FUNCTION IF EXISTS {schema}.{udf}({arg});".
+ _write_to_file(self.output_filehandle, "DROP FUNCTION IF EXISTS {schema}.{udf}({arg});".
format(schema=self._schema,
udf=udf,
arg=udf_arglist))
@@ -431,7 +431,7 @@ class ChangeHandler(UpgradeBase):
"""
for uda in self._uda:
for item in self._uda[uda]:
- _write_to_file("DROP AGGREGATE IF EXISTS {schema}.{uda}({arg});".
+ _write_to_file(self.output_filehandle, "DROP AGGREGATE IF EXISTS {schema}.{uda}({arg});".
format(schema=self._schema,
uda=uda,
arg=item['argument']))
@@ -442,7 +442,7 @@ class ChangeHandler(UpgradeBase):
@note We have special treatment for UDCs defined in the svec module
"""
for udc in self._udc:
- _write_to_file("DROP CAST IF EXISTS ({sourcetype} AS {targettype});".
+ _write_to_file(self.output_filehandle, "DROP CAST IF EXISTS ({sourcetype} AS {targettype});".
format(sourcetype=self._udc[udc]['sourcetype'],
targettype=self._udc[udc]['targettype']))
@@ -451,7 +451,7 @@ class ChangeHandler(UpgradeBase):
@brief Drop the madlib.training_info table, which should no longer be used since
the version 1.5
"""
- _write_to_file("DROP TABLE IF EXISTS {schema}.training_info;".
+ _write_to_file(self.output_filehandle, "DROP TABLE IF EXISTS {schema}.training_info;".
format(schema=self._schema))
def drop_changed_udo(self):
@@ -462,7 +462,7 @@ class ChangeHandler(UpgradeBase):
for value in self._udo[op]:
leftarg = value['leftarg'].replace('schema_madlib', self._schema)
rightarg = value['rightarg'].replace('schema_madlib', self._schema)
- _write_to_file("""
+ _write_to_file(self.output_filehandle, """
DROP OPERATOR IF EXISTS {schema}.{op} ({leftarg}, {rightarg});
""".format(schema=self._schema, **locals()))
@@ -473,7 +473,7 @@ class ChangeHandler(UpgradeBase):
for op_cls in self._udoc:
for value in self._udoc[op_cls]:
index = value['index']
- _write_to_file("""
+ _write_to_file(self.output_filehandle, """
DROP OPERATOR CLASS IF EXISTS {schema}.{op_cls} USING {index};
""".format(schema=self._schema, **locals()))
[2/2] madlib git commit: KNN: Make install-check asserts deterministic
Posted by ri...@apache.org.
KNN: Make install-check asserts deterministic
Knn install-check had a couple of unordered array_agg asserts which were not
deterministic. This commit ensures the order is deterministic.
Closes #279
Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/b88f6046
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/b88f6046
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/b88f6046
Branch: refs/heads/master
Commit: b88f60464645c5df443b7f371dcc8075e2a0bccb
Parents: 89bcdb7
Author: Nikhil Kak <nk...@pivotal.io>
Authored: Thu Jun 14 10:20:55 2018 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Fri Jun 15 08:24:43 2018 -0700
----------------------------------------------------------------------
src/ports/postgres/modules/knn/test/knn.sql_in | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/madlib/blob/b88f6046/src/ports/postgres/modules/knn/test/knn.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/knn/test/knn.sql_in b/src/ports/postgres/modules/knn/test/knn.sql_in
index 6a39c89..20348af 100644
--- a/src/ports/postgres/modules/knn/test/knn.sql_in
+++ b/src/ports/postgres/modules/knn/test/knn.sql_in
@@ -77,7 +77,7 @@ select assert(array_agg(prediction order by id)='{1,1,0,1,0,0}', 'Wrong output i
drop table if exists madlib_knn_result_classification;
select knn('knn_train_data','data','id','label','knn_test_data','data','id','madlib_knn_result_classification',3);
-select assert(array_agg(x)= '{1,2,3}','Wrong output in classification with k=3') from (select unnest(k_nearest_neighbours) as x from madlib_knn_result_classification where id = 1 order by x asc) y;
+select assert(array_agg(x order by id)= '{1,2,3}','Wrong output in classification with k=3') from (select unnest(k_nearest_neighbours) as x, id from madlib_knn_result_classification where id = 1 order by x asc) y;
drop table if exists madlib_knn_result_regression;
select knn('knn_train_data_reg','data','id','label','knn_test_data','data','id','madlib_knn_result_regression',4,False,'MADLIB_SCHEMA.squared_dist_norm2',False);
@@ -85,7 +85,7 @@ select assert(array_agg(prediction order by id)='{1,1,0.5,1,0.25,0.25}', 'Wrong
drop table if exists madlib_knn_result_regression;
select knn('knn_train_data_reg','data','id','label','knn_test_data','data','id','madlib_knn_result_regression',3,True);
-select assert(array_agg(x)= '{1,2,3}' , 'Wrong output in regression with k=3') from (select unnest(k_nearest_neighbours) as x from madlib_knn_result_regression where id = 1 order by x asc) y;
+select assert(array_agg(x order by id)= '{1,2,3}' , 'Wrong output in regression with k=3') from (select unnest(k_nearest_neighbours) as x, id from madlib_knn_result_regression where id = 1 order by x asc) y;
drop table if exists madlib_knn_result_classification;
select knn('knn_train_data','data','id','label','knn_test_data','data','id','madlib_knn_result_classification',3,False,NULL,False);