You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ok...@apache.org on 2020/04/02 18:01:05 UTC
[madlib] 01/04: Build: Update madpack scripts for pg11 and add
changelist
This is an automated email from the ASF dual-hosted git repository.
okislal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit e6bad2fac706fd7c1e49a5249215195555b9f1e3
Author: Orhan Kislal <ok...@apache.org>
AuthorDate: Fri Jan 17 12:26:32 2020 -0500
Build: Update madpack scripts for pg11 and add changelist
Co-authored-by: Amil Khanzada <am...@gmail.com>
---
src/madpack/changelist_1.16_1.17.yaml | 142 ++++++++++++++++++++++++++++++++++
src/madpack/diff_udf.sql | 120 ++++++++++++++--------------
src/madpack/madpack.py | 3 +
src/madpack/upgrade_util.py | 16 +++-
4 files changed, 221 insertions(+), 60 deletions(-)
diff --git a/src/madpack/changelist_1.16_1.17.yaml b/src/madpack/changelist_1.16_1.17.yaml
new file mode 100644
index 0000000..90f2e88
--- /dev/null
+++ b/src/madpack/changelist_1.16_1.17.yaml
@@ -0,0 +1,142 @@
+# ------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------------------------
+
+# Changelist for MADlib version rel/v1.16 to rel/v1.17.0
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+ madlib_keras_fit_multiple_model:
+ madlib_keras_gpu_info:
+ madlib_keras_model_selection:
+
+# Changes in the types (UDT) including removal and modification
+udt:
+
+# List of the UDF changes that affect the user externally. This includes change
+# in function name, return type, argument order or types, or removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+ - fit_final_multiple_model:
+ rettype: bytea
+ argument: bytea
+ - fit_transition:
+ rettype: bytea
+ argument: bytea, smallint[], real[], text, text, text, integer, integer[], integer[], integer, integer, bytea
+ - graph_apsp_get_path:
+ rettype: void
+ argument: text, integer, integer, text
+ - graph_sssp:
+ rettype: void
+ argument: text, text, text, text, integer, text
+ - graph_sssp:
+ rettype: void
+ argument: text, text, text, text, integer, text, text
+ - graph_sssp_get_path:
+ rettype: void
+ argument: text, integer, text
+ - internal_keras_eval_transition:
+ rettype: real[]
+ argument: real[], smallint[], real[], text, bytea, text, integer, integer[], integer[], integer, integer
+ - internal_keras_predict:
+ rettype: double precision[]
+ argument: real[], text, bytea, boolean, double precision, integer, integer[], integer[], integer, integer
+ - madlib_keras_evaluate:
+ rettype: void
+ argument: character varying, character varying, character varying, integer
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer, boolean
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer, boolean, character varying
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer, boolean, character varying, character varying
+ - madlib_keras_predict:
+ rettype: void
+ argument: character varying, character varying, character varying, character varying, character varying, character varying, integer
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text, text, boolean, boolean, text
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text, text, boolean, boolean
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text, text, boolean
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text, text
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text, boolean
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, boolean
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text
+
+
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+ - agg_array_concat:
+ rettype: anyarray
+ argument: anyarray
+ - fit_step:
+ rettype: bytea
+ argument: smallint[], real[], text, text, text, integer, integer[], integer[], integer, integer, bytea
+ - internal_keras_evaluate:
+ rettype: real[]
+ argument: smallint[], real[], text, bytea, text, integer, integer[], integer[], integer, integer
+
+# List of the UDC, UDO and UDOC changes.
+udc:
+
+# Changes in the operators (UDO)
+udo:
+
+# Changes in the operator classes (UDOC)
+udoc:
diff --git a/src/madpack/diff_udf.sql b/src/madpack/diff_udf.sql
index d73093c..3226409 100644
--- a/src/madpack/diff_udf.sql
+++ b/src/madpack/diff_udf.sql
@@ -15,67 +15,75 @@ CREATE OR REPLACE FUNCTION get_functions(table_name text, schema_name text,
type_filter text)
RETURNS VOID AS
$$
- import plpy
- plpy.execute("""
- CREATE TABLE {table_name} AS
- SELECT
- "schema", "name", filter_schema("retype", '{schema_name}') retype,
- filter_schema("argtypes", '{schema_name}') argtypes, "type"
- FROM
- (
+import plpy
+column_name = plpy.execute("""
+ SELECT column_name
+ FROM information_schema.columns
+ WHERE table_schema='pg_catalog' and table_name='pg_proc' and
+ (column_name='prokind' or column_name='proisagg')
+ """)[0]['column_name']
+proisagg_wrapper = "p.proisagg" if column_name == 'proisagg' else "p.prokind = 'a'"
+plpy.execute("""
+ CREATE TABLE {table_name} AS
+ SELECT
+ "schema", "name", filter_schema("retype", '{schema_name}') retype,
+ filter_schema("argtypes", '{schema_name}') argtypes, "type"
+ FROM
+ (
- SELECT n.nspname as "schema",
- p.proname as "name",
- CASE WHEN p.proretset THEN 'SETOF ' ELSE '' END ||
- pg_catalog.format_type(p.prorettype, NULL) as "retype",
- CASE WHEN proallargtypes IS NOT NULL THEN
- pg_catalog.array_to_string(ARRAY(
+ SELECT n.nspname as "schema",
+ p.proname as "name",
+ CASE WHEN p.proretset THEN 'SETOF ' ELSE '' END ||
+ pg_catalog.format_type(p.prorettype, NULL) as "retype",
+ CASE WHEN proallargtypes IS NOT NULL THEN
+ pg_catalog.array_to_string(ARRAY(
+ SELECT
+ pio || ptyp
+ FROM
+ (
SELECT
- pio || ptyp
- FROM
- (
- SELECT
- CASE
- WHEN p.proargmodes[s.i] = 'i' THEN ''
- WHEN p.proargmodes[s.i] = 'o' THEN 'OUT '
- WHEN p.proargmodes[s.i] = 'b' THEN 'INOUT '
- WHEN p.proargmodes[s.i] = 'v' THEN 'VARIADIC '
- END AS pio,
- --CASE
- -- WHEN COALESCE(p.proargnames[s.i], '') = '' THEN ''
- -- ELSE p.proargnames[s.i] || ' '
- --END ||
- pg_catalog.format_type(p.proallargtypes[s.i], NULL) AS ptyp
- FROM
- pg_catalog.generate_series(1, pg_catalog.array_upper(p.proallargtypes, 1)) AS s(i)
- ) qx
- WHERE pio = ''
- ), ', ')
- ELSE
- pg_catalog.array_to_string(ARRAY(
- SELECT
+ CASE
+ WHEN p.proargmodes[s.i] = 'i' THEN ''
+ WHEN p.proargmodes[s.i] = 'o' THEN 'OUT '
+ WHEN p.proargmodes[s.i] = 'b' THEN 'INOUT '
+ WHEN p.proargmodes[s.i] = 'v' THEN 'VARIADIC '
+ END AS pio,
--CASE
- -- WHEN COALESCE(p.proargnames[s.i+1], '') = '' THEN ''
- -- ELSE p.proargnames[s.i+1] || ' '
- -- END ||
- pg_catalog.format_type(p.proargtypes[s.i], NULL)
+ -- WHEN COALESCE(p.proargnames[s.i], '') = '' THEN ''
+ -- ELSE p.proargnames[s.i] || ' '
+ --END ||
+ pg_catalog.format_type(p.proallargtypes[s.i], NULL) AS ptyp
FROM
- pg_catalog.generate_series(0, pg_catalog.array_upper(p.proargtypes, 1)) AS s(i)
+ pg_catalog.generate_series(1, pg_catalog.array_upper(p.proallargtypes, 1)) AS s(i)
+ ) qx
+ WHERE pio = ''
), ', ')
- END AS "argtypes",
- CASE
- WHEN p.proisagg THEN 'agg'
- WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger'
- ELSE 'normal'
- END AS "type"
- FROM pg_catalog.pg_proc p
- LEFT JOIN pg_catalog.pg_namespace n
- ON n.oid = p.pronamespace
- WHERE n.nspname ~ '^({schema_name})$'
- ORDER BY 1, 2, 4
- ) q
- WHERE retype LIKE '{type_filter}' OR retype LIKE '{type_filter}[]'
- """.format(table_name=table_name, schema_name=schema_name, type_filter=type_filter))
+ ELSE
+ pg_catalog.array_to_string(ARRAY(
+ SELECT
+ --CASE
+ -- WHEN COALESCE(p.proargnames[s.i+1], '') = '' THEN ''
+ -- ELSE p.proargnames[s.i+1] || ' '
+ -- END ||
+ pg_catalog.format_type(p.proargtypes[s.i], NULL)
+ FROM
+ pg_catalog.generate_series(0, pg_catalog.array_upper(p.proargtypes, 1)) AS s(i)
+ ), ', ')
+ END AS "argtypes",
+ CASE
+ WHEN {proisagg_wrapper} THEN 'agg'
+ WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger'
+ ELSE 'normal'
+ END AS "type"
+ FROM pg_catalog.pg_proc p
+ LEFT JOIN pg_catalog.pg_namespace n
+ ON n.oid = p.pronamespace
+ WHERE n.nspname ~ '^({schema_name})$'
+ ORDER BY 1, 2, 4
+ ) q
+ WHERE retype LIKE '{type_filter}' OR retype LIKE '{type_filter}[]'
+ """.format(table_name=table_name, schema_name=schema_name,
+ type_filter=type_filter, proisagg_wrapper=proisagg_wrapper))
$$ LANGUAGE plpythonu;
diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py
index 86a7236..2af0480 100755
--- a/src/madpack/madpack.py
+++ b/src/madpack/madpack.py
@@ -701,6 +701,9 @@ def _process_py_sql_files_in_modules(modset, args_dict):
args_dict['create_obj_handle'],
args_dict['sc'])
elif calling_operation == INSTALL_DEV_CHECK:
+ # Skip certain tests for GP4.3
+ if dbver == '4.3ORCA' and module in ['deep_learning', 'kmeans']:
+ continue
_execute_per_module_install_dev_check_algo(
args_dict['schema'],
args_dict['test_user'],
diff --git a/src/madpack/upgrade_util.py b/src/madpack/upgrade_util.py
index 90511a3..9197b99 100644
--- a/src/madpack/upgrade_util.py
+++ b/src/madpack/upgrade_util.py
@@ -502,6 +502,9 @@ class ViewDependency(UpgradeBase):
"""
@brief Detect direct view dependencies on MADlib UDFs/UDAs
"""
+ proisagg_wrapper = "p.proisagg"
+ if self._portid == 'postgres' and self._dbver > 11:
+ proisagg_wrapper = "p.prokind = 'a'"
rows = self._run_sql("""
SELECT
view, nsp.nspname AS schema, procname, procoid, proisagg
@@ -513,7 +516,7 @@ class ViewDependency(UpgradeBase):
c.relnamespace AS namespace,
p.proname As procname,
p.oid AS procoid,
- p.proisagg AS proisagg
+ {proisagg_wrapper} AS proisagg
FROM
pg_class AS c,
pg_rewrite AS rw,
@@ -529,7 +532,8 @@ class ViewDependency(UpgradeBase):
) t1
WHERE
t1.namespace = nsp.oid
- """.format(schema_madlib_oid=self._schema_oid))
+ """.format(schema_madlib_oid=self._schema_oid,
+ proisagg_wrapper=proisagg_wrapper))
self._view2proc = defaultdict(list)
for row in rows:
@@ -1029,6 +1033,10 @@ class ScriptCleaner(UpgradeBase):
"""
# See _get_function_info for explanations.
+ proisagg_wrapper = "p.proisagg = true"
+ if self._portid == 'postgres' and self._dbver > 11:
+ proisagg_wrapper = "p.prokind = 'a'"
+
rows = self._run_sql("""
SELECT
max(proname) AS proname,
@@ -1047,12 +1055,12 @@ class ScriptCleaner(UpgradeBase):
pg_namespace AS nsp
WHERE
p.pronamespace = nsp.oid AND
- p.proisagg = true AND
+ {proisagg_wrapper} AND
nsp.nspname = '{schema}'
) AS f
GROUP BY
procoid
- """.format(schema=self._schema))
+ """.format(schema=self._schema, proisagg_wrapper=proisagg_wrapper))
self._existing_uda = defaultdict(list)
for row in rows:
# Consider about the overloaded aggregates