You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ok...@apache.org on 2020/04/02 18:01:04 UTC
[madlib] branch master updated (2f0bb2e -> af06387)
This is an automated email from the ASF dual-hosted git repository.
okislal pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git.
from 2f0bb2e correct disk space comment for gp5 and 6 in keras multi fit user docs
new e6bad2f Build: Update madpack scripts for pg11 and add changelist
new 88d3b8b Update Release Notes for 1.17
new 186a28b Build: Change 1.17-dev to 1.17
new af06387 Build: Make version numbering semantic
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
RELEASE_NOTES | 63 ++++++++-
deploy/DEB/postinst | 2 +-
deploy/postflight.sh | 2 +-
pom.xml | 2 +-
src/config/Version.yml | 2 +-
src/madpack/changelist_1.16_1.17.yaml | 142 +++++++++++++++++++++
src/madpack/diff_udf.sql | 120 +++++++++--------
src/madpack/madpack.py | 3 +
src/madpack/upgrade_util.py | 16 ++-
.../modules/deep_learning/madlib_keras.sql_in | 8 +-
.../madlib_keras_fit_multiple_model.sql_in | 6 +-
11 files changed, 292 insertions(+), 74 deletions(-)
create mode 100644 src/madpack/changelist_1.16_1.17.yaml
[madlib] 03/04: Build: Change 1.17-dev to 1.17
Posted by ok...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
okislal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit 186a28b6380bd44a87f5240ff9aabd836e023594
Author: Orhan Kislal <ok...@apache.org>
AuthorDate: Fri Jan 17 12:28:00 2020 -0500
Build: Change 1.17-dev to 1.17
Co-authored-by: Amil Khanzada <am...@gmail.com>
---
deploy/DEB/postinst | 2 +-
deploy/postflight.sh | 2 +-
pom.xml | 2 +-
src/config/Version.yml | 2 +-
src/ports/postgres/modules/deep_learning/madlib_keras.sql_in | 8 ++++----
.../modules/deep_learning/madlib_keras_fit_multiple_model.sql_in | 6 +++---
6 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/deploy/DEB/postinst b/deploy/DEB/postinst
index 379cd2e..297b944 100755
--- a/deploy/DEB/postinst
+++ b/deploy/DEB/postinst
@@ -22,7 +22,7 @@
# Source debconf library.
. /usr/share/debconf/confmodule
-MADLIB_VERSION="1.17-dev"
+MADLIB_VERSION="1.17"
MADLIB_INSTALL_PATH="InstallPathNotFound"
# Fetching configuration from debconf
diff --git a/deploy/postflight.sh b/deploy/postflight.sh
index 206050b..341093b 100755
--- a/deploy/postflight.sh
+++ b/deploy/postflight.sh
@@ -2,7 +2,7 @@
# $0 - Script Path, $1 - Package Path, $2 - Target Location, and $3 - Target Volume
-MADLIB_VERSION=1.17-dev
+MADLIB_VERSION=1.17
# Remove existing soft links
find $2/usr/local/madlib/bin -depth -type l -exec rm {} \; 2>/dev/null
diff --git a/pom.xml b/pom.xml
index 5d43fbf..0f40c46 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,7 @@
<groupId>org.apache.madlib</groupId>
<artifactId>madlib</artifactId>
- <version>1.17-dev</version>
+ <version>1.17</version>
<packaging>pom</packaging>
<build>
diff --git a/src/config/Version.yml b/src/config/Version.yml
index 8aa372c..ae6ad5c 100644
--- a/src/config/Version.yml
+++ b/src/config/Version.yml
@@ -1 +1 @@
-version: 1.17-dev
+version: 1.17
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
index b0c77a3..90e7a98 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras.sql_in
@@ -1089,7 +1089,7 @@ model_size | 0.7900390625
start_training_time | 2019-06-05 20:55:15.785034
end_training_time | 2019-06-05 20:55:25.373035
metrics_elapsed_time | {9.58799290657043}
-madlib_version | 1.16
+madlib_version | 1.17.0
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
dependent_vartype | character varying
@@ -1339,7 +1339,7 @@ model_size | 0.7900390625
start_training_time | 2019-06-05 20:58:23.224629
end_training_time | 2019-06-05 20:58:35.477499
metrics_elapsed_time | {4.69859290122986,8.2062520980835,10.8104848861694,12.2528700828552}
-madlib_version | 1.16
+madlib_version | 1.17.0
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
dependent_vartype | character varying
@@ -1445,7 +1445,7 @@ model_size | 0.7900390625
start_training_time | 2019-06-05 20:59:43.971792
end_training_time | 2019-06-05 20:59:51.654586
metrics_elapsed_time | {2.89326310157776,4.14273309707642,5.24781513214111,6.34498596191406,7.68279695510864}
-madlib_version | 1.16
+madlib_version | 1.17.0
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
dependent_vartype | character varying
@@ -1557,7 +1557,7 @@ model_size | 0.7900390625
start_training_time | 2019-06-05 21:01:03.998422
end_training_time | 2019-06-05 21:01:13.525838
metrics_elapsed_time | {9.52741599082947}
-madlib_version | 1.16
+madlib_version | 1.17.0
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
dependent_vartype | character varying
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in
index 9238652..8d68385 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.sql_in
@@ -1028,7 +1028,7 @@ name |
description |
start_training_time | 2019-12-16 18:54:33.826414
end_training_time | 2019-12-16 18:56:19.106321
-madlib_version | 1.17-dev
+madlib_version | 1.17.0
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
dependent_vartype | character varying
@@ -1189,7 +1189,7 @@ name | Sophie L.
description | Model selection for iris dataset
start_training_time | 2019-12-16 19:28:16.219137
end_training_time | 2019-12-16 19:30:19.238692
-madlib_version | 1.17-dev
+madlib_version | 1.17.0
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
dependent_vartype | character varying
@@ -1301,7 +1301,7 @@ name | Sophie L.
description | Simple MLP for iris dataset
start_training_time | 2019-12-16 20:07:41.488587
end_training_time | 2019-12-16 20:08:27.20651
-madlib_version | 1.17-dev
+madlib_version | 1.17.0
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
dependent_vartype | character varying
[madlib] 02/04: Update Release Notes for 1.17
Posted by ok...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
okislal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit 88d3b8bb8d0eb194c4396775132d31fd7c9575cf
Author: Orhan Kislal <ok...@apache.org>
AuthorDate: Tue Jan 21 17:58:52 2020 -0500
Update Release Notes for 1.17
---
RELEASE_NOTES | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 59 insertions(+), 2 deletions(-)
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 5b8e0ae..3a02ff5 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -12,10 +12,67 @@ Current list of bugs and issues can be found at https://issues.apache.org/jira/b
—-------------------------------------------------------------------------
MADlib v1.17:
-Release Date:
+Release Date: 2020-Mar-31
+
+New features
+ - DL: Add optional params to madlib_keras_fit_multiple_model (MADLIB-1397)
+ - DL: Fit and evaluate changes for asymmetric cluster config (MADLIB-1393)
+ - DL: Make param search fit() function work with existing evaluate and predict (MADLIB-1387)
+ - DL: ParamSearch: Add utility function for generating model selection table (MADLIB-1375)
+ - DL: Predict changes for asymmetric cluster config (MADLIB-1394)
+ - DL: Preprocessor should evenly distribute data on an arbitrary number of segments (MADLIB-1378)
+ - DL: Preprocessor support for asymmetric segment distribution (MADLIB-1392)
+ - DL: Remove model_arch_table column from the output of load_model_selection_table (MADLIB-1381)
+ - DL: Support DL predict without training on MADlib (MADLIB-1359)
+ - DL: Transfer learning for multi-model (MADLIB-1389)
+ - Kmeans: Add simple silhouette score for every point (MADLIB-1382)
+ - Kmeans: Select number of centroids in k-means (MADLIB-1380)
+ - PostgreSQL 12 support (MADLIB-1391)
+
+Improvements:
+ - Assoc rules: Add option to set number of posterior in association rules (MADLIB-1327)
+ - Correlation: Improve correlation and covariance memory usage with large number of groups (MADLIB-1301)
+ - DL: helper function for asymmetric cluster config (MADLIB-1390)
+ - DL: Mini-batch preprocessor for images - performance issue (MADLIB-1342)
+ - DL: Modify warm start logic for DL to handle case of missing weight (MADLIB-1400)
+ - DL: Param search for multiple models on MPP architecture (MADLIB-1386)
+ - DL: performance improvements to fit transition function (MADLIB-1418)
+ - Docs: Enhance Installation Guides (MADLIB-1399)
+ - Graph: SSSP should not show vertices in output table that are unreachable (MADLIB-1415)
+ - Knn - add zero check and output distance array (MADLIB-1370)
+ - LDA: Add stopping criteria on perplexity to LDA (MADLIB-1351)
+ - Summary: Last optional param in summary errors when NULL (MADLIB-1413)
+ - Summary: Summary function has dups for MFV for approximate results (MADLIB-1412)
+ - SVM: Change default num_components for SVM to max(100, 2*num_features) (MADLIB-1384)
+
+Bug fixes:
+ - DL: Deep Learning module does not work with tables in non-public schemas (MADLIB-1388)
+ - DL: Exception during madlib_keras_fit when model_arch_id is passed as NULL (MADLIB-1371)
+ - DL: fit and fit multiple fail with memory exception in gpdb6 (MADLIB-1405)
+ - DL: fit multiple takes up unnecessary disk space (MADLIB-1406)
+ - DL: Intermediate tables are not dropped (MADLIB-1404)
+ - DL: MADlib Keras operations create too many threads (MADLIB-1372)
+ - DL: metrics_elapsed_time for fit multi_model not captured correctly (MADLIB-1403)
+ - DL: predict fails with OOM in gpdb6 (MADLIB-1414)
+ - DL: Remove final function for fit multiple (MADLIB-1416)
+ - DL: Support schema qualified output tables for fit and fit_multiple (MADLIB-1417)
+ - Graph: APSP fails if both vertex id column and edge src column has the same name (MADLIB-1407)
+ - Graph: ASPS Path Function fails if src or dest column type is bigint (MADLIB-1408)
+ - Graph: Graph/wcc fails if the user specifies a schema for the output table (MADLIB-1411)
+ - Kmeans: k-means related functions must use same default distance function (MADLIB-1383)
+ - LDA: Term frequency and LDA - turn off notices (MADLIB-1395)
+ - MADlib cannot be built on PowerPC machines with Linux (MADLIB-1410)
+ - Pivot: Pivot documentation should say "out_table" instead of "output_table" (MADLIB-1376)
Other:
- - DL: Supported keras version is capped at 2.2.4, tensorflow version is capped at 1.14.
+ - DL: Support up to Keras version 2.2.4, Tensorflow version 1.14
+ - DL: If 'madlib_keras_fit_multiple_model()' is running on GPDB 5 and some versions of GPDB 6, the database will keep adding to the disk space (in proportion to model size) and will only release the disk space once the fit multiple query has completed execution. This is not the case for GPDB 6.5.0+ where disk space is released during the fit multiple query.
+ - DL: CUDA GPU memory cannot be released until the process holding it is terminated. This process holds the GPU memory until one of the following two things happen: query finishes and user logs out of the Postgres client/session; or, query finishes and user waits for the timeout set by `gp_vmem_idle_resource_timeout`. The default value for this timeout in Greenplum is 18 sec, but it can be changed.
+ - DL: pg_temp is not allowed as an output table schema for madlib_keras_fit_multiple_model().
+ - Build: Enable current versions of bison
+ - Build: Add cmake variable for gppkg filename
+ - Build: Add pull request template
+
—-------------------------------------------------------------------------
MADlib v1.16:
[madlib] 04/04: Build: Make version numbering semantic
Posted by ok...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
okislal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit af063879e876f318bba4b1025f78d64775959132
Author: Orhan Kislal <ok...@apache.org>
AuthorDate: Mon Mar 9 16:39:20 2020 -0400
Build: Make version numbering semantic
---
RELEASE_NOTES | 2 +-
deploy/DEB/postinst | 2 +-
deploy/postflight.sh | 2 +-
pom.xml | 2 +-
src/config/Version.yml | 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 3a02ff5..030d28c 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -10,7 +10,7 @@ commit history located at https://github.com/apache/madlib/commits/master.
Current list of bugs and issues can be found at https://issues.apache.org/jira/browse/MADLIB.
—-------------------------------------------------------------------------
-MADlib v1.17:
+MADlib v1.17.0:
Release Date: 2020-Mar-31
diff --git a/deploy/DEB/postinst b/deploy/DEB/postinst
index 297b944..319dbf6 100755
--- a/deploy/DEB/postinst
+++ b/deploy/DEB/postinst
@@ -22,7 +22,7 @@
# Source debconf library.
. /usr/share/debconf/confmodule
-MADLIB_VERSION="1.17"
+MADLIB_VERSION="1.17.0"
MADLIB_INSTALL_PATH="InstallPathNotFound"
# Fetching configuration from debconf
diff --git a/deploy/postflight.sh b/deploy/postflight.sh
index 341093b..d0e57a4 100755
--- a/deploy/postflight.sh
+++ b/deploy/postflight.sh
@@ -2,7 +2,7 @@
# $0 - Script Path, $1 - Package Path, $2 - Target Location, and $3 - Target Volume
-MADLIB_VERSION=1.17
+MADLIB_VERSION=1.17.0
# Remove existing soft links
find $2/usr/local/madlib/bin -depth -type l -exec rm {} \; 2>/dev/null
diff --git a/pom.xml b/pom.xml
index 0f40c46..4d0eec1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,7 @@
<groupId>org.apache.madlib</groupId>
<artifactId>madlib</artifactId>
- <version>1.17</version>
+ <version>1.17.0</version>
<packaging>pom</packaging>
<build>
diff --git a/src/config/Version.yml b/src/config/Version.yml
index ae6ad5c..2ecd442 100644
--- a/src/config/Version.yml
+++ b/src/config/Version.yml
@@ -1 +1 @@
-version: 1.17
+version: 1.17.0
[madlib] 01/04: Build: Update madpack scripts for pg11 and add
changelist
Posted by ok...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
okislal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit e6bad2fac706fd7c1e49a5249215195555b9f1e3
Author: Orhan Kislal <ok...@apache.org>
AuthorDate: Fri Jan 17 12:26:32 2020 -0500
Build: Update madpack scripts for pg11 and add changelist
Co-authored-by: Amil Khanzada <am...@gmail.com>
---
src/madpack/changelist_1.16_1.17.yaml | 142 ++++++++++++++++++++++++++++++++++
src/madpack/diff_udf.sql | 120 ++++++++++++++--------------
src/madpack/madpack.py | 3 +
src/madpack/upgrade_util.py | 16 +++-
4 files changed, 221 insertions(+), 60 deletions(-)
diff --git a/src/madpack/changelist_1.16_1.17.yaml b/src/madpack/changelist_1.16_1.17.yaml
new file mode 100644
index 0000000..90f2e88
--- /dev/null
+++ b/src/madpack/changelist_1.16_1.17.yaml
@@ -0,0 +1,142 @@
+# ------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------------------------
+
+# Changelist for MADlib version rel/v1.16 to rel/v1.17.0
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+ madlib_keras_fit_multiple_model:
+ madlib_keras_gpu_info:
+ madlib_keras_model_selection:
+
+# Changes in the types (UDT) including removal and modification
+udt:
+
+# List of the UDF changes that affect the user externally. This includes change
+# in function name, return type, argument order or types, or removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+ - fit_final_multiple_model:
+ rettype: bytea
+ argument: bytea
+ - fit_transition:
+ rettype: bytea
+ argument: bytea, smallint[], real[], text, text, text, integer, integer[], integer[], integer, integer, bytea
+ - graph_apsp_get_path:
+ rettype: void
+ argument: text, integer, integer, text
+ - graph_sssp:
+ rettype: void
+ argument: text, text, text, text, integer, text
+ - graph_sssp:
+ rettype: void
+ argument: text, text, text, text, integer, text, text
+ - graph_sssp_get_path:
+ rettype: void
+ argument: text, integer, text
+ - internal_keras_eval_transition:
+ rettype: real[]
+ argument: real[], smallint[], real[], text, bytea, text, integer, integer[], integer[], integer, integer
+ - internal_keras_predict:
+ rettype: double precision[]
+ argument: real[], text, bytea, boolean, double precision, integer, integer[], integer[], integer, integer
+ - madlib_keras_evaluate:
+ rettype: void
+ argument: character varying, character varying, character varying, integer
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer, boolean
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer, boolean, character varying
+ - madlib_keras_fit:
+ rettype: void
+ argument: character varying, character varying, character varying, integer, character varying, character varying, integer, integer, character varying, integer, boolean, character varying, character varying
+ - madlib_keras_predict:
+ rettype: void
+ argument: character varying, character varying, character varying, character varying, character varying, character varying, integer
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text, text, boolean, boolean, text
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text, text, boolean, boolean
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text, text, boolean
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text, text
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text, boolean
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, text
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text, boolean
+ - pivot:
+ rettype: void
+ argument: text, text, text, text, text
+
+
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+ - agg_array_concat:
+ rettype: anyarray
+ argument: anyarray
+ - fit_step:
+ rettype: bytea
+ argument: smallint[], real[], text, text, text, integer, integer[], integer[], integer, integer, bytea
+ - internal_keras_evaluate:
+ rettype: real[]
+ argument: smallint[], real[], text, bytea, text, integer, integer[], integer[], integer, integer
+
+# List of the UDC, UDO and UDOC changes.
+udc:
+
+# Changes in the operators (UDO)
+udo:
+
+# Changes in the operator classes (UDOC)
+udoc:
diff --git a/src/madpack/diff_udf.sql b/src/madpack/diff_udf.sql
index d73093c..3226409 100644
--- a/src/madpack/diff_udf.sql
+++ b/src/madpack/diff_udf.sql
@@ -15,67 +15,75 @@ CREATE OR REPLACE FUNCTION get_functions(table_name text, schema_name text,
type_filter text)
RETURNS VOID AS
$$
- import plpy
- plpy.execute("""
- CREATE TABLE {table_name} AS
- SELECT
- "schema", "name", filter_schema("retype", '{schema_name}') retype,
- filter_schema("argtypes", '{schema_name}') argtypes, "type"
- FROM
- (
+import plpy
+column_name = plpy.execute("""
+ SELECT column_name
+ FROM information_schema.columns
+ WHERE table_schema='pg_catalog' and table_name='pg_proc' and
+ (column_name='prokind' or column_name='proisagg')
+ """)[0]['column_name']
+proisagg_wrapper = "p.proisagg" if column_name == 'proisagg' else "p.prokind = 'a'"
+plpy.execute("""
+ CREATE TABLE {table_name} AS
+ SELECT
+ "schema", "name", filter_schema("retype", '{schema_name}') retype,
+ filter_schema("argtypes", '{schema_name}') argtypes, "type"
+ FROM
+ (
- SELECT n.nspname as "schema",
- p.proname as "name",
- CASE WHEN p.proretset THEN 'SETOF ' ELSE '' END ||
- pg_catalog.format_type(p.prorettype, NULL) as "retype",
- CASE WHEN proallargtypes IS NOT NULL THEN
- pg_catalog.array_to_string(ARRAY(
+ SELECT n.nspname as "schema",
+ p.proname as "name",
+ CASE WHEN p.proretset THEN 'SETOF ' ELSE '' END ||
+ pg_catalog.format_type(p.prorettype, NULL) as "retype",
+ CASE WHEN proallargtypes IS NOT NULL THEN
+ pg_catalog.array_to_string(ARRAY(
+ SELECT
+ pio || ptyp
+ FROM
+ (
SELECT
- pio || ptyp
- FROM
- (
- SELECT
- CASE
- WHEN p.proargmodes[s.i] = 'i' THEN ''
- WHEN p.proargmodes[s.i] = 'o' THEN 'OUT '
- WHEN p.proargmodes[s.i] = 'b' THEN 'INOUT '
- WHEN p.proargmodes[s.i] = 'v' THEN 'VARIADIC '
- END AS pio,
- --CASE
- -- WHEN COALESCE(p.proargnames[s.i], '') = '' THEN ''
- -- ELSE p.proargnames[s.i] || ' '
- --END ||
- pg_catalog.format_type(p.proallargtypes[s.i], NULL) AS ptyp
- FROM
- pg_catalog.generate_series(1, pg_catalog.array_upper(p.proallargtypes, 1)) AS s(i)
- ) qx
- WHERE pio = ''
- ), ', ')
- ELSE
- pg_catalog.array_to_string(ARRAY(
- SELECT
+ CASE
+ WHEN p.proargmodes[s.i] = 'i' THEN ''
+ WHEN p.proargmodes[s.i] = 'o' THEN 'OUT '
+ WHEN p.proargmodes[s.i] = 'b' THEN 'INOUT '
+ WHEN p.proargmodes[s.i] = 'v' THEN 'VARIADIC '
+ END AS pio,
--CASE
- -- WHEN COALESCE(p.proargnames[s.i+1], '') = '' THEN ''
- -- ELSE p.proargnames[s.i+1] || ' '
- -- END ||
- pg_catalog.format_type(p.proargtypes[s.i], NULL)
+ -- WHEN COALESCE(p.proargnames[s.i], '') = '' THEN ''
+ -- ELSE p.proargnames[s.i] || ' '
+ --END ||
+ pg_catalog.format_type(p.proallargtypes[s.i], NULL) AS ptyp
FROM
- pg_catalog.generate_series(0, pg_catalog.array_upper(p.proargtypes, 1)) AS s(i)
+ pg_catalog.generate_series(1, pg_catalog.array_upper(p.proallargtypes, 1)) AS s(i)
+ ) qx
+ WHERE pio = ''
), ', ')
- END AS "argtypes",
- CASE
- WHEN p.proisagg THEN 'agg'
- WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger'
- ELSE 'normal'
- END AS "type"
- FROM pg_catalog.pg_proc p
- LEFT JOIN pg_catalog.pg_namespace n
- ON n.oid = p.pronamespace
- WHERE n.nspname ~ '^({schema_name})$'
- ORDER BY 1, 2, 4
- ) q
- WHERE retype LIKE '{type_filter}' OR retype LIKE '{type_filter}[]'
- """.format(table_name=table_name, schema_name=schema_name, type_filter=type_filter))
+ ELSE
+ pg_catalog.array_to_string(ARRAY(
+ SELECT
+ --CASE
+ -- WHEN COALESCE(p.proargnames[s.i+1], '') = '' THEN ''
+ -- ELSE p.proargnames[s.i+1] || ' '
+ -- END ||
+ pg_catalog.format_type(p.proargtypes[s.i], NULL)
+ FROM
+ pg_catalog.generate_series(0, pg_catalog.array_upper(p.proargtypes, 1)) AS s(i)
+ ), ', ')
+ END AS "argtypes",
+ CASE
+ WHEN {proisagg_wrapper} THEN 'agg'
+ WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger'
+ ELSE 'normal'
+ END AS "type"
+ FROM pg_catalog.pg_proc p
+ LEFT JOIN pg_catalog.pg_namespace n
+ ON n.oid = p.pronamespace
+ WHERE n.nspname ~ '^({schema_name})$'
+ ORDER BY 1, 2, 4
+ ) q
+ WHERE retype LIKE '{type_filter}' OR retype LIKE '{type_filter}[]'
+ """.format(table_name=table_name, schema_name=schema_name,
+ type_filter=type_filter, proisagg_wrapper=proisagg_wrapper))
$$ LANGUAGE plpythonu;
diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py
index 86a7236..2af0480 100755
--- a/src/madpack/madpack.py
+++ b/src/madpack/madpack.py
@@ -701,6 +701,9 @@ def _process_py_sql_files_in_modules(modset, args_dict):
args_dict['create_obj_handle'],
args_dict['sc'])
elif calling_operation == INSTALL_DEV_CHECK:
+ # Skip certain tests for GP4.3
+ if dbver == '4.3ORCA' and module in ['deep_learning', 'kmeans']:
+ continue
_execute_per_module_install_dev_check_algo(
args_dict['schema'],
args_dict['test_user'],
diff --git a/src/madpack/upgrade_util.py b/src/madpack/upgrade_util.py
index 90511a3..9197b99 100644
--- a/src/madpack/upgrade_util.py
+++ b/src/madpack/upgrade_util.py
@@ -502,6 +502,9 @@ class ViewDependency(UpgradeBase):
"""
@brief Detect direct view dependencies on MADlib UDFs/UDAs
"""
+ proisagg_wrapper = "p.proisagg"
+ if self._portid == 'postgres' and self._dbver > 11:
+ proisagg_wrapper = "p.prokind = 'a'"
rows = self._run_sql("""
SELECT
view, nsp.nspname AS schema, procname, procoid, proisagg
@@ -513,7 +516,7 @@ class ViewDependency(UpgradeBase):
c.relnamespace AS namespace,
p.proname As procname,
p.oid AS procoid,
- p.proisagg AS proisagg
+ {proisagg_wrapper} AS proisagg
FROM
pg_class AS c,
pg_rewrite AS rw,
@@ -529,7 +532,8 @@ class ViewDependency(UpgradeBase):
) t1
WHERE
t1.namespace = nsp.oid
- """.format(schema_madlib_oid=self._schema_oid))
+ """.format(schema_madlib_oid=self._schema_oid,
+ proisagg_wrapper=proisagg_wrapper))
self._view2proc = defaultdict(list)
for row in rows:
@@ -1029,6 +1033,10 @@ class ScriptCleaner(UpgradeBase):
"""
# See _get_function_info for explanations.
+ proisagg_wrapper = "p.proisagg = true"
+ if self._portid == 'postgres' and self._dbver > 11:
+ proisagg_wrapper = "p.prokind = 'a'"
+
rows = self._run_sql("""
SELECT
max(proname) AS proname,
@@ -1047,12 +1055,12 @@ class ScriptCleaner(UpgradeBase):
pg_namespace AS nsp
WHERE
p.pronamespace = nsp.oid AND
- p.proisagg = true AND
+ {proisagg_wrapper} AND
nsp.nspname = '{schema}'
) AS f
GROUP BY
procoid
- """.format(schema=self._schema))
+ """.format(schema=self._schema, proisagg_wrapper=proisagg_wrapper))
self._existing_uda = defaultdict(list)
for row in rows:
# Consider about the overloaded aggregates