You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2017/12/18 06:40:24 UTC
[1/2] madlib git commit: Upgrade: Add upgrade to v1.3 + enhancements
Repository: madlib
Updated Branches:
refs/heads/master 32cce1a16 -> cefd15eac
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/upgrade_util.py
----------------------------------------------------------------------
diff --git a/src/madpack/upgrade_util.py b/src/madpack/upgrade_util.py
index c521f2e..53051fe 100644
--- a/src/madpack/upgrade_util.py
+++ b/src/madpack/upgrade_util.py
@@ -1,17 +1,23 @@
+from collections import defaultdict, Iterable
+import glob
+import os
import re
import yaml
-from collections import defaultdict
-import os
from utilities import is_rev_gte
from utilities import get_rev_num
+from utilities import run_query
+from utilities import get_dbver
-def run_sql(sql, portid, con_args):
- """
- @brief Wrapper function for run_query
- """
- from madpack import run_query
- return run_query(sql, True, con_args)
+if not __name__ == "__main__":
+ def run_sql(sql, portid, con_args):
+ """
+ @brief Wrapper function for run_query
+ """
+ return run_query(sql, con_args, True)
+else:
+ def run_sql(sql, portid, con_args):
+ return [{'dummy': 0}]
def get_signature_for_compare(schema, proname, rettype, argument):
@@ -28,26 +34,34 @@ class UpgradeBase:
"""
@brief Base class for handling the upgrade
"""
+
def __init__(self, schema, portid, con_args):
self._schema = schema.lower()
self._portid = portid
self._con_args = con_args
self._schema_oid = None
self._get_schema_oid()
+ self._dbver = get_dbver(self._con_args, self._portid)
"""
@brief Wrapper function for run_sql
"""
+
def _run_sql(self, sql):
return run_sql(sql, self._portid, self._con_args)
"""
@brief Get the oids of some objects from the catalog in the current version
"""
+
def _get_schema_oid(self):
- self._schema_oid = self._run_sql("""
- SELECT oid FROM pg_namespace WHERE nspname = '{schema}'
- """.format(schema=self._schema))[0]['oid']
+ res = self._run_sql("SELECT oid FROM pg_namespace WHERE nspname = '{0}'".
+ format(self._schema))[0]
+ if 'oid' in res:
+ self._schema_oid = res['oid']
+ else:
+ self._schema_oid = None
+ return self._schema_oid
def _get_function_info(self, oid):
"""
@@ -92,12 +106,19 @@ class ChangeHandler(UpgradeBase):
@brief This class reads changes from the configuration file and handles
the dropping of objects
"""
- def __init__(self, schema, portid, con_args, maddir, mad_dbrev, is_hawq2):
+
+ def __init__(self, schema, portid, con_args, maddir, mad_dbrev,
+ is_hawq2, upgrade_to=None):
UpgradeBase.__init__(self, schema, portid, con_args)
+
+ # FIXME: maddir includes the '/src' folder. It's supposed to be the
+ # parent of that directory.
self._maddir = maddir
self._mad_dbrev = mad_dbrev
self._is_hawq2 = is_hawq2
self._newmodule = {}
+ self._curr_rev = self._get_current_version() if not upgrade_to else upgrade_to
+
self._udt = {}
self._udf = {}
self._uda = {}
@@ -106,7 +127,19 @@ class ChangeHandler(UpgradeBase):
self._udoc = {}
self._load()
- def _load_config_param(self, config_iterable):
+ def _get_current_version(self):
+ """ Get current version of MADlib
+
+ This currently assumes that version is available in
+ '$MADLIB_HOME/src/config/Version.yml'
+ """
+ version_filepath = os.path.abspath(
+ os.path.join(self._maddir, 'config', 'Version.yml'))
+ with open(version_filepath) as ver_file:
+ version_str = str(yaml.load(ver_file)['version'])
+ return get_rev_num(version_str)
+
+ def _load_config_param(self, config_iterable, output_config_dict=None):
"""
Replace schema_madlib with the appropriate schema name and
make all function names lower case to ensure ease of comparison.
@@ -114,20 +147,18 @@ class ChangeHandler(UpgradeBase):
Args:
@param config_iterable is an iterable of dictionaries, each with
key = object name (eg. function name) and value = details
- for the object. The details for the object are assumed to
- be in a dictionary with following keys:
+ for the object. The details for the object are assumed
+ to be in a dictionary with following keys:
rettype: Return type
argument: List of arguments
Returns:
A dictionary that lists all specific objects (functions, aggregates, etc)
with object name as key and a list as value, where the list
- contains all the items present in
-
- another dictionary with objects details
- as the value.
+ contains all the items present in another dictionary with objects
+ details as the value.
"""
- _return_obj = defaultdict(list)
+ _return_obj = defaultdict(list) if not output_config_dict else output_config_dict
if config_iterable is not None:
for each_config in config_iterable:
for obj_name, obj_details in each_config.iteritems():
@@ -138,38 +169,105 @@ class ChangeHandler(UpgradeBase):
_return_obj[obj_name].append(formatted_obj)
return _return_obj
+ @classmethod
+ def _add_to_dict(cls, src_dict, dest_dict):
+ """ Update dictionary with contents of another dictionary
+
+ This function performs the same function as dict.update except it adds
+ to an existing value (instead of replacing it) if the value is an
+ Iterable.
+ """
+ if src_dict:
+ for k, v in src_dict.items():
+ if k in dest_dict:
+ if (isinstance(dest_dict[k], Iterable) and isinstance(v, Iterable)):
+ dest_dict[k] += v
+ elif isinstance(dest_dict[k], Iterable):
+ dest_dict[k].append(v)
+ else:
+ dest_dict[k] = v
+ else:
+ dest_dict[k] = v
+ return dest_dict
+
+ def _update_objects(self, config):
+ """ Update each upgrade object """
+ self._add_to_dict(config['new module'], self._newmodule)
+ self._add_to_dict(config['udt'], self._udt)
+ self._add_to_dict(config['udc'], self._udc)
+ self._add_to_dict(self._load_config_param(config['udf']), self._udf)
+ self._add_to_dict(self._load_config_param(config['uda']), self._uda)
+
+ def _get_relevant_filenames(self, upgrade_from):
+ """ Get all changelist files that together describe the upgrade process
+
+ Args:
+ @param upgrade_from: List. Version to upgrade from - the format is
+ expected to be per the output of get_rev_num
+
+ Details:
+ Changelist files are named in the format changelist_<src>_<dest>.yaml
+
+ When upgrading from 'upgrade_from_rev' to 'self._curr_rev', all
+ intermediate changelist files need to be followed to get all upgrade
+ steps. This function globs for such files and filters in changelists
+ that lie between the desired versions.
+
+ Additional verification: The function also ensures that a valid
+ upgrade path exists. Each version in the changelist files needs to
+ be seen twice (except upgrade_from and upgrade_to) for a valid path.
+ This is verified by performing an xor-like operation by
+ adding/deleting from a list.
+ """
+ output_filenames = []
+ upgrade_to = self._curr_rev
+
+ verify_list = [upgrade_from, upgrade_to]
+
+ # assuming that changelists are in the same directory as this file
+ glob_filter = os.path.abspath(
+ os.path.join(self._maddir, 'madpack', 'changelist*.yaml'))
+ all_changelists = glob.glob(glob_filter)
+ for each_ch in all_changelists:
+ # split file names to get dest versions
+ # Assumption: changelist format is
+ # changelist_<src>_<dest>.yaml
+ ch_basename = os.path.splitext(os.path.basename(each_ch))[0] # remove extension
+ ch_splits = ch_basename.split('_') # underscore delineates sections
+ if len(ch_splits) >= 3:
+ src_version, dest_version = [get_rev_num(i) for i in ch_splits[1:3]]
+
+ # file is part of upgrade if
+ # upgrade_to >= dest >= src >= upgrade_from
+ is_part_of_upgrade = (
+ is_rev_gte(src_version, upgrade_from) and
+ is_rev_gte(upgrade_to, dest_version))
+ if is_part_of_upgrade:
+ for ver in (src_version, dest_version):
+ if ver in verify_list:
+ verify_list.remove(ver)
+ else:
+ verify_list.append(ver)
+ abs_path = os.path.join(self._maddir, 'src', 'madpack', each_ch)
+ output_filenames.append(abs_path)
+
+ if verify_list:
+ # any version remaining in verify_list implies upgrade path is broken
+ raise RuntimeError("Upgrade from {0} to {1} broken due to missing "
+ "changelist files ({2}). ".
+ format(upgrade_from, upgrade_to, verify_list))
+ return output_filenames
+
def _load(self):
"""
@brief Load the configuration file
"""
-
rev = get_rev_num(self._mad_dbrev)
-
- # _mad_dbrev = 1.9.1
- if is_rev_gte([1,9,1],rev):
- filename = os.path.join(self._maddir, 'madpack',
- 'changelist_1.9.1_1.12.yaml')
- # _mad_dbrev = 1.10.0
- elif is_rev_gte([1,10],rev):
- filename = os.path.join(self._maddir, 'madpack',
- 'changelist_1.10.0_1.12.yaml')
- # _mad_dbrev = 1.11
- else:
- filename = os.path.join(self._maddir, 'madpack',
- 'changelist.yaml')
-
- config = yaml.load(open(filename))
-
- self._newmodule = config['new module'] if config['new module'] else {}
- self._udt = config['udt'] if config['udt'] else {}
- self._udc = config['udc'] if config['udc'] else {}
- self._udf = self._load_config_param(config['udf'])
- self._uda = self._load_config_param(config['uda'])
- # FIXME remove the following special handling for HAWQ after svec is
- # removed from catalog
- if self._portid != 'hawq' and not self._is_hawq2:
- self._udo = self._load_config_param(config['udo'])
- self._udoc = self._load_config_param(config['udoc'])
+ upgrade_filenames = self._get_relevant_filenames(rev)
+ for f in upgrade_filenames:
+ with open(f) as handle:
+ config = yaml.load(handle)
+ self._update_objects(config)
@property
def newmodule(self):
@@ -259,8 +357,9 @@ class ChangeHandler(UpgradeBase):
for opc, li in self._udoc.items():
for e in li:
changed_opcs.add((opc, e['index']))
-
- if self._portid == 'postgres':
+ gte_gpdb5 = (self._portid == 'greenplum' and
+ is_rev_gte(get_rev_num(self._dbver), get_rev_num('5.0')))
+ if (self._portid == 'postgres' or gte_gpdb5):
method_col = 'opcmethod'
else:
method_col = 'opcamid'
@@ -339,8 +438,8 @@ class ChangeHandler(UpgradeBase):
"""
for op in self._udo:
for value in self._udo[op]:
- leftarg=value['leftarg'].replace('schema_madlib', self._schema)
- rightarg=value['rightarg'].replace('schema_madlib', self._schema)
+ leftarg = value['leftarg'].replace('schema_madlib', self._schema)
+ rightarg = value['rightarg'].replace('schema_madlib', self._schema)
self._run_sql("""
DROP OPERATOR IF EXISTS {schema}.{op} ({leftarg}, {rightarg})
""".format(schema=self._schema, **locals()))
@@ -356,11 +455,13 @@ class ChangeHandler(UpgradeBase):
DROP OPERATOR CLASS IF EXISTS {schema}.{op_cls} USING {index}
""".format(schema=self._schema, **locals()))
+
class ViewDependency(UpgradeBase):
"""
@brief This class detects the direct/recursive view dependencies on MADLib
UDFs/UDAs/UDOs defined in the current version
"""
+
def __init__(self, schema, portid, con_args):
UpgradeBase.__init__(self, schema, portid, con_args)
self._view2proc = None
@@ -452,6 +553,7 @@ class ViewDependency(UpgradeBase):
"""
@brief Detect recursive view dependencies (view on view)
"""
+
def _detect_recursive_view_dependency(self):
rows = self._run_sql("""
SELECT
@@ -499,9 +601,9 @@ class ViewDependency(UpgradeBase):
@brief Filter out recursive view dependencies which are independent of
MADLib UDFs/UDAs
"""
+
def _filter_recursive_view_dependency(self):
# Get initial list
- import sys
checklist = []
checklist.extend(self._view2proc.keys())
checklist.extend(self._view2op.keys())
@@ -530,6 +632,7 @@ class ViewDependency(UpgradeBase):
"""
@brief Build the dependency graph (depender-to-dependee adjacency list)
"""
+
def _build_dependency_graph(self, hasProcDependency=False):
der2dee = self._view2view.copy()
for view in self._view2proc:
@@ -554,12 +657,14 @@ class ViewDependency(UpgradeBase):
"""
@brief Check dependencies
"""
+
def has_dependency(self):
return (len(self._view2proc) > 0) or (len(self._view2op) > 0)
"""
@brief Get the ordered views for creation
"""
+
def get_create_order_views(self):
graph = self._build_dependency_graph()
ordered_views = []
@@ -581,6 +686,7 @@ class ViewDependency(UpgradeBase):
"""
@brief Get the ordered views for dropping
"""
+
def get_drop_order_views(self):
ordered_views = self.get_create_order_views()
ordered_views.reverse()
@@ -678,10 +784,8 @@ class ViewDependency(UpgradeBase):
SET ROLE {owner};
CREATE OR REPLACE VIEW {schema}.{view} AS {definition};
RESET ROLE
- """.format(
- schema=schema, view=view,
- definition=definition,
- owner=owner))
+ """.format(schema=schema, view=view,
+ definition=definition, owner=owner))
def _node_to_str(self, node):
if len(node) == 2:
@@ -717,6 +821,7 @@ class TableDependency(UpgradeBase):
@brief This class detects the table dependencies on MADLib UDTs defined in the
current version
"""
+
def __init__(self, schema, portid, con_args):
UpgradeBase.__init__(self, schema, portid, con_args)
self._table2type = None
@@ -836,6 +941,7 @@ class ScriptCleaner(UpgradeBase):
@brief This class removes sql statements from a sql script which should not be
executed during the upgrade
"""
+
def __init__(self, schema, portid, con_args, change_handler):
UpgradeBase.__init__(self, schema, portid, con_args)
self._ch = change_handler
@@ -853,7 +959,9 @@ class ScriptCleaner(UpgradeBase):
"""
@brief Get the existing UDOCs in the current version
"""
- if self._portid == 'postgres':
+ gte_gpdb5 = (self._portid == 'greenplum' and
+ is_rev_gte(get_rev_num(self._dbver), get_rev_num('5.0')))
+ if (self._portid == 'postgres' or gte_gpdb5):
method_col = 'opcmethod'
else:
method_col = 'opcamid'
@@ -887,8 +995,8 @@ class ScriptCleaner(UpgradeBase):
self._existing_udo = defaultdict(list)
for row in rows:
self._existing_udo[row['oprname']].append(
- {'leftarg': row['oprleft'],
- 'rightarg': row['oprright']})
+ {'leftarg': row['oprleft'],
+ 'rightarg': row['oprright']})
def _get_existing_uda(self):
"""
@@ -928,8 +1036,8 @@ class ScriptCleaner(UpgradeBase):
for row in rows:
# Consider about the overloaded aggregates
self._existing_uda[row['proname']].append(
- {'rettype': row['rettype'],
- 'argument': row['argument']})
+ {'rettype': row['rettype'],
+ 'argument': row['argument']})
def _get_unchanged_operator_patterns(self):
"""
@@ -938,7 +1046,7 @@ class ScriptCleaner(UpgradeBase):
@return unchanged = existing - changed
"""
- self._get_existing_udo() # from the old version
+ self._get_existing_udo() # from the old version
operator_patterns = []
# for all, pass the changed ones, add others to ret
for each_udo, udo_details in self._existing_udo.items():
@@ -965,7 +1073,7 @@ class ScriptCleaner(UpgradeBase):
@return unchanged = existing - changed
"""
- self._get_existing_udoc() # from the old version
+ self._get_existing_udoc() # from the old version
opclass_patterns = []
# for all, pass the changed ones, add others to ret
for each_udoc, udoc_details in self._existing_udoc.items():
@@ -1055,6 +1163,7 @@ class ScriptCleaner(UpgradeBase):
"""
@breif Remove "drop/create type" statements in the sql script
"""
+
def _clean_type(self):
# remove 'drop type'
pattern = re.compile('DROP(\s+)TYPE(.*?);', re.DOTALL | re.IGNORECASE)
@@ -1076,6 +1185,7 @@ class ScriptCleaner(UpgradeBase):
"""
@brief Remove "drop/create cast" statements in the sql script
"""
+
def _clean_cast(self):
# remove 'drop cast'
pattern = re.compile('DROP(\s+)CAST(.*?);', re.DOTALL | re.IGNORECASE)
@@ -1102,6 +1212,7 @@ class ScriptCleaner(UpgradeBase):
"""
@brief Remove "drop/create operator" statements in the sql script
"""
+
def _clean_operator(self):
# remove 'drop operator'
pattern = re.compile('DROP\s+OPERATOR.*?PROCEDURE\s+=.*?;', re.DOTALL | re.IGNORECASE)
@@ -1117,6 +1228,7 @@ class ScriptCleaner(UpgradeBase):
"""
@brief Remove "drop/create operator class" statements in the sql script
"""
+
def _clean_opclass(self):
# remove 'drop operator class'
pattern = re.compile(r'DROP\s+OPERATOR\s*CLASS.*?;', re.DOTALL | re.IGNORECASE)
@@ -1132,6 +1244,7 @@ class ScriptCleaner(UpgradeBase):
"""
@brief Rewrite the type
"""
+
def _rewrite_type_in(self, arg):
type_mapper = {
'smallint': '(int2|smallint)',
@@ -1184,7 +1297,60 @@ class ScriptCleaner(UpgradeBase):
self._clean_function()
return self._sql
+
+import unittest
+
+
+class TestChangeHandler(unittest.TestCase):
+
+ def setUp(self):
+ self._dummy_schema = 'madlib'
+ self._dummy_portid = 1
+ self._dummy_con_args = 'x'
+ # maddir is the directory one level above current file
+ # dirname gives the directory of current file (madpack)
+ # join with pardir adds .. (e.g .../madpack/..)
+ # abspath concatenates by traversing the ..
+ self.maddir = os.path.abspath(
+ os.path.join(os.path.dirname(os.path.realpath(__file__)),
+ os.pardir))
+ self._dummy_hawq2 = False
+
+ def tearDown(self):
+ pass
+
+ def test_invalid_path(self):
+ with self.assertRaises(RuntimeError):
+ ChangeHandler(self._dummy_schema, self._dummy_portid,
+ self._dummy_con_args, self.maddir,
+ '1.9', self._dummy_hawq2,
+ upgrade_to=get_rev_num('1.12'))
+
+ def test_valid_path(self):
+ ch = ChangeHandler(self._dummy_schema, self._dummy_portid,
+ self._dummy_con_args, self.maddir,
+ '1.9.1', self._dummy_hawq2,
+ upgrade_to=get_rev_num('1.12'))
+ self.assertEqual(ch.newmodule.keys(),
+ ['knn', 'sssp', 'apsp', 'measures', 'stratified_sample',
+ 'encode_categorical', 'bfs', 'mlp', 'pagerank',
+ 'train_test_split', 'wcc'])
+ self.assertEqual(ch.udt, {'kmeans_result': None, 'kmeans_state': None})
+ self.assertEqual(ch.udf['forest_train'],
+ [{'argument': 'text, text, text, text, text, text, text, '
+ 'integer, integer, boolean, integer, integer, '
+ 'integer, integer, integer, text, boolean, '
+ 'double precision',
+ 'rettype': 'void'},
+ {'argument': 'text, text, text, text, text, text, text, '
+ 'integer, integer, boolean, integer, integer, '
+ 'integer, integer, integer, text, boolean',
+ 'rettype': 'void'},
+ {'argument': 'text, text, text, text, text, text, text, '
+ 'integer, integer, boolean, integer, integer, '
+ 'integer, integer, integer, text',
+ 'rettype': 'void'}])
+
+
if __name__ == '__main__':
- config = yaml.load(open('changelist.yaml'))
- for obj in ('new module', 'udt', 'udc', 'udf', 'uda', 'udo', 'udoc'):
- print config[obj]
+ unittest.main()
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/utilities.py
----------------------------------------------------------------------
diff --git a/src/madpack/utilities.py b/src/madpack/utilities.py
index e143d64..40a017a 100644
--- a/src/madpack/utilities.py
+++ b/src/madpack/utilities.py
@@ -22,10 +22,152 @@
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
from itertools import izip_longest
+import os
import re
+import sys
+import subprocess
import unittest
+# Some read-only variables
+this = os.path.basename(sys.argv[0]) # name of this script
+
+
+def error_(src_name, msg, stop=False):
+ """
+ Error message wrapper
+ @param msg error message
+ @param stop program exit flag
+ """
+ # Print to stdout
+ print("{0}: ERROR : {1}".format(src_name, msg))
+ # stack trace is not printed
+ if stop:
+ exit(2)
+# ------------------------------------------------------------------------------
+
+
+def info_(src_name, msg, verbose=True):
+ """
+ Info message wrapper (verbose)
+ @param msg info message
+ @param verbose prints only if True (prevents caller from performing a check)
+ """
+ if verbose:
+ print("{0}: INFO : {1}".format(src_name, msg))
+# ------------------------------------------------------------------------------
+
+
+def run_query(sql, con_args, show_error=True):
+ # Define sqlcmd
+ sqlcmd = 'psql'
+ delimiter = ' <$madlib_delimiter$> '
+
+ # Test the DB cmd line utility
+ std, err = subprocess.Popen(['which', sqlcmd], stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE).communicate()
+ if not std:
+ error_(this, "Command not found: %s" % sqlcmd, True)
+
+ # Run the query
+ runcmd = [sqlcmd,
+ '-h', con_args['host'].split(':')[0],
+ '-p', con_args['host'].split(':')[1],
+ '-d', con_args['database'],
+ '-U', con_args['user'],
+ '-F', delimiter,
+ '--no-password',
+ '--no-psqlrc',
+ '--no-align',
+ '-c', sql]
+ runenv = os.environ
+ if 'password' in con_args:
+ runenv["PGPASSWORD"] = con_args['password']
+ runenv["PGOPTIONS"] = '-c search_path=public -c client_min_messages=error'
+ std, err = subprocess.Popen(runcmd, env=runenv, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE).communicate()
+
+ if err:
+ if show_error:
+ error_("SQL command failed: \nSQL: %s \n%s" % (sql, err), False)
+ if 'password' in err:
+ raise EnvironmentError
+ else:
+ raise Exception
+
+ # Convert the delimited output into a dictionary
+ results = [] # list of rows
+ i = 0
+ for line in std.splitlines():
+ if i == 0:
+ cols = [name for name in line.split(delimiter)]
+ else:
+ row = {} # dict of col_name:col_value pairs
+ c = 0
+ for val in line.split(delimiter):
+ row[cols[c]] = val
+ c += 1
+ results.insert(i, row)
+ i += 1
+ # Drop the last line: "(X rows)"
+ try:
+ results.pop()
+ except Exception:
+ pass
+
+ return results
+# ------------------------------------------------------------------------------
+
+
+def get_madlib_dbrev(con_args, schema):
+ """
+ Read MADlib version from database
+ @param con_args database conection object
+ @param schema MADlib schema name
+ """
+ try:
+ n_madlib_versions = int(run_query(
+ """
+ SELECT count(*) AS cnt FROM pg_tables
+ WHERE schemaname='{0}' AND tablename='migrationhistory'
+ """.format(schema),
+ con_args,
+ True)[0]['cnt'])
+ if n_madlib_versions > 0:
+ madlib_version = run_query(
+ """
+ SELECT version
+ FROM {0}.migrationhistory
+ ORDER BY applied DESC LIMIT 1
+ """.format(schema),
+ con_args,
+ True)
+ if madlib_version:
+ return madlib_version[0]['version']
+ except Exception:
+ error_(this, "Failed reading MADlib db version", True)
+ return None
+# ------------------------------------------------------------------------------
+
+
+def get_dbver(con_args, portid):
+ """ Read version number from database (of form X.Y) """
+ try:
+ versionStr = run_query("SELECT pg_catalog.version()", con_args, True)[0]['version']
+ if portid == 'postgres':
+ match = re.search("PostgreSQL[a-zA-Z\s]*(\d+\.\d+)", versionStr)
+ elif portid == 'greenplum':
+ # for Greenplum the 3rd digit is necessary to differentiate
+ # 4.3.5+ from versions < 4.3.5
+ match = re.search("Greenplum[a-zA-Z\s]*(\d+\.\d+\.\d+)", versionStr)
+ elif portid == 'hawq':
+ match = re.search("HAWQ[a-zA-Z\s]*(\d+\.\d+)", versionStr)
+ return None if match is None else match.group(1)
+ except Exception:
+ error_(this, "Failed reading database version", True)
+# ------------------------------------------------------------------------------
+
+
def is_rev_gte(left, right):
""" Return if left >= right
[2/2] madlib git commit: Upgrade: Add upgrade to v1.3 + enhancements
Posted by ri...@apache.org.
Upgrade: Add upgrade to v1.3 + enhancements
Changes:
- Changed upgrade process to use incremental changelists
For example: upgrading from v1.11 to v1.13 will involve
combining v1.11 -> v1.12 changelist and
v1.12 -> v1.13 changelist.
This change eases the burden of pre-populating these changelist files.
- Added v1.12 -> v1.13 changelist file
- Added some unittests in upgrade_util.py
- Refactored madpack.py to move some functions to a common file
- Changed use of 'opcamid' to 'opcmethod' for Greenplum 5+
Closes #216
Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/cefd15ea
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/cefd15ea
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/cefd15ea
Branch: refs/heads/master
Commit: cefd15eaceb88d017a78d24ae7db3e69cf5941f7
Parents: 32cce1a
Author: Rahul Iyer <ri...@apache.org>
Authored: Sun Dec 17 22:32:50 2017 -0800
Committer: Rahul Iyer <ri...@apache.org>
Committed: Sun Dec 17 22:39:20 2017 -0800
----------------------------------------------------------------------
deploy/postflight.sh | 4 +-
pom.xml | 2 +-
src/config/Version.yml | 2 +-
src/madpack/changelist.yaml | 79 -----
src/madpack/changelist_1.10.0_1.11.yaml | 63 ++++
src/madpack/changelist_1.10.0_1.12.yaml | 89 ------
src/madpack/changelist_1.11_1.12.yaml | 79 +++++
src/madpack/changelist_1.12_1.13.yaml | 84 ++++++
src/madpack/changelist_1.9.1_1.10.0.yaml | 130 +++++++++
src/madpack/changelist_1.9.1_1.12.yaml | 165 -----------
src/madpack/diff_udt.sql | 25 +-
src/madpack/madpack.py | 404 +++++++++-----------------
src/madpack/upgrade_util.py | 298 ++++++++++++++-----
src/madpack/utilities.py | 142 +++++++++
14 files changed, 897 insertions(+), 669 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/deploy/postflight.sh
----------------------------------------------------------------------
diff --git a/deploy/postflight.sh b/deploy/postflight.sh
index e03afec..59fa77f 100755
--- a/deploy/postflight.sh
+++ b/deploy/postflight.sh
@@ -1,8 +1,8 @@
#!/bin/bash
-# $0 - Script Path, $1 - Package Path, $2 - Target Location, and $3 - Target Volumn
+# $0 - Script Path, $1 - Package Path, $2 - Target Location, and $3 - Target Volume
-MADLIB_VERSION=1.13-dev
+MADLIB_VERSION=1.13
find $2/usr/local/madlib/bin -type d -exec cp -RPf {} $2/usr/local/madlib/old_bin \; 2>/dev/null
find $2/usr/local/madlib/bin -depth -type d -exec rm -r {} \; 2>/dev/null
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 112b868..e1e3b0d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,7 @@
<groupId>org.apache.madlib</groupId>
<artifactId>madlib</artifactId>
- <version>1.13-dev</version>
+ <version>1.13</version>
<packaging>pom</packaging>
<build>
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/config/Version.yml
----------------------------------------------------------------------
diff --git a/src/config/Version.yml b/src/config/Version.yml
index 7c852f9..e2ed1a4 100644
--- a/src/config/Version.yml
+++ b/src/config/Version.yml
@@ -1 +1 @@
-version: 1.13-dev
+version: 1.13
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/changelist.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist.yaml b/src/madpack/changelist.yaml
deleted file mode 100644
index 77edeb8..0000000
--- a/src/madpack/changelist.yaml
+++ /dev/null
@@ -1,79 +0,0 @@
-# ------------------------------------------------------------------------------
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# ------------------------------------------------------------------------------
-
-# Changelist for MADlib version 1.11 to 1.12
-
-# This file contains all changes that were introduced in a new version of
-# MADlib. This changelist is used by the upgrade script to detect what objects
-# should be upgraded (while retaining all other objects from the previous version)
-
-# New modules (actually .sql_in files) added in upgrade version
-# For these files the sql_in code is retained as is with the functions in the
-# file installed on the upgrade version. All other files (that don't have
-# updates), are cleaned up to remove object replacements
-new module:
- # ----------------- Changes from 1.11 to 1.12 --------
- mlp:
- apsp:
- bfs:
- measures:
- wcc:
- stratified_sample:
- train_test_split:
-# Changes in the types (UDT) including removal and modification
-udt:
-
-# List of the UDF changes that affect the user externally. This includes change
-# in function name, return type, argument order or types, or removal of
-# the function. In each case, the original function is as good as removed and a
-# new function is created. In such cases, we should abort the upgrade if there
-# are user views dependent on this function, since the original function will
-# not be present in the upgraded version.
-udf:
- # ----------------- Changes from 1.11 to 1.12 ----------
- - tree_train:
- rettype: void
- argument: text, text, text, text, text, text, text, text, text, integer, integer, integer, integer, text, text, boolean
- - tree_train:
- rettype: void
- argument: text, text, text, text, text, text, text, text, text, integer, integer, integer, integer, text, text
- - forest_train:
- rettype: void
- argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text, boolean, double precision
- - forest_train:
- rettype: void
- argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text, boolean
- - forest_train:
- rettype: void
- argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text
- - _map_catlevel_to_int:
- rettype: integer[]
- argument: text[], text[], integer[]
-# Changes to aggregates (UDA) including removal and modification
-# Overloaded functions should be mentioned separately
-uda:
-
-# Casts (UDC) updated/removed
-udc:
-
-# Operators (UDO) removed/updated
-udo:
-
-# Operator Classes (UDOC) removed/updated
-udoc:
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/changelist_1.10.0_1.11.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.10.0_1.11.yaml b/src/madpack/changelist_1.10.0_1.11.yaml
new file mode 100644
index 0000000..28bdb08
--- /dev/null
+++ b/src/madpack/changelist_1.10.0_1.11.yaml
@@ -0,0 +1,63 @@
+# ------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------------------------
+
+# Changelist for MADlib version 1.10.0 to 1.11
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+ # ----------------- Changes from 1.10.0 to 1.11 --------
+ pagerank:
+# Changes in the types (UDT) including removal and modification
+udt:
+
+
+# List of the UDF changes that affect the user externally. This includes change
+# in function name, return type, argument order or types, or removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+ # ----------------- Changes from 1.10.0 to 1.11 ----------
+ - __build_tree:
+ rettype: void
+ argument: boolean, text, text, text, text, text, boolean, character varying[], character varying[], character varying[], text, text, integer, integer, integer, integer, text, smallint, text, integer
+ - graph_sssp_get_path:
+ rettype: integer[]
+ argument: text, integer
+
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+
+# Casts (UDC) updated/removed
+udc:
+
+# Operators (UDO) removed/updated
+udo:
+
+# Operator Classes (UDOC) removed/updated
+udoc:
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/changelist_1.10.0_1.12.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.10.0_1.12.yaml b/src/madpack/changelist_1.10.0_1.12.yaml
deleted file mode 100644
index 3cac569..0000000
--- a/src/madpack/changelist_1.10.0_1.12.yaml
+++ /dev/null
@@ -1,89 +0,0 @@
-# ------------------------------------------------------------------------------
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# ------------------------------------------------------------------------------
-
-# Changelist for MADlib version 1.10.0 to 1.12
-
-# This file contains all changes that were introduced in a new version of
-# MADlib. This changelist is used by the upgrade script to detect what objects
-# should be upgraded (while retaining all other objects from the previous version)
-
-# New modules (actually .sql_in files) added in upgrade version
-# For these files the sql_in code is retained as is with the functions in the
-# file installed on the upgrade version. All other files (that don't have
-# updates), are cleaned up to remove object replacements
-new module:
- # ----------------- Changes from 1.10.0 to 1.11 --------
- pagerank:
- # ----------------- Changes from 1.11 to 1.12 --------
- mlp:
- apsp:
- bfs:
- measures:
- wcc:
- stratified_sample:
- train_test_split:
-# Changes in the types (UDT) including removal and modification
-udt:
-
-
-# List of the UDF changes that affect the user externally. This includes change
-# in function name, return type, argument order or types, or removal of
-# the function. In each case, the original function is as good as removed and a
-# new function is created. In such cases, we should abort the upgrade if there
-# are user views dependent on this function, since the original function will
-# not be present in the upgraded version.
-udf:
- # ----------------- Changes from 1.10.0 to 1.11 ----------
- - __build_tree:
- rettype: void
- argument: boolean, text, text, text, text, text, boolean, character varying[], character varying[], character varying[], text, text, integer, integer, integer, integer, text, smallint, text, integer
- - graph_sssp_get_path:
- rettype: integer[]
- argument: text, integer
- # ----------------- Changes from 1.11 to 1.12 ----------
- - tree_train:
- rettype: void
- argument: text, text, text, text, text, text, text, text, text, integer, integer, integer, integer, text, text, boolean
- - tree_train:
- rettype: void
- argument: text, text, text, text, text, text, text, text, text, integer, integer, integer, integer, text, text
- - forest_train:
- rettype: void
- argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text, boolean, double precision
- - forest_train:
- rettype: void
- argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text, boolean
- - forest_train:
- rettype: void
- argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text
- - _map_catlevel_to_int:
- rettype: integer[]
- argument: text[], text[], integer[]
-# Changes to aggregates (UDA) including removal and modification
-# Overloaded functions should be mentioned separately
-uda:
-
-# Casts (UDC) updated/removed
-udc:
-
-# Operators (UDO) removed/updated
-udo:
-
-# Operator Classes (UDOC) removed/updated
-udoc:
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/changelist_1.11_1.12.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.11_1.12.yaml b/src/madpack/changelist_1.11_1.12.yaml
new file mode 100644
index 0000000..77edeb8
--- /dev/null
+++ b/src/madpack/changelist_1.11_1.12.yaml
@@ -0,0 +1,79 @@
+# ------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------------------------
+
+# Changelist for MADlib version 1.11 to 1.12
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+ # ----------------- Changes from 1.11 to 1.12 --------
+ mlp:
+ apsp:
+ bfs:
+ measures:
+ wcc:
+ stratified_sample:
+ train_test_split:
+# Changes in the types (UDT) including removal and modification
+udt:
+
+# List of the UDF changes that affect the user externally. This includes change
+# in function name, return type, argument order or types, or removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+ # ----------------- Changes from 1.11 to 1.12 ----------
+ - tree_train:
+ rettype: void
+ argument: text, text, text, text, text, text, text, text, text, integer, integer, integer, integer, text, text, boolean
+ - tree_train:
+ rettype: void
+ argument: text, text, text, text, text, text, text, text, text, integer, integer, integer, integer, text, text
+ - forest_train:
+ rettype: void
+ argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text, boolean, double precision
+ - forest_train:
+ rettype: void
+ argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text, boolean
+ - forest_train:
+ rettype: void
+ argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text
+ - _map_catlevel_to_int:
+ rettype: integer[]
+ argument: text[], text[], integer[]
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+
+# Casts (UDC) updated/removed
+udc:
+
+# Operators (UDO) removed/updated
+udo:
+
+# Operator Classes (UDOC) removed/updated
+udoc:
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/changelist_1.12_1.13.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.12_1.13.yaml b/src/madpack/changelist_1.12_1.13.yaml
new file mode 100644
index 0000000..a08ba9f
--- /dev/null
+++ b/src/madpack/changelist_1.12_1.13.yaml
@@ -0,0 +1,84 @@
+# ------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------------------------
+
+# Changelist for MADlib version 1.12 to 1.13
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+ # ----------------- Changes from 1.11 to 1.12 --------
+ hits:
+
+# Changes in the types (UDT) including removal and modification
+udt:
+
+# List of the UDF changes that affect the user externally. This includes change
+# in function name, return type, argument order or types, or removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+ # ----------------- Changes from 1.12 to 1.13 ----------
+ - __build_tree:
+ rettype: void
+ argument: boolean, text, text, text, text, text, boolean, character varying[], character varying[], character varying[], character varying[], text, text, integer, integer, integer, integer, text, smallint, text, integer
+ - mlp_classification:
+ rettype: void
+ argument: character varying, character varying, character varying, character varying
+ - mlp_igd_final:
+ rettype: mlp_step_result
+ argument: double precision[]
+ - mlp_igd_transition:
+ rettype: double precision[]
+ argument: double precision[], double precision[], double precision[], double precision[], double precision[], double precision, integer, integer, double precision, boolean, double precision[], integer, double precision, double precision[], double precision[]
+ - mlp_regression:
+ rettype: void
+ argument: character varying, character varying, character varying, character varying
+ - __knn_validate_src:
+ rettype: integer
+ argument: varchar, varchar, varchar, varchar, varchar, varchar, varchar, varchar, integer
+ - knn:
+ rettype: varchar
+ argument: varchar, varchar, varchar, varchar, varchar, varchar, varchar, varchar, integer
+ - knn:
+ rettype: varchar
+ argument: varchar, varchar, varchar, varchar, varchar, varchar, varchar, varchar
+
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+ - mlp_igd_step:
+ rettype: mlp_step_result
+ argument: double precision[], double precision[], double precision[], double precision[], double precision, integer, integer, double precision, boolean, double precision[], integer, double precision, double precision[], double precision[]
+
+# Casts (UDC) updated/removed
+udc:
+
+# Operators (UDO) removed/updated
+udo:
+
+# Operator Classes (UDOC) removed/updated
+udoc:
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/changelist_1.9.1_1.10.0.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.9.1_1.10.0.yaml b/src/madpack/changelist_1.9.1_1.10.0.yaml
new file mode 100644
index 0000000..2acf926
--- /dev/null
+++ b/src/madpack/changelist_1.9.1_1.10.0.yaml
@@ -0,0 +1,130 @@
+# ------------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------------------------
+
+# Changelist for MADlib version 1.9.1 to 1.10
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+ # ----------------- Changes from 1.9.1 to 1.10.0 ----------
+ sssp:
+ encode_categorical:
+ knn:
+
+# Changes in the types (UDT) including removal and modification
+udt:
+
+ # ----------------- Changes from 1.9.1 to 1.10.0 ----------
+ kmeans_result:
+ kmeans_state:
+
+# List of the UDF changes that affect the user externally. This includes change
+# in function name, return type, argument order or types, or removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+
+ # ----------------- Changes from 1.9.1 to 1.10.0 ----------
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying, character varying, integer, double precision
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying, character varying, integer
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying, character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[], character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, double precision[]
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, character varying, integer, double precision
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, character varying, integer
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying
+ - kmeans:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer, double precision, double precision
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer, double precision
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying
+ - kmeanspp:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer, double precision
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying, integer
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying, character varying
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer, character varying
+ - kmeans_random:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, integer
+ - internal_execute_using_kmeans_args:
+ rettype: schema_madlib.kmeans_result
+ argument: character varying, character varying, character varying, character varying, character varying, integer, double precision
+
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+
+# Casts (UDC) updated/removed
+udc:
+
+# Operators (UDO) removed/updated
+udo:
+
+# Operator Classes (UDOC) removed/updated
+udoc:
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/changelist_1.9.1_1.12.yaml
----------------------------------------------------------------------
diff --git a/src/madpack/changelist_1.9.1_1.12.yaml b/src/madpack/changelist_1.9.1_1.12.yaml
deleted file mode 100644
index 5ba331a..0000000
--- a/src/madpack/changelist_1.9.1_1.12.yaml
+++ /dev/null
@@ -1,165 +0,0 @@
-# ------------------------------------------------------------------------------
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# ------------------------------------------------------------------------------
-
-# Changelist for MADlib version 1.9.1 to 1.12
-
-# This file contains all changes that were introduced in a new version of
-# MADlib. This changelist is used by the upgrade script to detect what objects
-# should be upgraded (while retaining all other objects from the previous version)
-
-# New modules (actually .sql_in files) added in upgrade version
-# For these files the sql_in code is retained as is with the functions in the
-# file installed on the upgrade version. All other files (that don't have
-# updates), are cleaned up to remove object replacements
-new module:
- # ----------------- Changes from 1.9.1 to 1.10.0 ----------
- sssp:
- encode_categorical:
- knn:
- # ----------------- Changes from 1.10.0 to 1.11 --------
- pagerank:
- # ----------------- Changes from 1.11 to 1.12 --------
- mlp:
- apsp:
- bfs:
- measures:
- wcc:
- stratified_sample:
- train_test_split:
-# Changes in the types (UDT) including removal and modification
-udt:
-
- # ----------------- Changes from 1.9.1 to 1.10.0 ----------
- kmeans_result:
- kmeans_state:
-
-# List of the UDF changes that affect the user externally. This includes change
-# in function name, return type, argument order or types, or removal of
-# the function. In each case, the original function is as good as removed and a
-# new function is created. In such cases, we should abort the upgrade if there
-# are user views dependent on this function, since the original function will
-# not be present in the upgraded version.
-udf:
-
- # ----------------- Changes from 1.9.1 to 1.10.0 ----------
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying, integer, double precision
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying, integer
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[], character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, double precision[]
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying, integer, double precision
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying, integer
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying
- - kmeans:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision, double precision
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying
- - kmeanspp:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer, double precision
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying, integer
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying, character varying
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer, character varying
- - kmeans_random:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, integer
- - internal_execute_using_kmeans_args:
- rettype: schema_madlib.kmeans_result
- argument: character varying, character varying, character varying, character varying, character varying, integer, double precision
- # ----------------- Changes from 1.10.0 to 1.11 ----------
- - __build_tree:
- rettype: void
- argument: boolean, text, text, text, text, text, boolean, character varying[], character varying[], character varying[], text, text, integer, integer, integer, integer, text, smallint, text, integer
- - graph_sssp_get_path:
- rettype: integer[]
- argument: text, integer
- # ----------------- Changes from 1.11 to 1.12 ----------
- - tree_train:
- rettype: void
- argument: text, text, text, text, text, text, text, text, text, integer, integer, integer, integer, text, text, boolean
- - tree_train:
- rettype: void
- argument: text, text, text, text, text, text, text, text, text, integer, integer, integer, integer, text, text
- - forest_train:
- rettype: void
- argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text, boolean, double precision
- - forest_train:
- rettype: void
- argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text, boolean
- - forest_train:
- rettype: void
- argument: text, text, text, text, text, text, text, integer, integer, boolean, integer, integer, integer, integer, integer, text
- - _map_catlevel_to_int:
- rettype: integer[]
- argument: text[], text[], integer[]
-
-# Changes to aggregates (UDA) including removal and modification
-# Overloaded functions should be mentioned separately
-uda:
-
-# Casts (UDC) updated/removed
-udc:
-
-# Operators (UDO) removed/updated
-udo:
-
-# Operator Classes (UDOC) removed/updated
-udoc:
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/diff_udt.sql
----------------------------------------------------------------------
diff --git a/src/madpack/diff_udt.sql b/src/madpack/diff_udt.sql
index dbb0bfb..1070ea4 100644
--- a/src/madpack/diff_udt.sql
+++ b/src/madpack/diff_udt.sql
@@ -34,7 +34,8 @@ CREATE OR REPLACE FUNCTION detect_changed_types(
RETURNS TEXT[] AS
$$
import plpy
-
+ OLD_SCHEMA = 'madlib_old_vers'
+ NEW_SCHEMA = 'madlib'
rv = plpy.execute("""
SELECT name, old_relid, new_relid
FROM {common_udt_table}
@@ -44,22 +45,31 @@ $$
name = r['name']
old_relid = r['old_relid']
new_relid = r['new_relid']
- rv = plpy.execute("""
+ res = plpy.execute("""
SELECT
array_eq(old_type, new_type) AS changed
FROM
(
- SELECT array_agg(a.attname || pg_catalog.format_type(a.atttypid, a.atttypmod) || a.attnum order by a.attnum) AS old_type
+ SELECT array_agg(a.attname ||
+ regexp_replace(pg_catalog.format_type(a.atttypid, a.atttypmod),
+ '{old_schema}.', '') ||
+ a.attnum order by a.attnum) AS old_type
FROM pg_catalog.pg_attribute a
WHERE a.attrelid = '{old_relid}' AND a.attnum > 0 AND NOT a.attisdropped
) t1,
(
- SELECT array_agg(a.attname || pg_catalog.format_type(a.atttypid, a.atttypmod) || a.attnum order by a.attnum) AS new_type
+ SELECT array_agg(a.attname ||
+ regexp_replace(pg_catalog.format_type(a.atttypid, a.atttypmod),
+ '{new_schema}.', '') ||
+ a.attnum order by a.attnum) AS new_type
FROM pg_catalog.pg_attribute a
WHERE a.attrelid = '{new_relid}' AND a.attnum > 0 AND NOT a.attisdropped
) t2
- """.format(old_relid=old_relid, new_relid=new_relid))[0]['changed']
- if not rv:
+ """.format(old_relid=old_relid,
+ new_relid=new_relid,
+ old_schema=OLD_SCHEMA,
+ new_schema=NEW_SCHEMA))[0]['changed']
+ if not res:
changed_udt.append(name)
return changed_udt
$$ LANGUAGE plpythonu;
@@ -67,8 +77,11 @@ $$ LANGUAGE plpythonu;
-- Get UDTs
DROP TABLE IF EXISTS types_madlib;
DROP TABLE IF EXISTS types_madlib_old_vers;
+set search_path = public, madlib;
SELECT get_types('madlib');
+set search_path = public, madlib_old_vers;
SELECT get_types('madlib_old_vers');
+set search_path = public;
--SELECT name FROM types_madlib;
--SELECT name FROM types_madlib_v15;
http://git-wip-us.apache.org/repos/asf/madlib/blob/cefd15ea/src/madpack/madpack.py
----------------------------------------------------------------------
diff --git a/src/madpack/madpack.py b/src/madpack/madpack.py
index aa78a52..e81c258 100755
--- a/src/madpack/madpack.py
+++ b/src/madpack/madpack.py
@@ -14,8 +14,13 @@ import tempfile
import shutil
import upgrade_util as uu
+from utilities import error_
+from utilities import info_
from utilities import is_rev_gte
from utilities import get_rev_num
+from utilities import run_query
+from utilities import get_madlib_dbrev
+from utilities import get_dbver
# Required Python version
py_min_ver = [2, 6]
@@ -56,7 +61,6 @@ SUPPORTED_PORTS = ('postgres', 'greenplum', 'hawq')
# Global variables
portid = None # Target port ID (eg: pg90, gp40)
-dbconn = None # DB Connection object
dbver = None # DB version
con_args = {} # DB connection arguments
verbose = None # Verbose flag
@@ -79,93 +83,6 @@ def _make_dir(dir):
# ------------------------------------------------------------------------------
-def _error(msg, stop):
- """
- Error message wrapper
- @param msg error message
- @param stop program exit flag
- """
- # Print to stdout
- print this + ' : ERROR : ' + msg
- # stack trace is not printed
- if stop:
- exit(2)
-# ------------------------------------------------------------------------------
-
-
-def _info(msg, verbose=True):
- """
- Info message wrapper (verbose)
- @param msg info message
- @param verbose prints only if True
- """
- # Print to stdout
- if verbose:
- print this + ' : INFO : ' + msg
-# ------------------------------------------------------------------------------
-
-
-def run_query(sql, show_error, con_args=con_args):
- # Define sqlcmd
- sqlcmd = 'psql'
- delimiter = ' <$madlib_delimiter$> '
-
- # Test the DB cmd line utility
- std, err = subprocess.Popen(['which', sqlcmd], stdout=subprocess.PIPE,
- stderr=subprocess.PIPE).communicate()
- if std == '':
- _error("Command not found: %s" % sqlcmd, True)
-
- # Run the query
- runcmd = [sqlcmd,
- '-h', con_args['host'].split(':')[0],
- '-p', con_args['host'].split(':')[1],
- '-d', con_args['database'],
- '-U', con_args['user'],
- '-F', delimiter,
- '--no-password',
- '--no-psqlrc',
- '--no-align',
- '-c', sql]
- runenv = os.environ
- if 'password' in con_args:
- runenv["PGPASSWORD"] = con_args['password']
- runenv["PGOPTIONS"] = '-c search_path=public -c client_min_messages=error'
- std, err = subprocess.Popen(runcmd, env=runenv, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE).communicate()
-
- if err:
- if show_error:
- _error("SQL command failed: \nSQL: %s \n%s" % (sql, err), False)
- if 'password' in err:
- raise EnvironmentError
- else:
- raise Exception
-
- # Convert the delimited output into a dictionary
- results = [] # list of rows
- i = 0
- for line in std.splitlines():
- if i == 0:
- cols = [name for name in line.split(delimiter)]
- else:
- row = {} # dict of col_name:col_value pairs
- c = 0
- for val in line.split(delimiter):
- row[cols[c]] = val
- c += 1
- results.insert(i, row)
- i += 1
- # Drop the last line: "(X rows)"
- try:
- results.pop()
- except:
- pass
-
- return results
-# ------------------------------------------------------------------------------
-
-
def _internal_run_query(sql, show_error):
"""
Runs a SQL query on the target platform DB
@@ -175,7 +92,7 @@ def _internal_run_query(sql, show_error):
@param sql query text to execute
@param show_error displays the SQL error msg
"""
- return run_query(sql, show_error, con_args)
+ return run_query(sql, con_args, show_error)
# ------------------------------------------------------------------------------
@@ -234,7 +151,7 @@ def _run_sql_file(schema, maddir_mod_py, module, sqlfile,
# Check if the SQL file exists
if not os.path.isfile(sqlfile):
- _error("Missing module SQL file (%s)" % sqlfile, False)
+ error_(this, "Missing module SQL file (%s)" % sqlfile, False)
raise ValueError("Missing module SQL file (%s)" % sqlfile)
# Prepare the file using M4
@@ -261,12 +178,12 @@ def _run_sql_file(schema, maddir_mod_py, module, sqlfile,
'-I' + maddir_madpack,
sqlfile]
- _info("> ... parsing: " + " ".join(m4args), verbose)
+ info_(this, "> ... parsing: " + " ".join(m4args), verbose)
subprocess.call(m4args, stdout=f)
f.close()
except:
- _error("Failed executing m4 on %s" % sqlfile, False)
+ error_(this, "Failed executing m4 on %s" % sqlfile, False)
raise Exception
# Only update function definition
@@ -274,7 +191,7 @@ def _run_sql_file(schema, maddir_mod_py, module, sqlfile,
if upgrade:
# get filename from complete path without the extension
sub_module = os.path.splitext(os.path.basename(sqlfile))[0]
- _info(sub_module, False)
+ info_(this, sub_module, verbose)
if sub_module not in sc.get_change_handler().newmodule:
sql = open(tmpfile).read()
sql = sc.cleanup(sql)
@@ -287,7 +204,7 @@ def _run_sql_file(schema, maddir_mod_py, module, sqlfile,
std, err = subprocess.Popen(['which', sqlcmd], stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if not std:
- _error("Command not found: %s" % sqlcmd, True)
+ error_(this, "Command not found: %s" % sqlcmd, True)
runcmd = [sqlcmd, '-a',
'-v', 'ON_ERROR_STOP=1',
@@ -306,15 +223,15 @@ def _run_sql_file(schema, maddir_mod_py, module, sqlfile,
try:
log = open(logfile, 'w')
except:
- _error("Cannot create log file: %s" % logfile, False)
+ error_(this, "Cannot create log file: %s" % logfile, False)
raise Exception
# Run the SQL
try:
- _info("> ... executing " + tmpfile, verbose)
+ info_(this, "> ... executing " + tmpfile, verbose)
retval = subprocess.call(runcmd, env=runenv, stdout=log, stderr=log)
except:
- _error("Failed executing %s" % tmpfile, False)
+ error_(this, "Failed executing %s" % tmpfile, False)
raise Exception
finally:
log.close()
@@ -323,46 +240,6 @@ def _run_sql_file(schema, maddir_mod_py, module, sqlfile,
# ------------------------------------------------------------------------------
-def _get_madlib_dbrev(schema):
- """
- Read MADlib version from database
- @param dbconn database conection object
- @param schema MADlib schema name
- """
- try:
- row = _internal_run_query("SELECT count(*) AS cnt FROM pg_tables " +
- "WHERE schemaname='" + schema + "' AND " +
- "tablename='migrationhistory'", True)
- if int(row[0]['cnt']) > 0:
- row = _internal_run_query("""SELECT version FROM %s.migrationhistory
- ORDER BY applied DESC LIMIT 1""" % schema, True)
- if row:
- return row[0]['version']
- except:
- _error("Failed reading MADlib db version", True)
-
- return None
-# ------------------------------------------------------------------------------
-
-
-def _get_dbver():
- """ Read version number from database (of form X.Y) """
- try:
- versionStr = _internal_run_query("SELECT pg_catalog.version()", True)[0]['version']
- if portid == 'postgres':
- match = re.search("PostgreSQL[a-zA-Z\s]*(\d+\.\d+)", versionStr)
- elif portid == 'greenplum':
- # for Greenplum the 3rd digit is necessary to differentiate
- # 4.3.5+ from versions < 4.3.5
- match = re.search("Greenplum[a-zA-Z\s]*(\d+\.\d+\.\d+)", versionStr)
- elif portid == 'hawq':
- match = re.search("HAWQ[a-zA-Z\s]*(\d+\.\d+)", versionStr)
- return None if match is None else match.group(1)
- except:
- _error("Failed reading database version", True)
-# ------------------------------------------------------------------------------
-
-
def _check_db_port(portid):
"""
Make sure we are connected to the expected DB platform
@@ -372,7 +249,7 @@ def _check_db_port(portid):
try:
row = _internal_run_query("SELECT version() AS version", True)
except:
- _error("Cannot validate DB platform type", True)
+ error_(this, "Cannot validate DB platform type", True)
if row and row[0]['version'].lower().find(portid) >= 0:
if portid == 'postgres':
if row[0]['version'].lower().find('greenplum') < 0:
@@ -385,6 +262,7 @@ def _check_db_port(portid):
return False
# ------------------------------------------------------------------------------
+
def _print_revs(rev, dbrev, con_args, schema):
"""
Print version information
@@ -393,37 +271,38 @@ def _print_revs(rev, dbrev, con_args, schema):
@param con_args database connection arguments
@param schema MADlib schema name
"""
- _info("MADlib tools version = %s (%s)" % (str(rev), sys.argv[0]), True)
+ info_(this, "MADlib tools version = %s (%s)" % (str(rev), sys.argv[0]), True)
if con_args:
try:
- _info("MADlib database version = %s (host=%s, db=%s, schema=%s)"
+ info_(this, "MADlib database version = %s (host=%s, db=%s, schema=%s)"
% (dbrev, con_args['host'], con_args['database'], schema), True)
except:
- _info("MADlib database version = [Unknown] (host=%s, db=%s, schema=%s)"
+ info_(this, "MADlib database version = [Unknown] (host=%s, db=%s, schema=%s)"
% (dbrev, con_args['host'], con_args['database'], schema), True)
return
# ------------------------------------------------------------------------------
+
def _plpy_check(py_min_ver):
"""
Check pl/python existence and version
@param py_min_ver min Python version to run MADlib
"""
- _info("Testing PL/Python environment...", True)
+ info_(this, "Testing PL/Python environment...", True)
# Check PL/Python existence
rv = _internal_run_query("SELECT count(*) AS CNT FROM pg_language "
"WHERE lanname = 'plpythonu'", True)
if int(rv[0]['cnt']) > 0:
- _info("> PL/Python already installed", verbose)
+ info_(this, "> PL/Python already installed", verbose)
else:
- _info("> PL/Python not installed", verbose)
- _info("> Creating language PL/Python...", True)
+ info_(this, "> PL/Python not installed", verbose)
+ info_(this, "> Creating language PL/Python...", True)
try:
_internal_run_query("CREATE LANGUAGE plpythonu;", True)
except:
- _error("""Cannot create language plpythonu. Please check if you
+ error_(this, """Cannot create language plpythonu. Please check if you
have configured and installed portid (your platform) with
`--with-python` option. Stopping installation...""", False)
raise Exception
@@ -444,13 +323,13 @@ def _plpy_check(py_min_ver):
python = rv[0]['ver']
py_cur_ver = [int(i) for i in python.split('.')]
if py_cur_ver >= py_min_ver:
- _info("> PL/Python version: %s" % python, verbose)
+ info_(this, "> PL/Python version: %s" % python, verbose)
else:
- _error("PL/Python version too old: %s. You need %s or greater"
+ error_(this, "PL/Python version too old: %s. You need %s or greater"
% (python, '.'.join(str(i) for i in py_min_ver)), False)
raise Exception
_internal_run_query("DROP FUNCTION IF EXISTS plpy_version_for_madlib();", False)
- _info("> PL/Python environment OK (version: %s)" % python, True)
+ info_(this, "> PL/Python environment OK (version: %s)" % python, True)
# ------------------------------------------------------------------------------
@@ -461,7 +340,7 @@ def _db_install(schema, dbrev, testcase):
@param dbrev DB-level MADlib version
@param testcase command-line args for a subset of modules
"""
- _info("Installing MADlib into %s schema..." % schema.upper(), True)
+ info_(this, "Installing MADlib into %s schema..." % schema.upper(), True)
temp_schema = schema + '_v' + ''.join(map(str, get_rev_num(dbrev)))
# Check the status of MADlib objects in database
@@ -478,32 +357,37 @@ def _db_install(schema, dbrev, testcase):
if schema_writable and madlib_exists:
# work-around before UDT is available in HAWQ
if portid == 'hawq':
- _info("***************************************************************************", True)
- _info("* Schema MADLIB already exists", True)
- _info("* For HAWQ, MADlib objects will be overwritten to the 'MADLIB' schema", True)
- _info("* It may drop any database objects (tables, views, etc.) that depend on 'MADLIB' SCHEMA!!!!!!!!!!!!!", True)
- _info("***************************************************************************", True)
- _info("Would you like to continue? [Y/N]", True)
+ hawq_overwrite_msg = (
+ "***************************************************************************"
+ "* Schema MADLIB already exists"
+ "* MADlib objects will be overwritten to the 'MADLIB' schema"
+ "* It may drop any database objects (tables, views, etc.) that depend on 'MADLIB' SCHEMA"
+ "***************************************************************************"
+ "Would you like to continue? [Y/N]")
+ info_(this, hawq_overwrite_msg)
go = raw_input('>>> ').upper()
while go not in ('Y', 'YES', 'N', 'NO'):
go = raw_input('Yes or No >>> ').upper()
if go in ('N', 'NO'):
- _info('Installation stopped.', True)
+ info_(this, 'Installation stopped.', True)
return
# Rolling back in HAWQ will drop catalog functions. For exception, we
# simply push the exception to the caller to terminate the install
_db_create_objects(schema, None, testcase=testcase, hawq_debug=True)
else:
- _info("***************************************************************************", True)
- _info("* Schema %s already exists" % schema.upper(), True)
- _info("* Installer will rename it to %s" % temp_schema.upper(), True)
- _info("***************************************************************************", True)
- _info("Would you like to continue? [Y/N]", True)
+ schema_overwrite_msg = (
+ "***************************************************************************"
+ "* Schema {0} already exists"
+ "* Installer will rename it to {1}"
+ "***************************************************************************"
+ "Would you like to continue? [Y/N]".
+ format(schema.upper(), temp_schema.upper()))
+ info_(this, schema_overwrite_msg)
go = raw_input('>>> ').upper()
while go not in ('Y', 'YES', 'N', 'NO'):
go = raw_input('Yes or No >>> ').upper()
if go in ('N', 'NO'):
- _info('Installation stopped.', True)
+ info_(this, 'Installation stopped.', True)
return
# Rename MADlib schema
@@ -529,7 +413,7 @@ def _db_install(schema, dbrev, testcase):
try:
_db_create_objects(schema, None, testcase=testcase)
except:
- _error("Building database objects failed. "
+ error_(this, "Building database objects failed. "
"Before retrying: drop %s schema OR install MADlib into "
"a different schema." % schema.upper(), True)
@@ -542,7 +426,7 @@ def _db_install(schema, dbrev, testcase):
# simply push the exception to the caller to terminate the install
raise Exception("MADLIB schema is required for HAWQ")
- _info("> Schema %s does not exist" % schema.upper(), verbose)
+ info_(this, "> Schema %s does not exist" % schema.upper(), verbose)
# Create MADlib schema
try:
@@ -556,7 +440,7 @@ def _db_install(schema, dbrev, testcase):
except:
_db_rollback(schema, None)
- _info("MADlib %s installed successfully in %s schema." % (str(rev), schema.upper()), True)
+ info_(this, "MADlib %s installed successfully in %s schema." % (str(rev), schema.upper()))
# ------------------------------------------------------------------------------
@@ -567,37 +451,37 @@ def _db_upgrade(schema, dbrev):
@param dbrev DB-level MADlib version
"""
if is_rev_gte(get_rev_num(dbrev), get_rev_num(rev)):
- _info("Current MADlib version already up to date.", True)
+ info_(this, "Current MADlib version already up to date.", True)
return
- if is_rev_gte([1,9],get_rev_num(dbrev)):
- _error("""
- MADlib versions prior to v1.9.1 are not supported for upgrade.
- Please try upgrading to v1.9.1 and then upgrade to this version.
+ if is_rev_gte(get_rev_num('1.9.1'), get_rev_num(dbrev)):
+ error_(this, """
+ MADlib versions prior to v1.10 are not supported for upgrade.
+ Please try upgrading to v1.10 and then upgrade to this version.
""", True)
return
- _info("Upgrading MADlib into %s schema..." % schema.upper(), True)
- _info("\tDetecting dependencies...", True)
+ info_(this, "Upgrading MADlib into %s schema..." % schema.upper(), True)
+ info_(this, "\tDetecting dependencies...", True)
- _info("\tLoading change list...", True)
+ info_(this, "\tLoading change list...", True)
ch = uu.ChangeHandler(schema, portid, con_args, maddir, dbrev, is_hawq2)
- _info("\tDetecting table dependencies...", True)
+ info_(this, "\tDetecting table dependencies...", True)
td = uu.TableDependency(schema, portid, con_args)
- _info("\tDetecting view dependencies...", True)
+ info_(this, "\tDetecting view dependencies...", True)
vd = uu.ViewDependency(schema, portid, con_args)
abort = False
if td.has_dependency():
- _info("*" * 50, True)
- _info("\tFollowing user tables/indexes are dependent on MADlib objects:", True)
- _info(td.get_dependency_str(), True)
- _info("*" * 50, True)
+ info_(this, "*" * 50, True)
+ info_(this, "\tFollowing user tables/indexes are dependent on MADlib objects:", True)
+ info_(this, td.get_dependency_str(), True)
+ info_(this, "*" * 50, True)
cd_udt = [udt for udt in td.get_depended_udt() if udt in ch.udt]
if len(cd_udt) > 0:
- _error("""
+ error_(this, """
User has objects dependent on following updated MADlib types!
{0}
These objects need to be dropped before upgrading.
@@ -605,7 +489,7 @@ def _db_upgrade(schema, dbrev):
# we add special handling for 'linregr_result'
if 'linregr_result' in cd_udt:
- _info("""Dependency on 'linregr_result' could be due to objects
+ info_(this, """Dependency on 'linregr_result' could be due to objects
created from the output of the aggregate 'linregr'.
Please refer to the Linear Regression documentation
<http://madlib.apache.org/docs/latest/group__grp__linreg.html#warning>
@@ -617,7 +501,7 @@ def _db_upgrade(schema, dbrev):
d_udoc = td.get_depended_udoc_oids()
cd_udoc = [udoc for udoc in d_udoc if udoc in c_udoc]
if len(cd_udoc) > 0:
- _error("""
+ error_(this, """
User has objects dependent on the following updated MADlib operator classes!
oid={0}
These objects need to be dropped before upgrading.
@@ -625,16 +509,16 @@ def _db_upgrade(schema, dbrev):
abort = True
if vd.has_dependency():
- _info("*" * 50, True)
- _info("\tFollowing user views are dependent on MADlib objects:", True)
- _info(vd.get_dependency_graph_str(), True)
- _info("*" * 50, True)
+ info_(this, "*" * 50, True)
+ info_(this, "\tFollowing user views are dependent on MADlib objects:", True)
+ info_(this, vd.get_dependency_graph_str(), True)
+ info_(this, "*" * 50, True)
c_udf = ch.get_udf_signature()
d_udf = vd.get_depended_func_signature('UDF')
cd_udf = [udf for udf in d_udf if udf in c_udf]
if len(cd_udf) > 0:
- _error("""
+ error_(this, """
User has objects dependent on following updated MADlib functions!
{0}
These objects will fail to work with the updated functions and
@@ -646,7 +530,7 @@ def _db_upgrade(schema, dbrev):
d_uda = vd.get_depended_func_signature('UDA')
cd_uda = [uda for uda in d_uda if uda in c_uda]
if len(cd_uda) > 0:
- _error("""
+ error_(this, """
User has objects dependent on following updated MADlib aggregates!
{0}
These objects will fail to work with the new aggregates and
@@ -658,7 +542,7 @@ def _db_upgrade(schema, dbrev):
d_udo = vd.get_depended_opr_oids()
cd_udo = [udo for udo in d_udo if udo in c_udo]
if len(cd_udo) > 0:
- _error("""
+ error_(this, """
User has objects dependent on following updated MADlib operators!
oid={0}
These objects will fail to work with the new operators and
@@ -667,15 +551,15 @@ def _db_upgrade(schema, dbrev):
abort = True
if abort:
- _error("""------- Upgrade aborted. -------
+ error_(this, """------- Upgrade aborted. -------
Backup and drop all objects that depend on MADlib before trying upgrade again.
Use madpack reinstall to automatically drop these objects only if appropriate.""", True)
else:
- _info("No dependency problem found, continuing to upgrade ...", True)
+ info_(this, "No dependency problem found, continuing to upgrade ...", True)
- _info("\tReading existing UDAs/UDTs...", False)
+ info_(this, "\tReading existing UDAs/UDTs...", False)
sc = uu.ScriptCleaner(schema, portid, con_args, ch)
- _info("Script Cleaner initialized ...", False)
+ info_(this, "Script Cleaner initialized ...", False)
ch.drop_changed_uda()
ch.drop_changed_udoc()
@@ -686,7 +570,7 @@ def _db_upgrade(schema, dbrev):
ch.drop_traininginfo_4dt() # used types: oid, text, integer, float
_db_create_objects(schema, None, True, sc)
- _info("MADlib %s upgraded successfully in %s schema." % (str(rev), schema.upper()), True)
+ info_(this, "MADlib %s upgraded successfully in %s schema." % (str(rev), schema.upper()), True)
# ------------------------------------------------------------------------------
@@ -697,11 +581,11 @@ def _db_rename_schema(from_schema, to_schema):
@param to_schema new name for the schema
"""
- _info("> Renaming schema %s to %s" % (from_schema.upper(), to_schema.upper()), True)
+ info_(this, "> Renaming schema %s to %s" % (from_schema.upper(), to_schema.upper()), True)
try:
_internal_run_query("ALTER SCHEMA %s RENAME TO %s;" % (from_schema, to_schema), True)
except:
- _error('Cannot rename schema. Stopping installation...', False)
+ error_(this, 'Cannot rename schema. Stopping installation...', False)
raise Exception
# ------------------------------------------------------------------------------
@@ -713,11 +597,11 @@ def _db_create_schema(schema):
@param to_schema new name for the schema
"""
- _info("> Creating %s schema" % schema.upper(), True)
+ info_(this, "> Creating %s schema" % schema.upper(), True)
try:
_internal_run_query("CREATE SCHEMA %s;" % schema, True)
except:
- _info('Cannot create new schema. Rolling back installation...', True)
+ info_(this, 'Cannot create new schema. Rolling back installation...', True)
pass
# ------------------------------------------------------------------------------
@@ -735,42 +619,42 @@ def _db_create_objects(schema, old_schema, upgrade=False, sc=None, testcase="",
if not upgrade and not hawq_debug:
# Create MigrationHistory table
try:
- _info("> Creating %s.MigrationHistory table" % schema.upper(), True)
+ info_(this, "> Creating %s.MigrationHistory table" % schema.upper(), True)
_internal_run_query("DROP TABLE IF EXISTS %s.migrationhistory;" % schema, True)
sql = """CREATE TABLE %s.migrationhistory
(id serial, version varchar(255),
applied timestamp default current_timestamp);""" % schema
_internal_run_query(sql, True)
except:
- _error("Cannot crate MigrationHistory table", False)
+ error_(this, "Cannot crate MigrationHistory table", False)
raise Exception
# Copy MigrationHistory table for record keeping purposes
if old_schema:
try:
- _info("> Saving data from %s.MigrationHistory table" % old_schema.upper(), True)
+ info_(this, "> Saving data from %s.MigrationHistory table" % old_schema.upper(), True)
sql = """INSERT INTO %s.migrationhistory (version, applied)
SELECT version, applied FROM %s.migrationhistory
ORDER BY id;""" % (schema, old_schema)
_internal_run_query(sql, True)
except:
- _error("Cannot copy MigrationHistory table", False)
+ error_(this, "Cannot copy MigrationHistory table", False)
raise Exception
# Stamp the DB installation
try:
- _info("> Writing version info in MigrationHistory table", True)
+ info_(this, "> Writing version info in MigrationHistory table", True)
_internal_run_query("INSERT INTO %s.migrationhistory(version) "
"VALUES('%s')" % (schema, str(rev)), True)
except:
- _error("Cannot insert data into %s.migrationhistory table" % schema, False)
+ error_(this, "Cannot insert data into %s.migrationhistory table" % schema, False)
raise Exception
# Run migration SQLs
if upgrade:
- _info("> Creating/Updating objects for modules:", True)
+ info_(this, "> Creating/Updating objects for modules:", True)
else:
- _info("> Creating objects for modules:", True)
+ info_(this, "> Creating objects for modules:", True)
caseset = (set([test.strip() for test in testcase.split(',')])
if testcase != "" else set())
@@ -797,7 +681,7 @@ def _db_create_objects(schema, old_schema, upgrade=False, sc=None, testcase="",
if modset is not None and len(modset) > 0 and module not in modset:
continue
- _info("> - %s" % module, True)
+ info_(this, "> - %s" % module, True)
# Find the Python module dir (platform specific or generic)
if os.path.isdir(maddir + "/ports/" + portid + "/" + dbver + "/modules/" + module):
@@ -824,7 +708,7 @@ def _db_create_objects(schema, old_schema, upgrade=False, sc=None, testcase="",
sql_files = glob.glob(mask)
if not sql_files:
- _error("No files found in: %s" % mask, True)
+ error_(this, "No files found in: %s" % mask, True)
# Execute all SQL files for the module
for sqlfile in sql_files:
@@ -845,8 +729,8 @@ def _db_create_objects(schema, old_schema, upgrade=False, sc=None, testcase="",
sc)
# Check the exit status
if retval != 0:
- _error("TEST CASE RESULTed executing %s" % tmpfile, False)
- _error("Check the log at %s" % logfile, False)
+ error_(this, "TEST CASE RESULTed executing %s" % tmpfile, False)
+ error_(this, "Check the log at %s" % logfile, False)
raise Exception
# ------------------------------------------------------------------------------
@@ -857,23 +741,23 @@ def _db_rollback(drop_schema, keep_schema):
@param drop_schema name of the schema to drop
@param keep_schema name of the schema to rename and keep
"""
- _info("Rolling back the installation...", True)
+ info_(this, "Rolling back the installation...", True)
if not drop_schema:
- _error('No schema name to drop. Stopping rollback...', True)
+ error_(this, 'No schema name to drop. Stopping rollback...', True)
# Drop the current schema
- _info("> Dropping schema %s" % drop_schema.upper(), verbose)
+ info_(this, "> Dropping schema %s" % drop_schema.upper(), verbose)
try:
_internal_run_query("DROP SCHEMA %s CASCADE;" % (drop_schema), True)
except:
- _error("Cannot drop schema %s. Stopping rollback..." % drop_schema.upper(), True)
+ error_(this, "Cannot drop schema %s. Stopping rollback..." % drop_schema.upper(), True)
# Rename old to current schema
if keep_schema:
_db_rename_schema(keep_schema, drop_schema)
- _info("Rollback finished successfully.", True)
+ info_(this, "Rollback finished successfully.", True)
raise Exception
# ------------------------------------------------------------------------------
@@ -984,7 +868,7 @@ def main(argv):
global verbose
verbose = args.verbose
- _info("Arguments: " + str(args), verbose)
+ info_(this, "Arguments: " + str(args), verbose)
global keeplogs
keeplogs = args.keeplogs
@@ -993,7 +877,7 @@ def main(argv):
tmpdir = tempfile.mkdtemp('', 'madlib.', args.tmpdir)
except OSError, e:
tmpdir = e.filename
- _error("cannot create temporary directory: '%s'." % tmpdir, True)
+ error_(this, "cannot create temporary directory: '%s'." % tmpdir, True)
# Parse SCHEMA
if len(args.schema[0]) > 1:
@@ -1010,7 +894,7 @@ def main(argv):
ports[portid]
except:
portid = None
- _error("Can not find specs for port %s" % (args.platform[0]), True)
+ error_(this, "Can not find specs for port %s" % (args.platform[0]), True)
else:
portid = None
@@ -1041,7 +925,7 @@ def main(argv):
con_args['password'] = c_pass
# Try connecting to the database
- _info("Testing database connection...", verbose)
+ info_(this, "Testing database connection...", verbose)
try:
# check for password only if required
@@ -1050,11 +934,11 @@ def main(argv):
con_args['password'] = getpass.getpass("Password for user %s: " % c_user)
_internal_run_query("SELECT 1", False)
except:
- _error('Failed to connect to database', True)
+ error_(this, 'Failed to connect to database', True)
# Get DB version
global dbver
- dbver = _get_dbver()
+ dbver = get_dbver(con_args, portid)
global is_hawq2
if portid == "hawq" and is_rev_gte(get_rev_num(dbver), get_rev_num('2.0')):
is_hawq2 = True
@@ -1063,14 +947,14 @@ def main(argv):
# HAWQ < 2.0 has hard-coded schema name 'madlib'
if portid == 'hawq' and not is_hawq2 and schema.lower() != 'madlib':
- _error("*** Installation is currently restricted only to 'madlib' schema ***", True)
+ error_(this, "*** Installation is currently restricted only to 'madlib' schema ***", True)
# update maddir to use a relative path if available
global maddir
maddir = _get_relative_maddir(maddir, portid)
# Get MADlib version in DB
- dbrev = _get_madlib_dbrev(schema)
+ dbrev = get_madlib_dbrev(con_args, schema)
portdir = os.path.join(maddir, "ports", portid)
supportedVersions = [dirItem for dirItem in os.listdir(portdir)
@@ -1080,12 +964,12 @@ def main(argv):
dbver = ".".join(
map(str, max([versionStr.split('.')
for versionStr in supportedVersions])))
- _info("Could not parse version string reported by {DBMS}. Will "
+ info_(this, "Could not parse version string reported by {DBMS}. Will "
"default to newest supported version of {DBMS} "
"({version}).".format(DBMS=ports[portid]['name'],
version=dbver), True)
else:
- _info("Detected %s version %s." % (ports[portid]['name'], dbver),
+ info_(this, "Detected %s version %s." % (ports[portid]['name'], dbver),
True)
dbver_split = get_rev_num(dbver)
@@ -1108,20 +992,20 @@ def main(argv):
dbver = '4.3ORCA'
else:
# only need the first two digits for <= 4.3.4
- dbver = '.'.join(dbver_split[:2])
+ dbver = '.'.join(map(str, dbver_split[:2]))
elif portid == 'postgres':
if is_rev_gte(dbver_split, get_rev_num('10.0')):
# Postgres starting 10.0 uses semantic versioning. Hence,
# only need first digit for major version.
dbver = str(dbver_split[0])
if not os.path.isdir(os.path.join(portdir, dbver)):
- _error("This version is not among the %s versions for which "
+ error_(this, "This version is not among the %s versions for which "
"MADlib support files have been installed (%s)." %
(ports[portid]['name'], ", ".join(supportedVersions)), True)
# Validate that db platform is correct
if not _check_db_port(portid):
- _error("Invalid database platform specified.", True)
+ error_(this, "Invalid database platform specified.", True)
# Adjust MADlib directories for this port (if they exist)
global maddir_conf
@@ -1153,9 +1037,9 @@ def main(argv):
# Make sure we have the necessary parameters to continue
if args.command[0] != 'version':
if not portid:
- _error("Missing -p/--platform parameter.", True)
+ error_(this, "Missing -p/--platform parameter.", True)
if not con_args:
- _error("Unknown problem with database connection string: %s" % con_args, True)
+ error_(this, "Unknown problem with database connection string: %s" % con_args, True)
# COMMAND: version
if args.command[0] == 'version':
@@ -1163,11 +1047,11 @@ def main(argv):
# COMMAND: uninstall/reinstall
if args.command[0] in ('uninstall',) and (portid == 'hawq' and not is_hawq2):
- _error("madpack uninstall is currently not available for HAWQ", True)
+ error_(this, "madpack uninstall is currently not available for HAWQ", True)
if args.command[0] in ('uninstall', 'reinstall') and (portid != 'hawq' or is_hawq2):
if get_rev_num(dbrev) == [0]:
- _info("Nothing to uninstall. No version found in schema %s." % schema.upper(), True)
+ info_(this, "Nothing to uninstall. No version found in schema %s." % schema.upper(), True)
return
# Find any potential data to lose
@@ -1193,34 +1077,34 @@ def main(argv):
ORDER BY
n1.nspname, relname, attname, typname""" % schema.lower(), True)
- _info("*** Uninstalling MADlib ***", True)
- _info("***********************************************************************************", True)
- _info("* Schema %s and all database objects depending on it will be dropped!" % schema.upper(), True)
+ info_(this, "*** Uninstalling MADlib ***", True)
+ info_(this, "***********************************************************************************", True)
+ info_(this, "* Schema %s and all database objects depending on it will be dropped!" % schema.upper(), True)
if affected_objects:
- _info("* If you continue the following data will be lost (schema : table.column : type):", True)
+ info_(this, "* If you continue the following data will be lost (schema : table.column : type):", True)
for ao in affected_objects:
- _info('* - ' + ao['schema'] + ' : ' + ao['relation'] + '.' +
+ info_(this, '* - ' + ao['schema'] + ' : ' + ao['relation'] + '.' +
ao['column'] + ' : ' + ao['type'], True)
- _info("***********************************************************************************", True)
- _info("Would you like to continue? [Y/N]", True)
+ info_(this, "***********************************************************************************", True)
+ info_(this, "Would you like to continue? [Y/N]", True)
go = raw_input('>>> ').upper()
while go != 'Y' and go != 'N':
go = raw_input('Yes or No >>> ').upper()
# 2) Do the uninstall/drop
if go == 'N':
- _info('No problem. Nothing dropped.', True)
+ info_(this, 'No problem. Nothing dropped.', True)
return
elif go == 'Y':
- _info("> dropping schema %s" % schema.upper(), verbose)
+ info_(this, "> dropping schema %s" % schema.upper(), verbose)
try:
_internal_run_query("DROP SCHEMA %s CASCADE;" % (schema), True)
except:
- _error("Cannot drop schema %s." % schema.upper(), True)
+ error_(this, "Cannot drop schema %s." % schema.upper(), True)
- _info('Schema %s (and all dependent objects) has been dropped.' % schema.upper(), True)
- _info('MADlib uninstalled successfully.', True)
+ info_(this, 'Schema %s (and all dependent objects) has been dropped.' % schema.upper(), True)
+ info_(this, 'MADlib uninstalled successfully.', True)
else:
return
@@ -1232,20 +1116,20 @@ def main(argv):
print "Setting MADlib database version to be None for reinstall"
dbrev = None
- _info("*** Installing MADlib ***", True)
+ info_(this, "*** Installing MADlib ***", True)
# 1) Compare OS and DB versions.
# noop if OS <= DB.
_print_revs(rev, dbrev, con_args, schema)
if is_rev_gte(get_rev_num(dbrev), get_rev_num(rev)):
- _info("Current MADlib version already up to date.", True)
+ info_(this, "Current MADlib version already up to date.", True)
return
# proceed to create objects if nothing installed in DB or for HAWQ < 2.0
elif dbrev is None or (portid == 'hawq' and not is_hawq2):
pass
# error and refer to upgrade if OS > DB
else:
- _error("""Aborting installation: existing MADlib version detected in {0} schema
+ error_(this, """Aborting installation: existing MADlib version detected in {0} schema
To upgrade the {0} schema to MADlib v{1} please run the following command:
madpack upgrade -s {0} -p {2} [-c ...]
""".format(schema, rev, portid), True)
@@ -1255,16 +1139,16 @@ def main(argv):
_plpy_check(py_min_ver)
_db_install(schema, dbrev, args.testcase)
except:
- _error("MADlib installation failed.", True)
+ error_(this, "MADlib installation failed.", True)
# COMMAND: upgrade
if args.command[0] in ('upgrade', 'update'):
- _info("*** Upgrading MADlib ***", True)
- dbrev = _get_madlib_dbrev(schema)
+ info_(this, "*** Upgrading MADlib ***", True)
+ dbrev = get_madlib_dbrev(con_args, schema)
# 1) Check DB version. If None, nothing to upgrade.
if not dbrev:
- _info("MADlib is not installed in {schema} schema and there "
+ info_(this, "MADlib is not installed in {schema} schema and there "
"is nothing to upgrade. Please use install "
"instead.".format(schema=schema.upper()),
True)
@@ -1273,11 +1157,11 @@ def main(argv):
# 2) Compare OS and DB versions. Continue if OS > DB.
_print_revs(rev, dbrev, con_args, schema)
if is_rev_gte(get_rev_num(dbrev), get_rev_num(rev)):
- _info("Current MADlib version is already up-to-date.", True)
+ info_(this, "Current MADlib version is already up-to-date.", True)
return
if float('.'.join(dbrev.split('.')[0:2])) < 1.0:
- _info("The version gap is too large, upgrade is supported only for "
+ info_(this, "The version gap is too large, upgrade is supported only for "
"packages greater than or equal to v1.0.", True)
return
@@ -1290,7 +1174,7 @@ def main(argv):
print "Exception: " + str(e)
print sys.exc_info()
traceback.print_tb(sys.exc_info()[2])
- _error("MADlib upgrade failed.", True)
+ error_(this, "MADlib upgrade failed.", True)
# COMMAND: install-check
if args.command[0] == 'install-check':
@@ -1298,7 +1182,7 @@ def main(argv):
# 1) Compare OS and DB versions. Continue if OS = DB.
if get_rev_num(dbrev) != get_rev_num(rev):
_print_revs(rev, dbrev, con_args, schema)
- _info("Versions do not match. Install-check stopped.", True)
+ info_(this, "Versions do not match. Install-check stopped.", True)
return
# Create install-check user
@@ -1315,7 +1199,7 @@ def main(argv):
_internal_run_query("GRANT USAGE ON SCHEMA %s TO %s;" % (schema, test_user), True)
# 2) Run test SQLs
- _info("> Running test scripts for:", verbose)
+ info_(this, "> Running test scripts for:", verbose)
caseset = (set([test.strip() for test in args.testcase.split(',')])
if args.testcase != "" else set())
@@ -1345,7 +1229,7 @@ def main(argv):
# We can still run install-check on pmml with '-t' option.
if not modset and module in ['pmml']:
continue
- _info("> - %s" % module, verbose)
+ info_(this, "> - %s" % module, verbose)
# Make a temp dir for this module (if doesn't exist)
cur_tmpdir = tmpdir + '/' + module + '/test' # tmpdir is a global variable
@@ -1421,8 +1305,8 @@ def main(argv):
"|Time: %d milliseconds" % (milliseconds)
if result == 'FAIL':
- _error("Failed executing %s" % tmpfile, False)
- _error("Check the log at %s" % logfile, False)
+ error_(this, "Failed executing %s" % tmpfile, False)
+ error_(this, "Check the log at %s" % logfile, False)
# Cleanup test schema for the module
_internal_run_query("DROP SCHEMA IF EXISTS %s CASCADE;" % (test_schema), True)