[49/50] [abbrv] incubator-madlib git commit: Build: Fix module sort order for PGXN installation

[ri...@apache.org]

Build: Fix module sort order for PGXN installation

JIRA: MADLIB-1024

PGXN installation involves creating a single extension sql file that
contains all the SQL commands run during MADlib deployment. The modules
added into this extension file are to be placed in the right order,
taking dependencies into account.

MADlib has a function that compares a given file path with topologically
sorted modules to decide the order of concatenation to extension file.
This comparison is faulty since the module name is searched for in the
whole path, leading to false positive with modules that have another
module name as substring.  The specific bug was reported as 'svec_util'
being flagged in same order as 'svec'.

This commit fixes this issue taking advantage of the file path names being
of the form '.../modules/<module_name>/...', hence comparing the
whole module name.

Closes #106


Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/fa80240f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/fa80240f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/fa80240f

Branch: refs/heads/latest_release
Commit: fa80240f72a6551c2ee567d471afa499fd1d1efe
Parents: 0b8415e
Author: Rahul Iyer <ri...@apache.org>
Authored: Fri Feb 24 14:32:32 2017 -0800
Committer: Rahul Iyer <ri...@apache.org>
Committed: Mon Feb 27 13:13:18 2017 -0800

----------------------------------------------------------------------
 src/madpack/sort-module.py | 77 +++++++++++++++++++++++++++++++++++------
 1 file changed, 66 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/fa80240f/src/madpack/sort-module.py
----------------------------------------------------------------------
diff --git a/src/madpack/sort-module.py b/src/madpack/sort-module.py
index 7312e95..3c4691d 100644
--- a/src/madpack/sort-module.py
+++ b/src/madpack/sort-module.py
@@ -1,7 +1,24 @@
 #!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 """
-sort-module.py
+@file sort-module.py
 
 Sort input strings based on the module name in them such that
 the module dependencies are resolved.  Example:
@@ -13,21 +30,59 @@ Note this script assumes to be run at the exact current directory,
 so you need change directory first to run it.
 """
 
+import re
+import sys
+
 import configyml
 
-portspecs = configyml.get_modules("../config")
 
-def find_order(path):
+def get_modules_in_order():
+    """ Return a mapping between module_name and the order of the module.
+
+    Returns:
+        Dict. The output is of the form:
+            {'module_a': 1, 'module_b': 2, ...}
+    """
+    portspecs = configyml.get_modules("../config")
+    # get_modules returns a dictionary in following form:
+    #   { 'modules': [{'name': 'array_ops'}, {'name': 'bayes'}, ...,
+    #                 {'depends': ['array_ops'], 'name': 'stats'}, ... ]
+    #                 }
+    # The list of modules is pre-sorted using topological sort
+    module_order = dict()
     for i, moduleinfo in enumerate(portspecs['modules']):
-        modname = moduleinfo['name']
-        if modname in path:
-            return i
-    # return as the last if not found.
-    return len(portspecs['modules'])
+        module_order[moduleinfo['name']] = i
+    return module_order
+# ----------------------------------------------------------------------
+
+
+module_order = get_modules_in_order()
+
+# Not all modules are captured in the config file. For a missing module, assume
+# the sort rank is maximum (i.e. missing modules are installed last).
+MAX_RANK = len(module_order) + 1
+
+
+def find_order(path):
+    """ Return the position of a given file within the module order
+
+    Args:
+        @param: path: str, The path for a single SQL file. This path is assumed
+                to have the form: '.../modules/<module_name>/.../<file_name>.sql'
+    """
+    mod_name = re.match(r'.+/modules/(.+)/.+', path).group(1)
+    return module_order.get(mod_name, MAX_RANK)
+
+
+def main(file_paths):
+    """
+    Args:
+        @param: file_paths: List of paths to SQL files, where each path
+                is of the form: '.../modules/<module_name>/...'.
+    """
+    file_order = sorted(file_paths, key=find_order)
+    print " ".join(file_order)
 
-def main(args):
-    print " ".join(sorted(args, key = find_order))
 
 if __name__ == '__main__':
-    import sys
     main(sys.argv[1:])