You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@heron.apache.org by GitBox <gi...@apache.org> on 2018/05/18 18:17:08 UTC
[GitHub] nlu90 closed pull request #1093: heron package manager tool
nlu90 closed pull request #1093: heron package manager tool
URL: https://github.com/apache/incubator-heron/pull/1093
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/heron/packer/__init__.py b/heron/packer/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/heron/packer/src/__init__.py b/heron/packer/src/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/heron/packer/src/python/BUILD b/heron/packer/src/python/BUILD
new file mode 100644
index 0000000000..2e13b33ebd
--- /dev/null
+++ b/heron/packer/src/python/BUILD
@@ -0,0 +1,29 @@
+package(default_visibility = ["//visibility:public"])
+
+load("/tools/rules/pex_rules", "pex_library", "pex_binary")
+
+pex_library(
+ name = "heron-packer-lib",
+ srcs = [
+ "store/metastore.py",
+ "store/blobstore.py",
+ "common/utils.py",
+ "common/packer.py",
+ "common/constants.py",
+ "common/parser.py",
+ ],
+ deps = [
+ "//heron/common/src/python:common",
+ ],
+ reqs = ["pyyaml==3.10"],
+)
+
+pex_binary(
+ name = "heron-packer",
+ srcs = [
+ "main.py",
+ ],
+ deps = [
+ ":heron-packer-lib",
+ ]
+)
diff --git a/heron/packer/src/python/__init__.py b/heron/packer/src/python/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/heron/packer/src/python/common/__init__.py b/heron/packer/src/python/common/__init__.py
new file mode 100644
index 0000000000..d7e8c54ea0
--- /dev/null
+++ b/heron/packer/src/python/common/__init__.py
@@ -0,0 +1,10 @@
+"""common module"""
+
+from .packer import (
+ Packer,
+ HeronPacker
+)
+
+from .utils import *
+from .constants import *
+from .parser import *
diff --git a/heron/packer/src/python/common/constants.py b/heron/packer/src/python/common/constants.py
new file mode 100644
index 0000000000..75a7ce56f4
--- /dev/null
+++ b/heron/packer/src/python/common/constants.py
@@ -0,0 +1,17 @@
+"""constans used by package management tool"""
+
+CLUSTER = "cluster"
+COUNTER = "counter"
+DELETED = "deleted"
+DESC = "description"
+DEST_PATH = "dest_path"
+ENV = "env"
+EXTRA = "extra_fields"
+LATEST = "latest"
+LIVE = "live"
+PKG = "package"
+PKG_PATH = "pkg_path"
+RAW = "raw"
+ROLE = "role"
+STORE_PATH = "store_location"
+VERSION = "version"
diff --git a/heron/packer/src/python/common/packer.py b/heron/packer/src/python/common/packer.py
new file mode 100644
index 0000000000..b3af257696
--- /dev/null
+++ b/heron/packer/src/python/common/packer.py
@@ -0,0 +1,206 @@
+"""Package module defines the interfaces of package management tool needs to implement"""
+import sys
+
+from heron.common.src.python.color import Log
+from heron.packer.src.python.common import constants
+from heron.packer.src.python.common import utils
+
+class Packer(object):
+ """Package interface"""
+ def add_version(self, args):
+ """add a package with 'args' information"""
+ raise NotImplementedError("add_version is not implemented")
+
+ def download(self, args):
+ """download a package referenced by 'args'"""
+ raise NotImplementedError("download is not implemented")
+
+ def delete_version(self, args):
+ """delete a package referenced by 'args'"""
+ raise NotImplementedError("delete_version is not implemented")
+
+ def list_packages(self, args):
+ """list all packages referenced by 'args'"""
+ raise NotImplementedError("list_packages is not implemented")
+
+ def list_versions(self, args):
+ """list all versions of a package referenced by 'args'"""
+ raise NotImplementedError("list_versions is not implemented")
+
+ def set_live(self, args):
+ """set a package's version to 'live'"""
+ raise NotImplementedError("set_live is not implemented")
+
+ def unset_live(self, args):
+ """unset the 'live' tag on a package"""
+ raise NotImplementedError("unset_live is not implemented")
+
+ def show_version(self, args):
+ """show the basic information for package's version"""
+ raise NotImplementedError("show_version is not implemented")
+
+ def show_live(self, args):
+ """show the basic information for package's 'live' version"""
+ raise NotImplementedError("show_live is not implemented")
+
+ def show_latest(self, args):
+ """show the basic information for package's 'latest' version"""
+ raise NotImplementedError("show_latest is not implemented")
+
+ def clean(self, args):
+ """clean the state"""
+ raise NotImplementedError("clean is not implemented")
+
+class HeronPacker(Packer):
+ """A metastore and blobstore based implementation of Package"""
+ def __init__(self, metastore, blobstore):
+ self.metastore = metastore
+ self.blobstore = blobstore
+
+ def add_version(self, args):
+ role = args[constants.ROLE]
+ pkg_name = args[constants.PKG]
+ description = args[constants.DESC]
+ extra_info = args[constants.EXTRA]
+ file_path = args[constants.PKG_PATH]
+
+ # 1. upload file
+ successful, store_location = self.blobstore.upload(file_path)
+ if not successful:
+ Log.error("Failed to upload the package. Bailing out...")
+ sys.exit(1)
+
+ # 2. add package version info into metastore
+ successful, version = self.metastore.add_pkg_meta(
+ role, pkg_name, store_location, description, extra_info)
+ if not successful:
+ Log.error("Failed to add the meta data. Bailing out...")
+ sys.exit(1)
+
+ # 3. return a uri referencing this uploaded package in customized format
+ pkg_uri = self.metastore.get_pkg_uri(
+ self.blobstore.get_scheme(), role, pkg_name, version, extra_info)
+ Log.info("Uploaded package uri: %s" % pkg_uri)
+ return pkg_uri
+
+ def download(self, args):
+ role = args[constants.ROLE]
+ pkg_name = args[constants.PKG]
+ version = args[constants.VERSION]
+ extra_info = args[constants.EXTRA]
+ dest_path = args[constants.DEST_PATH]
+
+ # find package location
+ pkg_location = self.metastore.get_pkg_location(role, pkg_name, version, extra_info)
+ if pkg_location is None:
+ Log.error("Cannot find requested package. Bailing out...")
+ sys.exit(1)
+
+ # download
+ if self.blobstore.download(pkg_location, dest_path):
+ Log.info("Successfully downloaded package")
+ else:
+ Log.error("Failed to download pacakge")
+
+ def delete_version(self, args):
+ role = args[constants.ROLE]
+ pkg_name = args[constants.PKG]
+ version = args[constants.VERSION]
+ extra_info = args[constants.EXTRA]
+
+ # find package location
+ pkg_location = self.metastore.get_pkg_location(role, pkg_name, version, extra_info)
+ if pkg_location is None:
+ Log.error("Cannot find requested package. Bailing out...")
+ sys.exit(1)
+
+ # update metastore info with following 2 constraints:
+ # live cannot be deleted;
+ # latest needs to be updated if it's deleted
+ isSuccess = self.metastore.delete_pkg_meta(role, pkg_name, version, extra_info)
+ if not isSuccess:
+ Log.error("Failed to update metastore. Bailing out...")
+ sys.exit(1)
+
+ # delete the package
+ isSuccess = self.blobstore.delete(pkg_location)
+ if not isSuccess:
+ Log.error("meta info updated but blob file still exists. Can be cleaned with the clean cmd.")
+
+ def list_packages(self, args):
+ role = args[constants.ROLE]
+ extra_info = args[constants.EXTRA]
+ is_raw = args[constants.RAW]
+
+ # get packages from metastore
+ packages = self.metastore.get_packages(role, extra_info)
+
+ utils.print_list(packages, is_raw)
+
+ def list_versions(self, args):
+ role = args[constants.ROLE]
+ pkg_name = args[constants.PKG]
+ extra_info = args[constants.EXTRA]
+ is_raw = args[constants.RAW]
+
+ # get versions from metastore
+ versions = self.metastore.get_versions(role, pkg_name, extra_info)
+ utils.print_list(versions, is_raw)
+
+ def set_live(self, args):
+ role = args[constants.ROLE]
+ pkg_name = args[constants.PKG]
+ version = args[constants.VERSION]
+ extra_info = args[constants.EXTRA]
+
+ # set live in metastore
+ if self.metastore.set_tag(constants.LIVE, role, pkg_name, version, extra_info):
+ Log.info("Set live on %s/%s/%s success" % (role, pkg_name, version))
+ else:
+ Log.error("Set live on %s/%s/%s failed" % (role, pkg_name, version))
+
+ def unset_live(self, args):
+ role = args[constants.ROLE]
+ pkg_name = args[constants.PKG]
+ extra_info = args[constants.EXTRA]
+
+ # unset live in metastore
+ if self.metastore.unset_tag(constants.LIVE, role, pkg_name, extra_info):
+ Log.info("Unset live on %s/%s success" % (role, pkg_name))
+ else:
+ Log.error("Unset live one %s/%s failed" % (role, pkg_name))
+
+ def show_version(self, args):
+ role = args[constants.ROLE]
+ pkg_name = args[constants.PKG]
+ version = args[constants.VERSION]
+ extra_info = args[constants.EXTRA]
+ is_raw = args[constants.RAW]
+
+ # get package meta from metastore
+ pkg_info = self.metastore.get_pkg_meta(role, pkg_name, version, extra_info)
+ utils.print_dict(pkg_info, is_raw)
+
+ def show_live(self, args):
+ role = args[constants.ROLE]
+ pkg_name = args[constants.PKG]
+ extra_info = args[constants.EXTRA]
+ is_raw = args[constants.RAW]
+
+ # get live package information from metastore
+ live_pkg_info = self.metastore.get_pkg_meta(role, pkg_name, constants.LIVE, extra_info)
+ utils.print_dict(live_pkg_info, is_raw)
+
+ def show_latest(self, args):
+ role = args[constants.ROLE]
+ pkg_name = args[constants.PKG]
+ extra_info = args[constants.EXTRA]
+ is_raw = args[constants.RAW]
+
+ # get latest package information from metastore
+ latest_pkg_info = self.metastore.get_pkg_meta(role, pkg_name, constants.LATEST, extra_info)
+ utils.print_dict(latest_pkg_info, is_raw)
+
+ # clean the inconsistency state of package store.
+ def clean(self, args):
+ pass
diff --git a/heron/packer/src/python/common/parser.py b/heron/packer/src/python/common/parser.py
new file mode 100644
index 0000000000..c6d7c69143
--- /dev/null
+++ b/heron/packer/src/python/common/parser.py
@@ -0,0 +1,258 @@
+"""packer cli building module"""
+import argparse
+
+from heron.packer.src.python.common import constants
+from heron.common.src.python.color import Log
+
+help_epilog = '''Getting more help:
+ heron-package help <command> Prints help and options for <command>
+
+For detailed documentation, go to http://heronstreaming.io'''
+
+class SubcommandHelpFormatter(argparse.RawDescriptionHelpFormatter):
+ """Customized sub-command formatter"""
+ # pylint: disable=protected-access
+ def _format_action(self, action):
+ parts = super(SubcommandHelpFormatter, self)._format_action(action)
+ if action.nargs == argparse.PARSER:
+ parts = "\n".join(parts.split("\n")[1:])
+ return parts
+
+def get_subparser(parser, command):
+ """find subparser from 'parser' with name as 'command'"""
+ # pylint: disable=protected-access
+ subparsers_actions = [
+ action for action in parser._actions
+ if isinstance(action, argparse._SubParsersAction)
+ ]
+
+ # there will probably only be one subparser_action,
+ # but better save than sorry
+ for subparsers_action in subparsers_actions:
+ # get all subparsers
+ for choice, subparser in subparsers_action.choices.items():
+ if choice == command:
+ return subparser
+
+ return None
+
+def help_cmd(parser, namespace):
+ """help command implementation"""
+ command_help = namespace['help-command']
+
+ if command_help == 'help':
+ parser.print_help()
+ return True
+
+ subparser = get_subparser(parser, command_help)
+ if subparser:
+ print subparser.format_help()
+ return True
+ else:
+ Log.error("Unknown subcommand \'%s\'" % command_help)
+ return False
+
+def get_commands(packer):
+ """Bind cli commands with actual methods"""
+ PACKER_COMMANDS = {
+ "add_version": packer.add_version,
+ "download": packer.download,
+ "delete_version": packer.delete_version,
+ "list_packages": packer.list_packages,
+ "list_versions": packer.list_versions,
+ "show_version": packer.show_version,
+ "show_live": packer.show_live,
+ "show_latest": packer.show_latest,
+ "set_live": packer.set_live,
+ "unset_live": packer.unset_live,
+ }
+
+ return PACKER_COMMANDS
+
+def create_add_version_parser(parsers):
+ """Add add_version cmd to parser"""
+ add_version_parser = parsers.add_parser('add_version',
+ help='Upload a topology package',
+ add_help=False)
+ add_version_parser.add_argument(constants.ROLE, help='role name')
+ add_version_parser.add_argument(constants.PKG, help='package name')
+ add_version_parser.add_argument('--%s' % constants.EXTRA,
+ dest=constants.EXTRA,
+ default='',
+ help='extra user defined fields; separated by \\')
+ add_version_parser.add_argument(constants.PKG_PATH, help='jar or tar filepath')
+ add_version_parser.add_argument(constants.DESC, help='simple description about the jar')
+ add_version_parser.set_defaults(command='add_version')
+
+def create_download_parser(parsers):
+ """Add download cmd to parser"""
+ download_parser = parsers.add_parser('download',
+ help='Download a topology package',
+ add_help=False)
+ download_parser.add_argument(constants.ROLE, help='role name')
+ download_parser.add_argument(constants.PKG, help='packag name')
+ download_parser.add_argument(constants.VERSION, help='version number or LIVE, LATEST tag')
+ download_parser.add_argument('--%s' % constants.EXTRA,
+ dest=constants.EXTRA,
+ default='',
+ help='extra user defined fields; separated by \\')
+ download_parser.add_argument(constants.DEST_PATH, help='download file path')
+ download_parser.set_defaults(command='download')
+
+def create_delete_version_parser(parsers):
+ """Add delete_version cmd to parser"""
+ delete_version_parser = parsers.add_parser('delete_version',
+ help='Delete a topology pacakge',
+ add_help=False)
+ delete_version_parser.add_argument(constants.ROLE, help='role name')
+ delete_version_parser.add_argument(constants.PKG, help='packag name')
+ delete_version_parser.add_argument(constants.VERSION, help='version number or LIVE, LATEST tag')
+ delete_version_parser.add_argument('--%s' % constants.EXTRA,
+ dest=constants.EXTRA,
+ default='',
+ help='extra user defined fields; separated by \\')
+ delete_version_parser.set_defaults(command='delete_version')
+
+def create_list_packages_parser(parsers):
+ """Add list_packages cmd to parser"""
+ list_packages_parser = parsers.add_parser('list_packages',
+ help="List packages owned by a role",
+ add_help=False)
+ list_packages_parser.add_argument(constants.ROLE, help='role name')
+ list_packages_parser.add_argument('--%s' % constants.EXTRA,
+ dest=constants.EXTRA,
+ default='',
+ help='extra user defined fields; separated by \\')
+ list_packages_parser.add_argument('--%s' % constants.RAW,
+ dest=constants.RAW,
+ action='store_true',
+ help='print the result in raw format')
+ list_packages_parser.set_defaults(command='list_packages')
+
+def create_list_versions_parser(parsers):
+ """Add list_versions cmd to parser"""
+ list_versions_parser = parsers.add_parser('list_versions',
+ help="List a package's versions",
+ add_help=False)
+ list_versions_parser.add_argument(constants.ROLE, help='role name')
+ list_versions_parser.add_argument(constants.PKG, help='pacakge name')
+ list_versions_parser.add_argument('--%s' % constants.EXTRA,
+ dest=constants.EXTRA,
+ default='',
+ help='extra user defined fields; separated by \\')
+ list_versions_parser.add_argument('--%s' % constants.RAW,
+ dest=constants.RAW,
+ action='store_true',
+ help='print the result in raw format')
+ list_versions_parser.set_defaults(command='list_versions')
+
+def create_show_version_parser(parsers):
+ """Add show_version cmd to parser"""
+ show_version_parser = parsers.add_parser('show_version',
+ help="Show information for package's chosen version",
+ add_help=False)
+ show_version_parser.add_argument(constants.ROLE, help='role')
+ show_version_parser.add_argument(constants.PKG, help='package')
+ show_version_parser.add_argument(constants.VERSION, help='version number or LIVE, LATEST tag')
+ show_version_parser.add_argument('--%s' % constants.EXTRA,
+ dest=constants.EXTRA, default='',
+ help='extra user defined fields; separated by \\')
+ show_version_parser.add_argument('--%s' % constants.RAW,
+ dest=constants.RAW,
+ action='store_true',
+ help='print the result in raw format')
+ show_version_parser.set_defaults(command='show_version')
+
+def create_show_live_parser(parsers):
+ """Add show_live cmd to parser"""
+ show_live_parser = parsers.add_parser('show_live',
+ help="Show information for package's live version",
+ add_help=False)
+ show_live_parser.add_argument(constants.ROLE, help='role')
+ show_live_parser.add_argument(constants.PKG, help='package')
+ show_live_parser.add_argument('--%s' % constants.EXTRA,
+ dest=constants.EXTRA,
+ default='',
+ help='extra user defined fields; separated by \\')
+ show_live_parser.add_argument('--%s' % constants.RAW,
+ dest=constants.RAW,
+ action='store_true',
+ help='print the result in raw format')
+ show_live_parser.set_defaults(command='show_live')
+
+def create_show_latest_parser(parsers):
+ """Add show_latest cmd to parser"""
+ show_latest_parser = parsers.add_parser('show_latest',
+ help="Show information for pacakge's latest version",
+ add_help=False)
+ show_latest_parser.add_argument(constants.ROLE, help='role')
+ show_latest_parser.add_argument(constants.PKG, help='package')
+ show_latest_parser.add_argument('--%s' % constants.EXTRA,
+ dest=constants.EXTRA,
+ default='',
+ help='extra user defined fields; separated by \\')
+ show_latest_parser.add_argument('--%s' % constants.RAW,
+ dest=constants.RAW,
+ help='print the result in raw format')
+ show_latest_parser.set_defaults(command='show_latest')
+
+def create_set_live_parser(parsers):
+ """Add set_live cmd to parser"""
+ set_live_parser = parsers.add_parser('set_live',
+ help="Set 'live' tag on a package's version",
+ add_help=False)
+ set_live_parser.add_argument(constants.ROLE, help='role name')
+ set_live_parser.add_argument(constants.PKG, help='packag name')
+ set_live_parser.add_argument(constants.VERSION, help='version number or LIVE, LATEST tag')
+ set_live_parser.add_argument('--%s' % constants.EXTRA,
+ dest=constants.EXTRA,
+ default='',
+ help='extra user defined fields; separated by \\')
+ set_live_parser.set_defaults(command='set_live')
+
+def create_unset_live_parser(parsers):
+ """Add unset_live cmd to parser"""
+ unset_live_parser = parsers.add_parser('unset_live',
+ help="Unset 'live' tag on a pacakge's version",
+ add_help=False)
+ unset_live_parser.add_argument(constants.ROLE, help='role name')
+ unset_live_parser.add_argument(constants.PKG, help='packag name')
+ unset_live_parser.add_argument('--%s' % constants.EXTRA,
+ dest=constants.EXTRA,
+ default='',
+ help='extra user defined fields; separated by \\')
+ unset_live_parser.set_defaults(command='unset_live')
+
+def create_help_parser(parsers):
+ """Add help cmd to parser"""
+ help_parser = parsers.add_parser('help', help='Print help for commands', add_help=False)
+ help_parser.add_argument('help-command',
+ nargs='?',
+ default='help',
+ help='Provide help for a command')
+ help_parser.set_defaults(command='help')
+ # pylint: disable=protected-access
+ help_parser._positionals.title = "Required arguments"
+ help_parser._optionals.title = "Optional arguments"
+
+def create_parser():
+ """Create a command line parser for heron package management tool"""
+ parser = argparse.ArgumentParser(prog="heron-package",
+ epilog=help_epilog,
+ formatter_class=SubcommandHelpFormatter,
+ add_help=False)
+ subparsers = parser.add_subparsers(title="Available commands", metavar='<command> <options>')
+
+ create_add_version_parser(subparsers)
+ create_download_parser(subparsers)
+ create_delete_version_parser(subparsers)
+ create_list_packages_parser(subparsers)
+ create_list_versions_parser(subparsers)
+ create_show_version_parser(subparsers)
+ create_show_live_parser(subparsers)
+ create_show_latest_parser(subparsers)
+ create_set_live_parser(subparsers)
+ create_unset_live_parser(subparsers)
+ create_help_parser(subparsers)
+
+ return parser
diff --git a/heron/packer/src/python/common/utils.py b/heron/packer/src/python/common/utils.py
new file mode 100644
index 0000000000..40bbbd060b
--- /dev/null
+++ b/heron/packer/src/python/common/utils.py
@@ -0,0 +1,49 @@
+"""package utils module"""
+import hashlib
+import os
+import yaml
+
+BLOCK_SIZE = 4096
+CONF = "conf"
+PACKER_YAML = "packer.yaml"
+
+def get_md5(file_path):
+ """get the md5 checksum for a give 'file'"""
+ m = hashlib.md5()
+ with open(file_path, "rb") as input_file:
+ while True:
+ block = input_file.read(BLOCK_SIZE)
+ if not block:
+ break
+ m.update(block)
+ return m.hexdigest()
+
+def print_list(a_list, raw):
+ """print a raw list if 'raw' is True; line by line if 'raw' is False"""
+ if raw:
+ print a_list
+ else:
+ print "\n".join(a_list)
+
+def print_dict(a_dict, raw):
+ """print a raw dict if 'raw' is True; line by line if 'raw' is False"""
+ if raw:
+ print a_dict
+ else:
+ for key, value in a_dict.items():
+ print "%s: %s" % (key, value)
+
+def get_packer_conf_path():
+ """get the config file path"""
+ heron_dir = os.path.join("/".join(os.path.realpath(__file__).split('/')[:-7]))
+ conf_path = os.path.join(heron_dir, CONF, PACKER_YAML)
+ return conf_path
+
+def load_packer_conf():
+ """load yaml config file"""
+ conf_path = get_packer_conf_path()
+
+ with open(conf_path, "r") as conf_file:
+ conf = yaml.load(conf_file)
+
+ return conf
diff --git a/heron/packer/src/python/main.py b/heron/packer/src/python/main.py
new file mode 100644
index 0000000000..be5e38c4ed
--- /dev/null
+++ b/heron/packer/src/python/main.py
@@ -0,0 +1,39 @@
+"""Entry point for package management tool"""
+import sys
+
+from heron.package.src.python.common import packer
+from heron.package.src.python.common import parser
+from heron.package.src.python.common import utils
+from heron.package.src.python.store import blobstore
+from heron.package.src.python.store import metastore
+
+def get_packer(conf):
+ """get blobstore/metastore based package with config"""
+ blobstore_object = blobstore.get_blobstore(conf)
+ metastore_object = metastore.get_metastore(conf)
+
+ packer_object = packer.HeronPacker(metastore_object, blobstore_object)
+ return packer_object
+
+def main():
+ """main entry point"""
+ cmd_parser = parser.create_parser()
+
+ # if no argument is provided, print help and exit
+ if len(sys.argv[1:]) == 0:
+ cmd_parser.print_help()
+ sys.exit(1)
+
+ namespace = vars(cmd_parser.parse_args())
+
+ command = namespace['command']
+ if command == "help":
+ parser.help_cmd(cmd_parser, namespace)
+ else:
+ conf = utils.load_packer_conf()
+ packer = get_packer(conf)
+ packer_commands = parser.get_commands(packer)
+ packer_commands.get(command)(namespace)
+
+if __name__ == "__main__":
+ main()
diff --git a/heron/packer/src/python/store/__init__.py b/heron/packer/src/python/store/__init__.py
new file mode 100644
index 0000000000..0089b19515
--- /dev/null
+++ b/heron/packer/src/python/store/__init__.py
@@ -0,0 +1,11 @@
+"""store related module"""
+
+from .metastore import (
+ Metastore,
+ LocalMetastore
+)
+
+from .blobstore import (
+ Blobstore,
+ LocalBlobstore,
+)
diff --git a/heron/packer/src/python/store/blobstore.py b/heron/packer/src/python/store/blobstore.py
new file mode 100644
index 0000000000..52e24d14b1
--- /dev/null
+++ b/heron/packer/src/python/store/blobstore.py
@@ -0,0 +1,119 @@
+"""A blobstore module defines the interface for storing blob object"""
+import os
+import shutil
+import sys
+import time
+
+from heron.common.src.python.color import Log
+from heron.packer.src.python.common import utils
+
+BLOBSTORE_NAME = "heron.package.blobstore.name"
+BLOBSTORE_ROOT_PATH = "heron.package.blobstore.root_path"
+BLOBSTORE_SCHEME = "heron.package.blobstore.scheme"
+
+def get_blobstore(conf):
+ """Get blobstore instance with retrospection"""
+ blobstore = getattr(sys.modules[__name__], conf[BLOBSTORE_NAME])
+ instance = blobstore(conf)
+ return instance
+
+class Blobstore(object):
+ """Blobstore interface"""
+
+ def get_scheme(self):
+ """Return the storage scheme"""
+ raise NotImplementedError('get_scheme is not implemented')
+
+ def upload(self, file_path):
+ """Upload the file at file_path to blobstore and return (status, stored_location)"""
+ raise NotImplementedError('upload is not implemented')
+
+ def download(self, file_path, dest_path):
+ """Download a file from blobstore with file_path to dest_path"""
+ raise NotImplementedError('download is not implemented')
+
+ def delete(self, file_path):
+ """Delete a file from blobstore with file_path"""
+ raise NotImplementedError('delete is not implemented')
+
+class LocalBlobstore(Blobstore):
+ """Local file system based implementation of Blobstore"""
+ def __init__(self, conf):
+ # get root path for storing blob and scheme from config
+ self.root_path = conf[BLOBSTORE_ROOT_PATH]
+ self.scheme = conf[BLOBSTORE_SCHEME]
+
+ def get_scheme(self):
+ return self.scheme
+
+ def upload(self, file_path):
+ # verify the file to be uploaded exists
+ if not LocalBlobstore._is_valid_file(file_path):
+ Log.error("Upload file does not exist")
+ return False, ""
+
+ # get the path to store the file
+ dest_path = LocalBlobstore._resolve_path(self.root_path, file_path)
+
+ # copy file to destination
+ if LocalBlobstore._copy_file(file_path, dest_path):
+ return True, os.path.join(dest_path, os.path.basename(file_path))
+ else:
+ return False, ""
+
+ def download(self, file_path, dest_path):
+ if not LocalBlobstore._is_valid_file(file_path):
+ Log.error("Requested file does not exist")
+ return False
+
+ if not os.path.isdir(dest_path):
+ Log.error("File destination '%s' is not valid" % dest_path)
+ return False
+
+ if LocalBlobstore._copy_file(file_path, dest_path):
+ Log.info("%s is successfully downloaded" % os.path.basename(file_path))
+ return True
+ else:
+ return False
+
+ def delete(self, file_path):
+ if not LocalBlobstore._is_valid_file(file_path):
+ Log.error("Requested file does not exist")
+ return False
+
+ try:
+ # remove the file from local file system
+ os.remove(file_path)
+ Log.info("%s is successfully deleted" % os.path.basename(file_path))
+ except OSError as e:
+ Log.error("Failed to delete file: file path is not valid. Exception: %s" % e)
+ return False
+
+ return True
+
+ @staticmethod
+ def _copy_file(src, dst):
+ try:
+ # create directory if the destination directory doesn't exist
+ if not os.path.exists(dst):
+ os.makedirs(dst)
+
+ shutil.copy(src, dst)
+ except IOError as e:
+ Log.error("Unable to copy file %s to location %s. Exception: %s" % (src, dst, e))
+ return False
+
+ return True
+
+ @staticmethod
+ def _resolve_path(root_path, file_path):
+ # path format is: root_path/md5[0:4]/md5[4:]/timestamp/file_name
+ # md5 and timestamp are used to avoid collision
+ md5 = utils.get_md5(file_path)
+ dest_path = os.path.join(root_path, md5[0:4], md5[4:], str(int(time.time())))
+ return dest_path
+
+ @staticmethod
+ def _is_valid_file(file_path):
+ # validate the file path exists and is a file
+ return os.path.exists(file_path) and os.path.isfile(file_path)
diff --git a/heron/packer/src/python/store/metastore.py b/heron/packer/src/python/store/metastore.py
new file mode 100644
index 0000000000..92ae25e787
--- /dev/null
+++ b/heron/packer/src/python/store/metastore.py
@@ -0,0 +1,320 @@
+"""A metastore module defines the interface for storing meta information"""
+import json
+import os
+import sys
+
+from heron.common.src.python.color import Log
+from heron.packer.src.python.common import constants
+
+METASTORE_NAME = "heron.package.metastore.name"
+METASTORE_ROOT_PATH = "heron.package.metastore.root_path"
+
+def get_metastore(conf):
+ """Get metastore instance with retrospection"""
+ metastore = getattr(sys.modules[__name__], conf[METASTORE_NAME])
+ instance = metastore(conf)
+ return instance
+
+class Metastore(object):
+ """Metastore Interface"""
+
+ def get_packages(self, role, extra_info):
+ """Get all package names"""
+ raise NotImplementedError('get_packages is not implemented')
+
+ def get_versions(self, role, pkg_name, extra_info):
+ """Get all available version numbers for a package"""
+ raise NotImplementedError('get_versions is not implemented')
+
+ def add_pkg_meta(self, role, pkg_name, location, description, extra_info):
+ """Add meta data for a package's version"""
+ raise NotImplementedError('add_pkg_meta in not implemented')
+
+ def delete_pkg_meta(self, role, pkg_name, version, extra_info):
+ """Delete meta data for a package's version"""
+ raise NotImplementedError('delete_pkg_meta is not implemented')
+
+ def get_pkg_meta(self, role, pkg_name, version, extra_info):
+ """Get meta data for a package's version, "LIVE" and "LATEST" tag are supported"""
+ raise NotImplementedError('get_pkg_meta is not implemented')
+
+ def get_pkg_location(self, role, pkg_name, version, extra_info):
+ """Get package stored location"""
+ raise NotImplementedError('get_pkg_location is not implemented')
+
+ def set_tag(self, tag, role, pkg_name, version, extra_info):
+ """Set the tag on a package, only "LIVE" and "LATEST" tags are available"""
+ raise NotImplementedError('set_tag is not implemented')
+
+ def unset_tag(self, tag, role, pkg_name, extra_info):
+ """Clean the tag on a package"""
+ raise NotImplementedError('unset_tag is not implemented')
+
+ def get_pkg_uri(self, scheme, role, pkg_name, version, extra_info):
+ """Get pkg uri"""
+ raise NotImplementedError('get_pkg_uri is not implemented')
+
+class LocalMetastore(Metastore):
+ """Local file system based implementation of Metastore
+
+ The LocalMetastore stores meta information as a json file and has the following format:
+ {"role1":
+ {"package1":
+ {"live": None,
+ "latest": 1,
+ "counter": 3,
+ "1": {"deleted": True, "store_location": "foo/bar/p1.jar", "description": "foo bar"},
+ "2": {"deleted": False, ...},
+ ...
+ }
+ ....
+ }
+ ....
+ }
+ """
+ def __init__(self, conf):
+ # get meta file path from config
+ self.meta_file = conf[METASTORE_ROOT_PATH]
+
+ # create the meta file if missing
+ if not os.path.exists(self.meta_file):
+ with open(self.meta_file, "w") as meta:
+ meta.write("{}")
+
+ def get_packages(self, role, extra_info):
+ packages = []
+
+ meta_info = self._load_meta_file()
+
+ if role in meta_info:
+ # append all encountered package names
+ pkg_map = meta_info[role]
+ for key, _ in pkg_map.items():
+ packages.append(key)
+ else:
+ Log.error("role %s not found" % role)
+
+ return packages
+
+ def get_versions(self, role, pkg_name, extra_info):
+ versions = []
+
+ meta_info = self._load_meta_file()
+
+ if role in meta_info and pkg_name in meta_info[role]:
+ version_map = meta_info[role][pkg_name]
+ for key, value in version_map.items():
+ # ignore tag entries in the version_map
+ if key != constants.LIVE and \
+ key != constants.LATEST and \
+ key != constants.COUNTER:
+ version_msg = LocalMetastore._build_version_message(key, value)
+ versions.append(version_msg)
+ else:
+ Log.error("role/pkg (%s/%s) not found" % (role, pkg_name))
+
+ # dict entry is unordered, sorting for readability
+ versions.sort()
+
+ return versions
+
+ def add_pkg_meta(self, role, pkg_name, location, description, extra_info):
+ meta_info = self._load_meta_file()
+
+ # add role into meta_info if doesn't exist
+ if role not in meta_info:
+ meta_info[role] = {}
+
+ # add pkg_name into meta_info[role] if doesn't exist
+ if pkg_name not in meta_info[role]:
+ pkg_entry = {}
+ # live version
+ pkg_entry[constants.LIVE] = None
+ # largest valid version, 0 means no valid version
+ pkg_entry[constants.LATEST] = 0
+ # next available version
+ pkg_entry[constants.COUNTER] = 0
+ # add pkg_entry into meta_info
+ meta_info[role][pkg_name] = pkg_entry
+
+ # get the next available version number
+ meta_info[role][pkg_name][constants.COUNTER] += 1
+ # version will be used as a json map key, so it's converted to string
+ version = str(meta_info[role][pkg_name][constants.COUNTER])
+
+ # create the version info for a new version
+ version_entry = {}
+ version_entry[constants.STORE_PATH] = location
+ version_entry[constants.DELETED] = False
+ version_entry[constants.DESC] = description
+
+ # add new version entry
+ meta_info[role][pkg_name][version] = version_entry
+
+ # update latest version number
+ meta_info[role][pkg_name][constants.LATEST] = version
+
+ # update meta file
+ self._dump_meta_file(meta_info)
+
+ return True, version
+
+ def get_pkg_meta(self, role, pkg_name, version, extra_info):
+ meta_info = self._load_meta_file()
+
+ if not self._is_valid_pkg(meta_info, role, pkg_name):
+ Log.error("requested role/pkg (%s/%s) not found" % (role, pkg_name))
+ return None
+
+ # version must be less than LATEST version or be LIVE/LATEST tag
+ pkg_info = meta_info[role][pkg_name]
+ if not self._is_valid_version(pkg_info, version):
+ Log.error("requested version %s is not correct" % version)
+ return None
+
+ # translate to actual version if "LIVE" and "LATEST" tag provided
+ version = self._translate_version(version, pkg_info)
+
+ return pkg_info[version]
+
+ def get_pkg_location(self, role, pkg_name, version, extra_info):
+ version_info = self.get_pkg_meta(role, pkg_name, version, extra_info)
+ if version_info is None:
+ return None
+ else:
+ return version_info[constants.STORE_PATH]
+
+ def get_pkg_uri(self, scheme, role, pkg_name, version, extra_info):
+ return "%s://%s/%s/%s" % (scheme, role, pkg_name, version)
+
+ def delete_pkg_meta(self, role, pkg_name, version, extra_info):
+ meta_info = self._load_meta_file()
+
+ if not self._is_valid_pkg(meta_info, role, pkg_name):
+ Log.error("requested role/pkg (%s/%s) not found" % (role, pkg_name))
+ return False
+
+ pkg_info = meta_info[role][pkg_name]
+ if not self._is_valid_version(pkg_info, version):
+ Log.error("requested version %s is not correct" % version)
+ return False
+
+ # delete live is not allowed
+ if version == constants.LIVE or pkg_info[constants.LIVE] == version:
+ Log.error("Cannot delete a 'live' version")
+ return False
+
+ # delete latest requires update of the latest tag
+ if version == constants.LATEST or pkg_info[constants.LATEST] == version:
+ version = pkg_info[constants.LATEST]
+ # find the second latest version
+ pkg_info[constants.LATEST] = self._find_second_latest(pkg_info)
+
+ # check if the version is already deleted
+ if pkg_info[version][constants.DELETED]:
+ Log.error("version already deleted")
+ return False
+
+ # mark version deletion and update meta file
+ meta_info[role][pkg_name][version][constants.DELETED] = True
+ self._dump_meta_file(meta_info)
+
+ return True
+
+ @staticmethod
+ def _find_second_latest(pkg_info):
+ # return 0 if no valid second latest is found
+ if pkg_info[constants.LATEST] is None:
+ return "0"
+
+ latest = int(pkg_info[constants.LATEST])
+ for v in range(latest - 1, 0, -1):
+ version = str(v)
+ if not pkg_info[version][constants.DELETED]:
+ return version
+
+ return "0"
+
+ def set_tag(self, tag, role, pkg_name, version, extra_info):
+ return self._update_tag(tag, role, pkg_name, version)
+
+ def unset_tag(self, tag, role, pkg_name, extra_info):
+ return self._update_tag(tag, role, pkg_name, None, True)
+
+ def _update_tag(self, tag, role, pkg_name, version=None, is_reset=False):
+ meta_info = self._load_meta_file()
+
+ if not self._is_valid_pkg(meta_info, role, pkg_name):
+ Log.error("requested role/pkg (%s/%s) not found" % (role, pkg_name))
+ return False
+
+ pkg_info = meta_info[role][pkg_name]
+
+ # reset tag to initial value; set to provided version otherwise
+ if is_reset:
+ if tag == constants.LATEST:
+ meta_info[role][pkg_name][tag] = "0"
+ elif tag == constants.LIVE:
+ meta_info[role][pkg_name][tag] = None
+ else:
+ if not self._is_valid_version(pkg_info, version):
+ Log.error("requested version %s is not correct" % version)
+ return False
+
+ meta_info[role][pkg_name][tag] = version
+
+ self._dump_meta_file(meta_info)
+ return True
+
+ @staticmethod
+ def _is_valid_pkg(meta_info, role, pkg_name):
+ valid_pkg = False
+ if pkg_name in meta_info.get(role, {}):
+ valid_pkg = True
+
+ return valid_pkg
+
+ @staticmethod
+ def _is_valid_version(pkg_info, version):
+ # a valid version must be either a tag name or a version # less than LATEST
+ return version == constants.LATEST or \
+ version == constants.LIVE or \
+ int(pkg_info[constants.LATEST]) >= int(version)
+
+ @staticmethod
+ def _translate_version(version, pkg_info):
+ # if a version is a tag name, convert it to the actual version
+ if version == constants.LATEST:
+ if pkg_info[constants.LATEST] == 0:
+ Log.error("there's no latest version available for the pkg. Bailing out...")
+ version = pkg_info[constants.LATEST]
+
+ elif version == constants.LIVE:
+ if pkg_info[constants.LIVE] is None:
+ Log.error("there's no live version for the pkg. Bailing out...")
+ sys.exit(1)
+ version = pkg_info[constants.LIVE]
+
+ return version
+
+ def _load_meta_file(self):
+ with open(self.meta_file, "r") as meta:
+ meta_info = json.load(meta)
+
+ return meta_info
+
+ def _dump_meta_file(self, meta_info):
+ with open(self.meta_file, "w") as meta_file:
+ json.dump(meta_info, meta_file)
+
+ @staticmethod
+ def _build_version_message(version, info):
+ # current version message is in the following format:
+ # Version 1 DELETED
+ # "DESCRIPTION INFORMATION"
+ message = "Version %s" % version
+ if info[constants.DELETED]:
+ message += " DELETED"
+ message += "\n\t%s" % info[constants.DESC]
+
+ return message
\ No newline at end of file
diff --git a/heron/packer/tests/python/BUILD b/heron/packer/tests/python/BUILD
new file mode 100644
index 0000000000..463f2b8dcd
--- /dev/null
+++ b/heron/packer/tests/python/BUILD
@@ -0,0 +1,30 @@
+package(default_visibility = ["//visibility:public"])
+
+load("/tools/rules/pex_rules", "pex_test")
+
+pex_test(
+ name = "blobstore_unittest",
+ srcs = ["store/blobstore_unittest.py"],
+ deps = ["//heron/packer/src/python:heron-packer-lib"],
+ reqs = [
+ "py==1.4.27",
+ "pytest==2.6.4",
+ "unittest2==0.5.1",
+ "testfixtures==4.10.0",
+ ],
+ size = "small",
+)
+
+pex_test(
+ name = "metastore_unittest",
+ srcs = ["store/metastore_unittest.py"],
+ deps = ["//heron/packer/src/python:heron-packer-lib"],
+ reqs = [
+ "mock==1.0.1",
+ "py==1.4.27",
+ "pytest==2.6.4",
+ "unittest2==0.5.1",
+ "testfixtures==4.10.0",
+ ],
+ size = "small",
+)
\ No newline at end of file
diff --git a/heron/packer/tests/python/store/blobstore_unittest.py b/heron/packer/tests/python/store/blobstore_unittest.py
new file mode 100644
index 0000000000..3ab4974f3c
--- /dev/null
+++ b/heron/packer/tests/python/store/blobstore_unittest.py
@@ -0,0 +1,69 @@
+import os
+import unittest2 as unittest
+
+from heron.packer.src.python.store import blobstore
+from testfixtures import TempDirectory
+
+class LocalBlobstoreTest(unittest.TestCase):
+ def setUp(self):
+ self.tmp_dir = TempDirectory()
+ self.tmp_dir.makedir("blob")
+ self.conf = {
+ blobstore.BLOBSTORE_NAME: "LocalBlobstore",
+ blobstore.BLOBSTORE_ROOT_PATH: os.path.join(self.tmp_dir.path, "blob"),
+ blobstore.BLOBSTORE_SCHEME: "test_scheme",
+ }
+
+ self.blobstore = blobstore.LocalBlobstore(self.conf)
+ self.not_exists_file = "file/not/exists.txt"
+ self.not_valid_dir = "dir/not/valid"
+ self.test_file = "test.txt"
+
+ self.tmp_dir.write(self.test_file, b'foo upload test')
+
+ def tearDown(self):
+ TempDirectory.cleanup_all()
+
+ def test_upload(self):
+ # if file doesn't exist, return False and and empty string
+ status, location = self.blobstore.upload(self.not_exists_file)
+ self.assertFalse(status)
+ self.assertEqual(location, "")
+
+ # if file exists, return True and non empty string indicate the location
+ status, location = self.blobstore.upload(os.path.join(self.tmp_dir.path, self.test_file))
+ self.assertTrue(status)
+ self.assertIsNotNone(location)
+
+ def test_download(self):
+ # if downloaded file doesn't exist, return False and no operation
+ status = self.blobstore.download(self.not_exists_file, self.tmp_dir.path)
+ self.assertFalse(status)
+
+ # if dest path is not valid, return False and no operation
+ status = self.blobstore.download(
+ os.path.join(self.tmp_dir.path, self.test_file), self.not_valid_dir)
+ self.assertFalse(status)
+
+ # if file path and dest are valid, return True and file is downloaded
+ src_path = os.path.join(self.tmp_dir.path, self.test_file)
+ dest_path = os.path.join(self.tmp_dir.path, "download_test/")
+ self.tmp_dir.makedir(dest_path)
+ status = self.blobstore.download(src_path, dest_path)
+ self.assertTrue(status)
+ self.assertTrue(os.path.exists(dest_path))
+
+ def test_delete(self):
+ # if file doesn't exist, status is false
+ status = self.blobstore.delete(self.not_exists_file)
+ self.assertFalse(status)
+
+ # if file exists, verify status is true and the file is deleted
+ file_path = os.path.join(self.tmp_dir.path, self.test_file)
+ status = self.blobstore.delete(file_path)
+ self.assertTrue(status)
+ self.assertFalse(os.path.exists(file_path))
+
+ def test_get_scheme(self):
+ scheme = self.blobstore.get_scheme()
+ self.assertEqual(scheme, self.conf[blobstore.BLOBSTORE_SCHEME])
\ No newline at end of file
diff --git a/heron/packer/tests/python/store/metastore_unittest.py b/heron/packer/tests/python/store/metastore_unittest.py
new file mode 100644
index 0000000000..87720aa56f
--- /dev/null
+++ b/heron/packer/tests/python/store/metastore_unittest.py
@@ -0,0 +1,261 @@
+import os
+import unittest2 as unittest
+
+from heron.packer.src.python.common import constants
+from heron.packer.src.python.store import metastore
+from mock import patch
+from testfixtures import TempDirectory
+
+class LocalMetastoreTest(unittest.TestCase):
+ def setUp(self):
+ self.tmp_dir = TempDirectory()
+ self.meta_path = os.path.join(self.tmp_dir.path, "meta.json")
+ self.conf = {
+ metastore.METASTORE_NAME: "LocalMetastore",
+ metastore.METASTORE_ROOT_PATH: self.meta_path,
+ }
+ self.metastore = metastore.LocalMetastore(self.conf)
+ self.extra_info = "extra/information/stub"
+
+ def tearDown(self):
+ TempDirectory.cleanup_all()
+
+ @patch('heron.packer.src.python.store.metastore.LocalMetastore._load_meta_file')
+ def test_get_packages(self, mock_load_meta):
+ # mock the meta file
+ list_of_packages = ["p1", "p2", "p3", "p4"]
+ test_role = "test_role"
+ pkg_dict = dict((pkg, "Test") for pkg in list_of_packages)
+ mock_load_meta.return_value = {test_role: pkg_dict}
+ packages = self.metastore.get_packages(test_role, self.extra_info)
+ self.assertItemsEqual(packages, list_of_packages)
+
+ @patch('heron.packer.src.python.store.metastore.LocalMetastore._load_meta_file')
+ def test_get_versions(self, mock_load_meta):
+ # mock the meta file
+ test_role = "test_role"
+ test_pkg = "test_pkg"
+ version_dict = {
+ constants.LIVE: 2,
+ constants.LATEST: 2,
+ constants.COUNTER: 3,
+ "1": {constants.DELETED: True, constants.DESC: "version 1"},
+ "2": {constants.DELETED: False, constants.DESC: "version 2"}
+ }
+ mock_load_meta.return_value = {test_role:{test_pkg: version_dict}}
+ expected_versions = ['Version 1 DELETED\n\tversion 1', 'Version 2\n\tversion 2']
+
+ version_info = self.metastore.get_versions(test_role, test_pkg, self.extra_info)
+ self.assertItemsEqual(version_info, expected_versions)
+
+ @patch('heron.packer.src.python.store.metastore.LocalMetastore._dump_meta_file')
+ @patch('heron.packer.src.python.store.metastore.LocalMetastore._load_meta_file')
+ def test_add_pkg_meta(self, mock_load_meta, mock_dump_meta):
+ mock_load_meta.return_value = {}
+ mock_dump_meta.side_effect = None
+
+ role = "role"
+ pkg_name = "pkg_name"
+ description = "test package"
+ location = self.meta_path
+
+ status, version = self.metastore.add_pkg_meta(
+ role, pkg_name, location, description, self.extra_info)
+ self.assertTrue(status)
+ self.assertEqual("1", version)
+
+ @patch('heron.packer.src.python.store.metastore.LocalMetastore._load_meta_file')
+ def test_get_pkg_meta(self, mock_load_meta):
+ role = "test_role"
+ no_role = "no_role"
+ pkg_name = "test_pkg"
+ no_pkg_name = "no_test_pkg"
+ version = "1"
+ no_version = "100"
+
+ expect_meta = dict((v, str(v)) for v in range(1,3))
+
+ version_dict = {
+ constants.LATEST: "1",
+ constants.LIVE: "1",
+ constants.COUNTER: 1,
+ "1": expect_meta
+ }
+
+ mock_load_meta.return_value = {role: {pkg_name: version_dict}}
+
+ # invalid role provided
+ meta = self.metastore.get_pkg_meta(no_role, pkg_name, version, self.extra_info)
+ self.assertIsNone(meta)
+
+ # invalid package name provided
+ meta = self.metastore.get_pkg_meta(role, no_pkg_name, version, self.extra_info)
+ self.assertIsNone(meta)
+
+ # invalid version provided
+ meta = self.metastore.get_pkg_meta(role, pkg_name, no_version, self.extra_info)
+ self.assertIsNone(meta)
+
+ # correct information provided
+ meta = self.metastore.get_pkg_meta(role, pkg_name, version, self.extra_info)
+ self.assertItemsEqual(meta, expect_meta)
+
+ # LATEST tag provided
+ meta = self.metastore.get_pkg_meta(role, pkg_name, constants.LATEST, self.extra_info)
+ self.assertItemsEqual(meta, expect_meta)
+
+ # LIVE tag provided
+ meta = self.metastore.get_pkg_meta(role, pkg_name, constants.LIVE, self.extra_info)
+ self.assertItemsEqual(meta, expect_meta)
+
+ @patch('heron.packer.src.python.store.metastore.LocalMetastore._load_meta_file')
+ def test_get_pkg_location(self, mock_load_meta):
+ role = "role"
+ no_role = "no_role"
+ pkg_name = "pkg_name"
+ no_pkg_name = "no_pkg_name"
+ version = "1"
+ no_version = "100"
+
+ store_path = "test/store/path"
+ expect_meta = {constants.STORE_PATH: store_path}
+ version_dict = {
+ constants.LATEST: "1",
+ constants.LIVE: "1",
+ constants.COUNTER: 1,
+ "1": expect_meta
+ }
+
+ mock_load_meta.return_value = {role: {pkg_name: version_dict}}
+
+ # invalid role provided
+ path = self.metastore.get_pkg_location(no_role, pkg_name, version, self.extra_info)
+ self.assertIsNone(path)
+
+ # invalid package name provided
+ path = self.metastore.get_pkg_location(role, no_pkg_name, version, self.extra_info)
+ self.assertIsNone(path)
+
+ # invalid version provided
+ path = self.metastore.get_pkg_location(role, pkg_name, no_version, self.extra_info)
+ self.assertIsNone(path)
+
+ # correct information provided
+ path = self.metastore.get_pkg_location(role, pkg_name, version, self.extra_info)
+ self.assertItemsEqual(path, store_path)
+
+ # LATEST tag provided
+ path = self.metastore.get_pkg_location(role, pkg_name, constants.LATEST, self.extra_info)
+ self.assertItemsEqual(path, store_path)
+
+ # LIVE tag provided
+ path = self.metastore.get_pkg_location(role, pkg_name, constants.LIVE, self.extra_info)
+ self.assertItemsEqual(path, store_path)
+
+ def test_get_pkg_uri(self):
+ scheme = "test_scheme"
+ role = "test_role"
+ pkg_name = "test_pkg"
+ version = "version"
+ expected_uri = "%s://%s/%s/%s" % (scheme, role, pkg_name, version)
+ uri = self.metastore.get_pkg_uri(scheme, role, pkg_name, version, self.extra_info)
+ self.assertEqual(expected_uri, uri)
+
+ @patch('heron.packer.src.python.store.metastore.LocalMetastore._load_meta_file')
+ def test_delete_pkg_meta(self, mock_load_meta):
+ role = "role"
+ no_role = "no_role"
+ pkg_name = "pkg_name"
+ no_pkg_name = "no_pkg_name"
+ deleted_version = "1"
+ live_version = "2"
+ version = "3"
+ no_version = "100"
+
+ version_dict = {
+ constants.LIVE: live_version,
+ constants.LATEST: version,
+ constants.COUNTER: 3,
+ deleted_version: {constants.DELETED: True, constants.DESC: "version 1"},
+ live_version: {constants.DELETED: False, constants.DESC: "version 2"},
+ version: {constants.DELETED: False, constants.DESC: "version 3"},
+ }
+ mock_load_meta.return_value = {role:{pkg_name: version_dict}}
+
+ # invalid role provided
+ status = self.metastore.delete_pkg_meta(no_role, pkg_name, version, self.extra_info)
+ self.assertFalse(status)
+
+ # invalid package name provided
+ status = self.metastore.delete_pkg_meta(role, no_pkg_name, version, self.extra_info)
+ self.assertFalse(status)
+
+ # invalid version provided
+ status = self.metastore.delete_pkg_meta(role, pkg_name, no_version, self.extra_info)
+ self.assertFalse(status)
+
+ # live version provided
+ status = self.metastore.delete_pkg_meta(role, pkg_name, live_version, self.extra_info)
+ self.assertFalse(status)
+
+ # deleted version provided
+ status = self.metastore.delete_pkg_meta(no_role, pkg_name, deleted_version, self.extra_info)
+ self.assertFalse(status)
+
+ # correct version provided
+ status = self.metastore.delete_pkg_meta(role, pkg_name, version, self.extra_info)
+ self.assertTrue(status)
+
+ @patch('heron.packer.src.python.store.metastore.LocalMetastore._load_meta_file')
+ def test_set_tag(self, mock_load_meta):
+ role = "role"
+ no_role = "no_role"
+ pkg_name = "pkg_name"
+ no_pkg_name = "no_pkg_name"
+ version = "1"
+ no_version = "100"
+
+ version_dict = {
+ constants.LATEST: "1",
+ constants.LIVE: "1",
+ constants.COUNTER: 1,
+ "1": {"a": "info"},
+ }
+
+ mock_load_meta.return_value = {role: {pkg_name: version_dict}}
+
+ # invalid role provided
+ status = self.metastore.set_tag(constants.LIVE, no_role, pkg_name, version, self.extra_info)
+ self.assertFalse(status)
+
+ # invalid package name provided
+ status = self.metastore.set_tag(constants.LIVE, role, no_pkg_name, version, self.extra_info)
+ self.assertFalse(status)
+
+ # invalid version provided
+ status = self.metastore.set_tag(constants.LIVE, role, pkg_name, no_version, self.extra_info)
+ self.assertFalse(status)
+
+ # correct information provided
+ status = self.metastore.set_tag(constants.LIVE, role, pkg_name, version, self.extra_info)
+ self.assertTrue(status)
+
+ @patch('heron.packer.src.python.store.metastore.LocalMetastore._load_meta_file')
+ def test_unset_tag(self, mock_load_meta):
+ role = "role"
+ no_role = "no_role"
+ pkg_name = "pkg_name"
+ no_pkg_name = "no_pkg_name"
+ no_version = "100"
+
+ version_dict = {
+ constants.LATEST: "1",
+ constants.LIVE: "1",
+ constants.COUNTER: 1,
+ "1": {"a": "info"},
+ }
+
+ mock_load_meta.return_value = {role: {pkg_name: version_dict}}
+
+ status = self.metastore.unset_tag(constants.LIVE, role, pkg_name, self.extra_info)
+ self.assertTrue(status)
\ No newline at end of file
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services