You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by av...@apache.org on 2017/04/19 11:21:27 UTC
[09/12] incubator-ariatosca git commit: more review fixes
more review fixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/1cbd81b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/1cbd81b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/1cbd81b3
Branch: refs/heads/cli-tests
Commit: 1cbd81b3bb950dd589435af4bce6f0c1c1fc6411
Parents: c1f8eb6
Author: Ran Ziv <ra...@gigaspaces.com>
Authored: Wed Apr 19 13:19:26 2017 +0300
Committer: Ran Ziv <ra...@gigaspaces.com>
Committed: Wed Apr 19 13:19:26 2017 +0300
----------------------------------------------------------------------
aria/cli/commands/executions.py | 7 +-
aria/cli/commands/node_templates.py | 6 +-
aria/cli/commands/nodes.py | 7 +-
aria/cli/commands/plugins.py | 50 ++---------
aria/cli/commands/service_templates.py | 38 ++++-----
aria/cli/commands/services.py | 5 +-
aria/cli/commands/workflows.py | 6 +-
aria/cli/constants.py | 18 ----
aria/cli/core/aria.py | 10 +--
aria/cli/defaults.py | 20 +++++
aria/cli/helptexts.py | 2 +-
aria/cli/logger.py | 16 ++--
aria/cli/service_template_utils.py | 39 +++------
aria/cli/table.py | 90 +++++++++++++-------
aria/cli/utils.py | 88 ++++---------------
aria/modeling/service_common.py | 10 +--
aria/orchestrator/exceptions.py | 7 ++
aria/orchestrator/plugin.py | 23 +++++
aria/orchestrator/workflow_runner.py | 2 +-
aria/orchestrator/workflows/executor/celery.py | 4 +-
aria/orchestrator/workflows/executor/dry.py | 3 +-
aria/orchestrator/workflows/executor/process.py | 3 +-
aria/orchestrator/workflows/executor/thread.py | 3 +-
aria/utils/http.py | 62 ++++++++++++++
24 files changed, 258 insertions(+), 261 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/executions.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/executions.py b/aria/cli/commands/executions.py
index cd12ead..adec56b 100644
--- a/aria/cli/commands/executions.py
+++ b/aria/cli/commands/executions.py
@@ -51,7 +51,7 @@ def show(execution_id, model_storage, logger):
logger.info('Showing execution {0}'.format(execution_id))
execution = model_storage.execution.get(execution_id)
- print_data(EXECUTION_COLUMNS, execution.to_dict(), 'Execution:', max_width=50)
+ print_data(EXECUTION_COLUMNS, execution, 'Execution:', col_max_width=50)
# print execution parameters
logger.info('Execution Inputs:')
@@ -63,7 +63,6 @@ def show(execution_id, model_storage, logger):
logger.info('\t{0}: \t{1}'.format(input_name, input_value))
else:
logger.info('\tNo inputs')
- logger.info('')
@executions.command(name='list',
@@ -93,9 +92,9 @@ def list(service_name,
logger.info('Listing all executions...')
filters = {}
- executions_list = [e.to_dict() for e in model_storage.execution.list(
+ executions_list = model_storage.execution.list(
filters=filters,
- sort=utils.storage_sort_param(sort_by, descending))]
+ sort=utils.storage_sort_param(sort_by, descending)).items
print_data(EXECUTION_COLUMNS, executions_list, 'Executions:')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/node_templates.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/node_templates.py b/aria/cli/commands/node_templates.py
index c79e125..f0ca2c1 100644
--- a/aria/cli/commands/node_templates.py
+++ b/aria/cli/commands/node_templates.py
@@ -44,7 +44,7 @@ def show(node_template_id, model_storage, logger):
logger.info('Showing node template {0}'.format(node_template_id))
node_template = model_storage.node_template.get(node_template_id)
- print_data(NODE_TEMPLATE_COLUMNS, node_template.to_dict(), 'Node template:', max_width=50)
+ print_data(NODE_TEMPLATE_COLUMNS, node_template, 'Node template:', col_max_width=50)
# print node template properties
logger.info('Node template properties:')
@@ -86,8 +86,8 @@ def list(service_template_name, sort_by, descending, model_storage, logger):
logger.info('Listing all node templates...')
filters = {}
- node_templates_list = [nt.to_dict() for nt in model_storage.node_template.list(
+ node_templates_list = model_storage.node_template.list(
filters=filters,
- sort=utils.storage_sort_param(sort_by, descending))]
+ sort=utils.storage_sort_param(sort_by, descending)).items
print_data(NODE_TEMPLATE_COLUMNS, node_templates_list, 'Node templates:')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/nodes.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/nodes.py b/aria/cli/commands/nodes.py
index b1f2acc..0a00478 100644
--- a/aria/cli/commands/nodes.py
+++ b/aria/cli/commands/nodes.py
@@ -43,7 +43,7 @@ def show(node_id, model_storage, logger):
logger.info('Showing node {0}'.format(node_id))
node = model_storage.node.get(node_id)
- print_data(NODE_COLUMNS, node.to_dict(), 'Node:', 50)
+ print_data(NODE_COLUMNS, node, 'Node:', 50)
# print node attributes
logger.info('Node attributes:')
@@ -52,7 +52,6 @@ def show(node_id, model_storage, logger):
logger.info('\t{0}: {1}'.format(prop_name, prop_value))
else:
logger.info('\tNo attributes')
- logger.info('')
@nodes.command(name='list',
@@ -81,8 +80,8 @@ def list(service_name,
logger.info('Listing all nodes...')
filters = {}
- nodes_list = [node.to_dict() for node in model_storage.node.list(
+ nodes_list = model_storage.node.list(
filters=filters,
- sort=utils.storage_sort_param(sort_by, descending))]
+ sort=utils.storage_sort_param(sort_by, descending)).items
print_data(NODE_COLUMNS, nodes_list, 'Nodes:')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/plugins.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/plugins.py b/aria/cli/commands/plugins.py
index 41a272e..22552d6 100644
--- a/aria/cli/commands/plugins.py
+++ b/aria/cli/commands/plugins.py
@@ -13,11 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import zipfile
-
from .. import utils
from ..core import aria
-from ..exceptions import AriaCliError
from ..table import print_data
@@ -37,52 +34,21 @@ def plugins():
short_help='Validate a plugin')
@aria.argument('plugin-path')
@aria.options.verbose()
+@aria.pass_plugin_manager
@aria.pass_logger
-def validate(plugin_path, logger):
- """Validate a plugin
+def validate(plugin_path, plugin_manager, logger):
+ """Validate a plugin archive
- This will try to validate the plugin's archive is not corrupted.
- A valid plugin is a wagon (http://github.com/cloudify-cosomo/wagon)
+ A valid plugin is a wagon (http://github.com/cloudify-cosmo/wagon)
in the zip format (suffix may also be .wgn).
`PLUGIN_PATH` is the path to wagon archive to validate.
"""
logger.info('Validating plugin {0}...'.format(plugin_path))
-
- if not zipfile.is_zipfile(plugin_path):
- raise AriaCliError(
- 'Archive {0} is of an unsupported type. Only '
- 'zip/wgn is allowed'.format(plugin_path))
- with zipfile.ZipFile(plugin_path, 'r') as zip_file:
- infos = zip_file.infolist()
- try:
- package_name = infos[0].filename[:infos[0].filename.index('/')]
- package_json_path = "{0}/{1}".format(package_name, 'package.json')
- zip_file.getinfo(package_json_path)
- except (KeyError, ValueError, IndexError):
- raise AriaCliError(
- 'Failed to validate plugin {0} '
- '(package.json was not found in archive)'.format(plugin_path))
-
+ plugin_manager.validate_plugin(plugin_path)
logger.info('Plugin validated successfully')
-# @plugins.command(name='delete',
-# short_help='Delete a plugin')
-# @aria.argument('plugin-id')
-# @aria.options.verbose()
-# @aria.pass_model_storage
-# @aria.pass_logger
-# def delete(plugin_id, model_storage, logger):
-# """Delete a plugin
-#
-# `PLUGIN_ID` is the id of the plugin to delete.
-# """
-# logger.info('Deleting plugin {0}...'.format(plugin_id))
-# model_storage.plugin.delete(plugin_id=plugin_id)
-# logger.info('Plugin deleted')
-
-
@plugins.command(name='install',
short_help='Install a plugin')
@aria.argument('plugin-path')
@@ -114,7 +80,7 @@ def show(plugin_id, model_storage, logger):
"""
logger.info('Showing plugin {0}...'.format(plugin_id))
plugin = model_storage.plugin.get(plugin_id)
- print_data(PLUGIN_COLUMNS, plugin.to_dict(), 'Plugin:')
+ print_data(PLUGIN_COLUMNS, plugin, 'Plugin:')
@plugins.command(name='list',
@@ -128,6 +94,6 @@ def list(sort_by, descending, model_storage, logger):
"""List all plugins on the manager
"""
logger.info('Listing all plugins...')
- plugins_list = [p.to_dict() for p in model_storage.plugin.list(
- sort=utils.storage_sort_param(sort_by, descending))]
+ plugins_list = model_storage.plugin.list(
+ sort=utils.storage_sort_param(sort_by, descending)).items
print_data(PLUGIN_COLUMNS, plugins_list, 'Plugins:')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/service_templates.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/service_templates.py b/aria/cli/commands/service_templates.py
index 2ef37c0..189ba51 100644
--- a/aria/cli/commands/service_templates.py
+++ b/aria/cli/commands/service_templates.py
@@ -18,16 +18,16 @@ import os
from .. import csar
from .. import service_template_utils
+from .. import table
from .. import utils
from ..core import aria
-from ..table import print_data
from ...core import Core
from ...storage import exceptions as storage_exceptions
-DESCRIPTION_LIMIT = 20
+DESCRIPTION_FIELD_LENGTH_LIMIT = 20
SERVICE_TEMPLATE_COLUMNS = \
- ['id', 'name', 'main_file_name', 'created_at', 'updated_at']
+ ['id', 'name', 'description', 'main_file_name', 'created_at', 'updated_at']
@aria.group(name='service-templates')
@@ -51,11 +51,14 @@ def show(service_template_name, model_storage, logger):
"""
logger.info('Showing service template {0}...'.format(service_template_name))
service_template = model_storage.service_template.get_by_name(service_template_name)
- services = [d.to_dict() for d in service_template.services]
service_template_dict = service_template.to_dict()
- service_template_dict['#services'] = len(services)
+ service_template_dict['#services'] = len(service_template.services)
+
+ column_formatters = \
+ dict(description=table.trim_formatter_generator(DESCRIPTION_FIELD_LENGTH_LIMIT))
columns = SERVICE_TEMPLATE_COLUMNS + ['#services']
- print_data(columns, service_template_dict, 'Service-template:', max_width=50)
+ table.print_data(columns, service_template_dict, 'Service-template:',
+ column_formatters=column_formatters, col_max_width=50)
if service_template_dict['description'] is not None:
logger.info('Description:')
@@ -63,8 +66,8 @@ def show(service_template_name, model_storage, logger):
os.linesep))
logger.info('Existing services:')
- logger.info('{0}{1}'.format([s['name'] for s in services],
- os.linesep))
+ for service in service_template.services:
+ logger.info('\t{0}'.format(service.name))
@service_templates.command(name='list',
@@ -77,20 +80,15 @@ def show(service_template_name, model_storage, logger):
def list(sort_by, descending, model_storage, logger):
"""List all service templates
"""
- def trim_description(service_template):
- if service_template['description'] is not None:
- if len(service_template['description']) >= DESCRIPTION_LIMIT:
- service_template['description'] = '{0}..'.format(
- service_template['description'][:DESCRIPTION_LIMIT - 2])
- else:
- service_template['description'] = ''
- return service_template
logger.info('Listing all service templates...')
- service_templates_list = [trim_description(b.to_dict()) for b in
- model_storage.service_template.list(
- sort=utils.storage_sort_param(sort_by, descending))]
- print_data(SERVICE_TEMPLATE_COLUMNS, service_templates_list, 'Service templates:')
+ service_templates_list = model_storage.service_template.list(
+ sort=utils.storage_sort_param(sort_by, descending)).items
+
+ column_formatters = \
+ dict(description=table.trim_formatter_generator(DESCRIPTION_FIELD_LENGTH_LIMIT))
+ table.print_data(SERVICE_TEMPLATE_COLUMNS, service_templates_list, 'Service templates:',
+ column_formatters=column_formatters)
@service_templates.command(name='store',
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/services.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/services.py b/aria/cli/commands/services.py
index afa5e42..e1569a6 100644
--- a/aria/cli/commands/services.py
+++ b/aria/cli/commands/services.py
@@ -64,9 +64,9 @@ def list(service_template_name,
logger.info('Listing all services...')
filters = {}
- services_list = [d.to_dict() for d in model_storage.service.list(
+ services_list = model_storage.service.list(
sort=utils.storage_sort_param(sort_by=sort_by, descending=descending),
- filters=filters)]
+ filters=filters).items
print_data(SERVICE_COLUMNS, services_list, 'Services:')
@@ -177,4 +177,3 @@ def inputs(service_name, model_storage, logger):
logger.info(inputs_string.getvalue())
else:
logger.info('\tNo inputs')
- logger.info('')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/commands/workflows.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/workflows.py b/aria/cli/commands/workflows.py
index d380fac..5bd23b7 100644
--- a/aria/cli/commands/workflows.py
+++ b/aria/cli/commands/workflows.py
@@ -52,7 +52,7 @@ def show(workflow_name, service_name, model_storage, logger):
'service_template_name': service.service_template_name,
'service_name': service.name
}
- print_data(WORKFLOW_COLUMNS, workflow.to_dict(), 'Workflows:', defaults=defaults)
+ print_data(WORKFLOW_COLUMNS, workflow, 'Workflows:', defaults=defaults)
# print workflow inputs
required_inputs = dict()
@@ -78,7 +78,6 @@ def show(workflow_name, service_name, model_storage, logger):
else:
logger.info('\t\t{0}: \t{1}'.format(input_name,
input.value))
- logger.info('')
@workflows.command(name='list',
@@ -92,8 +91,7 @@ def list(service_name, model_storage, logger):
"""
logger.info('Listing workflows for service {0}...'.format(service_name))
service = model_storage.service.get_by_name(service_name)
- workflows_list = [wf.to_dict() for wf in
- sorted(service.workflows.values(), key=lambda w: w.name)]
+ workflows_list = sorted(service.workflows.values(), key=lambda w: w.name)
defaults = {
'service_template_name': service.service_template_name,
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/constants.py
----------------------------------------------------------------------
diff --git a/aria/cli/constants.py b/aria/cli/constants.py
deleted file mode 100644
index c68fb5e..0000000
--- a/aria/cli/constants.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-DEFAULT_SERVICE_TEMPLATE_FILENAME = 'service_template.yaml'
-HELP_TEXT_COLUMN_BUFFER = 5
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/core/aria.py
----------------------------------------------------------------------
diff --git a/aria/cli/core/aria.py b/aria/cli/core/aria.py
index fb5a81b..fe6dc4b 100644
--- a/aria/cli/core/aria.py
+++ b/aria/cli/core/aria.py
@@ -27,9 +27,9 @@ from ..env import (
env,
logger
)
+from .. import defaults
from .. import helptexts
from ..inputs import inputs_to_dict
-from ..constants import DEFAULT_SERVICE_TEMPLATE_FILENAME
from ...utils.exceptions import get_exception_as_string
from ... import __version__
@@ -316,13 +316,13 @@ class Options(object):
'--descending',
required=False,
is_flag=True,
- default=False,
+ default=defaults.SORT_DESCENDING_DEFAULT,
help=helptexts.DESCENDING)
self.service_template_filename = click.option(
'-n',
'--service-template-filename',
- default=DEFAULT_SERVICE_TEMPLATE_FILENAME,
+ default=defaults.SERVICE_TEMPLATE_FILENAME_DEFAULT,
help=helptexts.SERVICE_TEMPLATE_FILENAME)
@staticmethod
@@ -354,7 +354,7 @@ class Options(object):
help=help)
@staticmethod
- def task_max_attempts(default=30):
+ def task_max_attempts(default=defaults.TASK_MAX_ATTEMPTS_DEFAULT):
return click.option(
'--task-max-attempts',
type=int,
@@ -370,7 +370,7 @@ class Options(object):
help=helptexts.SORT_BY)
@staticmethod
- def task_retry_interval(default=30):
+ def task_retry_interval(default=defaults.TASK_RETRY_INTERVAL_DEFAULT):
return click.option(
'--task-retry-interval',
type=int,
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/defaults.py
----------------------------------------------------------------------
diff --git a/aria/cli/defaults.py b/aria/cli/defaults.py
new file mode 100644
index 0000000..6befd25
--- /dev/null
+++ b/aria/cli/defaults.py
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+SERVICE_TEMPLATE_FILENAME_DEFAULT = 'service_template.yaml'
+TASK_MAX_ATTEMPTS_DEFAULT = 30
+TASK_RETRY_INTERVAL_DEFAULT = 30
+SORT_DESCENDING_DEFAULT = False
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/helptexts.py
----------------------------------------------------------------------
diff --git a/aria/cli/helptexts.py b/aria/cli/helptexts.py
index 6e31f47..1a3f6c0 100644
--- a/aria/cli/helptexts.py
+++ b/aria/cli/helptexts.py
@@ -29,7 +29,7 @@ EXECUTION_ID = "The unique identifier for the execution"
SERVICE_TEMPLATE_PATH = "The path to the application's service template file"
SERVICE_TEMPLATE_FILENAME = (
"The name of the archive's main service template file. "
- "This is only relevant if uploading a non-csar archive")
+ "This is only relevant if uploading a (non-csar) archive")
INPUTS_PARAMS_USAGE = (
'(Can be provided as wildcard based paths '
'(*.yaml, /my_inputs/, etc..) to YAML files, a JSON string or as '
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/logger.py
----------------------------------------------------------------------
diff --git a/aria/cli/logger.py b/aria/cli/logger.py
index 2f012d9..1ffa918 100644
--- a/aria/cli/logger.py
+++ b/aria/cli/logger.py
@@ -25,7 +25,7 @@ MEDIUM_VERBOSE = 2
LOW_VERBOSE = 1
NO_VERBOSE = 0
-DEFAULT_LOGGER_CONFIG = {
+LOGGER_CONFIG_TEMPLATE = {
"version": 1,
"formatters": {
"file": {
@@ -57,7 +57,7 @@ class Logging(object):
def __init__(self, config):
self._log_file = None
self._verbosity_level = NO_VERBOSE
- self._all_loggers = []
+ self._all_loggers_names = []
self._configure_loggers(config)
self._lgr = logging.getLogger('aria.cli.main')
@@ -73,21 +73,21 @@ class Logging(object):
def verbosity_level(self):
return self._verbosity_level
- def is_high_verbose_level(self):
- return self.verbosity_level == HIGH_VERBOSE
-
@verbosity_level.setter
def verbosity_level(self, level):
self._verbosity_level = level
if self.is_high_verbose_level():
- for logger_name in self._all_loggers:
+ for logger_name in self._all_loggers_names:
logging.getLogger(logger_name).setLevel(logging.DEBUG)
+ def is_high_verbose_level(self):
+ return self.verbosity_level == HIGH_VERBOSE
+
def _configure_loggers(self, config):
loggers_config = config.logging.loggers
logfile = config.logging.filename
- logger_dict = copy.deepcopy(DEFAULT_LOGGER_CONFIG)
+ logger_dict = copy.deepcopy(LOGGER_CONFIG_TEMPLATE)
if logfile:
# set filename on file handler
logger_dict['handlers']['file']['filename'] = logfile
@@ -102,6 +102,7 @@ class Logging(object):
loggers = {}
for logger_name in loggers_config:
loggers[logger_name] = dict(handlers=list(logger_dict['handlers'].keys()))
+ self._all_loggers_names.append(logger_name)
logger_dict['loggers'] = loggers
# set level for all loggers
@@ -109,6 +110,5 @@ class Logging(object):
log = logging.getLogger(logger_name)
level = logging._levelNames[logging_level.upper()]
log.setLevel(level)
- self._all_loggers.append(logger_name)
dictconfig.dictConfig(logger_dict)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/service_template_utils.py
----------------------------------------------------------------------
diff --git a/aria/cli/service_template_utils.py b/aria/cli/service_template_utils.py
index 0300449..382cce1 100644
--- a/aria/cli/service_template_utils.py
+++ b/aria/cli/service_template_utils.py
@@ -26,28 +26,23 @@ def get(source, service_template_filename):
"""Get a source and return a path to the main service template file
The behavior based on then source argument content is:
- -
+ - local yaml file: return the file
- local archive:
extract it locally and return path service template file
- - local yaml file: return the file
- URL:
- - return it (download=False)
- - download and get service template from downloaded file (download=True)
+ - download and get service template from downloaded archive
- github repo:
- - map it to a URL and return it (download=False)
- - download and get service template from downloaded file (download=True)
+ - download and get service template from downloaded archive
Supported archive types are: csar, zip, tar, tar.gz and tar.bz2
:param source: Path/URL/github repo to archive/service-template file
:type source: str
- :param service_template_filename: Path to service template (if source is an archive file)
+ :param service_template_filename: Path to service template (if source is an archive [but
+ not a csar archive - with csars archives, this is read from the metadata file])
:type service_template_filename: str
- :param download: Download service template file if source is URL/github repo
- :type download: bool
- :return: Path to file (if archive/service-template file passed) or url
+ :return: Path to main service template file
:rtype: str
-
"""
if urlparse(source).scheme:
downloaded_file = utils.download_file(source)
@@ -115,26 +110,12 @@ def _map_to_github_url(source):
return url
-# def generate_id(service_template_path,
-# service_template_filename=DEFAULT_SERVICE_TEMPLATE_FILENAME):
-# """The name of the service template will be the name of the folder.
-# If service_template_filename is provided, it will be appended to the folder.
-# """
-# service_template_id = os.path.split(os.path.dirname(os.path.abspath(
-# service_template_path)))[-1]
-# if service_template_filename != DEFAULT_SERVICE_TEMPLATE_FILENAME:
-# filename, _ = os.path.splitext(os.path.basename(service_template_filename))
-# service_template_id = (service_template_id + '.' + filename)
-# return service_template_id.replace('_', '-')
-
-
def _is_archive(source):
return archive_utils.is_archive(source) or csar.is_csar_archive(source)
def _extract_csar_archive(archive):
- if csar.is_csar_archive(archive):
- reader = csar.read(source=archive)
- main_service_template_file_name = os.path.basename(reader.entry_definitions)
- return os.path.join(reader.destination,
- main_service_template_file_name)
+ reader = csar.read(source=archive)
+ main_service_template_file_name = os.path.basename(reader.entry_definitions)
+ return os.path.join(reader.destination,
+ main_service_template_file_name)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/table.py
----------------------------------------------------------------------
diff --git a/aria/cli/table.py b/aria/cli/table.py
index 36dcbea..11d791e 100644
--- a/aria/cli/table.py
+++ b/aria/cli/table.py
@@ -21,7 +21,25 @@ from prettytable import PrettyTable
from .env import logger
-def generate(cols, data, defaults=None):
+def print_data(columns, items, header_text,
+ column_formatters=None, col_max_width=None, defaults=None):
+ if items is None:
+ items = []
+ elif not isinstance(items, list):
+ items = [items]
+
+ pretty_table = _generate(columns, data=items, column_formatters=column_formatters,
+ defaults=defaults)
+ if col_max_width:
+ pretty_table.max_width = col_max_width
+ _log(header_text, pretty_table)
+
+
+def _log(title, table):
+ logger.info('{0}{1}{0}{2}{0}'.format(os.linesep, title, table))
+
+
+def _generate(cols, data, column_formatters=None, defaults=None):
"""
Return a new PrettyTable instance representing the list.
@@ -32,10 +50,16 @@ def generate(cols, data, defaults=None):
for example: ['id','name']
- data - An iterable of dictionaries, each dictionary must
- have key's corresponding to the cols items.
+ data - An iterable of dictionaries or objects, each element must
+ have keys or attributes corresponding to the cols items.
+
+ for example: [{'id':'123', 'name':'Pete'}]
- for example: [{'id':'123', 'name':'Pete']
+ column_formatters - A dictionary from a column name to a function that may manipulate
+ the values printed for this column.
+ (See below for a few built-in formatter examples)
+
+ for example: {'created_at': timestamp_formatter}
defaults - A dictionary specifying default values for
key's that don't exist in the data itself.
@@ -45,25 +69,26 @@ def generate(cols, data, defaults=None):
"""
def get_values_per_column(column, row_data):
- if column in row_data:
- if row_data[column] and isinstance(row_data[column], basestring):
- try:
- datetime.strptime(row_data[column][:10], '%Y-%m-%d')
- row_data[column] = \
- row_data[column].replace('T', ' ').replace('Z', ' ')
- except ValueError:
- # not a timestamp
- pass
- elif row_data[column] and isinstance(row_data[column], list):
- row_data[column] = ','.join(row_data[column])
- elif not row_data[column]:
- # if it's empty list, don't print []
- row_data[column] = ''
- return row_data[column]
+ if hasattr(row_data, column) or (isinstance(row_data, dict) and column in row_data):
+ val = row_data[column] if isinstance(row_data, dict) else getattr(row_data, column)
+
+ if val and isinstance(val, list):
+ val = [str(element) for element in val]
+ val = ','.join(val)
+ elif val is None or isinstance(val, list):
+ # don't print `[]` or `None` (but do print `0`, `False`, etc.)
+ val = ''
+
+ if column in column_formatters:
+ # calling the user's column formatter to manipulate the value
+ val = column_formatters[column](val)
+
+ return val
else:
return defaults[column]
- pretty_table = PrettyTable([col for col in cols])
+ column_formatters = column_formatters or dict()
+ pretty_table = PrettyTable(list(cols))
for datum in data:
values_row = []
@@ -74,17 +99,18 @@ def generate(cols, data, defaults=None):
return pretty_table
-def log(title, table):
- logger.info('{0}{1}{0}{2}{0}'.format(os.linesep, title, table))
-
+def timestamp_formatter(value):
+ try:
+ datetime.strptime(value[:10], '%Y-%m-%d')
+ return value.replace('T', ' ').replace('Z', ' ')
+ except ValueError:
+ # not a timestamp
+ return value
-def print_data(columns, items, header_text, max_width=None, defaults=None):
- if items is None:
- items = []
- elif not isinstance(items, list):
- items = [items]
- pretty_table = generate(columns, data=items, defaults=defaults)
- if max_width:
- pretty_table.max_width = max_width
- log(header_text, pretty_table)
+def trim_formatter_generator(max_length):
+ def trim_formatter(value):
+ if len(value) >= max_length:
+ value = '{0}..'.format(value[:max_length - 2])
+ return value
+ return trim_formatter
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/cli/utils.py
----------------------------------------------------------------------
diff --git a/aria/cli/utils.py b/aria/cli/utils.py
index 3cc68c9..852f24d 100644
--- a/aria/cli/utils.py
+++ b/aria/cli/utils.py
@@ -15,41 +15,19 @@
import os
import sys
-import string
-import random
-import tempfile
from StringIO import StringIO
from backports.shutil_get_terminal_size import get_terminal_size
-import requests
from .env import logger
from .exceptions import AriaCliError
-
-
-def dump_to_file(collection, file_path):
- with open(file_path, 'a') as f:
- f.write(os.linesep.join(collection))
- f.write(os.linesep)
-
-
-def is_virtual_env():
- return hasattr(sys, 'real_prefix')
+from ..utils import http
def storage_sort_param(sort_by, descending):
return {sort_by: 'desc' if descending else 'asc'}
-def generate_random_string(size=6,
- chars=string.ascii_uppercase + string.digits):
- return ''.join(random.choice(chars) for _ in range(size))
-
-
-def generate_suffixed_id(id):
- return '{0}_{1}'.format(id, generate_random_string())
-
-
def get_parameter_templates_as_string(parameter_templates):
params_string = StringIO()
@@ -64,43 +42,28 @@ def get_parameter_templates_as_string(parameter_templates):
return params_string.getvalue()
-def download_file(url, destination=None):
- """Download file.
-
- :param url: Location of the file to download
- :type url: str
- :param destination:
- Location where the file should be saved (autogenerated by default)
- :type destination: str | None
- :returns: Location where the file was saved
- :rtype: str
-
+def check_overriding_storage_exceptions(e, model_class, name):
"""
- chunk_size = 1024
-
- if not destination:
- file_descriptor, destination = tempfile.mkstemp()
- os.close(file_descriptor)
- logger.info('Downloading {0} to {1}...'.format(url, destination))
-
- try:
- response = requests.get(url, stream=True)
- except requests.exceptions.RequestException as ex:
- raise AriaCliError(
- 'Failed to download {0}. ({1})'.format(url, str(ex)))
+ This method checks whether the storage exception is a known type where we'd like to override
+ the exception message; If so, it raises a new error. Otherwise it simply returns.
+ """
+ assert isinstance(e, BaseException)
+ if 'UNIQUE constraint failed' in e.message:
+ new_message = \
+ 'Could not store {model_class} `{name}`{linesep}' \
+ 'There already a exists a {model_class} with the same name' \
+ .format(model_class=model_class, name=name, linesep=os.linesep)
+ trace = sys.exc_info()[2]
+ raise type(e), type(e)(new_message), trace # pylint: disable=raising-non-exception
- final_url = response.url
- if final_url != url:
- logger.debug('Redirected to {0}'.format(final_url))
+def download_file(url):
+ progress_bar = generate_progress_handler(url, 'Downloading')
try:
- with open(destination, 'wb') as destination_file:
- for chunk in response.iter_content(chunk_size):
- destination_file.write(chunk)
- except IOError as ex:
+ destination = http.download_file(url, logger=logger, progress_handler=progress_bar)
+ except Exception as e:
raise AriaCliError(
- 'Failed to download {0}. ({1})'.format(url, str(ex)))
-
+ 'Failed to download {0}. ({1})'.format(url, str(e)))
return destination
@@ -150,18 +113,3 @@ def generate_progress_handler(file_path, action='', max_bar_length=80):
sys.stdout.write(os.linesep)
return print_progress
-
-
-def check_overriding_storage_exceptions(e, model_class, name):
- """
- This method checks whether the storage exception is a known type where we'd like to override
- the exception message; If so, it raises a new error. Otherwise it simply returns.
- """
- assert isinstance(e, BaseException)
- if 'UNIQUE constraint failed' in e.message:
- new_message = \
- 'Could not store {model_class} `{name}`{linesep}' \
- 'There already a exists a {model_class} with the same name' \
- .format(model_class=model_class, name=name, linesep=os.linesep)
- trace = sys.exc_info()[2]
- raise type(e), type(e)(new_message), trace # pylint: disable=raising-non-exception
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/modeling/service_common.py
----------------------------------------------------------------------
diff --git a/aria/modeling/service_common.py b/aria/modeling/service_common.py
index e7fda29..1188f34 100644
--- a/aria/modeling/service_common.py
+++ b/aria/modeling/service_common.py
@@ -87,14 +87,8 @@ class ParameterBase(TemplateModelMixin):
if self.description:
console.puts(context.style.meta(self.description))
- @staticmethod
- def unwrap_dict(parameters_dict):
- """
- Takes a parameters dict and simplifies it into key-value dict
- :param parameters_dict: a parameter-name to parameter dict
- :return: a parameter-name to parameter value dict
- """
- return dict((k, v.value) for k, v in parameters_dict.iteritems())
+ def unwrap(self):
+ return self.name, self.value
@classmethod
def wrap(cls, name, value, description=None):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/exceptions.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/exceptions.py b/aria/orchestrator/exceptions.py
index fd3b66d..8d3dcc6 100644
--- a/aria/orchestrator/exceptions.py
+++ b/aria/orchestrator/exceptions.py
@@ -25,6 +25,13 @@ class OrchestratorError(AriaError):
pass
+class InvalidPluginError(AriaError):
+ """
+ Raised when an invalid plugin is validated unsuccessfully
+ """
+ pass
+
+
class PluginAlreadyExistsError(AriaError):
"""
Raised when a plugin with the same package name and package version already exists
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/plugin.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/plugin.py b/aria/orchestrator/plugin.py
index d526e9c..b79d7fc 100644
--- a/aria/orchestrator/plugin.py
+++ b/aria/orchestrator/plugin.py
@@ -17,6 +17,7 @@ import os
import tempfile
import subprocess
import sys
+import zipfile
from datetime import datetime
import wagon
@@ -69,6 +70,28 @@ class PluginManager(object):
self._plugins_dir,
'{0}-{1}'.format(plugin.package_name, plugin.package_version))
+ @staticmethod
+ def validate_plugin(source):
+ """
+ validate a plugin archive.
+ A valid plugin is a wagon (http://github.com/cloudify-cosmo/wagon)
+ in the zip format (suffix may also be .wgn).
+ """
+ if not zipfile.is_zipfile(source):
+ raise exceptions.InvalidPluginError(
+ 'Archive {0} is of an unsupported type. Only '
+ 'zip/wgn is allowed'.format(source))
+ with zipfile.ZipFile(source, 'r') as zip_file:
+ infos = zip_file.infolist()
+ try:
+ package_name = infos[0].filename[:infos[0].filename.index('/')]
+ package_json_path = "{0}/{1}".format(package_name, 'package.json')
+ zip_file.getinfo(package_json_path)
+ except (KeyError, ValueError, IndexError):
+ raise exceptions.InvalidPluginError(
+ 'Failed to validate plugin {0} '
+ '(package.json was not found in archive)'.format(source))
+
def _install_wagon(self, source, prefix):
pip_freeze_output = self._pip_freeze()
file_descriptor, constraint_path = tempfile.mkstemp(prefix='constraint-', suffix='.txt')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/workflow_runner.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflow_runner.py b/aria/orchestrator/workflow_runner.py
index 0051e8e..8779f06 100644
--- a/aria/orchestrator/workflow_runner.py
+++ b/aria/orchestrator/workflow_runner.py
@@ -80,7 +80,7 @@ class WorkflowRunner(object):
task_retry_interval=task_retry_interval)
# transforming the execution inputs to dict, to pass them to the workflow function
- execution_inputs_dict = models.Parameter.unwrap_dict(self.execution.inputs)
+ execution_inputs_dict = dict(inp.unwrap() for inp in self.execution.inputs.values())
self._tasks_graph = workflow_fn(ctx=workflow_context, **execution_inputs_dict)
executor = executor or ProcessExecutor(plugin_manager=plugin_manager)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/workflows/executor/celery.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/celery.py b/aria/orchestrator/workflows/executor/celery.py
index 3c98197..7bd9b7c 100644
--- a/aria/orchestrator/workflows/executor/celery.py
+++ b/aria/orchestrator/workflows/executor/celery.py
@@ -22,8 +22,6 @@ import Queue
from aria.orchestrator.workflows.executor import BaseExecutor
-from ....modeling.models import Parameter
-
class CeleryExecutor(BaseExecutor):
"""
@@ -46,7 +44,7 @@ class CeleryExecutor(BaseExecutor):
def execute(self, task):
self._tasks[task.id] = task
- inputs = Parameter.unwrap_dict(task.inputs.iteritems())
+ inputs = dict(inp.unwrap() for inp in task.inputs.values())
inputs['ctx'] = task.context
self._results[task.id] = self._app.send_task(
task.operation_mapping,
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/workflows/executor/dry.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/dry.py b/aria/orchestrator/workflows/executor/dry.py
index b14f5d7..d894b25 100644
--- a/aria/orchestrator/workflows/executor/dry.py
+++ b/aria/orchestrator/workflows/executor/dry.py
@@ -20,7 +20,6 @@ Dry executor
from datetime import datetime
from .base import BaseExecutor
-from ....modeling.models import Parameter
class DryExecutor(BaseExecutor):
@@ -38,7 +37,7 @@ class DryExecutor(BaseExecutor):
actor_type = type(task.actor).__name__.lower()
implementation = '{0} > '.format(task.plugin) if task.plugin else ''
implementation += task.implementation
- inputs = Parameter.unwrap_dict(task.inputs)
+ inputs = dict(inp.unwrap() for inp in task.inputs.values())
task.context.logger.info(
'Executing {actor_type} {task.actor.name} operation {task.interface_name} '
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/workflows/executor/process.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py
index 3c2b5fe..851d78e 100644
--- a/aria/orchestrator/workflows/executor/process.py
+++ b/aria/orchestrator/workflows/executor/process.py
@@ -48,7 +48,6 @@ from aria.utils import exceptions
from aria.orchestrator.workflows.executor import base
from aria.storage import instrumentation
from aria.modeling import types as modeling_types
-from aria.modeling.models import Parameter
_IS_WIN = os.name == 'nt'
@@ -149,7 +148,7 @@ class ProcessExecutor(base.BaseExecutor):
return {
'task_id': task.id,
'implementation': task.implementation,
- 'operation_inputs': Parameter.unwrap_dict(task.inputs),
+ 'operation_inputs': dict(inp.unwrap() for inp in task.inputs.values()),
'port': self._server_port,
'context': task.context.serialization_dict,
}
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/orchestrator/workflows/executor/thread.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/thread.py b/aria/orchestrator/workflows/executor/thread.py
index 8b443cc..f422592 100644
--- a/aria/orchestrator/workflows/executor/thread.py
+++ b/aria/orchestrator/workflows/executor/thread.py
@@ -23,7 +23,6 @@ import threading
from aria.utils import imports
from .base import BaseExecutor
-from ....modeling.models import Parameter
class ThreadExecutor(BaseExecutor):
@@ -60,7 +59,7 @@ class ThreadExecutor(BaseExecutor):
self._task_started(task)
try:
task_func = imports.load_attribute(task.implementation)
- inputs = Parameter.unwrap_dict(task.inputs)
+ inputs = dict(inp.unwrap() for inp in task.inputs.values())
task_func(ctx=task.context, **inputs)
self._task_succeeded(task)
except BaseException as e:
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1cbd81b3/aria/utils/http.py
----------------------------------------------------------------------
diff --git a/aria/utils/http.py b/aria/utils/http.py
new file mode 100644
index 0000000..7bdfd79
--- /dev/null
+++ b/aria/utils/http.py
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+
+import requests
+
+
+def download_file(url, destination=None, logger=None, progress_handler=None):
+ """Download file.
+
+ May raise IOError as well as requests.exceptions.RequestException
+ :param url: Location of the file to download
+ :type url: str
+ :param destination:
+ Location where the file should be saved (autogenerated by default)
+ :type destination: str | None
+ :returns: Location where the file was saved
+ :rtype: str
+
+ """
+ chunk_size = 1024
+
+ if not destination:
+ file_descriptor, destination = tempfile.mkstemp()
+ os.close(file_descriptor)
+ if logger:
+ logger.info('Downloading {0} to {1}...'.format(url, destination))
+
+ response = requests.get(url, stream=True)
+ final_url = response.url
+ if final_url != url and logger:
+ logger.debug('Redirected to {0}'.format(final_url))
+
+ read_bytes = 0
+ total_size = int(response.headers['Content-Length']) \
+ if 'Content-Length' in response.headers else None
+ try:
+ with open(destination, 'wb') as destination_file:
+ for chunk in response.iter_content(chunk_size):
+ destination_file.write(chunk)
+ if total_size and progress_handler:
+ # Only showing progress bar if we have the total content length
+ read_bytes += chunk_size
+ progress_handler(read_bytes, total_size)
+ finally:
+ response.close()
+
+ return destination