You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by mx...@apache.org on 2017/04/13 14:10:25 UTC
[1/7] incubator-ariatosca git commit: fixed ssh tests [Forced Update!]
Repository: incubator-ariatosca
Updated Branches:
refs/heads/ARIA-138-Make-logging-more-informative ad4e78add -> 661dafa9a (forced update)
fixed ssh tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/402eeb45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/402eeb45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/402eeb45
Branch: refs/heads/ARIA-138-Make-logging-more-informative
Commit: 402eeb45113b5ee873262dabe9e6f5812d5cac2d
Parents: e1b682b
Author: Ran Ziv <ra...@gigaspaces.com>
Authored: Thu Apr 13 11:39:16 2017 +0300
Committer: Ran Ziv <ra...@gigaspaces.com>
Committed: Thu Apr 13 11:39:16 2017 +0300
----------------------------------------------------------------------
tests/orchestrator/execution_plugin/test_ssh.py | 44 ++++++++++++--------
1 file changed, 27 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/402eeb45/tests/orchestrator/execution_plugin/test_ssh.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_ssh.py b/tests/orchestrator/execution_plugin/test_ssh.py
index d17def1..5305d00 100644
--- a/tests/orchestrator/execution_plugin/test_ssh.py
+++ b/tests/orchestrator/execution_plugin/test_ssh.py
@@ -217,29 +217,39 @@ class TestWithActualSSHServer(object):
@workflow
def mock_workflow(ctx, graph):
node = ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME)
+ inputs = {
+ 'script_path': script_path,
+ 'fabric_env': _FABRIC_ENV,
+ 'process': process,
+ 'use_sudo': use_sudo,
+ 'hide_output': hide_output,
+ 'custom_env_var': custom_input,
+ 'test_operation': '',
+ 'commands': commands
+ }
interface = mock.models.create_interface(
node.service,
'test',
'op',
- operation_kwargs=dict(implementation='{0}.{1}'.format(
- operations.__name__,
- operation.__name__))
+ operation_kwargs=dict(
+ implementation='{0}.{1}'.format(
+ operations.__name__,
+ operation.__name__),
+ inputs=inputs)
)
node.interfaces[interface.name] = interface
- graph.sequence(*[api.task.OperationTask.for_node(
- node=node,
- interface_name='test',
- operation_name='op',
- inputs={
- 'script_path': script_path,
- 'fabric_env': _FABRIC_ENV,
- 'process': process,
- 'use_sudo': use_sudo,
- 'hide_output': hide_output,
- 'custom_env_var': custom_input,
- 'test_operation': test_operation,
- 'commands': commands
- }) for test_operation in test_operations])
+
+ ops = []
+ for test_operation in test_operations:
+ op_inputs = inputs.copy()
+ op_inputs['test_operation'] = test_operation
+ ops.append(api.task.OperationTask.for_node(
+ node=node,
+ interface_name='test',
+ operation_name='op',
+ inputs=op_inputs))
+
+ graph.sequence(*ops)
return graph
tasks_graph = mock_workflow(ctx=self._workflow_context) # pylint: disable=no-value-for-parameter
eng = engine.Engine(
[4/7] incubator-ariatosca git commit: fixed pylint issues in tests
Posted by mx...@apache.org.
fixed pylint issues in tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/207cc093
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/207cc093
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/207cc093
Branch: refs/heads/ARIA-138-Make-logging-more-informative
Commit: 207cc093c6fe66fd9b1692a91f365fce1c72154a
Parents: 7f81c42
Author: Ran Ziv <ra...@gigaspaces.com>
Authored: Thu Apr 13 14:47:18 2017 +0300
Committer: Ran Ziv <ra...@gigaspaces.com>
Committed: Thu Apr 13 14:47:18 2017 +0300
----------------------------------------------------------------------
tests/cli/__init__.py | 2 +-
tests/cli/base_test.py | 1 -
tests/orchestrator/test_workflow_runner.py | 3 +--
3 files changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/207cc093/tests/cli/__init__.py
----------------------------------------------------------------------
diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py
index 13878a1..ae1e83e 100644
--- a/tests/cli/__init__.py
+++ b/tests/cli/__init__.py
@@ -11,4 +11,4 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/207cc093/tests/cli/base_test.py
----------------------------------------------------------------------
diff --git a/tests/cli/base_test.py b/tests/cli/base_test.py
index 783631b..55da476 100644
--- a/tests/cli/base_test.py
+++ b/tests/cli/base_test.py
@@ -55,4 +55,3 @@ def get_default_logger_config():
_default_logger_config = get_default_logger_config()
-
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/207cc093/tests/orchestrator/test_workflow_runner.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/test_workflow_runner.py b/tests/orchestrator/test_workflow_runner.py
index aa89ac5..54e940f 100644
--- a/tests/orchestrator/test_workflow_runner.py
+++ b/tests/orchestrator/test_workflow_runner.py
@@ -174,11 +174,10 @@ def test_execution_inputs_override_workflow_inputs(request):
inputs=dict((name, models.Parameter.wrap(name, val)) for name, val
in wf_inputs.iteritems()))
- with mock.patch('aria.orchestrator.workflow_runner.Engine') as mock_engine_cls:
+ with mock.patch('aria.orchestrator.workflow_runner.Engine'):
workflow_runner = _create_workflow_runner(
request, mock_workflow, inputs={'input2': 'overriding-value2', 'input3': 7})
- _, engine_kwargs = mock_engine_cls.call_args
assert len(workflow_runner.execution.inputs) == 3
# did not override input1 - expecting the default value from the workflow inputs
assert workflow_runner.execution.inputs['input1'].value == 'value1'
[7/7] incubator-ariatosca git commit: added load to logs,
enables to get the formatted string
Posted by mx...@apache.org.
added load to logs, enables to get the formatted string
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/661dafa9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/661dafa9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/661dafa9
Branch: refs/heads/ARIA-138-Make-logging-more-informative
Commit: 661dafa9a403fb581f8910026c7841cc4f503253
Parents: 9e2b70c
Author: max-orlov <ma...@gigaspaces.com>
Authored: Thu Apr 13 12:00:52 2017 +0300
Committer: max-orlov <ma...@gigaspaces.com>
Committed: Thu Apr 13 17:10:18 2017 +0300
----------------------------------------------------------------------
aria/cli/commands/executions.py | 4 +-
aria/cli/commands/logs.py | 2 +-
aria/cli/execution_logging.py | 82 +++++++++++++++++++++---------------
3 files changed, 50 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/661dafa9/aria/cli/commands/executions.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/executions.py b/aria/cli/commands/executions.py
index 4bdc481..bd421d2 100644
--- a/aria/cli/commands/executions.py
+++ b/aria/cli/commands/executions.py
@@ -146,13 +146,13 @@ def start(workflow_name,
try:
while execution_thread.is_alive():
for log in log_consumer:
- execution_logging.log(log)
+ execution_logging.load(log).log()
except KeyboardInterrupt:
_cancel_execution(workflow_runner, execution_thread, logger)
for log in log_consumer:
- execution_logging.log(log)
+ execution_logging.load(log).log()
# raise any errors from the execution thread (note these are not workflow execution errors)
execution_thread.raise_error_if_exists()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/661dafa9/aria/cli/commands/logs.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/logs.py b/aria/cli/commands/logs.py
index 4d5d4e2..e87ee3b 100644
--- a/aria/cli/commands/logs.py
+++ b/aria/cli/commands/logs.py
@@ -38,7 +38,7 @@ def list(execution_id, model_storage, logger):
any_logs = False
for log in log_consumer:
- execution_logging.log(log)
+ execution_logging.load(log).log()
any_logs = True
if not any_logs:
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/661dafa9/aria/cli/execution_logging.py
----------------------------------------------------------------------
diff --git a/aria/cli/execution_logging.py b/aria/cli/execution_logging.py
index 9f83310..2ec5c5e 100644
--- a/aria/cli/execution_logging.py
+++ b/aria/cli/execution_logging.py
@@ -14,45 +14,57 @@
# limitations under the License.
import os
+from contextlib import contextmanager
from . import logger
from .env import env
+DEFAULT_FORMATTING = {
+ logger.NO_VERBOSE: {'main_msg': '{item.msg}'},
+ logger.LOW_VERBOSE: {
+ 'main_msg': '{created_at} | {item.level[0]} | {item.msg}',
+ 'created_at': '%H:%M:%S'
+ }
+}
-def log(item):
- formats = {
- logger.NO_VERBOSE: {'main_msg': '{item.msg}'},
- logger.LOW_VERBOSE: {
- 'main_msg': '{created_at} | {item.level[0]} | {item.msg}',
- 'created_at': '%H:%M:%S'
- }
- }
+class _ExecutionLogging(object):
+
+ def __init__(self, item, formats=None):
+ self._item = item
+ self._formats = formats or DEFAULT_FORMATTING
+
+ def __repr__(self):
+ # Only NO_VERBOSE and LOW_VERBOSE are configurable formats. configuring
+ # the low verbose level should affect any higher level.
+ formats = self._formats[min(env.logging.verbosity_level, logger.LOW_VERBOSE)]
+
+ kwargs = dict(item=self._item)
+ if 'created_at' in formats:
+ kwargs['created_at'] = self._item.created_at.strftime(formats['created_at'])
+ if 'level' in formats:
+ kwargs['level'] = formats['level'].format(self._item.level)
+ if 'msg' in formats:
+ kwargs['msg'] = formats['msg'].format(self._item.msg)
+
+ if 'actor' in formats and self._item.task:
+ kwargs['actor'] = formats['actor'].format(self._item.task.actor)
+ if 'execution' in formats:
+ kwargs['execution'] = formats['execution'].format(self._item.execution)
+
+ # If no format was supplied just print out the original msg.
+ msg = formats.get('main_msg', '{item.msg}').format(**kwargs)
+
+ # Add the exception and the error msg.
+ if self._item.traceback and env.logging.verbosity_level >= logger.MEDIUM_VERBOSE:
+ msg += os.linesep + '------>'
+ for line in self._item.traceback.splitlines(True):
+ msg += '\t' + '|' + line
+
+ return msg
+
+ def log(self, *args, **kwargs):
+ return getattr(env.logging.logger, self._item.level.lower())(self)
+
- # Only NO_VERBOSE and LOW_VERBOSE are configurable formats. configuring
- # the low verbose level should affect any higher level.
- formats = formats[min(env.logging.verbosity_level, logger.LOW_VERBOSE)]
-
- kwargs = dict(item=item)
- if 'created_at' in formats:
- kwargs['created_at'] = item.created_at.strftime(formats['created_at'])
- if 'level' in formats:
- kwargs['level'] = formats['level'].format(item.level)
- if 'msg' in formats:
- kwargs['msg'] = formats['msg'].format(item.msg)
-
- if 'actor' in formats and item.task:
- kwargs['actor'] = formats['actor'].format(item.task.actor)
- if 'execution' in formats:
- kwargs['execution'] = formats['execution'].format(item.execution)
-
- # If no format was supplied just print out the original msg.
- msg = formats.get('main_msg', '{item.msg}').format(**kwargs)
-
- # Add the exception and the error msg.
- if item.traceback and env.logging.verbosity_level >= logger.MEDIUM_VERBOSE:
- msg += os.linesep + '------>'
- for line in item.traceback.splitlines(True):
- msg += '\t' + '|' + line
-
- return getattr(env.logging.logger, item.level.lower())(msg)
+load = _ExecutionLogging
[5/7] incubator-ariatosca git commit: added default NullHandler
support for py26
Posted by mx...@apache.org.
added default NullHandler support for py26
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/d6458c03
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/d6458c03
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/d6458c03
Branch: refs/heads/ARIA-138-Make-logging-more-informative
Commit: d6458c0391877f531471fec5764ee2d2dec21ec8
Parents: 207cc09
Author: max-orlov <ma...@gigaspaces.com>
Authored: Thu Apr 13 17:05:57 2017 +0300
Committer: max-orlov <ma...@gigaspaces.com>
Committed: Thu Apr 13 17:05:57 2017 +0300
----------------------------------------------------------------------
aria/logger.py | 9 +++++++++
aria/storage/core.py | 5 ++---
2 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d6458c03/aria/logger.py
----------------------------------------------------------------------
diff --git a/aria/logger.py b/aria/logger.py
index bbb6c7a..41e1ac8 100644
--- a/aria/logger.py
+++ b/aria/logger.py
@@ -21,6 +21,15 @@ import logging
from logging import handlers as logging_handlers
from datetime import datetime
+# NullHandler doesn't exist in < 27. this workaround is from
+# http://docs.python.org/release/2.6/library/logging.html#configuring-logging-for-a-library
+try:
+ from logging import NullHandler
+except ImportError:
+ class NullHandler(logging.Handler):
+ def emit(self, record):
+ pass
+
TASK_LOGGER_NAME = 'aria.executions.task'
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/d6458c03/aria/storage/core.py
----------------------------------------------------------------------
diff --git a/aria/storage/core.py b/aria/storage/core.py
index 0e900bb..8caca66 100644
--- a/aria/storage/core.py
+++ b/aria/storage/core.py
@@ -37,9 +37,8 @@ API:
* drivers - module, a pool of ARIA standard drivers.
* StorageDriver - class, abstract model implementation.
"""
-import logging
-from aria.logger import LoggerMixin
+from aria.logger import LoggerMixin, NullHandler
from . import sql_mapi
__all__ = (
@@ -75,7 +74,7 @@ class Storage(LoggerMixin):
# Set the logger handler of any storage object to NullHandler.
# This is since the absence of a handler shows up while using the CLI in the form of:
# `No handlers could be found for logger "aria.ResourceStorage"`.
- self.logger.addHandler(logging.NullHandler())
+ self.logger.addHandler(NullHandler())
self.api = api_cls
self.registered = {}
self._initiator = initiator
[3/7] incubator-ariatosca git commit: add logutils to support
configuring logging via dict
Posted by mx...@apache.org.
add logutils to support configuring logging via dict
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/7f81c423
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/7f81c423
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/7f81c423
Branch: refs/heads/ARIA-138-Make-logging-more-informative
Commit: 7f81c4236ff3518fc4918e5d4715149b057ccf05
Parents: 7b8a418
Author: max-orlov <ma...@gigaspaces.com>
Authored: Thu Apr 13 14:39:34 2017 +0300
Committer: max-orlov <ma...@gigaspaces.com>
Committed: Thu Apr 13 14:43:30 2017 +0300
----------------------------------------------------------------------
aria/cli/logger.py | 4 ++--
requirements.in | 4 ++++
requirements.txt | 17 ++++++++---------
3 files changed, 14 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7f81c423/aria/cli/logger.py
----------------------------------------------------------------------
diff --git a/aria/cli/logger.py b/aria/cli/logger.py
index cbb916d..2f012d9 100644
--- a/aria/cli/logger.py
+++ b/aria/cli/logger.py
@@ -17,7 +17,7 @@
import os
import copy
import logging
-import logging.config
+from logutils import dictconfig
HIGH_VERBOSE = 3
@@ -111,4 +111,4 @@ class Logging(object):
log.setLevel(level)
self._all_loggers.append(logger_name)
- logging.config.dictConfig(logger_dict)
+ dictconfig.dictConfig(logger_dict)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7f81c423/requirements.in
----------------------------------------------------------------------
diff --git a/requirements.in b/requirements.in
index 32e2d04..3950140 100644
--- a/requirements.in
+++ b/requirements.in
@@ -10,6 +10,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# In order to create the requirements.txt file, execute
+# pip-compile --output-file requirements.txt requirements.in (pip-tools package is needed).
+
PyYAML<3.13
requests>=2.3.0, <2.14.0
networkx>=1.9, <1.10 # version 1.10 dropped support of python 2.6
@@ -30,6 +33,7 @@ colorama>=0.3.3, < 0.3.5
PrettyTable>=0.7,<0.8
click_didyoumean==0.0.3
backports.shutil_get_terminal_size==1.0.0
+logutils==0.3.4.1
# Since the tool we are using to generate our requirements.txt, `pip-tools`,
# does not currently support conditional dependencies (;), we're adding our original
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7f81c423/requirements.txt
----------------------------------------------------------------------
diff --git a/requirements.txt b/requirements.txt
index aba6aa7..3accaa3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,21 +4,19 @@
#
# pip-compile --output-file requirements.txt requirements.in
#
-
-# ----------------------------------------------------------------------------------
# Since the tool we are using to generate our requirements.txt, `pip-tools`,
# does not currently support conditional dependencies (;), we're adding our original
-# conditional dependencies here manually.
+# conditional dependencies here as comments, and manually adding them to our
+# generated requirements.txt file.
# The relevant pip-tools issue: https://github.com/jazzband/pip-tools/issues/435
-importlib==1.0.4 ; python_version < '2.7'
-ordereddict==1.1 ; python_version < '2.7'
-total-ordering==0.1.0 ; python_version < '2.7'
+importlib ; python_version < '2.7'
+ordereddict ; python_version < '2.7'
+total-ordering ; python_version < '2.7' # only one version on pypi
# Fabric makes use of this library, but doesn't bring it :(
pypiwin32==219 ; sys_platform == 'win32'
# ----------------------------------------------------------------------------------
-appdirs==1.4.3 # via setuptools
args==0.1.0 # via clint
asn1crypto==0.22.0 # via cryptography
backports.shutil_get_terminal_size==1.0.0
@@ -39,10 +37,11 @@ ipaddress==1.0.18 # via cryptography
jinja2==2.8.1
jsonpickle==0.9.4
lockfile==0.12.2 # via cachecontrol
+logutils==0.3.4.1
markupsafe==1.0 # via jinja2
msgpack-python==0.4.8 # via cachecontrol
networkx==1.9.1
-packaging==16.8 # via cryptography, setuptools
+packaging==16.8 # via cryptography
paramiko==2.1.2 # via fabric
prettytable==0.7.2
pyasn1==0.2.3 # via paramiko
@@ -54,7 +53,7 @@ retrying==1.3.3
ruamel.ordereddict==0.4.9 # via ruamel.yaml
ruamel.yaml==0.11.15
shortuuid==0.5.0
-six==1.10.0 # via cryptography, packaging, retrying, setuptools
+six==1.10.0 # via cryptography, packaging, retrying
sqlalchemy==1.1.6
wagon==0.6.0
wheel==0.29.0 # via wagon
[2/7] incubator-ariatosca git commit: fixed pylint issues
Posted by mx...@apache.org.
fixed pylint issues
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/7b8a4187
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/7b8a4187
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/7b8a4187
Branch: refs/heads/ARIA-138-Make-logging-more-informative
Commit: 7b8a4187948683a63355d652636254de7abc3b1a
Parents: 402eeb4
Author: Ran Ziv <ra...@gigaspaces.com>
Authored: Thu Apr 13 12:23:26 2017 +0300
Committer: Ran Ziv <ra...@gigaspaces.com>
Committed: Thu Apr 13 12:23:26 2017 +0300
----------------------------------------------------------------------
aria/.pylintrc | 2 +-
aria/modeling/utils.py | 4 ++--
aria/orchestrator/context/common.py | 5 +++--
aria/orchestrator/context/operation.py | 2 --
aria/orchestrator/context/workflow.py | 4 +---
aria/utils/threading.py | 4 ++--
aria/utils/type.py | 2 +-
tests/.pylintrc | 2 +-
8 files changed, 11 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7b8a4187/aria/.pylintrc
----------------------------------------------------------------------
diff --git a/aria/.pylintrc b/aria/.pylintrc
index 7222605..7da8c56 100644
--- a/aria/.pylintrc
+++ b/aria/.pylintrc
@@ -77,7 +77,7 @@ confidence=
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,logging-format-interpolation,import-error,redefined-variable-type,broad-except,protected-access,global-statement,no-member
+disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,logging-format-interpolation,import-error,redefined-variable-type,broad-except,protected-access,global-statement,no-member,u
nused-argument
[REPORTS]
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7b8a4187/aria/modeling/utils.py
----------------------------------------------------------------------
diff --git a/aria/modeling/utils.py b/aria/modeling/utils.py
index acae065..917f495 100644
--- a/aria/modeling/utils.py
+++ b/aria/modeling/utils.py
@@ -26,7 +26,7 @@ from ..utils.type import validate_value_type
class ModelJSONEncoder(JSONEncoder):
- def default(self, o):
+ def default(self, o): # pylint: disable=method-hidden
from .mixins import ModelMixin
if isinstance(o, ModelMixin):
if hasattr(o, 'value'):
@@ -56,7 +56,7 @@ def create_inputs(inputs, template_inputs):
value=input_val)
input_models.append(parameter)
- return {input.name: input for input in input_models}
+ return dict((inp.name, inp) for inp in input_models)
def _merge_and_validate_inputs(inputs, template_inputs):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7b8a4187/aria/orchestrator/context/common.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/common.py b/aria/orchestrator/context/common.py
index 11b5eb9..15843db 100644
--- a/aria/orchestrator/context/common.py
+++ b/aria/orchestrator/context/common.py
@@ -29,7 +29,6 @@ from aria import (
)
from aria.storage import exceptions
-from ... import logger
from ...utils.uuid import generate_uuid
@@ -55,6 +54,7 @@ class BaseContext(object):
self,
name,
service_id,
+ execution_id,
model_storage,
resource_storage,
workdir=None,
@@ -65,12 +65,13 @@ class BaseContext(object):
self._model = model_storage
self._resource = resource_storage
self._service_id = service_id
+ self._execution_id = execution_id
self._workdir = workdir
self.logger = None
def _register_logger(self, level=None, task_id=None):
self.logger = self.PrefixedLogger(
- logging.getLogger(logger.TASK_LOGGER_NAME), self.logging_id, task_id=task_id)
+ logging.getLogger(aria_logger.TASK_LOGGER_NAME), self.logging_id, task_id=task_id)
self.logger.setLevel(level or logging.DEBUG)
if not self.logger.handlers:
self.logger.addHandler(aria_logger.create_console_log_handler())
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7b8a4187/aria/orchestrator/context/operation.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/operation.py b/aria/orchestrator/context/operation.py
index cbd186c..c7d8246 100644
--- a/aria/orchestrator/context/operation.py
+++ b/aria/orchestrator/context/operation.py
@@ -36,7 +36,6 @@ class BaseOperationContext(BaseContext):
service_id,
task_id,
actor_id,
- execution_id,
**kwargs):
super(BaseOperationContext, self).__init__(
name=name,
@@ -47,7 +46,6 @@ class BaseOperationContext(BaseContext):
self._task_id = task_id
self._actor_id = actor_id
self._thread_local = threading.local()
- self._execution_id = execution_id
self._register_logger(task_id=self.task.id)
def __repr__(self):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7b8a4187/aria/orchestrator/context/workflow.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/workflow.py b/aria/orchestrator/context/workflow.py
index ad4a2ff..667d22f 100644
--- a/aria/orchestrator/context/workflow.py
+++ b/aria/orchestrator/context/workflow.py
@@ -30,7 +30,6 @@ class WorkflowContext(BaseContext):
"""
def __init__(self,
workflow_name,
- execution_id,
parameters=None,
task_max_attempts=1,
task_retry_interval=0,
@@ -38,11 +37,10 @@ class WorkflowContext(BaseContext):
*args, **kwargs):
super(WorkflowContext, self).__init__(*args, **kwargs)
self._workflow_name = workflow_name
- self.parameters = parameters or {}
+ self._parameters = parameters or {}
self._task_max_attempts = task_max_attempts
self._task_retry_interval = task_retry_interval
self._task_ignore_failure = task_ignore_failure
- self._execution_id = execution_id
self._register_logger()
def __repr__(self):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7b8a4187/aria/utils/threading.py
----------------------------------------------------------------------
diff --git a/aria/utils/threading.py b/aria/utils/threading.py
index 06c48bc..bfd30f5 100644
--- a/aria/utils/threading.py
+++ b/aria/utils/threading.py
@@ -277,5 +277,5 @@ class ExceptionThread(Thread):
def raise_error_if_exists(self):
if self.is_error():
- type_, value, tb = self.exception
- raise type_, value, tb
+ type_, value, trace = self.exception
+ raise type_, value, trace
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7b8a4187/aria/utils/type.py
----------------------------------------------------------------------
diff --git a/aria/utils/type.py b/aria/utils/type.py
index fff0f2a..dad5427 100644
--- a/aria/utils/type.py
+++ b/aria/utils/type.py
@@ -40,7 +40,7 @@ def validate_value_type(value, type_name):
if type_ is None:
raise RuntimeError('No supported type_name was provided')
- if type(value) != type_:
+ if not isinstance(value, type_):
raise ValueError('Value {0} is not of type {1}'.format(value, type_name))
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7b8a4187/tests/.pylintrc
----------------------------------------------------------------------
diff --git a/tests/.pylintrc b/tests/.pylintrc
index 06409e9..eead6e8 100644
--- a/tests/.pylintrc
+++ b/tests/.pylintrc
@@ -77,7 +77,7 @@ confidence=
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,no-self-use,missing-docstring,attribute-defined-outside-init,redefined-outer-name,import-error,redefined-variable-type,broad
-except,protected-access,global-statement,too-many-locals,abstract-method,no-member
+disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,no-self-use,missing-docstring,attribute-defined-outside-init,redefined-outer-name,import-error,redefined-variable-type,broad
-except,protected-access,global-statement,too-many-locals,abstract-method,no-member,unused-argument
[REPORTS]
[6/7] incubator-ariatosca git commit:
ARIA-138-Make-logging-more-informative
Posted by mx...@apache.org.
ARIA-138-Make-logging-more-informative
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/9e2b70cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/9e2b70cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/9e2b70cf
Branch: refs/heads/ARIA-138-Make-logging-more-informative
Commit: 9e2b70cf1b239ada4de20dad27cf7a6e81ddb8f7
Parents: d6458c0
Author: max-orlov <ma...@gigaspaces.com>
Authored: Thu Apr 6 11:54:44 2017 +0300
Committer: max-orlov <ma...@gigaspaces.com>
Committed: Thu Apr 13 17:10:17 2017 +0300
----------------------------------------------------------------------
aria/cli/cli/aria.py | 2 +-
aria/cli/commands/__init__.py | 1 +
aria/cli/commands/executions.py | 12 +++-
aria/cli/commands/logs.py | 24 ++++----
aria/cli/execution_logging.py | 58 ++++++++++++++++++++
aria/cli/logger.py | 18 ++++++
aria/logger.py | 4 +-
aria/modeling/orchestration.py | 7 ++-
aria/orchestrator/context/common.py | 39 ++++++-------
aria/orchestrator/context/operation.py | 29 +---------
aria/orchestrator/context/workflow.py | 4 +-
aria/orchestrator/workflow_runner.py | 8 ++-
aria/orchestrator/workflows/core/engine.py | 2 +
aria/orchestrator/workflows/events_logging.py | 26 +++++----
aria/orchestrator/workflows/executor/base.py | 4 +-
aria/orchestrator/workflows/executor/process.py | 16 ++++--
aria/orchestrator/workflows/executor/thread.py | 8 ++-
examples/hello-world/scripts/stop.sh | 3 +
.../workflows/executor/test_executor.py | 2 +
19 files changed, 173 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/cli/cli/aria.py
----------------------------------------------------------------------
diff --git a/aria/cli/cli/aria.py b/aria/cli/cli/aria.py
index 0177134..0843b4a 100644
--- a/aria/cli/cli/aria.py
+++ b/aria/cli/cli/aria.py
@@ -18,7 +18,7 @@ import sys
import difflib
import StringIO
import traceback
-from functools import wraps
+from functools import wraps, partial
import click
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/cli/commands/__init__.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/__init__.py b/aria/cli/commands/__init__.py
index 7777791..9ca5e47 100644
--- a/aria/cli/commands/__init__.py
+++ b/aria/cli/commands/__init__.py
@@ -1,3 +1,4 @@
+
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/cli/commands/executions.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/executions.py b/aria/cli/commands/executions.py
index 396985a..4bdc481 100644
--- a/aria/cli/commands/executions.py
+++ b/aria/cli/commands/executions.py
@@ -13,7 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from aria.cli import execution_logging
from .. import utils
+from .. import logger as cli_logger
from ..table import print_data
from ..cli import aria
from ...modeling.models import Execution
@@ -139,13 +141,19 @@ def start(workflow_name,
logger.info('Starting {0}execution. Press Ctrl+C cancel'.format('dry ' if dry else ''))
execution_thread.start()
+
+ log_consumer = cli_logger.ModelLogConsumer(model_storage, workflow_runner.execution_id)
try:
while execution_thread.is_alive():
- # using join without a timeout blocks and ignores KeyboardInterrupt
- execution_thread.join(1)
+ for log in log_consumer:
+ execution_logging.log(log)
+
except KeyboardInterrupt:
_cancel_execution(workflow_runner, execution_thread, logger)
+ for log in log_consumer:
+ execution_logging.log(log)
+
# raise any errors from the execution thread (note these are not workflow execution errors)
execution_thread.raise_error_if_exists()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/cli/commands/logs.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/logs.py b/aria/cli/commands/logs.py
index f8873cd..4d5d4e2 100644
--- a/aria/cli/commands/logs.py
+++ b/aria/cli/commands/logs.py
@@ -12,13 +12,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-from .. import utils
+from ..logger import ModelLogConsumer
from ..cli import aria
+from .. import execution_logging
@aria.group(name='logs')
-@aria.options.verbose()
def logs():
"""Show logs from workflow executions
"""
@@ -31,19 +30,18 @@ def logs():
@aria.options.verbose()
@aria.pass_model_storage
@aria.pass_logger
-def list(execution_id,
- model_storage,
- logger):
+def list(execution_id, model_storage, logger):
"""Display logs for an execution
"""
logger.info('Listing logs for execution id {0}'.format(execution_id))
- logs_list = model_storage.log.list(filters=dict(execution_fk=execution_id),
- sort=utils.storage_sort_param('created_at', False))
- # TODO: print logs nicely
- if logs_list:
- for log in logs_list:
- print log
- else:
+ log_consumer = ModelLogConsumer(model_storage, execution_id)
+ any_logs = False
+
+ for log in log_consumer:
+ execution_logging.log(log)
+ any_logs = True
+
+ if not any_logs:
logger.info('\tNo logs')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/cli/execution_logging.py
----------------------------------------------------------------------
diff --git a/aria/cli/execution_logging.py b/aria/cli/execution_logging.py
new file mode 100644
index 0000000..9f83310
--- /dev/null
+++ b/aria/cli/execution_logging.py
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from . import logger
+from .env import env
+
+
+def log(item):
+
+ formats = {
+ logger.NO_VERBOSE: {'main_msg': '{item.msg}'},
+ logger.LOW_VERBOSE: {
+ 'main_msg': '{created_at} | {item.level[0]} | {item.msg}',
+ 'created_at': '%H:%M:%S'
+ }
+ }
+
+ # Only NO_VERBOSE and LOW_VERBOSE are configurable formats. configuring
+ # the low verbose level should affect any higher level.
+ formats = formats[min(env.logging.verbosity_level, logger.LOW_VERBOSE)]
+
+ kwargs = dict(item=item)
+ if 'created_at' in formats:
+ kwargs['created_at'] = item.created_at.strftime(formats['created_at'])
+ if 'level' in formats:
+ kwargs['level'] = formats['level'].format(item.level)
+ if 'msg' in formats:
+ kwargs['msg'] = formats['msg'].format(item.msg)
+
+ if 'actor' in formats and item.task:
+ kwargs['actor'] = formats['actor'].format(item.task.actor)
+ if 'execution' in formats:
+ kwargs['execution'] = formats['execution'].format(item.execution)
+
+ # If no format was supplied just print out the original msg.
+ msg = formats.get('main_msg', '{item.msg}').format(**kwargs)
+
+ # Add the exception and the error msg.
+ if item.traceback and env.logging.verbosity_level >= logger.MEDIUM_VERBOSE:
+ msg += os.linesep + '------>'
+ for line in item.traceback.splitlines(True):
+ msg += '\t' + '|' + line
+
+ return getattr(env.logging.logger, item.level.lower())(msg)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/cli/logger.py
----------------------------------------------------------------------
diff --git a/aria/cli/logger.py b/aria/cli/logger.py
index 2f012d9..6811f3a 100644
--- a/aria/cli/logger.py
+++ b/aria/cli/logger.py
@@ -112,3 +112,21 @@ class Logging(object):
self._all_loggers.append(logger_name)
dictconfig.dictConfig(logger_dict)
+
+
+class ModelLogConsumer(object):
+
+ def __init__(self, model_storage, execution_id, filters=None, sort=None):
+ self._last_visited_id = 0
+ self._model_storage = model_storage
+ self._execution_id = execution_id
+ self._additional_filters = filters or {}
+ self._sort = sort or {}
+
+ def __iter__(self):
+ filters = dict(execution_fk=self._execution_id, id=dict(gt=self._last_visited_id))
+ filters.update(self._additional_filters)
+
+ for log in self._model_storage.log.iter(filters=filters, sort=self._sort):
+ self._last_visited_id = log.id
+ yield log
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/logger.py
----------------------------------------------------------------------
diff --git a/aria/logger.py b/aria/logger.py
index 41e1ac8..5e3e925 100644
--- a/aria/logger.py
+++ b/aria/logger.py
@@ -177,10 +177,12 @@ class _SQLAlchemyHandler(logging.Handler):
log = self._cls(
execution_fk=self._execution_id,
task_fk=record.task_id,
- actor=record.prefix,
level=record.levelname,
msg=str(record.msg),
created_at=created_at,
+
+ # Not mandatory.
+ traceback=getattr(record, 'traceback', None)
)
self._session.add(log)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/modeling/orchestration.py
----------------------------------------------------------------------
diff --git a/aria/modeling/orchestration.py b/aria/modeling/orchestration.py
index 12a56f6..ca67628 100644
--- a/aria/modeling/orchestration.py
+++ b/aria/modeling/orchestration.py
@@ -393,7 +393,9 @@ class LogBase(ModelMixin):
level = Column(String)
msg = Column(String)
created_at = Column(DateTime, index=True)
- actor = Column(String)
+
+ # In case of failed execution
+ traceback = Column(Text)
# region foreign keys
@@ -408,5 +410,4 @@ class LogBase(ModelMixin):
# endregion
def __repr__(self):
- return "<{self.created_at}: [{self.level}] @{self.actor}> {msg}".format(
- self=self, msg=self.msg[:50])
+ return "{msg}".format(msg=self.msg)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/orchestrator/context/common.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/common.py b/aria/orchestrator/context/common.py
index 15843db..7097d40 100644
--- a/aria/orchestrator/context/common.py
+++ b/aria/orchestrator/context/common.py
@@ -38,43 +38,44 @@ class BaseContext(object):
"""
class PrefixedLogger(object):
- def __init__(self, logger, prefix='', task_id=None):
- self._logger = logger
- self._prefix = prefix
+ def __init__(self, base_logger, task_id=None):
+ self._logger = base_logger
self._task_id = task_id
def __getattr__(self, item):
if item.upper() in logging._levelNames:
- return partial(getattr(self._logger, item),
- extra={'prefix': self._prefix, 'task_id': self._task_id})
+ return partial(self._logger_with_task_id, _level=item)
else:
return getattr(self._logger, item)
- def __init__(
- self,
- name,
- service_id,
- execution_id,
- model_storage,
- resource_storage,
- workdir=None,
- **kwargs):
+ def _logger_with_task_id(self, *args, **kwargs):
+ level = kwargs.pop('_level')
+ kwargs.setdefault('extra', {})['task_id'] = self._task_id
+ return getattr(self._logger, level)(*args, **kwargs)
+
+ def __init__(self,
+ name,
+ service_id,
+ model_storage,
+ resource_storage,
+ execution_id,
+ workdir=None,
+ **kwargs):
super(BaseContext, self).__init__(**kwargs)
self._name = name
self._id = generate_uuid(variant='uuid')
self._model = model_storage
self._resource = resource_storage
self._service_id = service_id
- self._execution_id = execution_id
self._workdir = workdir
+ self._execution_id = execution_id
self.logger = None
def _register_logger(self, level=None, task_id=None):
self.logger = self.PrefixedLogger(
- logging.getLogger(aria_logger.TASK_LOGGER_NAME), self.logging_id, task_id=task_id)
+ logging.getLogger(logger.TASK_LOGGER_NAME), task_id=task_id)
self.logger.setLevel(level or logging.DEBUG)
if not self.logger.handlers:
- self.logger.addHandler(aria_logger.create_console_log_handler())
self.logger.addHandler(self._get_sqla_handler())
def _get_sqla_handler(self):
@@ -104,10 +105,6 @@ class BaseContext(object):
self.logger.removeHandler(handler)
@property
- def logging_id(self):
- raise NotImplementedError
-
- @property
def model(self):
"""
Access to the model storage
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/orchestrator/context/operation.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/operation.py b/aria/orchestrator/context/operation.py
index c7d8246..c383958 100644
--- a/aria/orchestrator/context/operation.py
+++ b/aria/orchestrator/context/operation.py
@@ -29,20 +29,8 @@ class BaseOperationContext(BaseContext):
Context object used during operation creation and execution
"""
- def __init__(self,
- name,
- model_storage,
- resource_storage,
- service_id,
- task_id,
- actor_id,
- **kwargs):
- super(BaseOperationContext, self).__init__(
- name=name,
- model_storage=model_storage,
- resource_storage=resource_storage,
- service_id=service_id,
- **kwargs)
+ def __init__(self, task_id, actor_id, **kwargs):
+ super(BaseOperationContext, self).__init__(**kwargs)
self._task_id = task_id
self._actor_id = actor_id
self._thread_local = threading.local()
@@ -55,10 +43,6 @@ class BaseOperationContext(BaseContext):
return '{name}({0})'.format(details, name=self.name)
@property
- def logging_id(self):
- raise NotImplementedError
-
- @property
def task(self):
"""
The task in the model storage
@@ -119,10 +103,6 @@ class NodeOperationContext(BaseOperationContext):
"""
@property
- def logging_id(self):
- return self.node.name or self.node.id
-
- @property
def node_template(self):
"""
the node of the current operation
@@ -145,11 +125,6 @@ class RelationshipOperationContext(BaseOperationContext):
"""
@property
- def logging_id(self):
- return '{0}->{1}'.format(self.source_node.name or self.source_node.id,
- self.target_node.name or self.target_node.id)
-
- @property
def source_node_template(self):
"""
The source node
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/orchestrator/context/workflow.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/workflow.py b/aria/orchestrator/context/workflow.py
index 667d22f..920b237 100644
--- a/aria/orchestrator/context/workflow.py
+++ b/aria/orchestrator/context/workflow.py
@@ -50,8 +50,8 @@ class WorkflowContext(BaseContext):
name=self.__class__.__name__, self=self))
@property
- def logging_id(self):
- return '{0}[{1}]'.format(self._workflow_name, self._execution_id)
+ def workflow_name(self):
+ return self._workflow_name
@property
def execution(self):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/orchestrator/workflow_runner.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflow_runner.py b/aria/orchestrator/workflow_runner.py
index 1cdf1de..c7d2c30 100644
--- a/aria/orchestrator/workflow_runner.py
+++ b/aria/orchestrator/workflow_runner.py
@@ -92,8 +92,12 @@ class WorkflowRunner(object):
tasks_graph=self._tasks_graph)
@property
+ def execution_id(self):
+ return self._execution_id
+
+ @property
def execution(self):
- return self._model_storage.execution.get(self._execution_id)
+ return self._model_storage.execution.get(self.execution_id)
@property
def service(self):
@@ -121,7 +125,7 @@ class WorkflowRunner(object):
execution.inputs = modeling_utils.create_inputs(inputs, workflow_inputs)
# TODO: these two following calls should execute atomically
- self._validate_no_active_executions(execution)
+ # self._validate_no_active_executions(execution)
self._model_storage.execution.put(execution)
return execution
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/orchestrator/workflows/core/engine.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/core/engine.py b/aria/orchestrator/workflows/core/engine.py
index 0503142..5c89f68 100644
--- a/aria/orchestrator/workflows/core/engine.py
+++ b/aria/orchestrator/workflows/core/engine.py
@@ -20,6 +20,7 @@ The workflow engine. Executes workflows
import time
from datetime import datetime
+import logging
import networkx
from aria import logger
@@ -40,6 +41,7 @@ class Engine(logger.LoggerMixin):
def __init__(self, executor, workflow_context, tasks_graph, **kwargs):
super(Engine, self).__init__(**kwargs)
+ self.logger.addHandler(logging.NullHandler())
self._workflow_context = workflow_context
self._execution_graph = networkx.DiGraph()
self._executor = executor
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/orchestrator/workflows/events_logging.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/events_logging.py b/aria/orchestrator/workflows/events_logging.py
index e831bfe..1bc5a1e 100644
--- a/aria/orchestrator/workflows/events_logging.py
+++ b/aria/orchestrator/workflows/events_logging.py
@@ -26,41 +26,43 @@ from .. import events
@events.start_task_signal.connect
def _start_task_handler(task, **kwargs):
- task.context.logger.debug('Event: Starting task: {task.name}'.format(task=task))
+ task.context.logger.info('{actor.name} {task.interface_name}.{task.operation_name} started...'
+ .format(actor=task.actor, task=task))
@events.on_success_task_signal.connect
def _success_task_handler(task, **kwargs):
- task.context.logger.debug('Event: Task success: {task.name}'.format(task=task))
+ task.context.logger.info('{actor.name} {task.interface_name}.{task.operation_name} successful'
+ .format(actor=task.actor, task=task))
@events.on_failure_task_signal.connect
-def _failure_operation_handler(task, exception, **kwargs):
- error = '{0}: {1}'.format(type(exception).__name__, exception)
- task.context.logger.error('Event: Task failure: {task.name} [{error}]'.format(
- task=task, error=error))
-
+def _failure_operation_handler(task, traceback, **kwargs):
+ task.context.logger.error(
+ '{actor.name} {task.interface_name}.{task.operation_name} failed'
+ .format(actor=task.actor, task=task), extra=dict(traceback=traceback)
+ )
@events.start_workflow_signal.connect
def _start_workflow_handler(context, **kwargs):
- context.logger.debug('Event: Starting workflow: {context.name}'.format(context=context))
+ context.logger.info("Starting '{ctx.workflow_name}' workflow execution".format(ctx=context))
@events.on_failure_workflow_signal.connect
def _failure_workflow_handler(context, **kwargs):
- context.logger.debug('Event: Workflow failure: {context.name}'.format(context=context))
+ context.logger.info("'{ctx.workflow_name}' workflow execution failed".format(ctx=context))
@events.on_success_workflow_signal.connect
def _success_workflow_handler(context, **kwargs):
- context.logger.debug('Event: Workflow success: {context.name}'.format(context=context))
+ context.logger.info("'{ctx.workflow_name}' workflow execution succeeded".format(ctx=context))
@events.on_cancelled_workflow_signal.connect
def _cancel_workflow_handler(context, **kwargs):
- context.logger.debug('Event: Workflow cancelled: {context.name}'.format(context=context))
+ context.logger.info("'{ctx.workflow_name}' workflow execution canceled".format(ctx=context))
@events.on_cancelling_workflow_signal.connect
def _cancelling_workflow_handler(context, **kwargs):
- context.logger.debug('Event: Workflow cancelling: {context.name}'.format(context=context))
+ context.logger.info("Cancelling '{ctx.workflow_name}' workflow execution".format(ctx=context))
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/orchestrator/workflows/executor/base.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/base.py b/aria/orchestrator/workflows/executor/base.py
index 4ae046d..39becef 100644
--- a/aria/orchestrator/workflows/executor/base.py
+++ b/aria/orchestrator/workflows/executor/base.py
@@ -44,8 +44,8 @@ class BaseExecutor(logger.LoggerMixin):
events.start_task_signal.send(task)
@staticmethod
- def _task_failed(task, exception):
- events.on_failure_task_signal.send(task, exception=exception)
+ def _task_failed(task, exception, traceback=None):
+ events.on_failure_task_signal.send(task, exception=exception, traceback=traceback)
@staticmethod
def _task_succeeded(task):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/orchestrator/workflows/executor/process.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py
index dc369ab..a5d3cce 100644
--- a/aria/orchestrator/workflows/executor/process.py
+++ b/aria/orchestrator/workflows/executor/process.py
@@ -25,6 +25,8 @@ import sys
# As part of the process executor implementation, subprocess are started with this module as their
# entry point. We thus remove this module's directory from the python path if it happens to be
# there
+from aria.modeling.models import Parameter
+
script_dir = os.path.dirname(__file__)
if script_dir in sys.path:
sys.path.remove(script_dir)
@@ -48,7 +50,6 @@ from aria.utils import exceptions
from aria.orchestrator.workflows.executor import base
from aria.storage import instrumentation
from aria.modeling import types as modeling_types
-from aria.modeling.models import Parameter
_IS_WIN = os.name == 'nt'
@@ -148,8 +149,10 @@ class ProcessExecutor(base.BaseExecutor):
def _create_arguments_dict(self, task):
return {
'task_id': task.id,
- 'implementation': task.implementation,
- 'operation_inputs': Parameter.unwrap_dict(task.inputs),
+ # 'implementation': task.implementation,
+ 'implementation': 'aria.orchestrator.execution_plugin.operations.run_script_locally',
+ # 'operation_inputs': Parameter.unwrap_dict(task.inputs),
+ 'operation_inputs': dict(script_path=task.implementation),
'port': self._server_port,
'context': task.context.serialization_dict,
}
@@ -234,9 +237,10 @@ class ProcessExecutor(base.BaseExecutor):
except BaseException as e:
e.message += 'Task failed due to {0}.'.format(request['exception']) + \
UPDATE_TRACKED_CHANGES_FAILED_STR
- self._task_failed(task, exception=e)
+ self._task_failed(
+ task, exception=e, traceback=exceptions.get_exception_as_string(*sys.exc_info()))
else:
- self._task_failed(task, exception=request['exception'])
+ self._task_failed(task, exception=request['exception'], traceback=request['traceback'])
def _handle_apply_tracked_changes_request(self, task_id, request, response):
task = self._tasks[task_id]
@@ -320,6 +324,7 @@ class _Messenger(object):
'type': type,
'task_id': self.task_id,
'exception': exceptions.wrap_if_needed(exception),
+ 'traceback': exceptions.get_exception_as_string(*sys.exc_info()),
'tracked_changes': tracked_changes
})
response = _recv_message(sock)
@@ -383,7 +388,6 @@ def _main():
# This is required for the instrumentation work properly.
# See docstring of `remove_mutable_association_listener` for further details
modeling_types.remove_mutable_association_listener()
-
with instrumentation.track_changes() as instrument:
try:
ctx = context_dict['context_cls'].deserialize_from_dict(**context_dict['context'])
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/aria/orchestrator/workflows/executor/thread.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/thread.py b/aria/orchestrator/workflows/executor/thread.py
index 8b443cc..ad5e41a 100644
--- a/aria/orchestrator/workflows/executor/thread.py
+++ b/aria/orchestrator/workflows/executor/thread.py
@@ -20,7 +20,9 @@ Thread based executor
import Queue
import threading
-from aria.utils import imports
+import sys
+
+from aria.utils import imports, exceptions
from .base import BaseExecutor
from ....modeling.models import Parameter
@@ -64,7 +66,9 @@ class ThreadExecutor(BaseExecutor):
task_func(ctx=task.context, **inputs)
self._task_succeeded(task)
except BaseException as e:
- self._task_failed(task, exception=e)
+ self._task_failed(task,
+ exception=e,
+ traceback=exceptions.get_exception_as_string(*sys.exc_info()))
# Daemon threads
except BaseException as e:
pass
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/examples/hello-world/scripts/stop.sh
----------------------------------------------------------------------
diff --git a/examples/hello-world/scripts/stop.sh b/examples/hello-world/scripts/stop.sh
index 5461caf..400726e 100755
--- a/examples/hello-world/scripts/stop.sh
+++ b/examples/hello-world/scripts/stop.sh
@@ -1,5 +1,8 @@
#!/bin/bash
+
+bla bla bla
+
set -e
TEMP_DIR="/tmp"
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/9e2b70cf/tests/orchestrator/workflows/executor/test_executor.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/executor/test_executor.py b/tests/orchestrator/workflows/executor/test_executor.py
index d84d1ec..e39a993 100644
--- a/tests/orchestrator/workflows/executor/test_executor.py
+++ b/tests/orchestrator/workflows/executor/test_executor.py
@@ -15,6 +15,7 @@
import logging
import uuid
+from collections import namedtuple
from contextlib import contextmanager
import pytest
@@ -119,6 +120,7 @@ class MockTask(object):
self.ignore_failure = False
self.interface_name = 'interface_name'
self.operation_name = 'operation_name'
+ self.actor = namedtuple('actor', 'name')(name='actor_name')
for state in models.Task.STATES:
setattr(self, state.upper(), state)