You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by mx...@apache.org on 2017/03/06 10:26:29 UTC
[2/2] incubator-ariatosca git commit:
ARIA-115-Log-model-should-have-an-Execution-field
ARIA-115-Log-model-should-have-an-Execution-field
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/c0d76ada
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/c0d76ada
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/c0d76ada
Branch: refs/heads/ARIA-115-Log-model-should-have-an-Execution-field
Commit: c0d76adaf37935a10a9f4d3b3fe4b508595192ab
Parents: 63b157c
Author: max-orlov <ma...@gigaspaces.com>
Authored: Sun Mar 5 19:18:21 2017 +0200
Committer: max-orlov <ma...@gigaspaces.com>
Committed: Mon Mar 6 12:26:23 2017 +0200
----------------------------------------------------------------------
aria/logger.py | 12 +++++++++---
aria/orchestrator/context/common.py | 16 +++++++++++++++-
aria/orchestrator/context/operation.py | 5 ++++-
aria/orchestrator/context/workflow.py | 4 ++--
aria/orchestrator/workflows/core/task.py | 1 +
aria/storage/modeling/orchestrator_elements.py | 8 ++++++++
tests/orchestrator/context/test_operation.py | 18 +++++++++++++-----
tests/test_logger.py | 1 +
8 files changed, 53 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c0d76ada/aria/logger.py
----------------------------------------------------------------------
diff --git a/aria/logger.py b/aria/logger.py
index 9fe05ae..6f0b84a 100644
--- a/aria/logger.py
+++ b/aria/logger.py
@@ -98,13 +98,17 @@ def create_console_log_handler(level=logging.DEBUG, formatter=None):
return console
-def create_sqla_log_handler(session, engine, log_cls, level=logging.DEBUG):
+def create_sqla_log_handler(session, engine, log_cls, execution_id, level=logging.DEBUG):
# This is needed since the engine and session are entirely new we need to reflect the db
# schema of the logging model into the engine and session.
log_cls.__table__.create(bind=engine, checkfirst=True)
- return _SQLAlchemyHandler(session=session, engine=engine, log_cls=log_cls, level=level)
+ return _SQLAlchemyHandler(session=session,
+ engine=engine,
+ log_cls=log_cls,
+ execution_id=execution_id,
+ level=level)
class _DefaultConsoleFormat(logging.Formatter):
@@ -148,16 +152,18 @@ def create_file_log_handler(
class _SQLAlchemyHandler(logging.Handler):
- def __init__(self, session, engine, log_cls, **kwargs):
+ def __init__(self, session, engine, log_cls, execution_id, **kwargs):
logging.Handler.__init__(self, **kwargs)
self._session = session
self._engine = engine
self._cls = log_cls
+ self._execution_id = execution_id
def emit(self, record):
created_at = datetime.strptime(logging.Formatter('%(asctime)s').formatTime(record),
'%Y-%m-%d %H:%M:%S,%f')
log = self._cls(
+ execution_fk=self._execution_id,
actor=record.prefix,
level=record.levelname,
msg=record.msg,
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c0d76ada/aria/orchestrator/context/common.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/common.py b/aria/orchestrator/context/common.py
index b34cd5d..1d228b6 100644
--- a/aria/orchestrator/context/common.py
+++ b/aria/orchestrator/context/common.py
@@ -17,6 +17,7 @@ A common context for both workflow and operation
"""
import logging
from contextlib import contextmanager
+from datetime import datetime
from functools import partial
from uuid import uuid4
@@ -62,6 +63,17 @@ class BaseContext(object):
self._workdir = workdir
self.logger = None
+ def _create_execution(self):
+ now = datetime.utcnow()
+ execution = self.model.execution.model_cls(
+ service_instance=self.service_instance,
+ workflow_name=self._workflow_name,
+ created_at=now,
+ parameters=self.parameters,
+ )
+ self.model.execution.put(execution)
+ return execution.id
+
def _register_logger(self, logger_name=None, level=None):
self.logger = self.PrefixedLogger(logging.getLogger(logger_name or self.__class__.__name__),
self.logging_id)
@@ -74,7 +86,9 @@ class BaseContext(object):
if self._model._initiator:
api_kwargs.update(self._model._initiator(**self._model._initiator_kwargs))
api_kwargs.update(**self._model._api_kwargs)
- return aria_logger.create_sqla_log_handler(log_cls=modeling.model.Log, **api_kwargs)
+ return aria_logger.create_sqla_log_handler(log_cls=modeling.model.Log,
+ execution_id=self._execution_id,
+ **api_kwargs)
def __repr__(self):
return (
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c0d76ada/aria/orchestrator/context/operation.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/operation.py b/aria/orchestrator/context/operation.py
index 3fb1786..cbaa462 100644
--- a/aria/orchestrator/context/operation.py
+++ b/aria/orchestrator/context/operation.py
@@ -34,6 +34,7 @@ class BaseOperationContext(BaseContext):
service_instance_id,
task_id,
actor_id,
+ execution_id,
**kwargs):
super(BaseOperationContext, self).__init__(
name=name,
@@ -44,6 +45,7 @@ class BaseOperationContext(BaseContext):
self._task_id = task_id
self._actor_id = actor_id
self._task = None
+ self._execution_id = execution_id
self._register_logger()
def __repr__(self):
@@ -89,7 +91,8 @@ class BaseOperationContext(BaseContext):
'actor_id': self._actor_id,
'workdir': self._workdir,
'model_storage': self.model.serialization_dict if self.model else None,
- 'resource_storage': self.resource.serialization_dict if self.resource else None
+ 'resource_storage': self.resource.serialization_dict if self.resource else None,
+ 'execution_id': self._execution_id
}
return {
'context_cls': context_cls,
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c0d76ada/aria/orchestrator/context/workflow.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/workflow.py b/aria/orchestrator/context/workflow.py
index 0afaa81..5de4b51 100644
--- a/aria/orchestrator/context/workflow.py
+++ b/aria/orchestrator/context/workflow.py
@@ -32,10 +32,10 @@ class WorkflowContext(BaseContext):
def __init__(self,
workflow_name,
parameters=None,
- execution_id=None,
task_max_attempts=1,
task_retry_interval=0,
task_ignore_failure=False,
+ execution_id=None,
*args, **kwargs):
super(WorkflowContext, self).__init__(*args, **kwargs)
self._workflow_name = workflow_name
@@ -45,7 +45,7 @@ class WorkflowContext(BaseContext):
self._task_ignore_failure = task_ignore_failure
# TODO: execution creation should happen somewhere else
# should be moved there, when such logical place exists
- self._execution_id = self._create_execution() if execution_id is None else execution_id
+ self._execution_id = execution_id or self._create_execution()
self._register_logger()
def __repr__(self):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c0d76ada/aria/orchestrator/workflows/core/task.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/core/task.py b/aria/orchestrator/workflows/core/task.py
index f19b1cf..9f63bcf 100644
--- a/aria/orchestrator/workflows/core/task.py
+++ b/aria/orchestrator/workflows/core/task.py
@@ -146,6 +146,7 @@ class OperationTask(BaseTask):
service_instance_id=self._workflow_context._service_instance_id,
task_id=operation_task.id,
actor_id=api_task.actor.id,
+ execution_id=self._workflow_context._execution_id,
workdir=self._workflow_context._workdir)
self._task_id = operation_task.id
self._update_fields = None
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c0d76ada/aria/storage/modeling/orchestrator_elements.py
----------------------------------------------------------------------
diff --git a/aria/storage/modeling/orchestrator_elements.py b/aria/storage/modeling/orchestrator_elements.py
index 854f00b..d06b5d0 100644
--- a/aria/storage/modeling/orchestrator_elements.py
+++ b/aria/storage/modeling/orchestrator_elements.py
@@ -471,6 +471,14 @@ class TaskBase(ModelMixin):
class LogBase(ModelMixin):
__tablename__ = 'log'
+ @declared_attr
+ def execution_fk(cls):
+ return cls.foreign_key('execution')
+
+ @declared_attr
+ def execution(cls):
+ return cls.many_to_one_relationship('execution')
+
level = Column(String)
msg = Column(String)
created_at = Column(DateTime, index=True)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c0d76ada/tests/orchestrator/context/test_operation.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_operation.py b/tests/orchestrator/context/test_operation.py
index 2b26bb1..8db2bc6 100644
--- a/tests/orchestrator/context/test_operation.py
+++ b/tests/orchestrator/context/test_operation.py
@@ -213,7 +213,7 @@ def test_plugin_workdir(ctx, thread_executor, tmpdir):
@pytest.fixture(params=[
- (thread.ThreadExecutor, dict()),
+ # (thread.ThreadExecutor, dict()),
(process.ProcessExecutor, dict(python_path=tests.ROOT_DIR))
])
def executor(request):
@@ -252,7 +252,7 @@ def test_node_operation_logging(ctx, executor):
)
execute(workflow_func=basic_workflow, workflow_context=ctx, executor=executor)
- _assert_loggins(ctx.model.log.list(), inputs)
+ _assert_loggins(ctx, inputs)
def test_relationship_operation_logging(ctx, executor):
@@ -283,14 +283,22 @@ def test_relationship_operation_logging(ctx, executor):
)
execute(workflow_func=basic_workflow, workflow_context=ctx, executor=executor)
- _assert_loggins(ctx.model.log.list(), inputs)
+ _assert_loggins(ctx, inputs)
-def _assert_loggins(logs, inputs):
+def _assert_loggins(ctx, inputs):
# The logs should contain the following: Workflow Start, Operation Start, custom operation
# log string (op_start), custom operation log string (op_end), Operation End, Workflow End.
- assert len(logs) == 6
+
+ executions = ctx.model.execution.list()
+ assert len(executions) == 1
+ execution = executions[0]
+
+ logs = ctx.model.log.list()
+ assert len(logs) == execution.logs.count() == 6
+ assert all(l in logs for l in execution.logs)
+ assert all(l.execution == execution for l in logs)
op_start_log = [l for l in logs if inputs['op_start'] in l.msg and l.level.lower() == 'info']
assert len(op_start_log) == 1
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c0d76ada/tests/test_logger.py
----------------------------------------------------------------------
diff --git a/tests/test_logger.py b/tests/test_logger.py
index 6457884..d6999fd 100644
--- a/tests/test_logger.py
+++ b/tests/test_logger.py
@@ -22,6 +22,7 @@ from aria.logger import (create_logger,
LoggerMixin,
_DefaultConsoleFormat)
+
def test_create_logger():
logger = create_logger()