You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by mx...@apache.org on 2017/01/30 14:14:59 UTC
[01/14] incubator-ariatosca git commit: ARIA-60 Fix use of thread
locals in parser context [Forced Update!]
Repository: incubator-ariatosca
Updated Branches:
refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields f86f433f7 -> 64a393ece (forced update)
ARIA-60 Fix use of thread locals in parser context
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/61b7094d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/61b7094d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/61b7094d
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: 61b7094d8fbd7cd34cebc14ab7cd537386143e57
Parents: dac4da7
Author: Tal Liron <ta...@gmail.com>
Authored: Sun Jan 15 16:45:35 2017 +0200
Committer: Ran Ziv <ra...@gigaspaces.com>
Committed: Sun Jan 15 16:46:05 2017 +0200
----------------------------------------------------------------------
aria/parser/consumption/context.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/61b7094d/aria/parser/consumption/context.py
----------------------------------------------------------------------
diff --git a/aria/parser/consumption/context.py b/aria/parser/consumption/context.py
index 8fb9bb6..a8b75df 100644
--- a/aria/parser/consumption/context.py
+++ b/aria/parser/consumption/context.py
@@ -24,6 +24,9 @@ from ..modeling import ModelingContext
from .style import Style
+_thread_locals = threading.local()
+
+
class ConsumptionContext(object):
"""
Properties:
@@ -44,8 +47,7 @@ class ConsumptionContext(object):
Gets the context attached to the current thread if there is one.
"""
- thread_locals = threading.local()
- return getattr(thread_locals, 'aria_consumption_context', None)
+ return getattr(_thread_locals, 'aria_consumption_context', None)
def __init__(self, set_thread_local=True):
self.args = []
@@ -65,8 +67,7 @@ class ConsumptionContext(object):
Attaches this context to the current thread.
"""
- thread_locals = threading.local()
- thread_locals.aria_consumption_context = self
+ _thread_locals.aria_consumption_context = self
def write(self, string):
"""
[08/14] incubator-ariatosca git commit: ARIA-72 Rework built-in
workflows to match TOSCA normative lifecycle
Posted by mx...@apache.org.
ARIA-72 Rework built-in workflows to match TOSCA normative lifecycle
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/db9ae9c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/db9ae9c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/db9ae9c7
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: db9ae9c70ae0703cb3f357bf53b95d1a351a8ab0
Parents: 0382b22
Author: Tal Liron <ta...@gmail.com>
Authored: Tue Jan 24 09:52:36 2017 +0200
Committer: Tal Liron <ta...@gmail.com>
Committed: Tue Jan 24 15:48:42 2017 +0200
----------------------------------------------------------------------
aria/orchestrator/workflows/builtin/__init__.py | 12 +-
.../workflows/builtin/execute_operation.py | 30 +-
aria/orchestrator/workflows/builtin/heal.py | 4 +-
aria/orchestrator/workflows/builtin/install.py | 42 +--
aria/orchestrator/workflows/builtin/start.py | 28 ++
aria/orchestrator/workflows/builtin/stop.py | 28 ++
.../orchestrator/workflows/builtin/uninstall.py | 41 +--
aria/orchestrator/workflows/builtin/utils.py | 70 +++++
.../orchestrator/workflows/builtin/workflows.py | 280 +++++++------------
tests/mock/operations.py | 48 +++-
.../orchestrator/workflows/builtin/__init__.py | 39 +--
.../orchestrator/workflows/builtin/test_heal.py | 2 +
tests/orchestrator/workflows/core/test_task.py | 6 +-
.../test_task_graph_into_exececution_graph.py | 2 +-
14 files changed, 342 insertions(+), 290 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/aria/orchestrator/workflows/builtin/__init__.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/builtin/__init__.py b/aria/orchestrator/workflows/builtin/__init__.py
index 0449a8e..d43a962 100644
--- a/aria/orchestrator/workflows/builtin/__init__.py
+++ b/aria/orchestrator/workflows/builtin/__init__.py
@@ -19,13 +19,17 @@ A set of builtin workflows
from .install import install
from .uninstall import uninstall
-from .execute_operation import execute_operation
-from .heal import heal
+from .start import start
+from .stop import stop
+
+
+BUILTIN_WORKFLOWS = ('install', 'uninstall', 'start', 'stop')
__all__ = [
+ 'BUILTIN_WORKFLOWS',
'install',
'uninstall',
- 'execute_operation',
- 'heal',
+ 'start',
+ 'stop'
]
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/aria/orchestrator/workflows/builtin/execute_operation.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/builtin/execute_operation.py b/aria/orchestrator/workflows/builtin/execute_operation.py
index ddbb8e7..e76993d 100644
--- a/aria/orchestrator/workflows/builtin/execute_operation.py
+++ b/aria/orchestrator/workflows/builtin/execute_operation.py
@@ -17,9 +17,8 @@
Builtin execute_operation workflow
"""
-from aria import workflow
-
-from .workflows import execute_operation_on_instance
+from ..api.task import OperationTask
+from ... import workflow
@workflow
@@ -68,7 +67,7 @@ def execute_operation(
# registering actual tasks to sequences
for node_instance in filtered_node_instances:
graph.add_tasks(
- execute_operation_on_instance(
+ _create_node_instance_task(
node_instance=node_instance,
operation=operation,
operation_kwargs=operation_kwargs,
@@ -102,3 +101,26 @@ def _filter_node_instances(context, node_ids=(), node_instance_ids=(), type_name
_is_node_instance_by_id(node_instance.id),
_is_node_by_type(node_instance.node.type_hierarchy))):
yield node_instance
+
+
+def _create_node_instance_task(
+ node_instance,
+ operation,
+ operation_kwargs,
+ allow_kwargs_override):
+ """
+ A workflow which executes a single operation
+ :param node_instance: the node instance to install
+ :param basestring operation: the operation name
+ :param dict operation_kwargs:
+ :param bool allow_kwargs_override:
+ :return:
+ """
+
+ if allow_kwargs_override is not None:
+ operation_kwargs['allow_kwargs_override'] = allow_kwargs_override
+
+ return OperationTask.node_instance(
+ instance=node_instance,
+ name=operation,
+ inputs=operation_kwargs)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/aria/orchestrator/workflows/builtin/heal.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/builtin/heal.py b/aria/orchestrator/workflows/builtin/heal.py
index 21da2ed..87ac492 100644
--- a/aria/orchestrator/workflows/builtin/heal.py
+++ b/aria/orchestrator/workflows/builtin/heal.py
@@ -13,13 +13,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# pylint: skip-file
+
"""
Builtin heal workflow
"""
from aria import workflow
-from .workflows import relationship_tasks, install_node_instance, uninstall_node_instance
+from .workflows import (install_node_instance, uninstall_node_instance)
from ..api import task
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/aria/orchestrator/workflows/builtin/install.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/builtin/install.py b/aria/orchestrator/workflows/builtin/install.py
index eb5b4e8..81d1da2 100644
--- a/aria/orchestrator/workflows/builtin/install.py
+++ b/aria/orchestrator/workflows/builtin/install.py
@@ -17,38 +17,18 @@
Builtin install workflow
"""
-from aria import workflow
-
from .workflows import install_node_instance
-from ..api import task
+from .utils import create_node_instance_task_dependencies
+from ..api.task import WorkflowTask
+from ... import workflow
@workflow
-def install(ctx, graph, node_instances=(), node_instance_sub_workflows=None):
- """
- The install workflow
- :param WorkflowContext ctx: the workflow context
- :param TaskGraph graph: the graph which will describe the workflow.
- :param node_instances: the node instances on which to run the workflow
- :param dict node_instance_sub_workflows: a dictionary of subworkflows with id as key and
- TaskGraph (or OperationContext) as value
- :return:
- """
- node_instance_sub_workflows = node_instance_sub_workflows or {}
- node_instances = node_instances or list(ctx.node_instances)
-
- # create install sub workflow for every node instance
- for node_instance in node_instances:
- node_instance_sub_workflow = task.WorkflowTask(install_node_instance,
- node_instance=node_instance)
- node_instance_sub_workflows[node_instance.id] = node_instance_sub_workflow
- graph.add_tasks(node_instance_sub_workflow)
-
- # create dependencies between the node instance sub workflow
- for node_instance in node_instances:
- node_instance_sub_workflow = node_instance_sub_workflows[node_instance.id]
- if node_instance.outbound_relationship_instances:
- dependencies = [
- node_instance_sub_workflows[relationship_instance.target_node_instance.id]
- for relationship_instance in node_instance.outbound_relationship_instances]
- graph.add_dependency(node_instance_sub_workflow, dependencies)
+def install(ctx, graph):
+ tasks_and_node_instances = []
+ for node_instance in ctx.model.node_instance.iter():
+ tasks_and_node_instances.append((
+ WorkflowTask(install_node_instance, node_instance=node_instance),
+ node_instance))
+ graph.add_tasks([task for task, _ in tasks_and_node_instances])
+ create_node_instance_task_dependencies(graph, tasks_and_node_instances)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/aria/orchestrator/workflows/builtin/start.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/builtin/start.py b/aria/orchestrator/workflows/builtin/start.py
new file mode 100644
index 0000000..a044d73
--- /dev/null
+++ b/aria/orchestrator/workflows/builtin/start.py
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Builtin start workflow
+"""
+
+from .workflows import start_node_instance
+from ..api.task import WorkflowTask
+from ... import workflow
+
+
+@workflow
+def start(ctx, graph):
+ for node_instance in ctx.model.node_instance.iter():
+ graph.add_tasks(WorkflowTask(start_node_instance, node_instance=node_instance))
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/aria/orchestrator/workflows/builtin/stop.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/builtin/stop.py b/aria/orchestrator/workflows/builtin/stop.py
new file mode 100644
index 0000000..584c0d3
--- /dev/null
+++ b/aria/orchestrator/workflows/builtin/stop.py
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Builtin stop workflow
+"""
+
+from .workflows import stop_node_instance
+from ..api.task import WorkflowTask
+from ... import workflow
+
+
+@workflow
+def stop(ctx, graph):
+ for node_instance in ctx.model.node_instance.iter():
+ graph.add_tasks(WorkflowTask(stop_node_instance, node_instance=node_instance))
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/aria/orchestrator/workflows/builtin/uninstall.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/builtin/uninstall.py b/aria/orchestrator/workflows/builtin/uninstall.py
index db1c0cc..bfcc9ec 100644
--- a/aria/orchestrator/workflows/builtin/uninstall.py
+++ b/aria/orchestrator/workflows/builtin/uninstall.py
@@ -17,37 +17,18 @@
Builtin uninstall workflow
"""
-from aria import workflow
-
from .workflows import uninstall_node_instance
-from ..api import task
+from .utils import create_node_instance_task_dependencies
+from ..api.task import WorkflowTask
+from ... import workflow
@workflow
-def uninstall(ctx, graph, node_instances=(), node_instance_sub_workflows=None):
- """
- The uninstall workflow
- :param WorkflowContext ctx: the workflow context
- :param TaskGraph graph: the graph which will describe the workflow.
- :param node_instances: the node instances on which to run the workflow
- :param dict node_instance_sub_workflows: a dictionary of subworkflows with id as key and
- TaskGraph (or OperationContext) as value
- :return:
- """
- node_instance_sub_workflows = node_instance_sub_workflows or {}
- node_instances = node_instances or list(ctx.node_instances)
-
- # create install sub workflow for every node instance
- for node_instance in node_instances:
- node_instance_sub_workflow = task.WorkflowTask(uninstall_node_instance,
- node_instance=node_instance)
- node_instance_sub_workflows[node_instance.id] = node_instance_sub_workflow
- graph.add_tasks(node_instance_sub_workflow)
-
- # create dependencies between the node instance sub workflow
- for node_instance in node_instances:
- node_instance_sub_workflow = node_instance_sub_workflows[node_instance.id]
- for relationship_instance in reversed(node_instance.outbound_relationship_instances):
- target_id = relationship_instance.target_node_instance.id
- graph.add_dependency(node_instance_sub_workflows[target_id],
- node_instance_sub_workflow)
+def uninstall(ctx, graph):
+ tasks_and_node_instances = []
+ for node_instance in ctx.model.node_instance.iter():
+ tasks_and_node_instances.append((
+ WorkflowTask(uninstall_node_instance, node_instance=node_instance),
+ node_instance))
+ graph.add_tasks([task for task, _ in tasks_and_node_instances])
+ create_node_instance_task_dependencies(graph, tasks_and_node_instances, reverse=True)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/aria/orchestrator/workflows/builtin/utils.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/builtin/utils.py b/aria/orchestrator/workflows/builtin/utils.py
new file mode 100644
index 0000000..8b7a8bc
--- /dev/null
+++ b/aria/orchestrator/workflows/builtin/utils.py
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..api.task import OperationTask
+
+
+def create_node_instance_task(operation_name, node_instance):
+ """
+ Returns a new operation task if the operation exists in the node instance, otherwise returns
+ None.
+ """
+
+ if operation_name in node_instance.node.operations:
+ return OperationTask.node_instance(instance=node_instance,
+ name=operation_name)
+ return None
+
+
+def create_relationship_instance_tasks(operation_name, operations_attr, node_instance):
+ """
+ Returns a list of operation tasks for each outbound relationship of the node instance if
+ the operation exists there.
+ """
+
+ sequence = []
+ for relationship_instance in node_instance.outbound_relationship_instances:
+ if operation_name in getattr(relationship_instance.relationship, operations_attr):
+ sequence.append(
+ OperationTask.relationship_instance(instance=relationship_instance,
+ name=operation_name,
+ operation_end=operations_attr))
+ return sequence
+
+
+def create_node_instance_task_dependencies(graph, tasks_and_node_instances, reverse=False):
+ """
+ Creates dependencies between tasks if there is an outbound relationship between their node
+ instances.
+ """
+
+ def get_task(node_instance_id):
+ for task, node_instance in tasks_and_node_instances:
+ if node_instance.id == node_instance_id:
+ return task
+ return None
+
+ for task, node_instance in tasks_and_node_instances:
+ dependencies = []
+ for relationship_instance in node_instance.outbound_relationship_instances:
+ dependency = get_task(relationship_instance.target_node_instance.id)
+ if dependency:
+ dependencies.append(dependency)
+ if dependencies:
+ if reverse:
+ for dependency in dependencies:
+ graph.add_dependency(dependency, task)
+ else:
+ graph.add_dependency(task, dependencies)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/aria/orchestrator/workflows/builtin/workflows.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/builtin/workflows.py b/aria/orchestrator/workflows/builtin/workflows.py
index 4f765b3..c2fbded 100644
--- a/aria/orchestrator/workflows/builtin/workflows.py
+++ b/aria/orchestrator/workflows/builtin/workflows.py
@@ -17,190 +17,122 @@
A set of builtin workflows.
"""
-from itertools import groupby
-
-from aria import workflow
-
-from ..api import task
+from .utils import (create_node_instance_task, create_relationship_instance_tasks)
+from ... import workflow
__all__ = (
'install_node_instance',
'uninstall_node_instance',
- 'execute_operation_on_instance',
+ 'start_node_instance',
+ 'stop_node_instance',
)
-# Install node instance workflow and sub workflows
-
@workflow(suffix_template='{node_instance.id}')
def install_node_instance(graph, node_instance, **kwargs):
- """
- A workflow which installs a node instance.
- :param TaskGraph graph: the tasks graph of which to edit
- :param node_instance: the node instance to install
- :return:
- """
- create_node_instance = task.OperationTask.node_instance(
- instance=node_instance,
- name='aria.interfaces.lifecycle.create')
-
- configure_node_instance = task.OperationTask.node_instance(
- instance=node_instance,
- name='aria.interfaces.lifecycle.configure')
- start_node_instance = task.OperationTask.node_instance(
- instance=node_instance,
- name='aria.interfaces.lifecycle.start')
-
- graph.sequence(
- create_node_instance,
- preconfigure_relationship(graph, node_instance),
- configure_node_instance,
- postconfigure_relationship(graph, node_instance),
- start_node_instance,
- establish_relationship(graph, node_instance)
- )
-
- return graph
-
-
-def preconfigure_relationship(graph, node_instance, **kwargs):
- """
-
- :param graph:
- :param node_instance:
- :return:
- """
- return relationships_tasks(
- graph=graph,
- operation_name='aria.interfaces.relationship_lifecycle.preconfigure',
- node_instance=node_instance)
-
-
-def postconfigure_relationship(graph, node_instance, **kwargs):
- """
-
- :param graph:
- :param node_instance:
- :return:
- """
- return relationships_tasks(
- graph=graph,
- operation_name='aria.interfaces.relationship_lifecycle.postconfigure',
- node_instance=node_instance)
-
-
-def establish_relationship(graph, node_instance, **kwargs):
- """
-
- :param graph:
- :param node_instance:
- :return:
- """
- return relationships_tasks(
- graph=graph,
- operation_name='aria.interfaces.relationship_lifecycle.establish',
- node_instance=node_instance)
-
-
-# Uninstall node instance workflow and subworkflows
+ sequence = []
+
+ # Create
+ sequence.append(
+ create_node_instance_task(
+ 'tosca.interfaces.node.lifecycle.Standard.create',
+ node_instance))
+
+ # Configure
+ sequence += \
+ create_relationship_instance_tasks(
+ 'tosca.interfaces.relationship.Configure.pre_configure_source',
+ 'source_operations',
+ node_instance)
+ sequence += \
+ create_relationship_instance_tasks(
+ 'tosca.interfaces.relationship.Configure.pre_configure_target',
+ 'target_operations',
+ node_instance)
+ sequence.append(
+ create_node_instance_task(
+ 'tosca.interfaces.node.lifecycle.Standard.configure',
+ node_instance))
+ sequence += \
+ create_relationship_instance_tasks(
+ 'tosca.interfaces.relationship.Configure.post_configure_source',
+ 'source_operations',
+ node_instance)
+ sequence += \
+ create_relationship_instance_tasks(
+ 'tosca.interfaces.relationship.Configure.post_configure_target',
+ 'target_operations',
+ node_instance)
+
+ # Start
+ sequence += _create_start_tasks(node_instance)
+
+ graph.sequence(*sequence)
+
@workflow(suffix_template='{node_instance.id}')
def uninstall_node_instance(graph, node_instance, **kwargs):
- """
- A workflow which uninstalls a node instance.
- :param TaskGraph graph: the tasks graph of which to edit
- :param node_instance: the node instance to uninstall
- :return:
- """
- stop_node_instance = task.OperationTask.node_instance(
- instance=node_instance,
- name='aria.interfaces.lifecycle.stop')
- delete_node_instance = task.OperationTask.node_instance(
- instance=node_instance,
- name='aria.interfaces.lifecycle.delete')
-
- graph.sequence(
- stop_node_instance,
- unlink_relationship(graph, node_instance),
- delete_node_instance
- )
-
-
-def unlink_relationship(graph, node_instance):
- """
-
- :param graph:
- :param node_instance:
- :return:
- """
- return relationships_tasks(
- graph=graph,
- operation_name='aria.interfaces.relationship_lifecycle.unlink',
- node_instance=node_instance
- )
-
-
-def execute_operation_on_instance(
- node_instance,
- operation,
- operation_kwargs,
- allow_kwargs_override):
- """
- A workflow which executes a single operation
- :param node_instance: the node instance to install
- :param basestring operation: the operation name
- :param dict operation_kwargs:
- :param bool allow_kwargs_override:
- :return:
- """
-
- if allow_kwargs_override is not None:
- operation_kwargs['allow_kwargs_override'] = allow_kwargs_override
-
- return task.OperationTask.node_instance(
- instance=node_instance,
- name=operation,
- inputs=operation_kwargs)
-
-
-def relationships_tasks(graph, operation_name, node_instance):
- """
- Creates a relationship task (source and target) for all of a node_instance relationships.
- :param basestring operation_name: the relationship operation name.
- :param WorkflowContext context:
- :param NodeInstance node_instance:
- :return:
- """
- relationships_groups = groupby(
- node_instance.outbound_relationship_instances,
- key=lambda relationship_instance: relationship_instance.target_node_instance.id)
-
- sub_tasks = []
- for _, (_, relationship_group) in enumerate(relationships_groups):
- for relationship_instance in relationship_group:
- relationship_operations = relationship_tasks(
- relationship_instance=relationship_instance,
- operation_name=operation_name)
- sub_tasks.append(relationship_operations)
-
- return graph.sequence(*sub_tasks)
-
-
-def relationship_tasks(relationship_instance, operation_name):
- """
- Creates a relationship task source and target.
- :param RelationshipInstance relationship_instance: the relationship instance itself
- :param operation_name:
- :return:
- """
- source_operation = task.OperationTask.relationship_instance(
- instance=relationship_instance,
- name=operation_name,
- operation_end=task.OperationTask.SOURCE_OPERATION)
- target_operation = task.OperationTask.relationship_instance(
- instance=relationship_instance,
- name=operation_name,
- operation_end=task.OperationTask.TARGET_OPERATION)
-
- return source_operation, target_operation
+ # Stop
+ sequence = _create_stop_tasks(node_instance)
+
+ # Delete
+ sequence.append(
+ create_node_instance_task(
+ 'tosca.interfaces.node.lifecycle.Standard.delete',
+ node_instance))
+
+ graph.sequence(*sequence)
+
+
+@workflow(suffix_template='{node_instance.id}')
+def start_node_instance(graph, node_instance, **kwargs):
+ graph.sequence(*_create_start_tasks(node_instance))
+
+
+@workflow(suffix_template='{node_instance.id}')
+def stop_node_instance(graph, node_instance, **kwargs):
+ graph.sequence(*_create_stop_tasks(node_instance))
+
+
+def _create_start_tasks(node_instance):
+ sequence = []
+ sequence.append(
+ create_node_instance_task(
+ 'tosca.interfaces.node.lifecycle.Standard.start',
+ node_instance))
+ sequence += \
+ create_relationship_instance_tasks(
+ 'tosca.interfaces.relationship.Configure.add_source',
+ 'source_operations',
+ node_instance)
+ sequence += \
+ create_relationship_instance_tasks(
+ 'tosca.interfaces.relationship.Configure.add_target',
+ 'target_operations',
+ node_instance)
+ sequence += \
+ create_relationship_instance_tasks(
+ 'tosca.interfaces.relationship.Configure.target_changed',
+ 'target_operations',
+ node_instance)
+ return sequence
+
+
+def _create_stop_tasks(node_instance):
+ sequence = []
+ sequence += \
+ create_relationship_instance_tasks(
+ 'tosca.interfaces.relationship.Configure.remove_target',
+ 'target_operations',
+ node_instance)
+ sequence += \
+ create_relationship_instance_tasks(
+ 'tosca.interfaces.relationship.Configure.target_changed',
+ 'target_operations',
+ node_instance)
+ sequence.append(
+ create_node_instance_task(
+ 'tosca.interfaces.node.lifecycle.Standard.stop',
+ node_instance))
+ return sequence
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/tests/mock/operations.py
----------------------------------------------------------------------
diff --git a/tests/mock/operations.py b/tests/mock/operations.py
index 407061f..5495f6a 100644
--- a/tests/mock/operations.py
+++ b/tests/mock/operations.py
@@ -14,20 +14,46 @@
# limitations under the License.
NODE_OPERATIONS_INSTALL = [
- 'aria.interfaces.lifecycle.create',
- 'aria.interfaces.lifecycle.configure',
- 'aria.interfaces.lifecycle.start',
- ]
+ 'tosca.interfaces.node.lifecycle.Standard.create',
+ 'tosca.interfaces.node.lifecycle.Standard.configure',
+ 'tosca.interfaces.node.lifecycle.Standard.start',
+]
+
NODE_OPERATIONS_UNINSTALL = [
- 'aria.interfaces.lifecycle.stop',
- 'aria.interfaces.lifecycle.delete',
+ 'tosca.interfaces.node.lifecycle.Standard.stop',
+ 'tosca.interfaces.node.lifecycle.Standard.delete',
]
+
NODE_OPERATIONS = NODE_OPERATIONS_INSTALL + NODE_OPERATIONS_UNINSTALL
RELATIONSHIP_OPERATIONS_INSTALL = [
- 'aria.interfaces.relationship_lifecycle.preconfigure',
- 'aria.interfaces.relationship_lifecycle.postconfigure',
- 'aria.interfaces.relationship_lifecycle.establish',
- ]
-RELATIONSHIP_OPERATIONS_UNINSTALL = ['aria.interfaces.relationship_lifecycle.unlink']
+ 'tosca.interfaces.relationship.Configure.pre_configure_source',
+ 'tosca.interfaces.relationship.Configure.pre_configure_target',
+ 'tosca.interfaces.relationship.Configure.add_source',
+ 'tosca.interfaces.relationship.Configure.add_target',
+]
+
+RELATIONSHIP_OPERATIONS_UNINSTALL = [
+ 'tosca.interfaces.relationship.Configure.remove_target',
+ 'tosca.interfaces.relationship.Configure.target_changed',
+]
+
RELATIONSHIP_OPERATIONS = RELATIONSHIP_OPERATIONS_INSTALL + RELATIONSHIP_OPERATIONS_UNINSTALL
+
+OPERATIONS_INSTALL = [
+ 'tosca.interfaces.node.lifecycle.Standard.create',
+ 'tosca.interfaces.relationship.Configure.pre_configure_source',
+ 'tosca.interfaces.relationship.Configure.pre_configure_target',
+ 'tosca.interfaces.node.lifecycle.Standard.configure',
+ 'tosca.interfaces.node.lifecycle.Standard.start',
+ 'tosca.interfaces.relationship.Configure.add_source',
+ 'tosca.interfaces.relationship.Configure.add_target',
+ 'tosca.interfaces.relationship.Configure.target_changed',
+]
+
+OPERATIONS_UNINSTALL = [
+ 'tosca.interfaces.relationship.Configure.remove_target',
+ 'tosca.interfaces.relationship.Configure.target_changed',
+ 'tosca.interfaces.node.lifecycle.Standard.stop',
+ 'tosca.interfaces.node.lifecycle.Standard.delete',
+]
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/tests/orchestrator/workflows/builtin/__init__.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/builtin/__init__.py b/tests/orchestrator/workflows/builtin/__init__.py
index 26ba82f..9f60e55 100644
--- a/tests/orchestrator/workflows/builtin/__init__.py
+++ b/tests/orchestrator/workflows/builtin/__init__.py
@@ -17,37 +17,14 @@ from tests import mock
def assert_node_install_operations(operations, with_relationships=False):
- if with_relationships:
- all_operations = [
- 'aria.interfaces.lifecycle.create',
- 'aria.interfaces.relationship_lifecycle.preconfigure',
- 'aria.interfaces.relationship_lifecycle.preconfigure',
- 'aria.interfaces.lifecycle.configure',
- 'aria.interfaces.relationship_lifecycle.postconfigure',
- 'aria.interfaces.relationship_lifecycle.postconfigure',
- 'aria.interfaces.lifecycle.start',
- 'aria.interfaces.relationship_lifecycle.establish',
- 'aria.interfaces.relationship_lifecycle.establish',
- ]
-
- for i, operation in enumerate(operations):
- assert operation.name.startswith(all_operations[i])
- else:
- for i, operation in enumerate(operations):
- assert operation.name.startswith(mock.operations.NODE_OPERATIONS_INSTALL[i])
+ all_operations = mock.operations.OPERATIONS_INSTALL if with_relationships else \
+ mock.operations.NODE_OPERATIONS_INSTALL
+ for i, operation in enumerate(operations):
+ assert operation.name.startswith(all_operations[i] + '.')
def assert_node_uninstall_operations(operations, with_relationships=False):
- if with_relationships:
- all_operations = [
- 'aria.interfaces.lifecycle.stop',
- 'aria.interfaces.relationship_lifecycle.unlink',
- 'aria.interfaces.relationship_lifecycle.unlink',
- 'aria.interfaces.lifecycle.delete',
- ]
-
- for i, operation in enumerate(operations):
- assert operation.name.startswith(all_operations[i])
- else:
- for i, operation in enumerate(operations):
- assert operation.name.startswith(mock.operations.NODE_OPERATIONS_UNINSTALL[i])
+ all_operations = mock.operations.OPERATIONS_UNINSTALL if with_relationships else \
+ mock.operations.NODE_OPERATIONS_UNINSTALL
+ for i, operation in enumerate(operations):
+ assert operation.name.startswith(all_operations[i] + '.')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/tests/orchestrator/workflows/builtin/test_heal.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/builtin/test_heal.py b/tests/orchestrator/workflows/builtin/test_heal.py
index ad281fd..b470790 100644
--- a/tests/orchestrator/workflows/builtin/test_heal.py
+++ b/tests/orchestrator/workflows/builtin/test_heal.py
@@ -31,6 +31,7 @@ def ctx(tmpdir):
storage.release_sqlite_storage(context.model)
+@pytest.mark.skip(reason='heal is not implemented for now')
def test_heal_dependent_node(ctx):
dependent_node_instance = \
ctx.model.node_instance.get_by_name(mock.models.DEPENDENT_NODE_INSTANCE_NAME)
@@ -60,6 +61,7 @@ def test_heal_dependent_node(ctx):
assert_node_install_operations(dependent_node_install_tasks, with_relationships=True)
+@pytest.mark.skip(reason='heal is not implemented for now')
def test_heal_dependency_node(ctx):
dependency_node_instance = \
ctx.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/tests/orchestrator/workflows/core/test_task.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/core/test_task.py b/tests/orchestrator/workflows/core/test_task.py
index 34088f5..061a3f2 100644
--- a/tests/orchestrator/workflows/core/test_task.py
+++ b/tests/orchestrator/workflows/core/test_task.py
@@ -42,7 +42,7 @@ class TestOperationTask(object):
with workflow_context.current.push(ctx):
api_task = api.task.OperationTask.node_instance(
instance=node_instance,
- name='aria.interfaces.lifecycle.create')
+ name='tosca.interfaces.node.lifecycle.Standard.create')
core_task = core.task.OperationTask(api_task=api_task)
return api_task, core_task
@@ -50,7 +50,7 @@ class TestOperationTask(object):
with workflow_context.current.push(ctx):
api_task = api.task.OperationTask.relationship_instance(
instance=relationship_instance,
- name='aria.interfaces.relationship_lifecycle.preconfigure',
+ name='tosca.interfaces.relationship.Configure.pre_configure_source',
operation_end=operation_end)
core_task = core.task.OperationTask(api_task=api_task)
return api_task, core_task
@@ -67,7 +67,7 @@ class TestOperationTask(object):
node.plugins = [{'name': plugin_name,
'package_name': 'p1',
'package_version': '0.1'}]
- node.operations['aria.interfaces.lifecycle.create'] = {'plugin': plugin_name}
+ node.operations['tosca.interfaces.node.lifecycle.Standard.create'] = {'plugin': plugin_name}
api_task, core_task = self._create_node_operation_task(ctx, node_instance)
storage_task = ctx.model.task.get_by_name(core_task.name)
assert storage_task.plugin_name == plugin_name
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/db9ae9c7/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py
index 18540f4..cd37bde 100644
--- a/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py
+++ b/tests/orchestrator/workflows/core/test_task_graph_into_exececution_graph.py
@@ -23,7 +23,7 @@ from tests import storage
def test_task_graph_into_execution_graph():
- operation_name = 'aria.interfaces.lifecycle.create'
+ operation_name = 'tosca.interfaces.node.lifecycle.Standard.create'
task_context = mock.context.simple(storage.get_sqlite_api_kwargs())
node_instance = \
task_context.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
[03/14] incubator-ariatosca git commit: ARIA-46 Execution plugin
Posted by mx...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/orchestrator/execution_plugin/test_ctx_proxy_server.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_ctx_proxy_server.py b/tests/orchestrator/execution_plugin/test_ctx_proxy_server.py
new file mode 100644
index 0000000..98ceff9
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/test_ctx_proxy_server.py
@@ -0,0 +1,359 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import sys
+import subprocess
+import StringIO
+
+import pytest
+
+from aria.orchestrator.execution_plugin import ctx_proxy
+
+
+class TestCtxProxy(object):
+
+ def test_attribute_access(self, server):
+ response = self.request(server, 'stub_attr', 'some_property')
+ assert response == 'some_value'
+
+ def test_sugared_attribute_access(self, server):
+ response = self.request(server, 'stub-attr', 'some-property')
+ assert response == 'some_value'
+
+ def test_dict_prop_access_get_key(self, server):
+ response = self.request(server, 'node', 'properties', 'prop1')
+ assert response == 'value1'
+
+ def test_dict_prop_access_get_key_nested(self, server):
+ response = self.request(server, 'node', 'properties', 'prop2.nested_prop1')
+ assert response == 'nested_value1'
+
+ def test_dict_prop_access_get_with_list_index(self, server):
+ response = self.request(server, 'node', 'properties', 'prop3[2].value')
+ assert response == 'value_2'
+
+ def test_dict_prop_access_set(self, server, ctx):
+ self.request(server, 'node', 'properties', 'prop4.key', 'new_value')
+ self.request(server, 'node', 'properties', 'prop3[2].value', 'new_value_2')
+ self.request(server, 'node', 'properties', 'prop4.some.new.path',
+ 'some_new_value')
+ assert ctx.node.properties['prop4']['key'] == 'new_value'
+ assert ctx.node.properties['prop3'][2]['value'] == 'new_value_2'
+ assert ctx.node.properties['prop4']['some']['new']['path'] == 'some_new_value'
+
+ def test_illegal_dict_access(self, server):
+ self.request(server, 'node', 'properties', 'prop4.key', 'new_value')
+ with pytest.raises(RuntimeError):
+ self.request(server, 'node', 'properties', 'prop4.key', 'new_value', 'what')
+
+ def test_method_invocation(self, server):
+ args = ['arg1', 'arg2', 'arg3']
+ response_args = self.request(server, 'stub-method', *args)
+ assert response_args == args
+
+ def test_method_invocation_no_args(self, server):
+ response = self.request(server, 'stub-method')
+ assert response == []
+
+ def test_method_invocation_kwargs(self, server):
+ arg1 = 'arg1'
+ arg2 = 'arg2'
+ arg4 = 'arg4_override'
+ arg5 = 'arg5'
+ kwargs = dict(
+ arg4=arg4,
+ arg5=arg5)
+ response = self.request(server, 'stub_args', arg1, arg2, kwargs)
+ assert response == dict(
+ arg1=arg1,
+ arg2=arg2,
+ arg3='arg3',
+ arg4=arg4,
+ args=[],
+ kwargs=dict(
+ arg5=arg5))
+
+ def test_empty_return_value(self, server):
+ response = self.request(server, 'stub_none')
+ assert response is None
+
+ def test_client_request_timeout(self, server):
+ with pytest.raises(IOError):
+ ctx_proxy.client._client_request(server.socket_url,
+ args=['stub-sleep', '0.5'],
+ timeout=0.1)
+
+ def test_processing_exception(self, server):
+ with pytest.raises(ctx_proxy.client._RequestError):
+ self.request(server, 'property_that_does_not_exist')
+
+ def test_not_json_serializable(self, server):
+ with pytest.raises(ctx_proxy.client._RequestError):
+ self.request(server, 'logger')
+
+ def test_no_string_arg(self, server):
+ args = ['stub_method', 1, 2]
+ response = self.request(server, *args)
+ assert response == args[1:]
+
+ class StubAttribute(object):
+ some_property = 'some_value'
+
+ class NodeAttribute(object):
+ def __init__(self, properties):
+ self.properties = properties
+
+ @staticmethod
+ def stub_method(*args):
+ return args
+
+ @staticmethod
+ def stub_sleep(seconds):
+ time.sleep(float(seconds))
+
+ @staticmethod
+ def stub_args(arg1, arg2, arg3='arg3', arg4='arg4', *args, **kwargs):
+ return dict(
+ arg1=arg1,
+ arg2=arg2,
+ arg3=arg3,
+ arg4=arg4,
+ args=args,
+ kwargs=kwargs)
+
+ @pytest.fixture
+ def ctx(self):
+ class MockCtx(object):
+ pass
+ ctx = MockCtx()
+ properties = {
+ 'prop1': 'value1',
+ 'prop2': {
+ 'nested_prop1': 'nested_value1'
+ },
+ 'prop3': [
+ {'index': 0, 'value': 'value_0'},
+ {'index': 1, 'value': 'value_1'},
+ {'index': 2, 'value': 'value_2'}
+ ],
+ 'prop4': {
+ 'key': 'value'
+ }
+ }
+ ctx.stub_none = None
+ ctx.stub_method = self.stub_method
+ ctx.stub_sleep = self.stub_sleep
+ ctx.stub_args = self.stub_args
+ ctx.stub_attr = self.StubAttribute()
+ ctx.node = self.NodeAttribute(properties)
+ return ctx
+
+ @pytest.fixture
+ def server(self, ctx):
+ result = ctx_proxy.server.CtxProxy(ctx)
+ yield result
+ result.close()
+
+ def request(self, server, *args):
+ return ctx_proxy.client._client_request(server.socket_url, args, timeout=5)
+
+
+class TestArgumentParsing(object):
+
+ def test_socket_url_arg(self):
+ self.expected.update(dict(socket_url='sock_url'))
+ ctx_proxy.client.main(['--socket-url', self.expected.get('socket_url')])
+
+ def test_socket_url_env(self):
+ expected_socket_url = 'env_sock_url'
+ os.environ['CTX_SOCKET_URL'] = expected_socket_url
+ self.expected.update(dict(socket_url=expected_socket_url))
+ ctx_proxy.client.main([])
+
+ def test_socket_url_missing(self):
+ del os.environ['CTX_SOCKET_URL']
+ with pytest.raises(RuntimeError):
+ ctx_proxy.client.main([])
+
+ def test_args(self):
+ self.expected.update(dict(args=['1', '2', '3']))
+ ctx_proxy.client.main(self.expected.get('args'))
+
+ def test_timeout(self):
+ self.expected.update(dict(timeout='10'))
+ ctx_proxy.client.main(['--timeout', self.expected.get('timeout')])
+ self.expected.update(dict(timeout='15'))
+ ctx_proxy.client.main(['-t', self.expected.get('timeout')])
+
+ def test_mixed_order(self):
+ self.expected.update(dict(
+ args=['1', '2', '3'], timeout='20', socket_url='mixed_socket_url'))
+ ctx_proxy.client.main(
+ ['-t', self.expected.get('timeout')] +
+ ['--socket-url', self.expected.get('socket_url')] +
+ self.expected.get('args'))
+ ctx_proxy.client.main(
+ ['-t', self.expected.get('timeout')] +
+ self.expected.get('args') +
+ ['--socket-url', self.expected.get('socket_url')])
+ ctx_proxy.client.main(
+ self.expected.get('args') +
+ ['-t', self.expected.get('timeout')] +
+ ['--socket-url', self.expected.get('socket_url')])
+
+ def test_json_args(self):
+ args = ['@1', '@[1,2,3]', '@{"key":"value"}']
+ expected_args = [1, [1, 2, 3], {'key': 'value'}]
+ self.expected.update(dict(args=expected_args))
+ ctx_proxy.client.main(args)
+
+ def test_json_arg_prefix(self):
+ args = ['_1', '@1']
+ expected_args = [1, '@1']
+ self.expected.update(dict(args=expected_args))
+ ctx_proxy.client.main(args + ['--json-arg-prefix', '_'])
+
+ def test_json_output(self):
+ self.assert_valid_output('string', 'string', '"string"')
+ self.assert_valid_output(1, '1', '1')
+ self.assert_valid_output([1, '2'], "[1, '2']", '[1, "2"]')
+ self.assert_valid_output({'key': 1},
+ "{'key': 1}",
+ '{"key": 1}')
+ self.assert_valid_output(False, '', 'false')
+ self.assert_valid_output(True, 'True', 'true')
+ self.assert_valid_output([], '', '[]')
+ self.assert_valid_output({}, '', '{}')
+
+ def assert_valid_output(self, response, ex_typed_output, ex_json_output):
+ self.mock_response = response
+ current_stdout = sys.stdout
+
+ def run(args, expected):
+ output = StringIO.StringIO()
+ sys.stdout = output
+ ctx_proxy.client.main(args)
+ assert output.getvalue() == expected
+
+ try:
+ run([], ex_typed_output)
+ run(['-j'], ex_json_output)
+ run(['--json-output'], ex_json_output)
+ finally:
+ sys.stdout = current_stdout
+
+ def mock_client_request(self, socket_url, args, timeout):
+ assert socket_url == self.expected.get('socket_url')
+ assert args == self.expected.get('args')
+ assert timeout == int(self.expected.get('timeout'))
+ return self.mock_response
+
+ @pytest.fixture(autouse=True)
+ def patch_client_request(self, mocker):
+ mocker.patch.object(ctx_proxy.client,
+ ctx_proxy.client._client_request.__name__,
+ self.mock_client_request)
+ mocker.patch.dict('os.environ', {'CTX_SOCKET_URL': 'stub'})
+
+ @pytest.fixture(autouse=True)
+ def defaults(self):
+ self.expected = dict(args=[], timeout=30, socket_url='stub')
+ self.mock_response = None
+
+
+class TestCtxEntryPoint(object):
+
+ def test_ctx_in_path(self):
+ p = subprocess.Popen(['ctx', '--help'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ p.communicate()
+ assert not p.wait()
+
+
+class TestPathDictAccess(object):
+ def test_simple_set(self):
+ obj = {}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ path_dict.set('foo', 42)
+ assert obj == {'foo': 42}
+
+ def test_nested_set(self):
+ obj = {'foo': {}}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ path_dict.set('foo.bar', 42)
+ assert obj == {'foo': {'bar': 42}}
+
+ def test_set_index(self):
+ obj = {'foo': [None, {'bar': 0}]}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ path_dict.set('foo[1].bar', 42)
+ assert obj == {'foo': [None, {'bar': 42}]}
+
+ def test_set_nonexistent_parent(self):
+ obj = {}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ path_dict.set('foo.bar', 42)
+ assert obj == {'foo': {'bar': 42}}
+
+ def test_set_nonexistent_parent_nested(self):
+ obj = {}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ path_dict.set('foo.bar.baz', 42)
+ assert obj == {'foo': {'bar': {'baz': 42}}}
+
+ def test_simple_get(self):
+ obj = {'foo': 42}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ result = path_dict.get('foo')
+ assert result == 42
+
+ def test_nested_get(self):
+ obj = {'foo': {'bar': 42}}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ result = path_dict.get('foo.bar')
+ assert result == 42
+
+ def test_nested_get_shadows_dotted_name(self):
+ obj = {'foo': {'bar': 42}, 'foo.bar': 58}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ result = path_dict.get('foo.bar')
+ assert result == 42
+
+ def test_index_get(self):
+ obj = {'foo': [0, 1]}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ result = path_dict.get('foo[1]')
+ assert result == 1
+
+ def test_get_nonexistent(self):
+ obj = {}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ with pytest.raises(RuntimeError):
+ path_dict.get('foo')
+
+ def test_get_by_index_not_list(self):
+ obj = {'foo': {0: 'not-list'}}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ with pytest.raises(RuntimeError):
+ path_dict.get('foo[0]')
+
+ def test_get_by_index_nonexistent_parent(self):
+ obj = {}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ with pytest.raises(RuntimeError):
+ path_dict.get('foo[1]')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/orchestrator/execution_plugin/test_global_ctx.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_global_ctx.py b/tests/orchestrator/execution_plugin/test_global_ctx.py
new file mode 100644
index 0000000..dad7547
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/test_global_ctx.py
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from aria.orchestrator import execution_plugin
+
+
+def test_python_script_scope():
+ assert execution_plugin.ctx is None
+ assert execution_plugin.inputs is None
+ ctx = object()
+ inputs = object()
+ with execution_plugin.python_script_scope(operation_ctx=ctx, operation_inputs=inputs):
+ assert execution_plugin.ctx is ctx
+ assert execution_plugin.inputs is inputs
+ assert execution_plugin.ctx is None
+ assert execution_plugin.inputs is None
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/orchestrator/execution_plugin/test_local.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_local.py b/tests/orchestrator/execution_plugin/test_local.py
new file mode 100644
index 0000000..497da48
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/test_local.py
@@ -0,0 +1,587 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+import pytest
+
+from aria import workflow
+from aria.orchestrator import events
+from aria.orchestrator.workflows import api
+from aria.orchestrator.workflows.exceptions import ExecutorException
+from aria.orchestrator.exceptions import TaskAbortException, TaskRetryException
+from aria.orchestrator.execution_plugin import operations
+from aria.orchestrator.execution_plugin.exceptions import ProcessException
+from aria.orchestrator.execution_plugin import local
+from aria.orchestrator.execution_plugin import constants
+from aria.orchestrator.workflows.executor import process
+from aria.orchestrator.workflows.core import engine
+
+from tests import mock, storage
+from tests.orchestrator.workflows.helpers import events_collector
+
+IS_WINDOWS = os.name == 'nt'
+
+
+class TestLocalRunScript(object):
+
+ def test_script_path_parameter(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties map.key value
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties map.key value
+ ''')
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ assert props['map']['key'] == 'value'
+
+ def test_process_env(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties map.key1 $key1
+ ctx node-instance runtime-properties map.key2 $key2
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties map.key1 %key1%
+ ctx node-instance runtime-properties map.key2 %key2%
+ ''')
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ process={
+ 'env': {
+ 'key1': 'value1',
+ 'key2': 'value2'
+ }
+ })
+ p_map = props['map']
+ assert p_map['key1'] == 'value1'
+ assert p_map['key2'] == 'value2'
+
+ def test_process_cwd(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties map.cwd $PWD
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties map.cwd %CD%
+ ''')
+ tmpdir = str(tmpdir)
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ process={
+ 'cwd': tmpdir
+ })
+ p_map = props['map']
+ assert p_map['cwd'] == tmpdir
+
+ def test_process_command_prefix(self, executor, workflow_context, tmpdir):
+ use_ctx = 'ctx node-instance runtime-properties map.key value'
+ python_script = ['import subprocess',
+ 'subprocess.Popen("{0}".split(' ')).communicate()[0]'.format(use_ctx)]
+ python_script = '\n'.join(python_script)
+ script_path = self._create_script(
+ tmpdir,
+ linux_script=python_script,
+ windows_script=python_script,
+ windows_suffix='',
+ linux_suffix='')
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ process={
+ 'env': {'TEST_KEY': 'value'},
+ 'command_prefix': 'python'
+ })
+ p_map = props['map']
+ assert p_map['key'] == 'value'
+
+ def test_process_args(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties map.arg1 "$1"
+ ctx node-instance runtime-properties map.arg2 $2
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties map.arg1 %1
+ ctx node-instance runtime-properties map.arg2 %2
+ ''')
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ process={
+ 'args': ['"arg with spaces"', 'arg2']
+ })
+ assert props['map']['arg1'] == 'arg with spaces'
+ assert props['map']['arg2'] == 'arg2'
+
+ def test_no_script_path(self, executor, workflow_context):
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=None)
+ assert isinstance(exception, TaskAbortException)
+ assert 'script_path' in exception.message
+
+ def test_script_error(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ echo 123123
+ command_that_does_not_exist
+ ''',
+ windows_script='''
+ @echo off
+ echo 123123
+ command_that_does_not_exist
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, ProcessException)
+ assert os.path.basename(script_path) in exception.command
+ assert exception.exit_code == 1 if IS_WINDOWS else 127
+ assert exception.stdout.strip() == '123123'
+ assert 'command_that_does_not_exist' in exception.stderr
+
+ def test_script_error_from_bad_ctx_request(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx property_that_does_not_exist
+ ''',
+ windows_script='''
+ ctx property_that_does_not_exist
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, ProcessException)
+ assert os.path.basename(script_path) in exception.command
+ assert exception.exit_code == 1
+ assert 'RequestError' in exception.stderr
+ assert 'property_that_does_not_exist' in exception.stderr
+
+ def test_python_script(self, executor, workflow_context, tmpdir):
+ script = '''
+from aria.orchestrator.execution_plugin import ctx, inputs
+if __name__ == '__main__':
+ ctx.node_instance.runtime_properties['key'] = inputs['key']
+'''
+ suffix = '.py'
+ script_path = self._create_script(
+ tmpdir,
+ linux_script=script,
+ windows_script=script,
+ linux_suffix=suffix,
+ windows_suffix=suffix)
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ inputs={'key': 'value'})
+ assert props['key'] == 'value'
+
+ @pytest.mark.parametrize(
+ 'value', ['string-value', [1, 2, 3], 999, 3.14, False,
+ {'complex1': {'complex2': {'key': 'value'}, 'list': [1, 2, 3]}}])
+ def test_inputs_as_environment_variables(self, executor, workflow_context, tmpdir, value):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties key "${input_as_env_var}"
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties key "%input_as_env_var%"
+ ''')
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ env_var=value)
+ expected = props['key'] if isinstance(value, basestring) else json.loads(props['key'])
+ assert expected == value
+
+ @pytest.mark.parametrize('value', ['override', {'key': 'value'}])
+ def test_explicit_env_variables_inputs_override(
+ self, executor, workflow_context, tmpdir, value):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties key "${input_as_env_var}"
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties key "%input_as_env_var%"
+ ''')
+
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ env_var='test-value',
+ process={
+ 'env': {
+ 'input_as_env_var': value
+ }
+ })
+ expected = props['key'] if isinstance(value, basestring) else json.loads(props['key'])
+ assert expected == value
+
+ def test_get_nonexistent_runtime_property(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties nonexistent
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties nonexistent
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, ProcessException)
+ assert os.path.basename(script_path) in exception.command
+ assert 'RequestError' in exception.stderr
+ assert 'nonexistent' in exception.stderr
+
+ def test_get_nonexistent_runtime_property_json(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx -j instance runtime-properties nonexistent
+ ''',
+ windows_script='''
+ ctx -j instance runtime-properties nonexistent
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, ProcessException)
+ assert os.path.basename(script_path) in exception.command
+ assert 'RequestError' in exception.stderr
+ assert 'nonexistent' in exception.stderr
+
+ def test_abort(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task abort abort-message
+ ''',
+ windows_script='''
+ ctx task abort abort-message
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, TaskAbortException)
+ assert exception.message == 'abort-message'
+
+ def test_retry(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task retry retry-message
+ ''',
+ windows_script='''
+ ctx task retry retry-message
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, TaskRetryException)
+ assert exception.message == 'retry-message'
+
+ def test_retry_with_interval(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task retry retry-message @100
+ ''',
+ windows_script='''
+ ctx task retry retry-message @100
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, TaskRetryException)
+ assert exception.message == 'retry-message'
+ assert exception.retry_interval == 100
+
+ def test_crash_abort_after_retry(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash
+ ctx task retry retry-message
+ ctx task abort should-raise-a-runtime-error
+ ''',
+ windows_script='''
+ ctx task retry retry-message
+ ctx task abort should-raise-a-runtime-error
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, TaskAbortException)
+ assert exception.message == constants.ILLEGAL_CTX_OPERATION_MESSAGE
+
+ def test_crash_retry_after_abort(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash
+ ctx task abort abort-message
+ ctx task retry should-raise-a-runtime-error
+ ''',
+ windows_script='''
+ ctx task abort abort-message
+ ctx task retry should-raise-a-runtime-error
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, TaskAbortException)
+ assert exception.message == constants.ILLEGAL_CTX_OPERATION_MESSAGE
+
+ def test_crash_abort_after_abort(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash
+ ctx task abort abort-message
+ ctx task abort should-raise-a-runtime-error
+ ''',
+ windows_script='''
+ ctx task abort abort-message
+ ctx task abort should-raise-a-runtime-error
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, TaskAbortException)
+ assert exception.message == constants.ILLEGAL_CTX_OPERATION_MESSAGE
+
+ def test_crash_retry_after_retry(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash
+ ctx task retry retry-message
+ ctx task retry should-raise-a-runtime-error
+ ''',
+ windows_script='''
+ ctx task retry retry-message
+ ctx task retry should-raise-a-runtime-error
+ ''')
+ exception = self._run_and_get_task_exception(
+ executor, workflow_context,
+ script_path=script_path)
+ assert isinstance(exception, TaskAbortException)
+ assert exception.message == constants.ILLEGAL_CTX_OPERATION_MESSAGE
+
+ def test_retry_returns_a_nonzero_exit_code(self, executor, workflow_context, tmpdir):
+ log_path = tmpdir.join('temp.log')
+ message = 'message'
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task retry '{0}' 2> {1}
+ echo should-not-run > {1}
+ '''.format(message, log_path),
+ windows_script='''
+ ctx task retry "{0}" 2> {1}
+ if %errorlevel% neq 0 exit /b %errorlevel%
+ echo should-not-run > {1}
+ '''.format(message, log_path))
+ with pytest.raises(ExecutorException):
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ assert log_path.read().strip() == message
+
+ def test_abort_returns_a_nonzero_exit_code(self, executor, workflow_context, tmpdir):
+ log_path = tmpdir.join('temp.log')
+ message = 'message'
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task abort '{0}' 2> {1}
+ echo should-not-run > {1}
+ '''.format(message, log_path),
+ windows_script='''
+ ctx task abort "{0}" 2> {1}
+ if %errorlevel% neq 0 exit /b %errorlevel%
+ echo should-not-run > {1}
+ '''.format(message, log_path))
+ with pytest.raises(ExecutorException):
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ assert log_path.read().strip() == message
+
+ def _create_script(self,
+ tmpdir,
+ linux_script,
+ windows_script,
+ windows_suffix='.bat',
+ linux_suffix=''):
+ suffix = windows_suffix if IS_WINDOWS else linux_suffix
+ script = windows_script if IS_WINDOWS else linux_script
+ script_path = tmpdir.join('script{0}'.format(suffix))
+ script_path.write(script)
+ return str(script_path)
+
+ def _run_and_get_task_exception(self, *args, **kwargs):
+ signal = events.on_failure_task_signal
+ with events_collector(signal) as collected:
+ with pytest.raises(ExecutorException):
+ self._run(*args, **kwargs)
+ return collected[signal][0]['kwargs']['exception']
+
+ def _run(self,
+ executor,
+ workflow_context,
+ script_path,
+ process=None,
+ env_var='value',
+ inputs=None):
+ local_script_path = script_path
+ script_path = os.path.basename(local_script_path) if local_script_path else None
+ if script_path:
+ workflow_context.resource.deployment.upload(
+ entry_id=str(workflow_context.deployment.id),
+ source=local_script_path,
+ path=script_path)
+
+ inputs = inputs or {}
+ inputs.update({
+ 'script_path': script_path,
+ 'process': process,
+ 'input_as_env_var': env_var
+ })
+
+ @workflow
+ def mock_workflow(ctx, graph):
+ op = 'test.op'
+ node_instance = ctx.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+ node_instance.node.operations[op] = {
+ 'operation': '{0}.{1}'.format(operations.__name__,
+ operations.run_script_locally.__name__)}
+ graph.add_tasks(api.task.OperationTask.node_instance(
+ instance=node_instance,
+ name=op,
+ inputs=inputs))
+ return graph
+ tasks_graph = mock_workflow(ctx=workflow_context) # pylint: disable=no-value-for-parameter
+ eng = engine.Engine(
+ executor=executor,
+ workflow_context=workflow_context,
+ tasks_graph=tasks_graph)
+ eng.execute()
+ return workflow_context.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties
+
+ @pytest.fixture
+ def executor(self):
+ result = process.ProcessExecutor()
+ yield result
+ result.close()
+
+ @pytest.fixture
+ def workflow_context(self, tmpdir):
+ workflow_context = mock.context.simple(
+ storage.get_sqlite_api_kwargs(str(tmpdir)),
+ resources_dir=str(tmpdir.join('resources')))
+ workflow_context.states = []
+ workflow_context.exception = None
+ yield workflow_context
+ storage.release_sqlite_storage(workflow_context.model)
+
+
+class BaseTestConfiguration(object):
+
+ @pytest.fixture(autouse=True)
+ def mock_execute(self, mocker):
+ def eval_func(**_):
+ self.called = 'eval'
+
+ def execute_func(process, **_):
+ self.process = process
+ self.called = 'execute'
+ self.process = {}
+ self.called = None
+ mocker.patch.object(local, '_execute_func', execute_func)
+ mocker.patch.object(local, '_eval_script_func', eval_func)
+
+ class Ctx(object):
+ @staticmethod
+ def download_resource(destination, *args, **kwargs):
+ return destination
+
+ def _run(self, script_path, process=None):
+ local.run_script(
+ script_path=script_path,
+ process=process,
+ ctx=self.Ctx)
+
+
+class TestPowerShellConfiguration(BaseTestConfiguration):
+
+ def test_implicit_powershell_call_with_ps1_extension(self):
+ self._run(script_path='script_path.ps1')
+ assert self.process['command_prefix'] == 'powershell'
+
+ def test_command_prefix_is_overridden_for_ps1_extension(self):
+ self._run(script_path='script_path.ps1',
+ process={'command_prefix': 'bash'})
+ assert self.process['command_prefix'] == 'bash'
+
+ def test_explicit_powershell_call(self):
+ self._run(script_path='script_path.ps1',
+ process={'command_prefix': 'powershell'})
+ assert self.process['command_prefix'] == 'powershell'
+
+
+class TestEvalPythonConfiguration(BaseTestConfiguration):
+
+ def test_explicit_eval_without_py_extension(self):
+ self._run(script_path='script_path',
+ process={'eval_python': True})
+ assert self.called == 'eval'
+
+ def test_explicit_eval_with_py_extension(self):
+ self._run(script_path='script_path.py',
+ process={'eval_python': True})
+ assert self.called == 'eval'
+
+ def test_implicit_eval(self):
+ self._run(script_path='script_path.py')
+ assert self.called == 'eval'
+
+ def test_explicit_execute_without_py_extension(self):
+ self._run(script_path='script_path',
+ process={'eval_python': False})
+ assert self.called == 'execute'
+
+ def test_explicit_execute_with_py_extension(self):
+ self._run(script_path='script_path.py',
+ process={'eval_python': False})
+ assert self.called == 'execute'
+
+ def test_implicit_execute(self):
+ self._run(script_path='script_path')
+ assert self.called == 'execute'
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/orchestrator/execution_plugin/test_ssh.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_ssh.py b/tests/orchestrator/execution_plugin/test_ssh.py
new file mode 100644
index 0000000..6b5c783
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/test_ssh.py
@@ -0,0 +1,481 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import contextlib
+import json
+import logging
+import os
+
+import pytest
+
+import fabric.api
+from fabric.contrib import files
+from fabric import context_managers
+
+from aria.storage import model
+from aria.orchestrator import events
+from aria.orchestrator import workflow
+from aria.orchestrator.workflows import api
+from aria.orchestrator.workflows.executor import process
+from aria.orchestrator.workflows.core import engine
+from aria.orchestrator.workflows.exceptions import ExecutorException
+from aria.orchestrator.exceptions import TaskAbortException, TaskRetryException
+from aria.orchestrator.execution_plugin import operations
+from aria.orchestrator.execution_plugin import constants
+from aria.orchestrator.execution_plugin.exceptions import ProcessException, TaskException
+from aria.orchestrator.execution_plugin.ssh import operations as ssh_operations
+
+from tests import mock, storage, resources
+from tests.orchestrator.workflows.helpers import events_collector
+
+_CUSTOM_BASE_DIR = '/tmp/new-aria-ctx'
+
+_FABRIC_ENV = {
+ 'host_string': 'localhost',
+ 'user': 'travis',
+ 'password': 'travis'
+}
+
+
+@pytest.mark.skipif(not os.environ.get('TRAVIS'), reason='actual ssh server required')
+class TestWithActualSSHServer(object):
+
+ def test_run_script_basic(self):
+ expected_runtime_property_value = 'some_value'
+ props = self._execute(env={'test_value': expected_runtime_property_value})
+ assert props['test_value'] == expected_runtime_property_value
+
+ @pytest.mark.skip(reason='sudo privileges are required')
+ def test_run_script_as_sudo(self):
+ self._execute(use_sudo=True)
+ with self._ssh_env():
+ assert files.exists('/opt/test_dir')
+ fabric.api.sudo('rm -rf /opt/test_dir')
+
+ def test_run_script_default_base_dir(self):
+ props = self._execute()
+ assert props['work_dir'] == '{0}/work'.format(constants.DEFAULT_BASE_DIR)
+
+ @pytest.mark.skip(reason='Re-enable once output from process executor can be captured')
+ @pytest.mark.parametrize('hide_groups', [[], ['everything']])
+ def test_run_script_with_hide(self, hide_groups):
+ self._execute(hide_output=hide_groups)
+ output = 'TODO'
+ expected_log_message = ('[localhost] run: source {0}/scripts/'
+ .format(constants.DEFAULT_BASE_DIR))
+ if hide_groups:
+ assert expected_log_message not in output
+ else:
+ assert expected_log_message in output
+
+ def test_run_script_process_config(self):
+ expected_env_value = 'test_value_env'
+ expected_arg1_value = 'test_value_arg1'
+ expected_arg2_value = 'test_value_arg2'
+ expected_cwd = '/tmp'
+ expected_base_dir = _CUSTOM_BASE_DIR
+ props = self._execute(
+ env={'test_value_env': expected_env_value},
+ process={
+ 'args': [expected_arg1_value, expected_arg2_value],
+ 'cwd': expected_cwd,
+ 'base_dir': expected_base_dir
+ })
+ assert props['env_value'] == expected_env_value
+ assert len(props['bash_version']) > 0
+ assert props['arg1_value'] == expected_arg1_value
+ assert props['arg2_value'] == expected_arg2_value
+ assert props['cwd'] == expected_cwd
+ assert props['ctx_path'] == '{0}/ctx'.format(expected_base_dir)
+
+ def test_run_script_command_prefix(self):
+ props = self._execute(process={'command_prefix': 'bash -i'})
+ assert 'i' in props['dollar_dash']
+
+ def test_run_script_reuse_existing_ctx(self):
+ expected_test_value_1 = 'test_value_1'
+ expected_test_value_2 = 'test_value_2'
+ props = self._execute(
+ test_operations=['{0}_1'.format(self.test_name),
+ '{0}_2'.format(self.test_name)],
+ env={'test_value1': expected_test_value_1,
+ 'test_value2': expected_test_value_2})
+ assert props['test_value1'] == expected_test_value_1
+ assert props['test_value2'] == expected_test_value_2
+
+ def test_run_script_download_resource_plain(self, tmpdir):
+ resource = tmpdir.join('resource')
+ resource.write('content')
+ self._upload(str(resource), 'test_resource')
+ props = self._execute()
+ assert props['test_value'] == 'content'
+
+ def test_run_script_download_resource_and_render(self, tmpdir):
+ resource = tmpdir.join('resource')
+ resource.write('{{ctx.deployment.name}}')
+ self._upload(str(resource), 'test_resource')
+ props = self._execute()
+ assert props['test_value'] == self._workflow_context.deployment.name
+
+ @pytest.mark.parametrize('value', ['string-value', [1, 2, 3], {'key': 'value'}])
+ def test_run_script_inputs_as_env_variables_no_override(self, value):
+ props = self._execute(custom_input=value)
+ return_value = props['test_value']
+ expected = return_value if isinstance(value, basestring) else json.loads(return_value)
+ assert value == expected
+
+ @pytest.mark.parametrize('value', ['string-value', [1, 2, 3], {'key': 'value'}])
+ def test_run_script_inputs_as_env_variables_process_env_override(self, value):
+ props = self._execute(custom_input='custom-input-value',
+ env={'custom_env_var': value})
+ return_value = props['test_value']
+ expected = return_value if isinstance(value, basestring) else json.loads(return_value)
+ assert value == expected
+
+ def test_run_script_error_in_script(self):
+ exception = self._execute_and_get_task_exception()
+ assert isinstance(exception, TaskException)
+
+ def test_run_script_abort_immediate(self):
+ exception = self._execute_and_get_task_exception()
+ assert isinstance(exception, TaskAbortException)
+ assert exception.message == 'abort-message'
+
+ def test_run_script_retry(self):
+ exception = self._execute_and_get_task_exception()
+ assert isinstance(exception, TaskRetryException)
+ assert exception.message == 'retry-message'
+
+ def test_run_script_abort_error_ignored_by_script(self):
+ exception = self._execute_and_get_task_exception()
+ assert isinstance(exception, TaskAbortException)
+ assert exception.message == 'abort-message'
+
+ def test_run_commands(self):
+ temp_file_path = '/tmp/very_temporary_file'
+ with self._ssh_env():
+ if files.exists(temp_file_path):
+ fabric.api.run('rm {0}'.format(temp_file_path))
+ self._execute(commands=['touch {0}'.format(temp_file_path)])
+ with self._ssh_env():
+ assert files.exists(temp_file_path)
+ fabric.api.run('rm {0}'.format(temp_file_path))
+
+ @pytest.fixture(autouse=True)
+ def _setup(self, request, workflow_context, executor, capfd):
+ self._workflow_context = workflow_context
+ self._executor = executor
+ self._capfd = capfd
+ self.test_name = request.node.originalname or request.node.name
+ with self._ssh_env():
+ for directory in [constants.DEFAULT_BASE_DIR, _CUSTOM_BASE_DIR]:
+ if files.exists(directory):
+ fabric.api.run('rm -rf {0}'.format(directory))
+
+ @contextlib.contextmanager
+ def _ssh_env(self):
+ with self._capfd.disabled():
+ with context_managers.settings(fabric.api.hide('everything'),
+ **_FABRIC_ENV):
+ yield
+
+ def _execute(self,
+ env=None,
+ use_sudo=False,
+ hide_output=None,
+ process=None,
+ custom_input='',
+ test_operations=None,
+ commands=None):
+ process = process or {}
+ if env:
+ process.setdefault('env', {}).update(env)
+
+ test_operations = test_operations or [self.test_name]
+
+ local_script_path = os.path.join(resources.DIR, 'scripts', 'test_ssh.sh')
+ script_path = os.path.basename(local_script_path)
+ self._upload(local_script_path, script_path)
+
+ if commands:
+ operation = operations.run_commands_with_ssh
+ else:
+ operation = operations.run_script_with_ssh
+
+ @workflow
+ def mock_workflow(ctx, graph):
+ op = 'test.op'
+ node_instance = ctx.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+ node_instance.node.operations[op] = {
+ 'operation': '{0}.{1}'.format(operations.__name__, operation.__name__)}
+ graph.sequence(*[api.task.OperationTask.node_instance(
+ instance=node_instance,
+ name=op,
+ inputs={
+ 'script_path': script_path,
+ 'fabric_env': _FABRIC_ENV,
+ 'process': process,
+ 'use_sudo': use_sudo,
+ 'hide_output': hide_output,
+ 'custom_env_var': custom_input,
+ 'test_operation': test_operation,
+ 'commands': commands
+ }) for test_operation in test_operations])
+ return graph
+ tasks_graph = mock_workflow(ctx=self._workflow_context) # pylint: disable=no-value-for-parameter
+ eng = engine.Engine(
+ executor=self._executor,
+ workflow_context=self._workflow_context,
+ tasks_graph=tasks_graph)
+ eng.execute()
+ return self._workflow_context.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties
+
+ def _execute_and_get_task_exception(self, *args, **kwargs):
+ signal = events.on_failure_task_signal
+ with events_collector(signal) as collected:
+ with pytest.raises(ExecutorException):
+ self._execute(*args, **kwargs)
+ return collected[signal][0]['kwargs']['exception']
+
+ def _upload(self, source, path):
+ self._workflow_context.resource.deployment.upload(
+ entry_id=str(self._workflow_context.deployment.id),
+ source=source,
+ path=path)
+
+ @pytest.fixture
+ def executor(self):
+ result = process.ProcessExecutor()
+ yield result
+ result.close()
+
+ @pytest.fixture
+ def workflow_context(self, tmpdir):
+ workflow_context = mock.context.simple(
+ storage.get_sqlite_api_kwargs(str(tmpdir)),
+ resources_dir=str(tmpdir.join('resources')))
+ workflow_context.states = []
+ workflow_context.exception = None
+ yield workflow_context
+ storage.release_sqlite_storage(workflow_context.model)
+
+
+class TestFabricEnvHideGroupsAndRunCommands(object):
+
+ def test_fabric_env_default_override(self):
+ # first sanity for no override
+ self._run()
+ assert self.mock.settings_merged['timeout'] == constants.FABRIC_ENV_DEFAULTS['timeout']
+ # now override
+ invocation_fabric_env = self.default_fabric_env.copy()
+ timeout = 1000000
+ invocation_fabric_env['timeout'] = timeout
+ self._run(fabric_env=invocation_fabric_env)
+ assert self.mock.settings_merged['timeout'] == timeout
+
+ def test_implicit_host_string(self, mocker):
+ expected_ip = '1.1.1.1'
+ mocker.patch.object(self._Ctx.task.runs_on, 'ip', expected_ip)
+ fabric_env = self.default_fabric_env.copy()
+ del fabric_env['host_string']
+ self._run(fabric_env=fabric_env)
+ assert self.mock.settings_merged['host_string'] == expected_ip
+
+ def test_explicit_host_string(self):
+ fabric_env = self.default_fabric_env.copy()
+ host_string = 'explicit_host_string'
+ fabric_env['host_string'] = host_string
+ self._run(fabric_env=fabric_env)
+ assert self.mock.settings_merged['host_string'] == host_string
+
+ def test_override_warn_only(self):
+ fabric_env = self.default_fabric_env.copy()
+ self._run(fabric_env=fabric_env)
+ assert self.mock.settings_merged['warn_only'] is True
+ fabric_env = self.default_fabric_env.copy()
+ fabric_env['warn_only'] = False
+ self._run(fabric_env=fabric_env)
+ assert self.mock.settings_merged['warn_only'] is False
+
+ def test_missing_host_string(self):
+ with pytest.raises(TaskAbortException) as exc_ctx:
+ fabric_env = self.default_fabric_env.copy()
+ del fabric_env['host_string']
+ self._run(fabric_env=fabric_env)
+ assert '`host_string` not supplied' in str(exc_ctx.value)
+
+ def test_missing_user(self):
+ with pytest.raises(TaskAbortException) as exc_ctx:
+ fabric_env = self.default_fabric_env.copy()
+ del fabric_env['user']
+ self._run(fabric_env=fabric_env)
+ assert '`user` not supplied' in str(exc_ctx.value)
+
+ def test_missing_key_or_password(self):
+ with pytest.raises(TaskAbortException) as exc_ctx:
+ fabric_env = self.default_fabric_env.copy()
+ del fabric_env['key_filename']
+ self._run(fabric_env=fabric_env)
+ assert 'Access credentials not supplied' in str(exc_ctx.value)
+
+ def test_hide_in_settings_and_non_viable_groups(self):
+ groups = ('running', 'stdout')
+ self._run(hide_output=groups)
+ assert set(self.mock.settings_merged['hide_output']) == set(groups)
+ with pytest.raises(TaskAbortException) as exc_ctx:
+ self._run(hide_output=('running', 'bla'))
+ assert '`hide_output` must be a subset of' in str(exc_ctx.value)
+
+ def test_run_commands(self):
+ def test(use_sudo):
+ commands = ['command1', 'command2']
+ self._run(
+ commands=commands,
+ use_sudo=use_sudo)
+ assert all(item in self.mock.settings_merged.items() for
+ item in self.default_fabric_env.items())
+ assert self.mock.settings_merged['warn_only'] is True
+ assert self.mock.settings_merged['use_sudo'] == use_sudo
+ assert self.mock.commands == commands
+ self.mock.settings_merged = {}
+ self.mock.commands = []
+ test(use_sudo=False)
+ test(use_sudo=True)
+
+ def test_failed_command(self):
+ with pytest.raises(ProcessException) as exc_ctx:
+ self._run(commands=['fail'])
+ exception = exc_ctx.value
+ assert exception.stdout == self.MockCommandResult.stdout
+ assert exception.stderr == self.MockCommandResult.stderr
+ assert exception.command == self.MockCommandResult.command
+ assert exception.exit_code == self.MockCommandResult.return_code
+
+ class MockCommandResult(object):
+ stdout = 'mock_stdout'
+ stderr = 'mock_stderr'
+ command = 'mock_command'
+ return_code = 1
+
+ def __init__(self, failed):
+ self.failed = failed
+
+ class MockFabricApi(object):
+
+ def __init__(self):
+ self.commands = []
+ self.settings_merged = {}
+
+ @contextlib.contextmanager
+ def settings(self, *args, **kwargs):
+ self.settings_merged.update(kwargs)
+ if args:
+ groups = args[0]
+ self.settings_merged.update({'hide_output': groups})
+ yield
+
+ def run(self, command):
+ self.commands.append(command)
+ self.settings_merged['use_sudo'] = False
+ return TestFabricEnvHideGroupsAndRunCommands.MockCommandResult(command == 'fail')
+
+ def sudo(self, command):
+ self.commands.append(command)
+ self.settings_merged['use_sudo'] = True
+ return TestFabricEnvHideGroupsAndRunCommands.MockCommandResult(command == 'fail')
+
+ def hide(self, *groups):
+ return groups
+
+ def exists(self, *args, **kwargs):
+ raise RuntimeError
+
+ class _Ctx(object):
+ class Stub(object):
+ @staticmethod
+ def abort(message=None):
+ model.Task.abort(message)
+ ip = None
+ task = Stub
+ task.runs_on = Stub
+ logger = logging.getLogger()
+
+ @pytest.fixture(autouse=True)
+ def _setup(self, mocker):
+ self.default_fabric_env = {
+ 'host_string': 'test',
+ 'user': 'test',
+ 'key_filename': 'test',
+ }
+ self.mock = self.MockFabricApi()
+ mocker.patch('fabric.api', self.mock)
+
+ def _run(self,
+ commands=(),
+ fabric_env=None,
+ process=None,
+ use_sudo=False,
+ hide_output=None):
+ operations.run_commands_with_ssh(
+ ctx=self._Ctx,
+ commands=commands,
+ process=process,
+ fabric_env=fabric_env or self.default_fabric_env,
+ use_sudo=use_sudo,
+ hide_output=hide_output)
+
+
+class TestUtilityFunctions(object):
+
+ def test_paths(self):
+ base_dir = '/path'
+ local_script_path = '/local/script/path.py'
+ paths = ssh_operations._Paths(base_dir=base_dir,
+ local_script_path=local_script_path)
+ assert paths.local_script_path == local_script_path
+ assert paths.remote_ctx_dir == base_dir
+ assert paths.base_script_path == 'path.py'
+ assert paths.remote_ctx_path == '/path/ctx'
+ assert paths.remote_scripts_dir == '/path/scripts'
+ assert paths.remote_work_dir == '/path/work'
+ assert paths.remote_env_script_path.startswith('/path/scripts/env-path.py-')
+ assert paths.remote_script_path.startswith('/path/scripts/path.py-')
+
+ def test_write_environment_script_file(self):
+ base_dir = '/path'
+ local_script_path = '/local/script/path.py'
+ paths = ssh_operations._Paths(base_dir=base_dir,
+ local_script_path=local_script_path)
+ env = {'one': "'1'"}
+ local_socket_url = 'local_socket_url'
+ remote_socket_url = 'remote_socket_url'
+ env_script_lines = set([l for l in ssh_operations._write_environment_script_file(
+ process={'env': env},
+ paths=paths,
+ local_socket_url=local_socket_url,
+ remote_socket_url=remote_socket_url
+ ).getvalue().split('\n') if l])
+ expected_env_script_lines = set([
+ 'export PATH=/path:$PATH',
+ 'export PYTHONPATH=/path:$PYTHONPATH',
+ 'chmod +x /path/ctx',
+ 'chmod +x {0}'.format(paths.remote_script_path),
+ 'export CTX_SOCKET_URL={0}'.format(remote_socket_url),
+ 'export LOCAL_CTX_SOCKET_URL={0}'.format(local_socket_url),
+ 'export one=\'1\''
+ ])
+ assert env_script_lines == expected_env_script_lines
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/orchestrator/workflows/api/test_task.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/api/test_task.py b/tests/orchestrator/workflows/api/test_task.py
index 58e387f..601c437 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -16,6 +16,7 @@
import pytest
+from aria.storage import model
from aria.orchestrator import context
from aria.orchestrator.workflows import api
@@ -72,6 +73,7 @@ class TestOperationTask(object):
assert api_task.plugin == {'name': 'plugin',
'package_name': 'package',
'package_version': '0.1'}
+ assert api_task.runs_on == model.Task.RUNS_ON_NODE_INSTANCE
def test_source_relationship_operation_task_creation(self, ctx):
operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
@@ -104,6 +106,7 @@ class TestOperationTask(object):
assert api_task.plugin == {'name': 'plugin',
'package_name': 'package',
'package_version': '0.1'}
+ assert api_task.runs_on == model.Task.RUNS_ON_SOURCE
def test_target_relationship_operation_task_creation(self, ctx):
operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
@@ -136,6 +139,7 @@ class TestOperationTask(object):
assert api_task.plugin == {'name': 'plugin',
'package_name': 'package',
'package_version': '0.1'}
+ assert api_task.runs_on == model.Task.RUNS_ON_TARGET
def test_operation_task_default_values(self, ctx):
dependency_node_instance = ctx.model.node_instance.get_by_name(
@@ -151,6 +155,7 @@ class TestOperationTask(object):
assert task.max_attempts == ctx._task_max_attempts
assert task.ignore_failure == ctx._task_ignore_failure
assert task.plugin == {}
+ assert task.runs_on is None
class TestWorkflowTask(object):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/orchestrator/workflows/core/test_task.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/core/test_task.py b/tests/orchestrator/workflows/core/test_task.py
index 5381f5d..020de32 100644
--- a/tests/orchestrator/workflows/core/test_task.py
+++ b/tests/orchestrator/workflows/core/test_task.py
@@ -38,7 +38,7 @@ def ctx(tmpdir):
class TestOperationTask(object):
- def _create_operation_task(self, ctx, node_instance):
+ def _create_node_operation_task(self, ctx, node_instance):
with workflow_context.current.push(ctx):
api_task = api.task.OperationTask.node_instance(
instance=node_instance,
@@ -46,21 +46,31 @@ class TestOperationTask(object):
core_task = core.task.OperationTask(api_task=api_task)
return api_task, core_task
- def test_operation_task_creation(self, ctx):
+ def _create_relationship_operation_task(self, ctx, relationship_instance, operation_end):
+ with workflow_context.current.push(ctx):
+ api_task = api.task.OperationTask.relationship_instance(
+ instance=relationship_instance,
+ name='aria.interfaces.relationship_lifecycle.preconfigure',
+ operation_end=operation_end)
+ core_task = core.task.OperationTask(api_task=api_task)
+ return api_task, core_task
+
+ def test_node_operation_task_creation(self, ctx):
storage_plugin = mock.models.get_plugin(package_name='p1', package_version='0.1')
storage_plugin_other = mock.models.get_plugin(package_name='p0', package_version='0.0')
ctx.model.plugin.put(storage_plugin_other)
ctx.model.plugin.put(storage_plugin)
- node_instance = \
- ctx.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+ node_instance = ctx.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
node = node_instance.node
node.plugins = [{'name': 'plugin1',
'package_name': 'p1',
'package_version': '0.1'}]
node.operations['aria.interfaces.lifecycle.create'] = {'plugin': 'plugin1'}
- api_task, core_task = self._create_operation_task(ctx, node_instance)
+ api_task, core_task = self._create_node_operation_task(ctx, node_instance)
storage_task = ctx.model.task.get_by_name(core_task.name)
assert storage_task.execution_name == ctx.execution.name
+ assert storage_task.runs_on.id == core_task.context.node_instance.id
assert core_task.model_task == storage_task
assert core_task.name == api_task.name
assert core_task.operation_mapping == api_task.operation_mapping
@@ -68,11 +78,25 @@ class TestOperationTask(object):
assert core_task.inputs == api_task.inputs == storage_task.inputs
assert core_task.plugin == storage_plugin
+ def test_source_relationship_operation_task_creation(self, ctx):
+ relationship_instance = ctx.model.relationship_instance.list()[0]
+ _, core_task = self._create_relationship_operation_task(
+ ctx, relationship_instance,
+ api.task.OperationTask.SOURCE_OPERATION)
+ assert core_task.model_task.runs_on.id == relationship_instance.source_node_instance.id
+
+ def test_target_relationship_operation_task_creation(self, ctx):
+ relationship_instance = ctx.model.relationship_instance.list()[0]
+ _, core_task = self._create_relationship_operation_task(
+ ctx, relationship_instance,
+ api.task.OperationTask.TARGET_OPERATION)
+ assert core_task.model_task.runs_on.id == relationship_instance.target_node_instance.id
+
def test_operation_task_edit_locked_attribute(self, ctx):
node_instance = \
ctx.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
- _, core_task = self._create_operation_task(ctx, node_instance)
+ _, core_task = self._create_node_operation_task(ctx, node_instance)
now = datetime.utcnow()
with pytest.raises(exceptions.TaskException):
core_task.status = core_task.STARTED
@@ -89,7 +113,7 @@ class TestOperationTask(object):
node_instance = \
ctx.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
- _, core_task = self._create_operation_task(ctx, node_instance)
+ _, core_task = self._create_node_operation_task(ctx, node_instance)
future_time = datetime.utcnow() + timedelta(seconds=3)
with core_task._update():
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/orchestrator/workflows/helpers.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/helpers.py b/tests/orchestrator/workflows/helpers.py
new file mode 100644
index 0000000..8e3f9b1
--- /dev/null
+++ b/tests/orchestrator/workflows/helpers.py
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from contextlib import contextmanager
+
+
+@contextmanager
+def events_collector(*signals):
+ handlers = {}
+ collected = {}
+
+ def handler_factory(key):
+ def handler(*args, **kwargs):
+ signal_events = collected.setdefault(key, [])
+ signal_events.append({'args': args, 'kwargs': kwargs})
+ handlers[signal] = handler
+ return handler
+
+ for signal in signals:
+ signal.connect(handler_factory(signal))
+ try:
+ yield collected
+ finally:
+ for signal in signals:
+ signal.disconnect(handlers[signal])
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/resources/scripts/test_ssh.sh
----------------------------------------------------------------------
diff --git a/tests/resources/scripts/test_ssh.sh b/tests/resources/scripts/test_ssh.sh
new file mode 100644
index 0000000..6f18278
--- /dev/null
+++ b/tests/resources/scripts/test_ssh.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -u
+set -e
+
+test_run_script_basic() {
+ ctx node-instance runtime-properties test_value $test_value
+}
+
+test_run_script_as_sudo() {
+ mkdir -p /opt/test_dir
+}
+
+test_run_script_default_base_dir() {
+ ctx node-instance runtime-properties work_dir $PWD
+}
+
+test_run_script_with_hide() {
+ true
+}
+
+test_run_script_process_config() {
+ ctx node-instance runtime-properties env_value $test_value_env
+ ctx node-instance runtime-properties bash_version $BASH_VERSION
+ ctx node-instance runtime-properties arg1_value $1
+ ctx node-instance runtime-properties arg2_value $2
+ ctx node-instance runtime-properties cwd $PWD
+ ctx node-instance runtime-properties ctx_path $(which ctx)
+}
+
+test_run_script_command_prefix() {
+ ctx node-instance runtime-properties dollar_dash $-
+}
+
+test_run_script_reuse_existing_ctx_1() {
+ ctx node-instance runtime-properties test_value1 $test_value1
+}
+
+test_run_script_reuse_existing_ctx_2() {
+ ctx node-instance runtime-properties test_value2 $test_value2
+}
+
+test_run_script_download_resource_plain() {
+ local destination=$(mktemp)
+ ctx download-resource ${destination} test_resource
+ ctx node-instance runtime-properties test_value "$(cat ${destination})"
+}
+
+test_run_script_download_resource_and_render() {
+ local destination=$(mktemp)
+ ctx download-resource-and-render ${destination} test_resource
+ ctx node-instance runtime-properties test_value "$(cat ${destination})"
+}
+
+test_run_script_inputs_as_env_variables_no_override() {
+ ctx node-instance runtime-properties test_value "$custom_env_var"
+}
+
+test_run_script_inputs_as_env_variables_process_env_override() {
+ ctx node-instance runtime-properties test_value "$custom_env_var"
+}
+
+test_run_script_error_in_script() {
+ ctx property-that-does-not-exist
+}
+
+test_run_script_abort_immediate() {
+ ctx task abort abort-message
+}
+
+test_run_script_retry() {
+ ctx task retry retry-message
+}
+
+test_run_script_abort_error_ignored_by_script() {
+ set +e
+ ctx task abort abort-message
+}
+
+# Injected by test
+${test_operation} $@
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/test_logger.py
----------------------------------------------------------------------
diff --git a/tests/test_logger.py b/tests/test_logger.py
index 0199068..70f08bb 100644
--- a/tests/test_logger.py
+++ b/tests/test_logger.py
@@ -109,7 +109,7 @@ def test_loggermixin(capsys):
test_string = 'loggermixing_test_string'
- create_logger(handlers=[create_console_log_handler()])
+ logger = create_logger(handlers=[create_console_log_handler()])
custom_class = type('CustomClass', (LoggerMixin,), {}).with_logger()
custom_class.logger.debug(test_string)
@@ -117,6 +117,9 @@ def test_loggermixin(capsys):
_, err = capsys.readouterr()
assert test_string in err
+ for handler in logger.handlers:
+ logger.removeHandler(handler)
+
# TODO: figure out what up with pickle
# class_pickled = pickle.dumps(custom_class)
# class_unpickled = pickle.loads(class_pickled)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/utils/test_exceptions.py
----------------------------------------------------------------------
diff --git a/tests/utils/test_exceptions.py b/tests/utils/test_exceptions.py
new file mode 100644
index 0000000..5d030e2
--- /dev/null
+++ b/tests/utils/test_exceptions.py
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import jsonpickle
+
+from aria.utils import exceptions
+
+_ARG1 = 'arg-1'
+_ARG2 = 'arg-2'
+
+
+class TestWrapIfNeeded(object):
+
+ def test_no_wrapping_required1(self):
+ e = JsonPickleableException1(_ARG1, _ARG2)
+ assert exceptions.wrap_if_needed(e) is e
+
+ def test_no_wrapping_required2(self):
+ e = JsonPickleableException1(arg1=_ARG1, arg2=_ARG2)
+ assert exceptions.wrap_if_needed(e) is e
+
+ def test_no_wrapping_required3(self):
+ e = JsonPickleableException2(arg1=_ARG1, arg2=_ARG2)
+ assert exceptions.wrap_if_needed(e) is e
+
+ def test_wrapping_required1(self):
+ e = NonJsonPickleableException(_ARG1, _ARG2)
+ wrapped_e = exceptions.wrap_if_needed(e)
+ wrapped_e = jsonpickle.loads(jsonpickle.dumps(wrapped_e))
+ assert isinstance(wrapped_e, exceptions._WrappedException)
+ assert wrapped_e.exception_type == type(e).__name__
+ assert wrapped_e.exception_str == str(e)
+
+ def test_wrapping_required2(self):
+ e = NonJsonPickleableException(arg1=_ARG1, arg2=_ARG2)
+ wrapped_e = exceptions.wrap_if_needed(e)
+ wrapped_e = jsonpickle.loads(jsonpickle.dumps(wrapped_e))
+ assert isinstance(wrapped_e, exceptions._WrappedException)
+ assert wrapped_e.exception_type == type(e).__name__
+ assert wrapped_e.exception_str == str(e)
+
+
+class JsonPickleableException1(Exception):
+ def __init__(self, arg1, arg2):
+ super(JsonPickleableException1, self).__init__(arg1, arg2)
+ self.arg1 = arg1
+ self.arg2 = arg2
+
+
+class JsonPickleableException2(Exception):
+ def __init__(self, arg1=None, arg2=None):
+ super(JsonPickleableException2, self).__init__()
+ self.arg1 = arg1
+ self.arg2 = arg2
+
+
+class NonJsonPickleableException(Exception):
+ def __init__(self, arg1, arg2):
+ super(NonJsonPickleableException, self).__init__()
+ self.arg1 = arg1
+ self.arg2 = arg2
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tox.ini
----------------------------------------------------------------------
diff --git a/tox.ini b/tox.ini
index 68f9ffa..fa4bd5c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,6 +14,13 @@
envlist=py27,py26,pywin,pylint_code,pylint_tests
[testenv]
+passenv =
+ TRAVIS
+ PYTHON
+ PYTHON_VERSION
+ PYTHON_ARCH
+setenv =
+ INSTALL_CTX=1
deps =
-rrequirements.txt
-rtests/requirements.txt
[14/14] incubator-ariatosca git commit: added support for strict
lists and dictionaries
Posted by mx...@apache.org.
added support for strict lists and dictionaries
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/64a393ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/64a393ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/64a393ec
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: 64a393eceed453675c6ef74a5f6fa88c291d55fc
Parents: 4447829
Author: mxmrlv <mx...@gmail.com>
Authored: Sun Jan 15 19:07:29 2017 +0200
Committer: mxmrlv <mx...@gmail.com>
Committed: Mon Jan 30 16:14:47 2017 +0200
----------------------------------------------------------------------
aria/storage/type.py | 152 ++++++++++++++++-
tests/storage/__init__.py | 17 +-
tests/storage/test_model_storage.py | 195 +--------------------
tests/storage/test_structures.py | 281 +++++++++++++++++++++++++++++++
4 files changed, 449 insertions(+), 196 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/64a393ec/aria/storage/type.py
----------------------------------------------------------------------
diff --git a/aria/storage/type.py b/aria/storage/type.py
index dd2a5ce..1b28f33 100644
--- a/aria/storage/type.py
+++ b/aria/storage/type.py
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
+from collections import namedtuple
+from types import NoneType
from sqlalchemy import (
TypeDecorator,
@@ -61,10 +63,68 @@ class List(_MutableType):
return list
+class _StrictDictMixin(object):
+ ANY_TYPE = 'any_type'
+
+ _key_cls = ANY_TYPE
+ _value_cls = ANY_TYPE
+
+ @classmethod
+ def coerce(cls, key, value):
+ "Convert plain dictionaries to MutableDict."
+ try:
+ if not isinstance(value, cls):
+ if isinstance(value, dict):
+ for k, v in value.items():
+ cls._assert_strict_key(k)
+ cls._assert_strict_value(v)
+ return cls(value)
+ return mutable.MutableDict.coerce(key, value)
+ else:
+ return value
+ except ValueError as e:
+ raise exceptions.StorageError('SQL Storage error: {0}'.format(str(e)))
+
+ def __setitem__(self, key, value):
+ self._assert_strict_key(key)
+ self._assert_strict_value(value)
+ super(_StrictDictMixin, self).__setitem__(key, value)
+
+ def setdefault(self, key, value):
+ self._assert_strict_key(key)
+ self._assert_strict_value(value)
+ super(_StrictDictMixin, self).setdefault(key, value)
+
+ def update(self, *a, **kw):
+ if isinstance(a, dict):
+ for k, v in a.items():
+ self._assert_strict_key(k)
+ self._assert_strict_value(v)
+ for k, v in kw.items():
+ self._assert_strict_key(k)
+ self._assert_strict_value(v)
+ super(_StrictDictMixin, self).update(*a, **kw)
+
+ @classmethod
+ def _assert_strict_key(cls, key):
+ if not isinstance(key, cls._key_cls):
+ raise exceptions.StorageError("Key type was set strictly to {0}, but was {1}".format(
+ cls._key_cls, type(key)
+ ))
+
+ @classmethod
+ def _assert_strict_value(cls, value):
+ if not isinstance(value, cls._value_cls):
+ raise exceptions.StorageError("Value type was set strictly to {0}, but was {1}".format(
+ cls._value_cls, type(value)
+ ))
+
+
class _MutableDict(mutable.MutableDict):
"""
Enables tracking for dict values.
"""
+
@classmethod
def coerce(cls, key, value):
"Convert plain dictionaries to MutableDict."
@@ -74,16 +134,106 @@ class _MutableDict(mutable.MutableDict):
raise exceptions.StorageError('SQL Storage error: {0}'.format(str(e)))
+class _StrictListMixin(object):
+ ANY_TYPE = 'any_type'
+
+ _item_cls = ANY_TYPE
+
+ @classmethod
+ def coerce(cls, key, value):
+ "Convert plain dictionaries to MutableDict."
+ try:
+ if not isinstance(value, cls):
+ if isinstance(value, list):
+ for item in value:
+ cls._assert_item(item)
+ return cls(value)
+ return mutable.MutableList.coerce(key, value)
+ else:
+ return value
+ except ValueError as e:
+ raise exceptions.StorageError('SQL Storage error: {0}'.format(str(e)))
+
+ def __setitem__(self, index, value):
+ """Detect list set events and emit change events."""
+ self._assert_item(value)
+ super(_StrictListMixin, self).__setitem__(index, value)
+
+ def append(self, x):
+ self._assert_item(x)
+ super(_StrictListMixin, self).append(x)
+
+ def extend(self, x):
+ self._assert_item(x)
+ super(_StrictListMixin, self).extend(x)
+
+ def insert(self, i, x):
+ self._assert_item(x)
+ super(_StrictListMixin, self).insert(i, x)
+
+ @classmethod
+ def _assert_item(cls, item):
+ if not isinstance(item, cls._item_cls):
+ raise exceptions.StorageError("Key type was set strictly to {0}, but was {1}".format(
+ cls._item_cls, type(item)
+ ))
+
+
class _MutableList(mutable.MutableList):
@classmethod
def coerce(cls, key, value):
"Convert plain dictionaries to MutableDict."
+
try:
- return mutable.MutableList.coerce(key, value)
+ if not isinstance(value, cls):
+ if isinstance(value, list):
+ return cls(value)
+
+ return mutable.Mutable.coerce(key, value)
+ else:
+ return value
+
except ValueError as e:
raise exceptions.StorageError('SQL Storage error: {0}'.format(str(e)))
+_strict_dict_id = namedtuple('strict_dict_id', 'key_cls, value_cls')
+_strict_map = {}
+
+
+def StrictDict(key_cls, value_cls):
+ strict_dict_map_key = _strict_dict_id(key_cls=key_cls, value_cls=value_cls)
+ if strict_dict_map_key not in _strict_map:
+ strict_dict_cls = type(
+ 'StrictDict_{0}_{1}'.format(key_cls.__name__, value_cls.__name__),
+ (Dict, ),
+ {}
+ )
+ type(
+ 'StrictMutableDict_{0}_{1}'.format(key_cls.__name__, value_cls.__name__),
+ (_StrictDictMixin, _MutableDict),
+ {'_key_cls': key_cls, '_value_cls': value_cls}
+ ).associate_with(strict_dict_cls)
+ _strict_map[strict_dict_map_key] = strict_dict_cls
+
+ return _strict_map[strict_dict_map_key]
+
+
+def StrictList(item_cls):
+ if item_cls not in _strict_map:
+ strict_list_cls = type(
+ 'StrictList_{0}'.format(item_cls.__name__),
+ (List, ),
+ {}
+ )
+ type(
+ 'StrictMutableList_{0}'.format(item_cls.__name__),
+ (_StrictListMixin, _MutableList),
+ {'_item_cls': item_cls}
+ ).associate_with(strict_list_cls)
+ _strict_map[item_cls] = strict_list_cls
+
+ return _strict_map[item_cls]
def _mutable_association_listener(mapper, cls):
for prop in mapper.column_attrs:
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/64a393ec/tests/storage/__init__.py
----------------------------------------------------------------------
diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py
index 9101fd0..c772acb 100644
--- a/tests/storage/__init__.py
+++ b/tests/storage/__init__.py
@@ -17,7 +17,13 @@ import platform
from tempfile import mkdtemp
from shutil import rmtree
-from aria.storage import model
+from sqlalchemy import Column, Text, Integer
+
+from aria.storage import (
+ model,
+ structure,
+ type as aria_type,
+)
from sqlalchemy import (
create_engine,
orm)
@@ -25,6 +31,15 @@ from sqlalchemy.orm import scoped_session
from sqlalchemy.pool import StaticPool
+class MockModel(model.DeclarativeBase, structure.ModelMixin): #pylint: disable=abstract-method
+ __tablename__ = 'mock_models'
+ model_dict = Column(aria_type.Dict)
+ model_list = Column(aria_type.List)
+ value = Column(Integer)
+ name = Column(Text)
+
+
+
class TestFileSystem(object):
def setup_method(self):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/64a393ec/tests/storage/test_model_storage.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py
index 0e8d1a0..d1596e3 100644
--- a/tests/storage/test_model_storage.py
+++ b/tests/storage/test_model_storage.py
@@ -15,31 +15,16 @@
import pytest
-from sqlalchemy import Column, Text, Integer
-
from aria.storage import (
ModelStorage,
model,
exceptions,
sql_mapi,
- structure,
- type as aria_type,
)
from aria import application_model_storage
from ..storage import get_sqlite_api_kwargs, release_sqlite_storage
-from ..mock import (
- context as mock_context,
- models,
- operations
-)
-
-class MockModel(model.DeclarativeBase, structure.ModelMixin): #pylint: disable=abstract-method
- __tablename__ = 'mock_models'
- model_dict = Column(aria_type.Dict)
- model_list = Column(aria_type.List)
- value = Column(Integer)
- name = Column(Text)
+from . import MockModel
@pytest.fixture
@@ -50,11 +35,6 @@ def storage():
release_sqlite_storage(base_storage)
-@pytest.fixture
-def context():
- return mock_context.simple(get_sqlite_api_kwargs())
-
-
@pytest.fixture(scope='module', autouse=True)
def module_cleanup():
model.DeclarativeBase.metadata.remove(MockModel.__table__) #pylint: disable=no-member
@@ -79,65 +59,6 @@ def test_model_storage(storage):
storage.mock_model.get(mock_model.id)
-def test_inner_dict_update(storage):
- inner_dict = {'inner_value': 1}
-
- mock_model = MockModel(model_dict={'inner_dict': inner_dict, 'value': 0})
- storage.mock_model.put(mock_model)
-
- storage_mm = storage.mock_model.get(mock_model.id)
- assert storage_mm == mock_model
-
- storage_mm.model_dict['inner_dict']['inner_value'] = 2
- storage_mm.model_dict['value'] = -1
- storage.mock_model.update(storage_mm)
- storage_mm = storage.mock_model.get(storage_mm.id)
-
- assert storage_mm.model_dict['inner_dict']['inner_value'] == 2
- assert storage_mm.model_dict['value'] == -1
-
-
-def test_inner_list_update(storage):
- mock_model = MockModel(model_list=[0, [1]])
- storage.mock_model.put(mock_model)
-
- storage_mm = storage.mock_model.get(mock_model.id)
- assert storage_mm == mock_model
-
- storage_mm.model_list[1][0] = 'new_inner_value'
- storage_mm.model_list[0] = 'new_value'
- storage.mock_model.update(storage_mm)
- storage_mm = storage.mock_model.get(storage_mm.id)
-
- assert storage_mm.model_list[1][0] == 'new_inner_value'
- assert storage_mm.model_list[0] == 'new_value'
-
-
-def test_model_to_dict(context):
- deployment = context.deployment
- deployment_dict = deployment.to_dict()
-
- expected_keys = [
- 'created_at',
- 'description',
- 'inputs',
- 'groups',
- 'permalink',
- 'policy_triggers',
- 'policy_types',
- 'outputs',
- 'scaling_groups',
- 'updated_at',
- 'workflows',
- 'blueprint_name',
- ]
-
- for expected_key in expected_keys:
- assert expected_key in deployment_dict
-
- assert 'blueprint_fk' not in deployment_dict
-
-
def test_application_storage_factory():
storage = application_model_storage(sql_mapi.SQLAlchemyModelAPI,
api_kwargs=get_sqlite_api_kwargs())
@@ -152,117 +73,3 @@ def test_application_storage_factory():
assert storage.execution
release_sqlite_storage(storage)
-
-
-def test_relationship_model_ordering(context):
- deployment = context.model.deployment.get_by_name(models.DEPLOYMENT_NAME)
- source_node = context.model.node.get_by_name(models.DEPENDENT_NODE_NAME)
- source_node_instance = context.model.node_instance.get_by_name(
- models.DEPENDENT_NODE_INSTANCE_NAME)
- target_node = context.model.node.get_by_name(models.DEPENDENCY_NODE_NAME)
- target_node_instance = context.model.node_instance.get_by_name(
- models.DEPENDENCY_NODE_INSTANCE_NAME)
- new_node = model.Node(
- name='new_node',
- type='test_node_type',
- type_hierarchy=[],
- number_of_instances=1,
- planned_number_of_instances=1,
- deploy_number_of_instances=1,
- properties={},
- operations=dict((key, {}) for key in operations.NODE_OPERATIONS),
- min_number_of_instances=1,
- max_number_of_instances=1,
- deployment=deployment
- )
- source_to_new_relationship = model.Relationship(
- source_node=source_node,
- target_node=new_node,
- source_interfaces={},
- source_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
- target_interfaces={},
- target_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
- type='rel_type',
- type_hierarchy=[],
- properties={},
- )
- new_node_instance = model.NodeInstance(
- name='new_node_instance',
- runtime_properties={},
- version=None,
- node=new_node,
- state='',
- scaling_groups=[]
- )
- source_to_new_relationship_instance = model.RelationshipInstance(
- relationship=source_to_new_relationship,
- source_node_instance=source_node_instance,
- target_node_instance=new_node_instance,
- )
-
- new_to_target_relationship = model.Relationship(
- source_node=new_node,
- target_node=target_node,
- source_interfaces={},
- source_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
- target_interfaces={},
- target_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
- type='rel_type',
- type_hierarchy=[],
- properties={},
- )
- new_to_target_relationship_instance = model.RelationshipInstance(
- relationship=new_to_target_relationship,
- source_node_instance=new_node_instance,
- target_node_instance=target_node_instance,
- )
-
-
- context.model.node.put(new_node)
- context.model.node_instance.put(new_node_instance)
- context.model.relationship.put(source_to_new_relationship)
- context.model.relationship.put(new_to_target_relationship)
- context.model.relationship_instance.put(source_to_new_relationship_instance)
- context.model.relationship_instance.put(new_to_target_relationship_instance)
-
- def flip_and_assert(node_instance, direction):
- """
- Reversed the order of relationships and assert effects took place.
- :param node_instance: the node instance to operatate on
- :param direction: the type of relationships to flip (inbound/outbount)
- :return:
- """
- assert direction in ('inbound', 'outbound')
-
- relationships = getattr(node_instance.node, direction + '_relationships')
- relationship_instances = getattr(node_instance, direction + '_relationship_instances')
- assert len(relationships) == 2
- assert len(relationship_instances) == 2
-
- first_rel, second_rel = relationships
- first_rel_instance, second_rel_instance = relationship_instances
- assert getattr(first_rel, relationships.ordering_attr) == 0
- assert getattr(second_rel, relationships.ordering_attr) == 1
- assert getattr(first_rel_instance, relationship_instances.ordering_attr) == 0
- assert getattr(second_rel_instance, relationship_instances.ordering_attr) == 1
-
- reversed_relationships = list(reversed(relationships))
- reversed_relationship_instances = list(reversed(relationship_instances))
-
- assert relationships != reversed_relationships
- assert relationship_instances != reversed_relationship_instances
-
- relationships[:] = reversed_relationships
- relationship_instances[:] = reversed_relationship_instances
- context.model.node_instance.update(node_instance)
-
- assert relationships == reversed_relationships
- assert relationship_instances == reversed_relationship_instances
-
- assert getattr(first_rel, relationships.ordering_attr) == 1
- assert getattr(second_rel, relationships.ordering_attr) == 0
- assert getattr(first_rel_instance, relationship_instances.ordering_attr) == 1
- assert getattr(second_rel_instance, relationship_instances.ordering_attr) == 0
-
- flip_and_assert(source_node_instance, 'outbound')
- flip_and_assert(target_node_instance, 'inbound')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/64a393ec/tests/storage/test_structures.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_structures.py b/tests/storage/test_structures.py
new file mode 100644
index 0000000..e6d2093
--- /dev/null
+++ b/tests/storage/test_structures.py
@@ -0,0 +1,281 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+import sqlalchemy
+
+from aria.storage import (
+ ModelStorage,
+ sql_mapi,
+ model,
+ type,
+ exceptions
+)
+
+from ..storage import get_sqlite_api_kwargs, release_sqlite_storage, structure
+from . import MockModel
+from ..mock import (
+ models,
+ operations,
+ context as mock_context
+)
+
+
+@pytest.fixture
+def storage():
+ base_storage = ModelStorage(sql_mapi.SQLAlchemyModelAPI, api_kwargs=get_sqlite_api_kwargs())
+ base_storage.register(MockModel)
+ yield base_storage
+ release_sqlite_storage(base_storage)
+
+
+@pytest.fixture(scope='module', autouse=True)
+def module_cleanup():
+ model.DeclarativeBase.metadata.remove(MockModel.__table__) #pylint: disable=no-member
+
+
+@pytest.fixture
+def context():
+ return mock_context.simple(get_sqlite_api_kwargs())
+
+
+def test_inner_dict_update(storage):
+ inner_dict = {'inner_value': 1}
+
+ mock_model = MockModel(model_dict={'inner_dict': inner_dict, 'value': 0})
+ storage.mock_model.put(mock_model)
+
+ storage_mm = storage.mock_model.get(mock_model.id)
+ assert storage_mm == mock_model
+
+ storage_mm.model_dict['inner_dict']['inner_value'] = 2
+ storage_mm.model_dict['value'] = -1
+ storage.mock_model.update(storage_mm)
+ storage_mm = storage.mock_model.get(storage_mm.id)
+
+ assert storage_mm.model_dict['inner_dict']['inner_value'] == 2
+ assert storage_mm.model_dict['value'] == -1
+
+
+def test_inner_list_update(storage):
+ mock_model = MockModel(model_list=[0, [1]])
+ storage.mock_model.put(mock_model)
+
+ storage_mm = storage.mock_model.get(mock_model.id)
+ assert storage_mm == mock_model
+
+ storage_mm.model_list[1][0] = 'new_inner_value'
+ storage_mm.model_list[0] = 'new_value'
+ storage.mock_model.update(storage_mm)
+ storage_mm = storage.mock_model.get(storage_mm.id)
+
+ assert storage_mm.model_list[1][0] == 'new_inner_value'
+ assert storage_mm.model_list[0] == 'new_value'
+
+
+def test_model_to_dict(context):
+ deployment = context.deployment
+ deployment_dict = deployment.to_dict()
+
+ expected_keys = [
+ 'created_at',
+ 'description',
+ 'inputs',
+ 'groups',
+ 'permalink',
+ 'policy_triggers',
+ 'policy_types',
+ 'outputs',
+ 'scaling_groups',
+ 'updated_at',
+ 'workflows',
+ 'blueprint_name',
+ ]
+
+ for expected_key in expected_keys:
+ assert expected_key in deployment_dict
+
+ assert 'blueprint_fk' not in deployment_dict
+
+
+def test_relationship_model_ordering(context):
+ deployment = context.model.deployment.get_by_name(models.DEPLOYMENT_NAME)
+ source_node = context.model.node.get_by_name(models.DEPENDENT_NODE_NAME)
+ source_node_instance = context.model.node_instance.get_by_name(
+ models.DEPENDENT_NODE_INSTANCE_NAME)
+ target_node = context.model.node.get_by_name(models.DEPENDENCY_NODE_NAME)
+ target_node_instance = context.model.node_instance.get_by_name(
+ models.DEPENDENCY_NODE_INSTANCE_NAME)
+ new_node = model.Node(
+ name='new_node',
+ type='test_node_type',
+ type_hierarchy=[],
+ number_of_instances=1,
+ planned_number_of_instances=1,
+ deploy_number_of_instances=1,
+ properties={},
+ operations=dict((key, {}) for key in operations.NODE_OPERATIONS),
+ min_number_of_instances=1,
+ max_number_of_instances=1,
+ deployment=deployment
+ )
+ source_to_new_relationship = model.Relationship(
+ source_node=source_node,
+ target_node=new_node,
+ source_interfaces={},
+ source_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
+ target_interfaces={},
+ target_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
+ type='rel_type',
+ type_hierarchy=[],
+ properties={},
+ )
+ new_node_instance = model.NodeInstance(
+ name='new_node_instance',
+ runtime_properties={},
+ version=None,
+ node=new_node,
+ state='',
+ scaling_groups=[]
+ )
+ source_to_new_relationship_instance = model.RelationshipInstance(
+ relationship=source_to_new_relationship,
+ source_node_instance=source_node_instance,
+ target_node_instance=new_node_instance,
+ )
+
+ new_to_target_relationship = model.Relationship(
+ source_node=new_node,
+ target_node=target_node,
+ source_interfaces={},
+ source_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
+ target_interfaces={},
+ target_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
+ type='rel_type',
+ type_hierarchy=[],
+ properties={},
+ )
+ new_to_target_relationship_instance = model.RelationshipInstance(
+ relationship=new_to_target_relationship,
+ source_node_instance=new_node_instance,
+ target_node_instance=target_node_instance,
+ )
+
+
+ context.model.node.put(new_node)
+ context.model.node_instance.put(new_node_instance)
+ context.model.relationship.put(source_to_new_relationship)
+ context.model.relationship.put(new_to_target_relationship)
+ context.model.relationship_instance.put(source_to_new_relationship_instance)
+ context.model.relationship_instance.put(new_to_target_relationship_instance)
+
+ def flip_and_assert(node_instance, direction):
+ """
+ Reversed the order of relationships and assert effects took place.
+ :param node_instance: the node instance to operatate on
+ :param direction: the type of relationships to flip (inbound/outbount)
+ :return:
+ """
+ assert direction in ('inbound', 'outbound')
+
+ relationships = getattr(node_instance.node, direction + '_relationships')
+ relationship_instances = getattr(node_instance, direction + '_relationship_instances')
+ assert len(relationships) == 2
+ assert len(relationship_instances) == 2
+
+ first_rel, second_rel = relationships
+ first_rel_instance, second_rel_instance = relationship_instances
+ assert getattr(first_rel, relationships.ordering_attr) == 0
+ assert getattr(second_rel, relationships.ordering_attr) == 1
+ assert getattr(first_rel_instance, relationship_instances.ordering_attr) == 0
+ assert getattr(second_rel_instance, relationship_instances.ordering_attr) == 1
+
+ reversed_relationships = list(reversed(relationships))
+ reversed_relationship_instances = list(reversed(relationship_instances))
+
+ assert relationships != reversed_relationships
+ assert relationship_instances != reversed_relationship_instances
+
+ relationships[:] = reversed_relationships
+ relationship_instances[:] = reversed_relationship_instances
+ context.model.node_instance.update(node_instance)
+
+ assert relationships == reversed_relationships
+ assert relationship_instances == reversed_relationship_instances
+
+ assert getattr(first_rel, relationships.ordering_attr) == 1
+ assert getattr(second_rel, relationships.ordering_attr) == 0
+ assert getattr(first_rel_instance, relationship_instances.ordering_attr) == 1
+ assert getattr(second_rel_instance, relationship_instances.ordering_attr) == 0
+
+ flip_and_assert(source_node_instance, 'outbound')
+ flip_and_assert(target_node_instance, 'inbound')
+
+
+class StrictClass(model.DeclarativeBase, structure.ModelMixin):
+ __tablename__ = 'class_with_strict_dict'
+
+ strict_dict = sqlalchemy.Column(type.StrictDict(basestring, basestring))
+ strict_list = sqlalchemy.Column(type.StrictList(basestring))
+
+
+def test_strict_dict():
+
+ strict_class = StrictClass()
+
+ def assert_strict(sd, with_assigment=False):
+ if with_assigment:
+ with pytest.raises(exceptions.StorageError):
+ sd.strict_dict['key'] = None
+
+ with pytest.raises(exceptions.StorageError):
+ sd.strict_dict = {'key': None}
+
+ if with_assigment:
+ with pytest.raises(exceptions.StorageError):
+ sd.strict_dict[None] = 'value'
+
+ with pytest.raises(exceptions.StorageError):
+ sd.strict_dict = {None: 'value'}
+
+ if with_assigment:
+ with pytest.raises(exceptions.StorageError):
+ sd.strict_dict[None] = None
+
+ with pytest.raises(exceptions.StorageError):
+ sd.strict_dict = {None: None}
+
+ assert_strict(strict_class)
+ strict_class.strict_dict = {'key': 'value'}
+ assert strict_class.strict_dict == {'key': 'value'}
+ assert_strict(strict_class, with_assigment=True)
+
+
+def test_strict_list():
+ strict_class = StrictClass()
+
+ def assert_strict(sd, with_assigment=False):
+ if with_assigment:
+ with pytest.raises(exceptions.StorageError):
+ sd.strict_list[0] = None
+
+ with pytest.raises(exceptions.StorageError):
+ sd.strict_list = [None]
+
+ assert_strict(strict_class)
+ strict_class.strict_list = ['item']
+ assert strict_class.strict_list == ['item']
+ assert_strict(strict_class, with_assigment=True)
[06/14] incubator-ariatosca git commit: ARIA-70 fix task plugin_name
Posted by mx...@apache.org.
ARIA-70 fix task plugin_name
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/eaf99748
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/eaf99748
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/eaf99748
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: eaf9974840c101a98d3a626723e37f4990e12ecb
Parents: 8ecf5bb
Author: Dan Kilman <da...@gigaspaces.com>
Authored: Sun Jan 22 20:43:40 2017 +0200
Committer: Dan Kilman <da...@gigaspaces.com>
Committed: Sun Jan 22 20:43:40 2017 +0200
----------------------------------------------------------------------
aria/orchestrator/workflows/core/task.py | 1 +
aria/storage/base_model.py | 5 +----
tests/orchestrator/workflows/core/test_task.py | 6 ++++--
3 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/eaf99748/aria/orchestrator/workflows/core/task.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/core/task.py b/aria/orchestrator/workflows/core/task.py
index 1deb66a..4017ed0 100644
--- a/aria/orchestrator/workflows/core/task.py
+++ b/aria/orchestrator/workflows/core/task.py
@@ -135,6 +135,7 @@ class OperationTask(BaseTask):
retry_interval=api_task.retry_interval,
ignore_failure=api_task.ignore_failure,
plugin=plugins[0] if plugins else None,
+ plugin_name=plugin.get('name'),
execution=self._workflow_context.execution,
runs_on=api_task.runs_on
)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/eaf99748/aria/storage/base_model.py
----------------------------------------------------------------------
diff --git a/aria/storage/base_model.py b/aria/storage/base_model.py
index 8566e66..f7d0e5b 100644
--- a/aria/storage/base_model.py
+++ b/aria/storage/base_model.py
@@ -662,10 +662,6 @@ class TaskBase(ModelMixin):
return cls.one_to_many_relationship('plugin_fk')
@declared_attr
- def plugin_name(cls):
- return association_proxy('plugin', 'name')
-
- @declared_attr
def execution_fk(cls):
return cls.foreign_key(ExecutionBase, nullable=True)
@@ -723,6 +719,7 @@ class TaskBase(ModelMixin):
# Operation specific fields
operation_mapping = Column(String)
inputs = Column(Dict)
+ plugin_name = Column(String)
_runs_on = Column(Enum(*RUNS_ON, name='runs_on'), name='runs_on')
@property
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/eaf99748/tests/orchestrator/workflows/core/test_task.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/core/test_task.py b/tests/orchestrator/workflows/core/test_task.py
index 020de32..34088f5 100644
--- a/tests/orchestrator/workflows/core/test_task.py
+++ b/tests/orchestrator/workflows/core/test_task.py
@@ -63,12 +63,14 @@ class TestOperationTask(object):
node_instance = ctx.model.node_instance.get_by_name(
mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
node = node_instance.node
- node.plugins = [{'name': 'plugin1',
+ plugin_name = 'plugin1'
+ node.plugins = [{'name': plugin_name,
'package_name': 'p1',
'package_version': '0.1'}]
- node.operations['aria.interfaces.lifecycle.create'] = {'plugin': 'plugin1'}
+ node.operations['aria.interfaces.lifecycle.create'] = {'plugin': plugin_name}
api_task, core_task = self._create_node_operation_task(ctx, node_instance)
storage_task = ctx.model.task.get_by_name(core_task.name)
+ assert storage_task.plugin_name == plugin_name
assert storage_task.execution_name == ctx.execution.name
assert storage_task.runs_on.id == core_task.context.node_instance.id
assert core_task.model_task == storage_task
[12/14] incubator-ariatosca git commit: ARIA-36 plugin workdir
Posted by mx...@apache.org.
ARIA-36 plugin workdir
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/84699172
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/84699172
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/84699172
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: 84699172270e9d538958a9ff0dbb874243f962c6
Parents: e3056d4
Author: Dan Kilman <da...@gigaspaces.com>
Authored: Wed Jan 25 16:22:41 2017 +0200
Committer: Dan Kilman <da...@gigaspaces.com>
Committed: Thu Jan 26 12:03:49 2017 +0200
----------------------------------------------------------------------
aria/orchestrator/context/common.py | 2 ++
aria/orchestrator/context/operation.py | 15 +++++++++-
aria/orchestrator/context/serialization.py | 1 +
aria/orchestrator/workflows/core/task.py | 3 +-
aria/utils/file.py | 28 ++++++++++++++++++
tests/orchestrator/context/test_operation.py | 35 ++++++++++++++++++++++-
tests/orchestrator/context/test_serialize.py | 13 +++++++--
7 files changed, 91 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84699172/aria/orchestrator/context/common.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/common.py b/aria/orchestrator/context/common.py
index 691c17d..6ab27ef 100644
--- a/aria/orchestrator/context/common.py
+++ b/aria/orchestrator/context/common.py
@@ -34,6 +34,7 @@ class BaseContext(logger.LoggerMixin):
deployment_id,
model_storage,
resource_storage,
+ workdir=None,
**kwargs):
super(BaseContext, self).__init__(**kwargs)
self._name = name
@@ -41,6 +42,7 @@ class BaseContext(logger.LoggerMixin):
self._model = model_storage
self._resource = resource_storage
self._deployment_id = deployment_id
+ self._workdir = workdir
def __repr__(self):
return (
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84699172/aria/orchestrator/context/operation.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/operation.py b/aria/orchestrator/context/operation.py
index b33d107..23a6fd4 100644
--- a/aria/orchestrator/context/operation.py
+++ b/aria/orchestrator/context/operation.py
@@ -17,7 +17,7 @@
Workflow and operation contexts
"""
-
+from aria.utils import file
from .common import BaseContext
@@ -60,6 +60,19 @@ class BaseOperationContext(BaseContext):
self._task = self.model.task.get(self._task_id)
return self._task
+ @property
+ def plugin_workdir(self):
+ """
+ A work directory that is unique to the plugin and the deployment id
+ """
+ if not self.task.plugin_name:
+ return None
+ plugin_workdir = '{0}/plugins/{1}/{2}'.format(self._workdir,
+ self.deployment.id,
+ self.task.plugin_name)
+ file.makedirs(plugin_workdir)
+ return plugin_workdir
+
class NodeOperationContext(BaseOperationContext):
"""
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84699172/aria/orchestrator/context/serialization.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/serialization.py b/aria/orchestrator/context/serialization.py
index 93cb38a..760818f 100644
--- a/aria/orchestrator/context/serialization.py
+++ b/aria/orchestrator/context/serialization.py
@@ -26,6 +26,7 @@ def operation_context_to_dict(context):
'deployment_id': context._deployment_id,
'task_id': context._task_id,
'actor_id': context._actor_id,
+ 'workdir': context._workdir
}
if context.model:
model = context.model
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84699172/aria/orchestrator/workflows/core/task.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/core/task.py b/aria/orchestrator/workflows/core/task.py
index 4017ed0..f65fc0d 100644
--- a/aria/orchestrator/workflows/core/task.py
+++ b/aria/orchestrator/workflows/core/task.py
@@ -146,7 +146,8 @@ class OperationTask(BaseTask):
resource_storage=self._workflow_context.resource,
deployment_id=self._workflow_context._deployment_id,
task_id=operation_task.id,
- actor_id=api_task.actor.id)
+ actor_id=api_task.actor.id,
+ workdir=self._workflow_context._workdir)
self._task_id = operation_task.id
self._update_fields = None
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84699172/aria/utils/file.py
----------------------------------------------------------------------
diff --git a/aria/utils/file.py b/aria/utils/file.py
new file mode 100644
index 0000000..b515f70
--- /dev/null
+++ b/aria/utils/file.py
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import os
+
+
+def makedirs(path):
+ """An extension of os.makedirs that doesn't fail if the directory already exists"""
+ if os.path.isdir(path):
+ return
+ try:
+ os.makedirs(path)
+ except IOError as e:
+ if e.errno != errno.EEXIST:
+ raise
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84699172/tests/orchestrator/context/test_operation.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_operation.py b/tests/orchestrator/context/test_operation.py
index b5f52a3..b0918d1 100644
--- a/tests/orchestrator/context/test_operation.py
+++ b/tests/orchestrator/context/test_operation.py
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
+
import pytest
from aria import (
@@ -35,7 +37,8 @@ global_test_holder = {}
@pytest.fixture
def ctx(tmpdir):
- context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+ context = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)),
+ workdir=str(tmpdir.join('workdir')))
yield context
storage.release_sqlite_storage(context.model)
@@ -173,6 +176,30 @@ def test_invalid_task_operation_id(ctx, executor):
assert op_node_instance_id != other_node_instance.id
+def test_plugin_workdir(ctx, executor, tmpdir):
+ op = 'test.op'
+ plugin_name = 'mock_plugin'
+ node = ctx.model.node.get_by_name(mock.models.DEPENDENCY_NODE_NAME)
+ node.operations[op] = {'operation': '{0}.{1}'.format(__name__, _test_plugin_workdir.__name__),
+ 'plugin': plugin_name}
+ node.plugins = [{'name': plugin_name}]
+ ctx.model.node.update(node)
+ node_instance = ctx.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+
+ filename = 'test_file'
+ content = 'file content'
+ inputs = {'filename': filename, 'content': content}
+
+ @workflow
+ def basic_workflow(graph, **_):
+ graph.add_tasks(api.task.OperationTask.node_instance(
+ name=op, instance=node_instance, inputs=inputs))
+
+ execute(workflow_func=basic_workflow, workflow_context=ctx, executor=executor)
+ expected_file = tmpdir.join('workdir', 'plugins', str(ctx.deployment.id), plugin_name, filename)
+ assert expected_file.read() == content
+
+
@operation
def my_operation(ctx, **_):
global_test_holder[ctx.name] = ctx
@@ -183,6 +210,12 @@ def get_node_instance_id(ctx, **_):
global_test_holder[ctx.name] = ctx.node_instance.id
+@operation
+def _test_plugin_workdir(ctx, filename, content):
+ with open(os.path.join(ctx.plugin_workdir, filename), 'w') as f:
+ f.write(content)
+
+
@pytest.fixture(autouse=True)
def cleanup():
global_test_holder.clear()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/84699172/tests/orchestrator/context/test_serialize.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_serialize.py b/tests/orchestrator/context/test_serialize.py
index ed0afcd..76930b1 100644
--- a/tests/orchestrator/context/test_serialize.py
+++ b/tests/orchestrator/context/test_serialize.py
@@ -49,9 +49,13 @@ def test_illegal_serialize_of_memory_model_storage(memory_model_storage):
@workflow
def _mock_workflow(ctx, graph):
+ op = 'test.op'
+ plugin_name = 'mock_plugin'
node_instance = ctx.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
- node_instance.node.operations['test.op'] = {'operation': _operation_mapping()}
- task = api.task.OperationTask.node_instance(instance=node_instance, name='test.op')
+ node = node_instance.node
+ node.operations[op] = {'operation': _operation_mapping(), 'plugin': plugin_name}
+ node.plugins = [{'name': plugin_name}]
+ task = api.task.OperationTask.node_instance(instance=node_instance, name=op)
graph.add_tasks(task)
return graph
@@ -72,6 +76,8 @@ def _mock_operation(ctx):
# Here we test that the resource storage was properly re-created
test_file_content = ctx.resource.blueprint.read(TEST_FILE_ENTRY_ID, TEST_FILE_NAME)
assert test_file_content == TEST_FILE_CONTENT
+ # a non empty plugin workdir tells us that we kept the correct base_workdir
+ assert ctx.plugin_workdir is not None
def _operation_mapping():
@@ -88,7 +94,8 @@ def executor():
@pytest.fixture
def context(tmpdir):
result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)),
- resources_dir=str(tmpdir.join('resources')))
+ resources_dir=str(tmpdir.join('resources')),
+ workdir=str(tmpdir.join('workdir')))
yield result
storage.release_sqlite_storage(result.model)
[07/14] incubator-ariatosca git commit: ARIA-74 Add process executor
extension registration hooks
Posted by mx...@apache.org.
ARIA-74 Add process executor extension registration hooks
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/0382b229
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/0382b229
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/0382b229
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: 0382b229c8eddcbf95ea7792b2bdcd9d9dda303b
Parents: eaf9974
Author: Dan Kilman <da...@gigaspaces.com>
Authored: Mon Jan 23 16:55:41 2017 +0200
Committer: Dan Kilman <da...@gigaspaces.com>
Committed: Mon Jan 23 19:21:32 2017 +0200
----------------------------------------------------------------------
aria/extension.py | 15 ++++
aria/orchestrator/plugin.py | 9 ++-
aria/orchestrator/workflows/executor/process.py | 5 ++
.../executor/test_process_executor_extension.py | 80 ++++++++++++++++++++
4 files changed, 106 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/0382b229/aria/extension.py
----------------------------------------------------------------------
diff --git a/aria/extension.py b/aria/extension.py
index ddb7c25..4dba74f 100644
--- a/aria/extension.py
+++ b/aria/extension.py
@@ -118,8 +118,23 @@ class _ParserExtensionRegistration(_ExtensionRegistration):
parser = _ParserExtensionRegistration()
+class _ProcessExecutorExtensionRegistration(_ExtensionRegistration):
+ """Process executor extension class decorator"""
+
+ @_registrar
+ def decorate(self):
+ """
+ The operation function executed by the process executor will be decorated with the function
+ returned from decorate().
+ """
+ return []
+
+process_executor = _ProcessExecutorExtensionRegistration()
+
+
def init():
"""
Initialize all registrars by calling all registered functions
"""
parser.init()
+ process_executor.init()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/0382b229/aria/orchestrator/plugin.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/plugin.py b/aria/orchestrator/plugin.py
index 3005756..381504e 100644
--- a/aria/orchestrator/plugin.py
+++ b/aria/orchestrator/plugin.py
@@ -39,14 +39,17 @@ class PluginManager(object):
"""
metadata = wagon.show(source)
cls = self._model.plugin.model_cls
+
+ os_props = metadata['build_server_os_properties']
+
plugin = cls(
archive_name=metadata['archive_name'],
supported_platform=metadata['supported_platform'],
supported_py_versions=metadata['supported_python_versions'],
# Remove suffix colon after upgrading wagon to > 0.5.0
- distribution=metadata['build_server_os_properties']['distribution:'],
- distribution_release=metadata['build_server_os_properties']['distribution_version'],
- distribution_version=metadata['build_server_os_properties']['distribution_release'],
+ distribution=os_props.get('distribution:') or os_props.get('distribution'),
+ distribution_release=os_props['distribution_version'],
+ distribution_version=os_props['distribution_release'],
package_name=metadata['package_name'],
package_version=metadata['package_version'],
package_source=metadata['package_source'],
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/0382b229/aria/orchestrator/workflows/executor/process.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py
index 2cc9178..9fa0302 100644
--- a/aria/orchestrator/workflows/executor/process.py
+++ b/aria/orchestrator/workflows/executor/process.py
@@ -39,6 +39,8 @@ import Queue
import jsonpickle
+import aria
+from aria.extension import process_executor
from aria.utils import imports
from aria.utils import exceptions
from aria.orchestrator.workflows.executor import base
@@ -291,6 +293,9 @@ def _main():
try:
ctx = serialization.operation_context_from_dict(context_dict)
task_func = imports.load_attribute(operation_mapping)
+ aria.install_aria_extensions()
+ for decorate in process_executor.decorate():
+ task_func = decorate(task_func)
task_func(ctx=ctx, **operation_inputs)
messenger.succeeded(tracked_changes=instrument.tracked_changes)
except BaseException as e:
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/0382b229/tests/orchestrator/workflows/executor/test_process_executor_extension.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/executor/test_process_executor_extension.py b/tests/orchestrator/workflows/executor/test_process_executor_extension.py
new file mode 100644
index 0000000..4a8ef57
--- /dev/null
+++ b/tests/orchestrator/workflows/executor/test_process_executor_extension.py
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from aria import extension
+from aria.orchestrator.workflows import api
+from aria.orchestrator.workflows.core import engine
+from aria.orchestrator.workflows.executor import process
+from aria.orchestrator import workflow, operation
+
+import tests
+from tests import mock
+from tests import storage
+
+
+def test_decorate_extension(context, executor):
+ inputs = {'input1': 1, 'input2': 2}
+
+ def get_node_instance(ctx):
+ return ctx.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+
+ @workflow
+ def mock_workflow(ctx, graph):
+ node_instance = get_node_instance(ctx)
+ op = 'test.op'
+ op_dict = {'operation': '{0}.{1}'.format(__name__, _mock_operation.__name__)}
+ node_instance.node.operations['test.op'] = op_dict
+ task = api.task.OperationTask.node_instance(instance=node_instance, name=op, inputs=inputs)
+ graph.add_tasks(task)
+ return graph
+ graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter
+ eng = engine.Engine(executor=executor, workflow_context=context, tasks_graph=graph)
+ eng.execute()
+ out = get_node_instance(context).runtime_properties['out']
+ assert out['wrapper_inputs'] == inputs
+ assert out['function_inputs'] == inputs
+
+
+@extension.process_executor
+class MockProcessExecutorExtension(object):
+
+ def decorate(self):
+ def decorator(function):
+ def wrapper(ctx, **operation_inputs):
+ ctx.node_instance.runtime_properties['out'] = {'wrapper_inputs': operation_inputs}
+ function(ctx=ctx, **operation_inputs)
+ return wrapper
+ return decorator
+
+
+@operation
+def _mock_operation(ctx, **operation_inputs):
+ ctx.node_instance.runtime_properties['out']['function_inputs'] = operation_inputs
+
+
+@pytest.fixture
+def executor():
+ result = process.ProcessExecutor(python_path=[tests.ROOT_DIR])
+ yield result
+ result.close()
+
+
+@pytest.fixture
+def context(tmpdir):
+ result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+ yield result
+ storage.release_sqlite_storage(result.model)
[10/14] incubator-ariatosca git commit: ARIA-12 New "Runner" class
for simple workflow executions
Posted by mx...@apache.org.
ARIA-12 New "Runner" class for simple workflow executions
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/372f2a15
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/372f2a15
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/372f2a15
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: 372f2a15011800d3503a8688800ce18fc02aa04e
Parents: 4c75aeb
Author: Tal Liron <ta...@gmail.com>
Authored: Tue Jan 24 17:57:50 2017 +0200
Committer: Tal Liron <ta...@gmail.com>
Committed: Wed Jan 25 15:05:55 2017 +0200
----------------------------------------------------------------------
aria/orchestrator/runner.py | 135 +++++++++++++++++++++++++++++++++
tests/mock/__init__.py | 2 +-
tests/mock/context.py | 36 +--------
tests/mock/topology.py | 96 +++++++++++++++++++++++
tests/orchestrator/test_runner.py | 74 ++++++++++++++++++
5 files changed, 309 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/372f2a15/aria/orchestrator/runner.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/runner.py b/aria/orchestrator/runner.py
new file mode 100644
index 0000000..16acc19
--- /dev/null
+++ b/aria/orchestrator/runner.py
@@ -0,0 +1,135 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Workflow runner
+"""
+
+import platform
+import tempfile
+import os
+
+from sqlalchemy import (create_engine, orm) # @UnresolvedImport
+from sqlalchemy.pool import StaticPool # @UnresolvedImport
+
+from .context.workflow import WorkflowContext
+from .workflows.core.engine import Engine
+from .workflows.executor.thread import ThreadExecutor
+from ..storage import model
+from ..storage.sql_mapi import SQLAlchemyModelAPI
+from ..storage.filesystem_rapi import FileSystemResourceAPI
+from .. import (application_model_storage, application_resource_storage)
+
+
+SQLITE_IN_MEMORY = 'sqlite:///:memory:'
+
+
+class Runner(object):
+ """
+ Runs workflows on a deployment. By default uses temporary storage (either on disk or in memory)
+ but can also be used with existing storage.
+
+ Handles the initialization of the storage engine and provides convenience methods for
+ sub-classes to create tasks.
+
+ :param path: path to Sqlite database file; use '' (the default) to use a temporary file,
+ and None to use an in-memory database
+ :type path: string
+ """
+
+ def __init__(self, workflow_name, workflow_fn, inputs, initialize_model_storage_fn,
+ deployment_id, storage_path='', is_storage_temporary=True):
+ if storage_path == '':
+ # Temporary file storage
+ the_file, storage_path = tempfile.mkstemp(suffix='.db', prefix='aria-')
+ os.close(the_file)
+
+ self._storage_path = storage_path
+ self._is_storage_temporary = is_storage_temporary
+
+ workflow_context = self.create_workflow_context(workflow_name, deployment_id,
+ initialize_model_storage_fn)
+
+ tasks_graph = workflow_fn(ctx=workflow_context, **inputs)
+
+ self._engine = Engine(
+ executor=ThreadExecutor(),
+ workflow_context=workflow_context,
+ tasks_graph=tasks_graph)
+
+ def run(self):
+ try:
+ self._engine.execute()
+ finally:
+ self.cleanup()
+
+ def create_workflow_context(self, workflow_name, deployment_id, initialize_model_storage_fn):
+ model_storage = self.create_sqlite_model_storage()
+ initialize_model_storage_fn(model_storage)
+ resource_storage = self.create_fs_resource_storage()
+ return WorkflowContext(
+ name=workflow_name,
+ model_storage=model_storage,
+ resource_storage=resource_storage,
+ deployment_id=deployment_id,
+ workflow_name=self.__class__.__name__,
+ task_max_attempts=1,
+ task_retry_interval=1)
+
+ def create_sqlite_model_storage(self): # pylint: disable=no-self-use
+ self.cleanup()
+
+ # Engine
+ if self._storage_path is None:
+ # In memory
+ # Causes serious threading problems:
+ # https://gehrcke.de/2015/05/in-memory-sqlite-database-and-flask-a-threading-trap/
+ sqlite_engine = create_engine(
+ SQLITE_IN_MEMORY,
+ connect_args={'check_same_thread': False},
+ poolclass=StaticPool)
+ else:
+ path_prefix = '' if 'Windows' in platform.system() else '/'
+ sqlite_engine = create_engine(
+ 'sqlite:///%s%s' % (path_prefix, self._storage_path))
+
+ # Models
+ model.DeclarativeBase.metadata.create_all(bind=sqlite_engine) # @UndefinedVariable
+
+ # Session
+ sqlite_session_factory = orm.sessionmaker(bind=sqlite_engine)
+ if self._storage_path is None:
+ sqlite_session = sqlite_session_factory()
+ else:
+ # File-based storage only
+ sqlite_session = orm.scoped_session(session_factory=sqlite_session_factory)
+
+ # Storage
+ sqlite_kwargs = dict(engine=sqlite_engine, session=sqlite_session)
+ return application_model_storage(
+ SQLAlchemyModelAPI,
+ api_kwargs=sqlite_kwargs)
+
+ def create_fs_resource_storage(self, directory='.'): # pylint: disable=no-self-use
+ fs_kwargs = dict(directory=directory)
+ return application_resource_storage(
+ FileSystemResourceAPI,
+ api_kwargs=fs_kwargs)
+
+ def cleanup(self):
+ if self._is_storage_temporary \
+ and (self._storage_path is not None) \
+ and os.path.isfile(self._storage_path):
+ os.remove(self._storage_path)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/372f2a15/tests/mock/__init__.py
----------------------------------------------------------------------
diff --git a/tests/mock/__init__.py b/tests/mock/__init__.py
index 14541d0..9004b4c 100644
--- a/tests/mock/__init__.py
+++ b/tests/mock/__init__.py
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from . import models, context, operations
+from . import models, context, topology, operations
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/372f2a15/tests/mock/context.py
----------------------------------------------------------------------
diff --git a/tests/mock/context.py b/tests/mock/context.py
index 5559675..ec4bfb8 100644
--- a/tests/mock/context.py
+++ b/tests/mock/context.py
@@ -19,43 +19,13 @@ from aria.storage.filesystem_rapi import FileSystemResourceAPI
from aria.storage.sql_mapi import SQLAlchemyModelAPI
from . import models
+from .topology import create_simple_topology_two_nodes
def simple(mapi_kwargs, resources_dir=None, **kwargs):
model_storage = aria.application_model_storage(SQLAlchemyModelAPI, api_kwargs=mapi_kwargs)
- blueprint = models.get_blueprint()
- model_storage.blueprint.put(blueprint)
- deployment = models.get_deployment(blueprint)
- model_storage.deployment.put(deployment)
- #################################################################################
- # Creating a simple deployment with node -> node as a graph
-
- dependency_node = models.get_dependency_node(deployment)
- model_storage.node.put(dependency_node)
- storage_dependency_node = model_storage.node.get(dependency_node.id)
-
- dependency_node_instance = models.get_dependency_node_instance(storage_dependency_node)
- model_storage.node_instance.put(dependency_node_instance)
- storage_dependency_node_instance = model_storage.node_instance.get(dependency_node_instance.id)
-
- dependent_node = models.get_dependent_node(deployment)
- model_storage.node.put(dependent_node)
- storage_dependent_node = model_storage.node.get(dependent_node.id)
-
- dependent_node_instance = models.get_dependent_node_instance(storage_dependent_node)
- model_storage.node_instance.put(dependent_node_instance)
- storage_dependent_node_instance = model_storage.node_instance.get(dependent_node_instance.id)
-
- relationship = models.get_relationship(storage_dependent_node, storage_dependency_node)
- model_storage.relationship.put(relationship)
- storage_relationship = model_storage.relationship.get(relationship.id)
- relationship_instance = models.get_relationship_instance(
- relationship=storage_relationship,
- target_instance=storage_dependency_node_instance,
- source_instance=storage_dependent_node_instance
- )
- model_storage.relationship_instance.put(relationship_instance)
+ deployment_id = create_simple_topology_two_nodes(model_storage)
# pytest tmpdir
if resources_dir:
@@ -70,7 +40,7 @@ def simple(mapi_kwargs, resources_dir=None, **kwargs):
name='simple_context',
model_storage=model_storage,
resource_storage=resource_storage,
- deployment_id=deployment.id,
+ deployment_id=deployment_id,
workflow_name=models.WORKFLOW_NAME,
task_max_attempts=models.TASK_MAX_ATTEMPTS,
task_retry_interval=models.TASK_RETRY_INTERVAL
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/372f2a15/tests/mock/topology.py
----------------------------------------------------------------------
diff --git a/tests/mock/topology.py b/tests/mock/topology.py
new file mode 100644
index 0000000..e219c33
--- /dev/null
+++ b/tests/mock/topology.py
@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+
+from aria.storage import model
+
+from . import models
+
+
+def create_simple_topology_single_node(model_storage, deployment_id, create_operation):
+ now = datetime.utcnow()
+
+ blueprint = model.Blueprint(name='mock-blueprint',
+ created_at=now,
+ updated_at=now,
+ plan={},
+ main_file_name='mock-file')
+ model_storage.blueprint.put(blueprint)
+
+ deployment = model.Deployment(name='mock-deployment-%d' % deployment_id,
+ blueprint_fk=blueprint.id,
+ created_at=now,
+ updated_at=now)
+ model_storage.deployment.put(deployment)
+
+ node = model.Node(name='mock-node',
+ type='tosca.nodes.Compute',
+ operations={
+ 'tosca.interfaces.node.lifecycle.Standard.create': {
+ 'operation': create_operation,
+ 'inputs': {
+ 'key': 'create',
+ 'value': True}}},
+ number_of_instances=1,
+ planned_number_of_instances=1,
+ deploy_number_of_instances=1,
+ min_number_of_instances=1,
+ max_number_of_instances=1,
+ deployment_fk=deployment.id)
+ model_storage.node.put(node)
+
+ node_instance = model.NodeInstance(name='mock-node-instance',
+ state='',
+ node_fk=node.id)
+ model_storage.node_instance.put(node_instance)
+
+
+def create_simple_topology_two_nodes(model_storage):
+ blueprint = models.get_blueprint()
+ model_storage.blueprint.put(blueprint)
+ deployment = models.get_deployment(blueprint)
+ model_storage.deployment.put(deployment)
+
+ #################################################################################
+ # Creating a simple deployment with node -> node as a graph
+
+ dependency_node = models.get_dependency_node(deployment)
+ model_storage.node.put(dependency_node)
+ storage_dependency_node = model_storage.node.get(dependency_node.id)
+
+ dependency_node_instance = models.get_dependency_node_instance(storage_dependency_node)
+ model_storage.node_instance.put(dependency_node_instance)
+ storage_dependency_node_instance = model_storage.node_instance.get(dependency_node_instance.id)
+
+ dependent_node = models.get_dependent_node(deployment)
+ model_storage.node.put(dependent_node)
+ storage_dependent_node = model_storage.node.get(dependent_node.id)
+
+ dependent_node_instance = models.get_dependent_node_instance(storage_dependent_node)
+ model_storage.node_instance.put(dependent_node_instance)
+ storage_dependent_node_instance = model_storage.node_instance.get(dependent_node_instance.id)
+
+ relationship = models.get_relationship(storage_dependent_node, storage_dependency_node)
+ model_storage.relationship.put(relationship)
+ storage_relationship = model_storage.relationship.get(relationship.id)
+ relationship_instance = models.get_relationship_instance(
+ relationship=storage_relationship,
+ target_instance=storage_dependency_node_instance,
+ source_instance=storage_dependent_node_instance
+ )
+ model_storage.relationship_instance.put(relationship_instance)
+
+ return deployment.id
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/372f2a15/tests/orchestrator/test_runner.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/test_runner.py b/tests/orchestrator/test_runner.py
new file mode 100644
index 0000000..1d46e91
--- /dev/null
+++ b/tests/orchestrator/test_runner.py
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from aria import workflow
+from aria.orchestrator import operation
+from aria.orchestrator.workflows.api.task import OperationTask
+from aria.orchestrator.runner import Runner
+
+from tests import mock
+
+import pytest
+
+
+OPERATION_RESULTS = {}
+
+
+@operation
+def mock_create_operation(ctx, key, value, **kwargs): # pylint: disable=unused-argument
+ OPERATION_RESULTS[key] = value
+
+
+@pytest.fixture(autouse=True)
+def cleanup():
+ OPERATION_RESULTS.clear()
+
+
+def test_runner_no_tasks():
+ @workflow
+ def workflow_fn(ctx, graph): # pylint: disable=unused-argument
+ pass
+
+ _test_runner(workflow_fn)
+
+
+def test_runner_tasks():
+ @workflow
+ def workflow_fn(ctx, graph):
+ for node_instance in ctx.model.node_instance.iter():
+ graph.add_tasks(
+ OperationTask.node_instance(instance=node_instance,
+ name='tosca.interfaces.node.lifecycle.Standard.create'))
+
+ _test_runner(workflow_fn)
+
+ assert OPERATION_RESULTS.get('create') is True
+
+
+def _initialize_model_storage_fn(model_storage):
+ mock.topology.create_simple_topology_single_node(
+ model_storage,
+ 1,
+ '%s.%s' % (__name__, mock_create_operation.__name__)
+ )
+
+
+def _test_runner(workflow_fn):
+ runner = Runner(workflow_name='runner workflow',
+ workflow_fn=workflow_fn,
+ inputs={},
+ initialize_model_storage_fn=_initialize_model_storage_fn,
+ deployment_id=1)
+ runner.run()
[09/14] incubator-ariatosca git commit: ARIA-68 Update runtime
properties during operation
Posted by mx...@apache.org.
ARIA-68 Update runtime properties during operation
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/4c75aeba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/4c75aeba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/4c75aeba
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: 4c75aeba322761471b247dc4e29692cde20043e9
Parents: db9ae9c
Author: Dan Kilman <da...@gigaspaces.com>
Authored: Tue Jan 24 16:58:10 2017 +0200
Committer: Dan Kilman <da...@gigaspaces.com>
Committed: Wed Jan 25 14:06:55 2017 +0200
----------------------------------------------------------------------
aria/orchestrator/workflows/executor/process.py | 93 ++++++++++++++------
aria/storage/instrumentation.py | 18 +++-
.../test_process_executor_tracked_changes.py | 62 ++++++++++++-
tests/storage/test_instrumentation.py | 38 ++++++++
4 files changed, 179 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/4c75aeba/aria/orchestrator/workflows/executor/process.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py
index 9fa0302..7d990fa 100644
--- a/aria/orchestrator/workflows/executor/process.py
+++ b/aria/orchestrator/workflows/executor/process.py
@@ -29,6 +29,7 @@ script_dir = os.path.dirname(__file__)
if script_dir in sys.path:
sys.path.remove(script_dir)
+import contextlib
import io
import threading
import socket
@@ -136,37 +137,39 @@ class ProcessExecutor(base.BaseExecutor):
while not self._stopped:
try:
# Accept messages written to the server socket
- message = self._recv_message()
- message_type = message['type']
- if message_type == 'closed':
- break
- task_id = message['task_id']
- if message_type == 'started':
- self._task_started(self._tasks[task_id])
- elif message_type == 'succeeded':
- task = self._remove_task(task_id)
- instrumentation.apply_tracked_changes(
- tracked_changes=message['tracked_changes'],
- model=task.context.model)
- self._task_succeeded(task)
- elif message_type == 'failed':
- task = self._remove_task(task_id)
- instrumentation.apply_tracked_changes(
- tracked_changes=message['tracked_changes'],
- model=task.context.model)
- self._task_failed(task, exception=message['exception'])
- else:
- raise RuntimeError('Invalid state')
+ with contextlib.closing(self._server_socket.accept()[0]) as connection:
+ message = self._recv_message(connection)
+ message_type = message['type']
+ if message_type == 'closed':
+ break
+ task_id = message['task_id']
+ if message_type == 'started':
+ self._task_started(self._tasks[task_id])
+ elif message_type == 'apply_tracked_changes':
+ task = self._tasks[task_id]
+ instrumentation.apply_tracked_changes(
+ tracked_changes=message['tracked_changes'],
+ model=task.context.model)
+ elif message_type == 'succeeded':
+ task = self._remove_task(task_id)
+ instrumentation.apply_tracked_changes(
+ tracked_changes=message['tracked_changes'],
+ model=task.context.model)
+ self._task_succeeded(task)
+ elif message_type == 'failed':
+ task = self._remove_task(task_id)
+ instrumentation.apply_tracked_changes(
+ tracked_changes=message['tracked_changes'],
+ model=task.context.model)
+ self._task_failed(task, exception=message['exception'])
+ else:
+ raise RuntimeError('Invalid state')
except BaseException as e:
self.logger.debug('Error in process executor listener: {0}'.format(e))
- def _recv_message(self):
- connection, _ = self._server_socket.accept()
- try:
- message_len, = struct.unpack(_INT_FMT, self._recv_bytes(connection, _INT_SIZE))
- return jsonpickle.loads(self._recv_bytes(connection, message_len))
- finally:
- connection.close()
+ def _recv_message(self, connection):
+ message_len, = struct.unpack(_INT_FMT, self._recv_bytes(connection, _INT_SIZE))
+ return jsonpickle.loads(self._recv_bytes(connection, message_len))
@staticmethod
def _recv_bytes(connection, count):
@@ -247,6 +250,9 @@ class _Messenger(object):
"""Task failed message"""
self._send_message(type='failed', tracked_changes=tracked_changes, exception=exception)
+ def apply_tracked_changes(self, tracked_changes):
+ self._send_message(type='apply_tracked_changes', tracked_changes=tracked_changes)
+
def closed(self):
"""Executor closed message"""
self._send_message(type='closed')
@@ -263,10 +269,40 @@ class _Messenger(object):
})
sock.send(struct.pack(_INT_FMT, len(data)))
sock.sendall(data)
+ # send message will block until the server side closes the connection socket
+ # because we want it to be synchronous
+ sock.recv(1)
finally:
sock.close()
+def _patch_session(ctx, messenger, instrument):
+ # model will be None only in tests that test the executor component directly
+ if not ctx.model:
+ return
+
+ # We arbitrarily select the ``node_instance`` mapi to extract the session from it.
+ # could have been any other mapi just as well
+ session = ctx.model.node_instance._session
+ original_refresh = session.refresh
+
+ def patched_refresh(target):
+ instrument.clear(target)
+ original_refresh(target)
+
+ def patched_commit():
+ messenger.apply_tracked_changes(instrument.tracked_changes)
+ instrument.clear()
+
+ # when autoflush is set to true (the default), refreshing an object will trigger
+ # an auto flush by sqlalchemy, this autoflush will attempt to commit changes made so
+ # far on the session. this is not the desired behavior in the subprocess
+ session.autoflush = False
+
+ session.commit = patched_commit
+ session.refresh = patched_refresh
+
+
def _main():
arguments_json_path = sys.argv[1]
with open(arguments_json_path) as f:
@@ -292,6 +328,7 @@ def _main():
with instrumentation.track_changes() as instrument:
try:
ctx = serialization.operation_context_from_dict(context_dict)
+ _patch_session(ctx=ctx, messenger=messenger, instrument=instrument)
task_func = imports.load_attribute(operation_mapping)
aria.install_aria_extensions()
for decorate in process_executor.decorate():
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/4c75aeba/aria/storage/instrumentation.py
----------------------------------------------------------------------
diff --git a/aria/storage/instrumentation.py b/aria/storage/instrumentation.py
index d71190d..537dbb5 100644
--- a/aria/storage/instrumentation.py
+++ b/aria/storage/instrumentation.py
@@ -75,7 +75,7 @@ class _Instrumentation(object):
def _register_set_attribute_listener(self, instrumented_attribute, attribute_type):
def listener(target, value, *_):
- mapi_name = api.generate_lower_name(target.__class__)
+ mapi_name = self._mapi_name(target.__class__)
tracked_instances = self.tracked_changes.setdefault(mapi_name, {})
tracked_attributes = tracked_instances.setdefault(target.id, {})
if value is None:
@@ -90,7 +90,7 @@ class _Instrumentation(object):
def _register_instance_listeners(self, instrumented_class, instrumented_attributes):
def listener(target, *_):
- mapi_name = api.generate_lower_name(instrumented_class)
+ mapi_name = self._mapi_name(instrumented_class)
tracked_instances = self.tracked_changes.setdefault(mapi_name, {})
tracked_attributes = tracked_instances.setdefault(target.id, {})
for attribute_name, attribute_type in instrumented_attributes.items():
@@ -108,6 +108,14 @@ class _Instrumentation(object):
sqlalchemy.event.listen(*listener_args)
self.listeners.append(listener_args)
+ def clear(self, target=None):
+ if target:
+ mapi_name = self._mapi_name(target.__class__)
+ tracked_instances = self.tracked_changes.setdefault(mapi_name, {})
+ tracked_instances.pop(target.id, None)
+ else:
+ self.tracked_changes.clear()
+
def restore(self):
"""Remove all listeners registered by this instrumentation"""
for listener_args in self.listeners:
@@ -120,6 +128,10 @@ class _Instrumentation(object):
def __exit__(self, exc_type, exc_val, exc_tb):
self.restore()
+ @staticmethod
+ def _mapi_name(instrumented_class):
+ return api.generate_lower_name(instrumented_class)
+
class _Value(object):
# You may wonder why is this a full blown class and not a named tuple. The reason is that
@@ -155,3 +167,5 @@ def apply_tracked_changes(tracked_changes, model):
if not instance:
instance = mapi.get(instance_id)
setattr(instance, attribute_name, value.current)
+ if instance:
+ mapi.update(instance)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/4c75aeba/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
index 1564292..bd1fa96 100644
--- a/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
+++ b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import copy
+
import pytest
from aria.orchestrator.workflows import api
@@ -52,18 +54,51 @@ def _update_runtime_properties(context):
context.node_instance.runtime_properties.update(_TEST_RUNTIME_PROPERTIES)
-def _run_workflow(context, executor, op_func):
+def test_refresh_state_of_tracked_attributes(context, executor):
+ out = _run_workflow(context=context, executor=executor, op_func=_mock_refreshing_operation)
+ assert out['initial'] == out['after_refresh']
+ assert out['initial'] != out['after_change']
+
+
+def test_apply_tracked_changes_during_an_operation(context, executor):
+ inputs = {
+ 'committed': {'some': 'new', 'properties': 'right here'},
+ 'changed_but_refreshed': {'some': 'newer', 'properties': 'right there'}
+ }
+
+ expected_initial = context.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties
+
+ out = _run_workflow(context=context, executor=executor, op_func=_mock_updating_operation,
+ inputs=inputs)
+
+ expected_after_update = expected_initial.copy()
+ expected_after_update.update(inputs['committed'])
+ expected_after_change = expected_after_update.copy()
+ expected_after_change.update(inputs['changed_but_refreshed'])
+ expected_after_refresh = expected_after_update
+
+ assert out['initial'] == expected_initial
+ assert out['after_update'] == expected_after_update
+ assert out['after_change'] == expected_after_change
+ assert out['after_refresh'] == expected_after_refresh
+
+
+def _run_workflow(context, executor, op_func, inputs=None):
@workflow
def mock_workflow(ctx, graph):
node_instance = ctx.model.node_instance.get_by_name(
mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
node_instance.node.operations['test.op'] = {'operation': _operation_mapping(op_func)}
- task = api.task.OperationTask.node_instance(instance=node_instance, name='test.op')
+ task = api.task.OperationTask.node_instance(instance=node_instance, name='test.op',
+ inputs=inputs or {})
graph.add_tasks(task)
return graph
graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter
eng = engine.Engine(executor=executor, workflow_context=context, tasks_graph=graph)
eng.execute()
+ return context.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties.get('out')
@operation
@@ -77,6 +112,29 @@ def _mock_fail_operation(ctx):
raise RuntimeError
+@operation
+def _mock_refreshing_operation(ctx):
+ out = {'initial': copy.deepcopy(ctx.node_instance.runtime_properties)}
+ ctx.node_instance.runtime_properties.update({'some': 'new', 'properties': 'right here'})
+ out['after_change'] = copy.deepcopy(ctx.node_instance.runtime_properties)
+ ctx.model.node_instance.refresh(ctx.node_instance)
+ out['after_refresh'] = copy.deepcopy(ctx.node_instance.runtime_properties)
+ ctx.node_instance.runtime_properties['out'] = out
+
+
+@operation
+def _mock_updating_operation(ctx, committed, changed_but_refreshed):
+ out = {'initial': copy.deepcopy(ctx.node_instance.runtime_properties)}
+ ctx.node_instance.runtime_properties.update(committed)
+ ctx.model.node_instance.update(ctx.node_instance)
+ out['after_update'] = copy.deepcopy(ctx.node_instance.runtime_properties)
+ ctx.node_instance.runtime_properties.update(changed_but_refreshed)
+ out['after_change'] = copy.deepcopy(ctx.node_instance.runtime_properties)
+ ctx.model.node_instance.refresh(ctx.node_instance)
+ out['after_refresh'] = copy.deepcopy(ctx.node_instance.runtime_properties)
+ ctx.node_instance.runtime_properties['out'] = out
+
+
def _operation_mapping(func):
return '{name}.{func.__name__}'.format(name=__name__, func=func)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/4c75aeba/tests/storage/test_instrumentation.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_instrumentation.py b/tests/storage/test_instrumentation.py
index 8e7f9aa..8b826e9 100644
--- a/tests/storage/test_instrumentation.py
+++ b/tests/storage/test_instrumentation.py
@@ -232,6 +232,44 @@ class TestInstrumentation(object):
assert instance2_1.dict1 == {'initial': 'value', 'new': 'value'}
assert instance2_2.list1 == ['initial', 'new_value']
+ def test_clear_instance(self, storage):
+ instance1 = MockModel1(name='name1')
+ instance2 = MockModel1(name='name2')
+ for instance in [instance1, instance2]:
+ storage.mock_model_1.put(instance)
+ instrument = self._track_changes({MockModel1.dict1: dict})
+ instance1.dict1 = {'new': 'value'}
+ instance2.dict1 = {'new2': 'value2'}
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {
+ instance1.id: {'dict1': Value(STUB, {'new': 'value'})},
+ instance2.id: {'dict1': Value(STUB, {'new2': 'value2'})}
+ }
+ }
+ instrument.clear(instance1)
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {
+ instance2.id: {'dict1': Value(STUB, {'new2': 'value2'})}
+ }
+ }
+
+ def test_clear_all(self, storage):
+ instance1 = MockModel1(name='name1')
+ instance2 = MockModel1(name='name2')
+ for instance in [instance1, instance2]:
+ storage.mock_model_1.put(instance)
+ instrument = self._track_changes({MockModel1.dict1: dict})
+ instance1.dict1 = {'new': 'value'}
+ instance2.dict1 = {'new2': 'value2'}
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {
+ instance1.id: {'dict1': Value(STUB, {'new': 'value'})},
+ instance2.id: {'dict1': Value(STUB, {'new2': 'value2'})}
+ }
+ }
+ instrument.clear()
+ assert instrument.tracked_changes == {}
+
def _track_changes(self, instrumented):
instrument = instrumentation.track_changes(instrumented)
instruments_holder.append(instrument)
[11/14] incubator-ariatosca git commit: ARIA-73 Transfer parser's
TOSCA model to storage
Posted by mx...@apache.org.
ARIA-73 Transfer parser's TOSCA model to storage
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/e3056d44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/e3056d44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/e3056d44
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: e3056d44961cf838bcce65004a8a0076825abb56
Parents: 372f2a1
Author: Tal Liron <ta...@gmail.com>
Authored: Wed Jan 25 15:25:33 2017 +0200
Committer: Tal Liron <ta...@gmail.com>
Committed: Wed Jan 25 15:25:33 2017 +0200
----------------------------------------------------------------------
aria/parser/modeling/storage.py | 224 +++++++++++++++++++++++++++++++++++
1 file changed, 224 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/e3056d44/aria/parser/modeling/storage.py
----------------------------------------------------------------------
diff --git a/aria/parser/modeling/storage.py b/aria/parser/modeling/storage.py
new file mode 100644
index 0000000..46c3a7c
--- /dev/null
+++ b/aria/parser/modeling/storage.py
@@ -0,0 +1,224 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+from threading import RLock
+
+from ...storage import model
+from ...orchestrator import operation
+from ...utils.console import puts, Colored
+from ...utils.formatting import safe_repr
+
+
+def initialize_storage(context, model_storage, deployment_id):
+ blueprint = create_blueprint(context)
+ model_storage.blueprint.put(blueprint)
+
+ deployment = create_deployment(context, blueprint, deployment_id)
+ model_storage.deployment.put(deployment)
+
+ # Create nodes and node instances
+ for node_template in context.modeling.model.node_templates.itervalues():
+ node = create_node(context, deployment, node_template)
+ model_storage.node.put(node)
+
+ for a_node in context.modeling.instance.find_nodes(node_template.name):
+ node_instance = create_node_instance(node, a_node)
+ model_storage.node_instance.put(node_instance)
+
+ # Create relationships
+ for node_template in context.modeling.model.node_templates.itervalues():
+ for index, requirement_template in enumerate(node_template.requirement_templates):
+ # We are currently limited only to requirements for specific node templates!
+ if requirement_template.target_node_template_name:
+ source = model_storage.node.get_by_name(node_template.name)
+ target = model_storage.node.get_by_name(
+ requirement_template.target_node_template_name)
+ relationship = create_relationship(context, source, target,
+ requirement_template.relationship_template)
+ model_storage.relationship.put(relationship)
+
+ for node in context.modeling.instance.find_nodes(node_template.name):
+ for relationship_model in node.relationships:
+ if relationship_model.source_requirement_index == index:
+ source_instance = \
+ model_storage.node_instance.get_by_name(node.id)
+ target_instance = \
+ model_storage.node_instance.get_by_name(
+ relationship_model.target_node_id)
+ relationship_instance = \
+ create_relationship_instance(relationship, source_instance,
+ target_instance)
+ model_storage.relationship_instance.put(relationship_instance)
+
+
+def create_blueprint(context):
+ now = datetime.utcnow()
+ main_file_name = unicode(context.presentation.location)
+ try:
+ name = context.modeling.model.metadata.values.get('template_name')
+ except AttributeError:
+ name = None
+ return model.Blueprint(
+ plan={},
+ name=name or main_file_name,
+ description=context.modeling.model.description or '',
+ created_at=now,
+ updated_at=now,
+ main_file_name=main_file_name)
+
+
+def create_deployment(context, blueprint, deployment_id):
+ now = datetime.utcnow()
+ return model.Deployment(
+ name='%s_%s' % (blueprint.name, deployment_id),
+ blueprint_fk=blueprint.id,
+ description=context.modeling.instance.description or '',
+ created_at=now,
+ updated_at=now,
+ workflows={},
+ inputs={},
+ groups={},
+ permalink='',
+ policy_triggers={},
+ policy_types={},
+ outputs={},
+ scaling_groups={})
+
+
+def create_node(context, deployment, node_template):
+ operations = create_operations(context, node_template.interface_templates, '_dry_node')
+ return model.Node(
+ name=node_template.name,
+ type=node_template.type_name,
+ type_hierarchy=[],
+ number_of_instances=node_template.default_instances,
+ planned_number_of_instances=node_template.default_instances,
+ deploy_number_of_instances=node_template.default_instances,
+ properties={},
+ operations=operations,
+ min_number_of_instances=node_template.min_instances,
+ max_number_of_instances=node_template.max_instances or 100,
+ deployment_fk=deployment.id)
+
+
+def create_relationship(context, source, target, relationship_template):
+ if relationship_template:
+ source_operations = create_operations(context,
+ relationship_template.source_interface_templates,
+ '_dry_relationship')
+ target_operations = create_operations(context,
+ relationship_template.target_interface_templates,
+ '_dry_relationship')
+ else:
+ source_operations = {}
+ target_operations = {}
+ return model.Relationship(
+ source_node_fk=source.id,
+ target_node_fk=target.id,
+ source_interfaces={},
+ source_operations=source_operations,
+ target_interfaces={},
+ target_operations=target_operations,
+ type='rel_type',
+ type_hierarchy=[],
+ properties={})
+
+
+def create_node_instance(node, node_model):
+ return model.NodeInstance(
+ name=node_model.id,
+ runtime_properties={},
+ version=None,
+ node_fk=node.id,
+ state='',
+ scaling_groups=[])
+
+
+def create_relationship_instance(relationship, source_instance, target_instance):
+ return model.RelationshipInstance(
+ relationship_fk=relationship.id,
+ source_node_instance_fk=source_instance.id,
+ target_node_instance_fk=target_instance.id)
+
+
+def create_operations(context, interfaces, fn_name):
+ operations = {}
+ for interface in interfaces.itervalues():
+ operations[interface.type_name] = {}
+ for oper in interface.operation_templates.itervalues():
+ name = '%s.%s' % (interface.type_name, oper.name)
+ operations[name] = {
+ 'operation': '%s.%s' % (__name__, fn_name),
+ 'inputs': {
+ '_plugin': None,
+ '_implementation': None}}
+ if oper.implementation:
+ plugin, implementation = _parse_implementation(context, oper.implementation)
+ operations[name]['inputs']['_plugin'] = plugin
+ operations[name]['inputs']['_implementation'] = implementation
+
+ return operations
+
+
+def _parse_implementation(context, implementation):
+ index = implementation.find('>')
+ if index == -1:
+ return 'execution', implementation
+ plugin = implementation[:index].strip()
+
+ # TODO: validation should happen in parser
+ if (plugin != 'execution') and (_get_plugin(context, plugin) is None):
+ raise ValueError('unknown plugin: "%s"' % plugin)
+
+ implementation = implementation[index+1:].strip()
+ return plugin, implementation
+
+
+def _get_plugin(context, plugin_name):
+ def is_plugin(type_name):
+ return context.modeling.policy_types.get_role(type_name) == 'plugin'
+
+ for policy in context.modeling.instance.policies.itervalues():
+ if (policy.name == plugin_name) and is_plugin(policy.type_name):
+ return policy
+
+ return None
+
+
+_TERMINAL_LOCK = RLock()
+
+
+@operation
+def _dry_node(ctx, _plugin, _implementation, **kwargs):
+ with _TERMINAL_LOCK:
+ print '> node instance: %s' % Colored.red(ctx.node_instance.name)
+ _dump_implementation(_plugin, _implementation)
+
+
+@operation
+def _dry_relationship(ctx, _plugin, _implementation, **kwargs):
+ with _TERMINAL_LOCK:
+ puts('> relationship instance: %s -> %s' % (
+ Colored.red(ctx.relationship_instance.source_node_instance.name),
+ Colored.red(ctx.relationship_instance.target_node_instance.name)))
+ _dump_implementation(_plugin, _implementation)
+
+
+def _dump_implementation(plugin, implementation):
+ if plugin:
+ print ' plugin: %s' % Colored.magenta(plugin)
+ if implementation:
+ print ' implementation: %s' % Colored.yellow(safe_repr(implementation))
[13/14] incubator-ariatosca git commit: ARIA-61 Initial version of
CLI "workflow" command
Posted by mx...@apache.org.
ARIA-61 Initial version of CLI "workflow" command
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/44478293
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/44478293
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/44478293
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: 44478293c0f16ed1c60fb5c8b8b12e73954fddfe
Parents: 8469917
Author: Tal Liron <ta...@gmail.com>
Authored: Wed Jan 25 16:30:16 2017 +0200
Committer: Tal Liron <ta...@gmail.com>
Committed: Thu Jan 26 12:38:16 2017 +0200
----------------------------------------------------------------------
aria/cli/args_parser.py | 141 +++++----
aria/cli/cli.py | 10 +-
aria/cli/commands.py | 313 ++++++++++++-------
aria/cli/csar.py | 2 +-
aria/orchestrator/__init__.py | 2 +-
aria/orchestrator/decorators.py | 3 +
aria/parser/modeling/__init__.py | 4 +-
aria/parser/modeling/types.py | 28 +-
.../simple_v1_0/modeling/__init__.py | 1 +
9 files changed, 313 insertions(+), 191 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/44478293/aria/cli/args_parser.py
----------------------------------------------------------------------
diff --git a/aria/cli/args_parser.py b/aria/cli/args_parser.py
index 8eacf05..e661620 100644
--- a/aria/cli/args_parser.py
+++ b/aria/cli/args_parser.py
@@ -68,6 +68,7 @@ def config_parser(parser=None):
add_init_parser(sub_parser)
add_execute_parser(sub_parser)
add_parse_parser(sub_parser)
+ add_workflow_parser(sub_parser)
add_spec_parser(sub_parser)
add_csar_create_parser(sub_parser)
add_csar_open_parser(sub_parser)
@@ -76,6 +77,73 @@ def config_parser(parser=None):
@sub_parser_decorator(
+ name='parse',
+ help='Parse a blueprint',
+ formatter_class=SmartFormatter)
+def add_parse_parser(parse):
+ """
+ ``parse`` command parser configuration
+ """
+ parse.add_argument(
+ 'uri',
+ help='URI or file path to service template')
+ parse.add_argument(
+ 'consumer',
+ nargs='?',
+ default='validate',
+ help='"validate" (default), "presentation", "model", "types", "instance", or consumer '
+ 'class name (full class path or short name)')
+ parse.add_argument(
+ '--loader-source',
+ default='aria.parser.loading.DefaultLoaderSource',
+ help='loader source class for the parser')
+ parse.add_argument(
+ '--reader-source',
+ default='aria.parser.reading.DefaultReaderSource',
+ help='reader source class for the parser')
+ parse.add_argument(
+ '--presenter-source',
+ default='aria.parser.presentation.DefaultPresenterSource',
+ help='presenter source class for the parser')
+ parse.add_argument(
+ '--presenter',
+ help='force use of this presenter class in parser')
+ parse.add_argument(
+ '--prefix', nargs='*',
+ help='prefixes for imports')
+ parse.add_flag_argument(
+ 'debug',
+ help_true='print debug info',
+ help_false='don\'t print debug info')
+ parse.add_flag_argument(
+ 'cached-methods',
+ help_true='enable cached methods',
+ help_false='disable cached methods',
+ default=True)
+
+
+@sub_parser_decorator(
+ name='workflow',
+ help='Run a workflow on a blueprint',
+ formatter_class=SmartFormatter)
+def add_workflow_parser(workflow):
+ """
+ ``workflow`` command parser configuration
+ """
+ workflow.add_argument(
+ 'uri',
+ help='URI or file path to service template')
+ workflow.add_argument(
+ '-w', '--workflow',
+ default='install',
+ help='The workflow name')
+ workflow.add_argument(
+ '-d', '--deployment-id',
+ required=False,
+ help='A unique ID for the deployment')
+
+
+@sub_parser_decorator(
name='init',
help='Initialize environment',
formatter_class=SmartFormatter)
@@ -146,65 +214,6 @@ def add_execute_parser(execute):
@sub_parser_decorator(
- name='parse',
- help='Parse a blueprint',
- formatter_class=SmartFormatter)
-def add_parse_parser(parse):
- """
- ``parse`` command parser configuration
- """
- parse.add_argument(
- 'uri',
- help='URI or file path to profile')
- parse.add_argument(
- 'consumer',
- nargs='?',
- default='instance',
- help='consumer class name (full class path or short name)')
- parse.add_argument(
- '--loader-source',
- default='aria.parser.loading.DefaultLoaderSource',
- help='loader source class for the parser')
- parse.add_argument(
- '--reader-source',
- default='aria.parser.reading.DefaultReaderSource',
- help='reader source class for the parser')
- parse.add_argument(
- '--presenter-source',
- default='aria.parser.presentation.DefaultPresenterSource',
- help='presenter source class for the parser')
- parse.add_argument(
- '--presenter',
- help='force use of this presenter class in parser')
- parse.add_argument(
- '--prefix', nargs='*',
- help='prefixes for imports')
- parse.add_flag_argument(
- 'debug',
- help_true='print debug info',
- help_false='don\'t print debug info')
- parse.add_flag_argument(
- 'cached-methods',
- help_true='enable cached methods',
- help_false='disable cached methods',
- default=True)
-
-
-@sub_parser_decorator(
- name='spec',
- help='Specification tool',
- formatter_class=SmartFormatter)
-def add_spec_parser(spec):
- """
- ``spec`` command parser configuration
- """
- spec.add_argument(
- '--csv',
- action='store_true',
- help='output as CSV')
-
-
-@sub_parser_decorator(
name='csar-create',
help='Create a CSAR file from a TOSCA service template directory',
formatter_class=SmartFormatter)
@@ -243,3 +252,17 @@ def add_csar_validate_parser(parse):
parse.add_argument(
'source',
help='CSAR file location')
+
+
+@sub_parser_decorator(
+ name='spec',
+ help='Specification tool',
+ formatter_class=SmartFormatter)
+def add_spec_parser(spec):
+ """
+ ``spec`` command parser configuration
+ """
+ spec.add_argument(
+ '--csv',
+ action='store_true',
+ help='output as CSV')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/44478293/aria/cli/cli.py
----------------------------------------------------------------------
diff --git a/aria/cli/cli.py b/aria/cli/cli.py
index c5830d5..20ace2c 100644
--- a/aria/cli/cli.py
+++ b/aria/cli/cli.py
@@ -29,13 +29,14 @@ from ..logger import (
from ..utils.exceptions import print_exception
from .args_parser import config_parser
from .commands import (
+ ParseCommand,
+ WorkflowCommand,
InitCommand,
ExecuteCommand,
- ParseCommand,
- SpecCommand,
CSARCreateCommand,
CSAROpenCommand,
CSARValidateCommand,
+ SpecCommand,
)
__version__ = '0.1.0'
@@ -49,13 +50,14 @@ class AriaCli(LoggerMixin):
def __init__(self, *args, **kwargs):
super(AriaCli, self).__init__(*args, **kwargs)
self.commands = {
+ 'parse': ParseCommand.with_logger(base_logger=self.logger),
+ 'workflow': WorkflowCommand.with_logger(base_logger=self.logger),
'init': InitCommand.with_logger(base_logger=self.logger),
'execute': ExecuteCommand.with_logger(base_logger=self.logger),
- 'parse': ParseCommand.with_logger(base_logger=self.logger),
- 'spec': SpecCommand.with_logger(base_logger=self.logger),
'csar-create': CSARCreateCommand.with_logger(base_logger=self.logger),
'csar-open': CSAROpenCommand.with_logger(base_logger=self.logger),
'csar-validate': CSARValidateCommand.with_logger(base_logger=self.logger),
+ 'spec': SpecCommand.with_logger(base_logger=self.logger),
}
def __enter__(self):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/44478293/aria/cli/commands.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands.py b/aria/cli/commands.py
index 1cd765f..0890cd1 100644
--- a/aria/cli/commands.py
+++ b/aria/cli/commands.py
@@ -26,15 +26,10 @@ import tempfile
from glob import glob
from importlib import import_module
-from yaml import safe_load, YAMLError
+from ruamel import yaml # @UnresolvedImport
from .. import extension
-from .. import (application_model_storage, application_resource_storage)
from ..logger import LoggerMixin
-from ..storage import (FileSystemModelDriver, FileSystemResourceDriver)
-from ..orchestrator.context.workflow import WorkflowContext
-from ..orchestrator.workflows.core.engine import Engine
-from ..orchestrator.workflows.executor.process import ProcessExecutor
from ..parser import iter_specifications
from ..parser.consumption import (
ConsumptionContext,
@@ -47,29 +42,27 @@ from ..parser.consumption import (
Instance
)
from ..parser.loading import LiteralLocation, UriLocation
+from ..parser.modeling import initialize_storage
from ..utils.application import StorageManager
from ..utils.caching import cachedmethod
from ..utils.console import (puts, Colored, indent)
from ..utils.imports import (import_fullname, import_modules)
-from . import csar
+from ..utils.collections import OrderedDict
+from ..orchestrator import WORKFLOW_DECORATOR_RESERVED_ARGUMENTS
+from ..orchestrator.runner import Runner
+from ..orchestrator.workflows.builtin import BUILTIN_WORKFLOWS
+
from .exceptions import (
AriaCliFormatInputsError,
AriaCliYAMLInputsError,
AriaCliInvalidInputsError
)
-from .storage import (
- local_resource_storage,
- create_local_storage,
- local_model_storage,
- create_user_space,
- user_space,
- local_storage,
-)
+from . import csar
class BaseCommand(LoggerMixin):
"""
- Base class for CLI commands
+ Base class for CLI commands.
"""
def __repr__(self):
@@ -101,9 +94,9 @@ class BaseCommand(LoggerMixin):
try:
parsed_dict.update(json.loads(input_string))
except BaseException:
- parsed_dict.update((input.split('=')
- for input in input_string.split(';')
- if input))
+ parsed_dict.update((i.split('=')
+ for i in input_string.split(';')
+ if i))
except Exception as exc:
raise AriaCliFormatInputsError(str(exc), inputs=input_string)
@@ -111,8 +104,8 @@ class BaseCommand(LoggerMixin):
self.logger.info('Processing inputs source: {0}'.format(input_path))
try:
with open(input_path) as input_file:
- content = safe_load(input_file)
- except YAMLError as exc:
+ content = yaml.safe_load(input_file)
+ except yaml.YAMLError as exc:
raise AriaCliYAMLInputsError(
'"{0}" is not a valid YAML. {1}'.format(input_path, str(exc)))
if isinstance(content, dict):
@@ -136,9 +129,153 @@ class BaseCommand(LoggerMixin):
return parsed_dict
+class ParseCommand(BaseCommand):
+ """
+ :code:`parse` command.
+
+ Given a blueprint, emits information in human-readable, JSON, or YAML format from various phases
+ of the ARIA parser.
+ """
+
+ def __call__(self, args_namespace, unknown_args):
+ super(ParseCommand, self).__call__(args_namespace, unknown_args)
+
+ if args_namespace.prefix:
+ for prefix in args_namespace.prefix:
+ extension.parser.uri_loader_prefix().append(prefix)
+
+ cachedmethod.ENABLED = args_namespace.cached_methods
+
+ context = ParseCommand.create_context_from_namespace(args_namespace)
+ context.args = unknown_args
+
+ consumer = ConsumerChain(context, (Read, Validate))
+
+ consumer_class_name = args_namespace.consumer
+ dumper = None
+ if consumer_class_name == 'validate':
+ dumper = None
+ elif consumer_class_name == 'presentation':
+ dumper = consumer.consumers[0]
+ elif consumer_class_name == 'model':
+ consumer.append(Model)
+ elif consumer_class_name == 'types':
+ consumer.append(Model, Types)
+ elif consumer_class_name == 'instance':
+ consumer.append(Model, Inputs, Instance)
+ else:
+ consumer.append(Model, Inputs, Instance)
+ consumer.append(import_fullname(consumer_class_name))
+
+ if dumper is None:
+ # Default to last consumer
+ dumper = consumer.consumers[-1]
+
+ consumer.consume()
+
+ if not context.validation.dump_issues():
+ dumper.dump()
+ exit(1)
+
+ @staticmethod
+ def create_context_from_namespace(namespace, **kwargs):
+ args = vars(namespace).copy()
+ args.update(kwargs)
+ return ParseCommand.create_context(**args)
+
+ @staticmethod
+ def create_context(uri,
+ loader_source,
+ reader_source,
+ presenter_source,
+ presenter,
+ debug,
+ **kwargs):
+ context = ConsumptionContext()
+ context.loading.loader_source = import_fullname(loader_source)()
+ context.reading.reader_source = import_fullname(reader_source)()
+ context.presentation.location = UriLocation(uri) if isinstance(uri, basestring) else uri
+ context.presentation.presenter_source = import_fullname(presenter_source)()
+ context.presentation.presenter_class = import_fullname(presenter)
+ context.presentation.print_exceptions = debug
+ return context
+
+
+class WorkflowCommand(BaseCommand):
+ """
+ :code:`workflow` command.
+ """
+
+ WORKFLOW_POLICY_INTERNAL_PROPERTIES = ('function', 'implementation', 'dependencies')
+
+ def __call__(self, args_namespace, unknown_args):
+ super(WorkflowCommand, self).__call__(args_namespace, unknown_args)
+
+ deployment_id = args_namespace.deployment_id or 1
+ context = self._parse(args_namespace.uri)
+ workflow_fn, inputs = self._get_workflow(context, args_namespace.workflow)
+ self._run(context, args_namespace.workflow, workflow_fn, inputs, deployment_id)
+
+ def _parse(self, uri):
+ # Parse
+ context = ConsumptionContext()
+ context.presentation.location = UriLocation(uri)
+ consumer = ConsumerChain(context, (Read, Validate, Model, Inputs, Instance))
+ consumer.consume()
+
+ if context.validation.dump_issues():
+ exit(1)
+
+ return context
+
+ def _get_workflow(self, context, workflow_name):
+ if workflow_name in BUILTIN_WORKFLOWS:
+ workflow_fn = import_fullname('aria.orchestrator.workflows.builtin.%s' % workflow_name)
+ inputs = {}
+ else:
+ try:
+ policy = context.modeling.instance.policies[workflow_name]
+ except KeyError:
+ raise AttributeError('workflow policy does not exist: "%s"' % workflow_name)
+ if context.modeling.policy_types.get_role(policy.type_name) != 'workflow':
+ raise AttributeError('policy is not a workflow: "%s"' % workflow_name)
+
+ try:
+ sys.path.append(policy.properties['implementation'].value)
+ except KeyError:
+ pass
+
+ workflow_fn = import_fullname(policy.properties['function'].value)
+
+ for k in policy.properties:
+ if k in WORKFLOW_DECORATOR_RESERVED_ARGUMENTS:
+ raise AttributeError('workflow policy "%s" defines a reserved property: "%s"' %
+ (workflow_name, k))
+
+ inputs = OrderedDict([
+ (k, v.value) for k, v in policy.properties.iteritems()
+ if k not in WorkflowCommand.WORKFLOW_POLICY_INTERNAL_PROPERTIES
+ ])
+
+ return workflow_fn, inputs
+
+ def _run(self, context, workflow_name, workflow_fn, inputs, deployment_id):
+ # Storage
+ def _initialize_storage(model_storage):
+ initialize_storage(context, model_storage, deployment_id)
+
+ # Create runner
+ runner = Runner(workflow_name, workflow_fn, inputs, _initialize_storage, deployment_id)
+
+ # Run
+ runner.run()
+
+
class InitCommand(BaseCommand):
"""
- ``init`` command implementation
+ :code:`init` command.
+
+ Broken. Currently maintained for reference.
"""
_IN_VIRTUAL_ENV = hasattr(sys, 'real_prefix')
@@ -216,7 +353,9 @@ class InitCommand(BaseCommand):
class ExecuteCommand(BaseCommand):
"""
- ``execute`` command implementation
+ :code:`execute` command.
+
+ Broken. Currently maintained for reference.
"""
def __call__(self, args_namespace, unknown_args):
@@ -310,98 +449,7 @@ class ExecuteCommand(BaseCommand):
raise
-class ParseCommand(BaseCommand):
- def __call__(self, args_namespace, unknown_args):
- super(ParseCommand, self).__call__(args_namespace, unknown_args)
-
- if args_namespace.prefix:
- for prefix in args_namespace.prefix:
- extension.parser.uri_loader_prefix().append(prefix)
-
- cachedmethod.ENABLED = args_namespace.cached_methods
-
- context = ParseCommand.create_context_from_namespace(args_namespace)
- context.args = unknown_args
-
- consumer = ConsumerChain(context, (Read, Validate))
-
- consumer_class_name = args_namespace.consumer
- dumper = None
- if consumer_class_name == 'presentation':
- dumper = consumer.consumers[0]
- elif consumer_class_name == 'model':
- consumer.append(Model)
- elif consumer_class_name == 'types':
- consumer.append(Model, Types)
- elif consumer_class_name == 'instance':
- consumer.append(Model, Inputs, Instance)
- else:
- consumer.append(Model, Inputs, Instance)
- consumer.append(import_fullname(consumer_class_name))
-
- if dumper is None:
- # Default to last consumer
- dumper = consumer.consumers[-1]
-
- consumer.consume()
-
- if not context.validation.dump_issues():
- dumper.dump()
-
- @staticmethod
- def create_context_from_namespace(namespace, **kwargs):
- args = vars(namespace).copy()
- args.update(kwargs)
- return ParseCommand.create_context(**args)
-
- @staticmethod
- def create_context(uri,
- loader_source,
- reader_source,
- presenter_source,
- presenter,
- debug,
- **kwargs):
- context = ConsumptionContext()
- context.loading.loader_source = import_fullname(loader_source)()
- context.reading.reader_source = import_fullname(reader_source)()
- context.presentation.location = UriLocation(uri) if isinstance(uri, basestring) else uri
- context.presentation.presenter_source = import_fullname(presenter_source)()
- context.presentation.presenter_class = import_fullname(presenter)
- context.presentation.print_exceptions = debug
- return context
-
-
-class SpecCommand(BaseCommand):
- def __call__(self, args_namespace, unknown_args):
- super(SpecCommand, self).__call__(args_namespace, unknown_args)
-
- # Make sure that all @dsl_specification decorators are processed
- for pkg in extension.parser.specification_package():
- import_modules(pkg)
-
- # TODO: scan YAML documents as well
-
- if args_namespace.csv:
- writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL)
- writer.writerow(('Specification', 'Section', 'Code', 'URL'))
- for spec, sections in iter_specifications():
- for section, details in sections:
- writer.writerow((spec, section, details['code'], details['url']))
-
- else:
- for spec, sections in iter_specifications():
- puts(Colored.cyan(spec))
- with indent(2):
- for section, details in sections:
- puts(Colored.blue(section))
- with indent(2):
- for k, v in details.iteritems():
- puts('%s: %s' % (Colored.magenta(k), v))
-
-
class BaseCSARCommand(BaseCommand):
-
@staticmethod
def _parse_and_dump(reader):
context = ConsumptionContext()
@@ -439,7 +487,6 @@ class BaseCSARCommand(BaseCommand):
class CSARCreateCommand(BaseCSARCommand):
-
def __call__(self, args_namespace, unknown_args):
super(CSARCreateCommand, self).__call__(args_namespace, unknown_args)
csar.write(
@@ -451,7 +498,6 @@ class CSARCreateCommand(BaseCSARCommand):
class CSAROpenCommand(BaseCSARCommand):
-
def __call__(self, args_namespace, unknown_args):
super(CSAROpenCommand, self).__call__(args_namespace, unknown_args)
self._read(
@@ -460,7 +506,40 @@ class CSAROpenCommand(BaseCSARCommand):
class CSARValidateCommand(BaseCSARCommand):
-
def __call__(self, args_namespace, unknown_args):
super(CSARValidateCommand, self).__call__(args_namespace, unknown_args)
self._validate(args_namespace.source)
+
+
+class SpecCommand(BaseCommand):
+ """
+ :code:`spec` command.
+
+ Emits all uses of :code:`@dsl_specification` in the codebase, in human-readable or CSV format.
+ """
+
+ def __call__(self, args_namespace, unknown_args):
+ super(SpecCommand, self).__call__(args_namespace, unknown_args)
+
+ # Make sure that all @dsl_specification decorators are processed
+ for pkg in extension.parser.specification_package():
+ import_modules(pkg)
+
+ # TODO: scan YAML documents as well
+
+ if args_namespace.csv:
+ writer = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL)
+ writer.writerow(('Specification', 'Section', 'Code', 'URL'))
+ for spec, sections in iter_specifications():
+ for section, details in sections:
+ writer.writerow((spec, section, details['code'], details['url']))
+
+ else:
+ for spec, sections in iter_specifications():
+ puts(Colored.cyan(spec))
+ with indent(2):
+ for section, details in sections:
+ puts(Colored.blue(section))
+ with indent(2):
+ for k, v in details.iteritems():
+ puts('%s: %s' % (Colored.magenta(k), v))
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/44478293/aria/cli/csar.py
----------------------------------------------------------------------
diff --git a/aria/cli/csar.py b/aria/cli/csar.py
index 933df17..b185f46 100644
--- a/aria/cli/csar.py
+++ b/aria/cli/csar.py
@@ -19,7 +19,7 @@ import tempfile
import zipfile
import requests
-from ruamel import yaml
+from ruamel import yaml # @UnresolvedImport
META_FILE = 'TOSCA-Metadata/TOSCA.meta'
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/44478293/aria/orchestrator/__init__.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/__init__.py b/aria/orchestrator/__init__.py
index 90d6442..097ee1d 100644
--- a/aria/orchestrator/__init__.py
+++ b/aria/orchestrator/__init__.py
@@ -15,7 +15,7 @@
"""
Aria orchestrator
"""
-from .decorators import workflow, operation
+from .decorators import workflow, operation, WORKFLOW_DECORATOR_RESERVED_ARGUMENTS
from . import (
context,
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/44478293/aria/orchestrator/decorators.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/decorators.py b/aria/orchestrator/decorators.py
index 6e8ab39..bbe43f4 100644
--- a/aria/orchestrator/decorators.py
+++ b/aria/orchestrator/decorators.py
@@ -26,6 +26,9 @@ from . import context
from .workflows.api import task_graph
+WORKFLOW_DECORATOR_RESERVED_ARGUMENTS = ('ctx', 'graph')
+
+
def workflow(func=None, suffix_template=''):
"""
Workflow decorator
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/44478293/aria/parser/modeling/__init__.py
----------------------------------------------------------------------
diff --git a/aria/parser/modeling/__init__.py b/aria/parser/modeling/__init__.py
index bc4373c..a1efd9f 100644
--- a/aria/parser/modeling/__init__.py
+++ b/aria/parser/modeling/__init__.py
@@ -24,6 +24,7 @@ from .model_elements import (ServiceModel, NodeTemplate, RequirementTemplate, Ca
GroupPolicyTemplate, GroupPolicyTriggerTemplate, MappingTemplate,
SubstitutionTemplate, InterfaceTemplate, OperationTemplate)
from .types import TypeHierarchy, Type, RelationshipType, PolicyType, PolicyTriggerType
+from .storage import initialize_storage
__all__ = (
'CannotEvaluateFunctionException',
@@ -65,4 +66,5 @@ __all__ = (
'Type',
'RelationshipType',
'PolicyType',
- 'PolicyTriggerType')
+ 'PolicyTriggerType',
+ 'initialize_storage')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/44478293/aria/parser/modeling/types.py
----------------------------------------------------------------------
diff --git a/aria/parser/modeling/types.py b/aria/parser/modeling/types.py
index e6249ef..0a232fc 100644
--- a/aria/parser/modeling/types.py
+++ b/aria/parser/modeling/types.py
@@ -29,6 +29,7 @@ class Type(object):
self.name = name
self.description = None
+ self.role = None
self.children = StrictList(value_class=Type)
def get_parent(self, name):
@@ -40,6 +41,13 @@ class Type(object):
return parent
return None
+ def is_descendant(self, base_name, name):
+ base = self.get_descendant(base_name)
+ if base is not None:
+ if base.get_descendant(name) is not None:
+ return True
+ return False
+
def get_descendant(self, name):
if self.name == name:
return self
@@ -49,24 +57,28 @@ class Type(object):
return found
return None
- def is_descendant(self, base_name, name):
- base = self.get_descendant(base_name)
- if base is not None:
- if base.get_descendant(name) is not None:
- return True
- return False
-
def iter_descendants(self):
for child in self.children:
yield child
for descendant in child.iter_descendants():
yield descendant
+ def get_role(self, name):
+ def _get_role(the_type):
+ if the_type is None:
+ return None
+ elif the_type.role is None:
+ return _get_role(self.get_parent(the_type.name))
+ return the_type.role
+
+ return _get_role(self.get_descendant(name))
+
@property
def as_raw(self):
return OrderedDict((
('name', self.name),
- ('description', self.description)))
+ ('description', self.description),
+ ('role', self.role)))
def dump(self, context):
if self.name:
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/44478293/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py
----------------------------------------------------------------------
diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py
index 5c3206b..811da19 100644
--- a/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py
+++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/__init__.py
@@ -330,6 +330,7 @@ def create_types(context, root, types, normalize=None):
model = Type(the_type._name)
if the_type.description:
model.description = the_type.description.value
+ model.role = the_type._get_extension('role')
if parent_type is None:
root.children.append(model)
else:
[02/14] incubator-ariatosca git commit: ARIA-63 Implement attribute
tracking for subprocesses
Posted by mx...@apache.org.
ARIA-63 Implement attribute tracking for subprocesses
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/385b209d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/385b209d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/385b209d
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: 385b209db2bf797cdd9877c3d031f3b258b8f79f
Parents: 61b7094
Author: Dan Kilman <da...@gigaspaces.com>
Authored: Sun Jan 15 17:42:23 2017 +0200
Committer: Dan Kilman <da...@gigaspaces.com>
Committed: Tue Jan 17 12:46:04 2017 +0200
----------------------------------------------------------------------
aria/orchestrator/workflows/executor/process.py | 48 ++--
aria/storage/instrumentation.py | 157 +++++++++++
aria/storage/type.py | 38 ++-
tests/.pylintrc | 2 +-
.../test_process_executor_tracked_changes.py | 95 +++++++
tests/storage/test_instrumentation.py | 275 +++++++++++++++++++
6 files changed, 595 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/aria/orchestrator/workflows/executor/process.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py
index e0a8aeb..51596c3 100644
--- a/aria/orchestrator/workflows/executor/process.py
+++ b/aria/orchestrator/workflows/executor/process.py
@@ -42,6 +42,8 @@ import jsonpickle
from aria.utils import imports
from aria.orchestrator.workflows.executor import base
from aria.orchestrator.context import serialization
+from aria.storage import instrumentation
+from aria.storage import type as storage_type
_IS_WIN = os.name == 'nt'
@@ -139,10 +141,17 @@ class ProcessExecutor(base.BaseExecutor):
if message_type == 'started':
self._task_started(self._tasks[task_id])
elif message_type == 'succeeded':
- self._task_succeeded(self._remove_task(task_id))
+ task = self._remove_task(task_id)
+ instrumentation.apply_tracked_changes(
+ tracked_changes=message['tracked_changes'],
+ model=task.context.model)
+ self._task_succeeded(task)
elif message_type == 'failed':
- self._task_failed(self._remove_task(task_id),
- exception=message['exception'])
+ task = self._remove_task(task_id)
+ instrumentation.apply_tracked_changes(
+ tracked_changes=message['tracked_changes'],
+ model=task.context.model)
+ self._task_failed(task, exception=message['exception'])
else:
raise RuntimeError('Invalid state')
except BaseException as e:
@@ -227,26 +236,27 @@ class _Messenger(object):
"""Task started message"""
self._send_message(type='started')
- def succeeded(self):
+ def succeeded(self, tracked_changes):
"""Task succeeded message"""
- self._send_message(type='succeeded')
+ self._send_message(type='succeeded', tracked_changes=tracked_changes)
- def failed(self, exception):
+ def failed(self, tracked_changes, exception):
"""Task failed message"""
- self._send_message(type='failed', exception=exception)
+ self._send_message(type='failed', tracked_changes=tracked_changes, exception=exception)
def closed(self):
"""Executor closed message"""
self._send_message(type='closed')
- def _send_message(self, type, exception=None):
+ def _send_message(self, type, tracked_changes=None, exception=None):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', self.port))
try:
data = jsonpickle.dumps({
'type': type,
'task_id': self.task_id,
- 'exception': exception
+ 'exception': exception,
+ 'tracked_changes': tracked_changes
})
sock.send(struct.pack(_INT_FMT, len(data)))
sock.sendall(data)
@@ -271,13 +281,19 @@ def _main():
operation_mapping = arguments['operation_mapping']
operation_inputs = arguments['operation_inputs']
context_dict = arguments['context']
- try:
- ctx = serialization.operation_context_from_dict(context_dict)
- task_func = imports.load_attribute(operation_mapping)
- task_func(ctx=ctx, **operation_inputs)
- messenger.succeeded()
- except BaseException as e:
- messenger.failed(exception=e)
+
+ # This is required for the instrumentation work properly.
+ # See docstring of `remove_mutable_association_listener` for further details
+ storage_type.remove_mutable_association_listener()
+
+ with instrumentation.track_changes() as instrument:
+ try:
+ ctx = serialization.operation_context_from_dict(context_dict)
+ task_func = imports.load_attribute(operation_mapping)
+ task_func(ctx=ctx, **operation_inputs)
+ messenger.succeeded(tracked_changes=instrument.tracked_changes)
+ except BaseException as e:
+ messenger.failed(exception=e, tracked_changes=instrument.tracked_changes)
if __name__ == '__main__':
_main()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/aria/storage/instrumentation.py
----------------------------------------------------------------------
diff --git a/aria/storage/instrumentation.py b/aria/storage/instrumentation.py
new file mode 100644
index 0000000..d71190d
--- /dev/null
+++ b/aria/storage/instrumentation.py
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+
+import sqlalchemy.event
+
+from . import api
+from . import model as _model
+
+_STUB = object()
+_INSTRUMENTED = {
+ _model.NodeInstance.runtime_properties: dict
+}
+
+
+def track_changes(instrumented=None):
+ """Track changes in the specified model columns
+
+ This call will register event listeners using sqlalchemy's event mechanism. The listeners
+ instrument all returned objects such that the attributes specified in ``instrumented``, will
+ be replaced with a value that is stored in the returned instrumentation context
+ ``tracked_changes`` property.
+
+ Why should this be implemented when sqlalchemy already does a fantastic job at tracking changes
+ you ask? Well, when sqlalchemy is used with sqlite, due to how sqlite works, only one process
+ can hold a write lock to the database. This does not work well when ARIA runs tasks in
+ subprocesses (by the process executor) and these tasks wish to change some state as well. These
+ tasks certainly deserve a chance to do so!
+
+ To enable this, the subprocess calls ``track_changes()`` before any state changes are made.
+ At the end of the subprocess execution, it should return the ``tracked_changes`` attribute of
+ the instrumentation context returned from this call, to the parent process. The parent process
+ will then call ``apply_tracked_changes()`` that resides in this module as well.
+ At that point, the changes will actually be written back to the database.
+
+ :param instrumented: A dict from model columns to their python native type
+ :return: The instrumentation context
+ """
+ return _Instrumentation(instrumented or _INSTRUMENTED)
+
+
+class _Instrumentation(object):
+
+ def __init__(self, instrumented):
+ self.tracked_changes = {}
+ self.listeners = []
+ self._track_changes(instrumented)
+
+ def _track_changes(self, instrumented):
+ instrumented_classes = {}
+ for instrumented_attribute, attribute_type in instrumented.items():
+ self._register_set_attribute_listener(
+ instrumented_attribute=instrumented_attribute,
+ attribute_type=attribute_type)
+ instrumented_class = instrumented_attribute.parent.entity
+ instrumented_class_attributes = instrumented_classes.setdefault(instrumented_class, {})
+ instrumented_class_attributes[instrumented_attribute.key] = attribute_type
+ for instrumented_class, instrumented_attributes in instrumented_classes.items():
+ self._register_instance_listeners(
+ instrumented_class=instrumented_class,
+ instrumented_attributes=instrumented_attributes)
+
+ def _register_set_attribute_listener(self, instrumented_attribute, attribute_type):
+ def listener(target, value, *_):
+ mapi_name = api.generate_lower_name(target.__class__)
+ tracked_instances = self.tracked_changes.setdefault(mapi_name, {})
+ tracked_attributes = tracked_instances.setdefault(target.id, {})
+ if value is None:
+ current = None
+ else:
+ current = copy.deepcopy(attribute_type(value))
+ tracked_attributes[instrumented_attribute.key] = _Value(_STUB, current)
+ return current
+ listener_args = (instrumented_attribute, 'set', listener)
+ sqlalchemy.event.listen(*listener_args, retval=True)
+ self.listeners.append(listener_args)
+
+ def _register_instance_listeners(self, instrumented_class, instrumented_attributes):
+ def listener(target, *_):
+ mapi_name = api.generate_lower_name(instrumented_class)
+ tracked_instances = self.tracked_changes.setdefault(mapi_name, {})
+ tracked_attributes = tracked_instances.setdefault(target.id, {})
+ for attribute_name, attribute_type in instrumented_attributes.items():
+ if attribute_name not in tracked_attributes:
+ initial = getattr(target, attribute_name)
+ if initial is None:
+ current = None
+ else:
+ current = copy.deepcopy(attribute_type(initial))
+ tracked_attributes[attribute_name] = _Value(initial, current)
+ target.__dict__[attribute_name] = tracked_attributes[attribute_name].current
+ for listener_args in [(instrumented_class, 'load', listener),
+ (instrumented_class, 'refresh', listener),
+ (instrumented_class, 'refresh_flush', listener)]:
+ sqlalchemy.event.listen(*listener_args)
+ self.listeners.append(listener_args)
+
+ def restore(self):
+ """Remove all listeners registered by this instrumentation"""
+ for listener_args in self.listeners:
+ if sqlalchemy.event.contains(*listener_args):
+ sqlalchemy.event.remove(*listener_args)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.restore()
+
+
+class _Value(object):
+ # You may wonder why is this a full blown class and not a named tuple. The reason is that
+ # jsonpickle that is used to serialize the tracked_changes, does not handle named tuples very
+ # well. At the very least, I could not get it to behave.
+
+ def __init__(self, initial, current):
+ self.initial = initial
+ self.current = current
+
+ def __eq__(self, other):
+ if not isinstance(other, _Value):
+ return False
+ return self.initial == other.initial and self.current == other.current
+
+ def __hash__(self):
+ return hash(self.initial) ^ hash(self.current)
+
+
+def apply_tracked_changes(tracked_changes, model):
+ """Write tracked changes back to the database using provided model storage
+
+ :param tracked_changes: The ``tracked_changes`` attribute of the instrumentation context
+ returned by calling ``track_changes()``
+ :param model: The model storage used to actually apply the changes
+ """
+ for mapi_name, tracked_instances in tracked_changes.items():
+ mapi = getattr(model, mapi_name)
+ for instance_id, tracked_attributes in tracked_instances.items():
+ instance = None
+ for attribute_name, value in tracked_attributes.items():
+ if value.initial != value.current:
+ if not instance:
+ instance = mapi.get(instance_id)
+ setattr(instance, attribute_name, value.current)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/aria/storage/type.py
----------------------------------------------------------------------
diff --git a/aria/storage/type.py b/aria/storage/type.py
index ab50b0f..dd2a5ce 100644
--- a/aria/storage/type.py
+++ b/aria/storage/type.py
@@ -16,7 +16,8 @@ import json
from sqlalchemy import (
TypeDecorator,
- VARCHAR
+ VARCHAR,
+ event
)
from sqlalchemy.ext import mutable
@@ -84,5 +85,36 @@ class _MutableList(mutable.MutableList):
raise exceptions.StorageError('SQL Storage error: {0}'.format(str(e)))
-_MutableList.associate_with(List)
-_MutableDict.associate_with(Dict)
+def _mutable_association_listener(mapper, cls):
+ for prop in mapper.column_attrs:
+ column_type = prop.columns[0].type
+ if isinstance(column_type, Dict):
+ _MutableDict.associate_with_attribute(getattr(cls, prop.key))
+ if isinstance(column_type, List):
+ _MutableList.associate_with_attribute(getattr(cls, prop.key))
+_LISTENER_ARGS = (mutable.mapper, 'mapper_configured', _mutable_association_listener)
+
+
+def _register_mutable_association_listener():
+ event.listen(*_LISTENER_ARGS)
+
+
+def remove_mutable_association_listener():
+ """
+ Remove the event listener that associates ``Dict`` and ``List`` column types with
+ ``MutableDict`` and ``MutableList``, respectively.
+
+ This call must happen before any model instance is instantiated.
+ This is because once it does, that would trigger the listener we are trying to remove.
+ Once it is triggered, many other listeners will then be registered.
+ At that point, it is too late.
+
+ The reason this function exists is that the association listener, interferes with ARIA change
+ tracking instrumentation, so a way to disable it is required.
+
+ Note that the event listener this call removes is registered by default.
+ """
+ if event.contains(*_LISTENER_ARGS):
+ event.remove(*_LISTENER_ARGS)
+
+_register_mutable_association_listener()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/tests/.pylintrc
----------------------------------------------------------------------
diff --git a/tests/.pylintrc b/tests/.pylintrc
index 23251af..5de0691 100644
--- a/tests/.pylintrc
+++ b/tests/.pylintrc
@@ -77,7 +77,7 @@ confidence=
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,no-self-use,missing-docstring,attribute-defined-outside-init,redefined-outer-name,import-error,redefined-variable-type,broad
-except,protected-access,global-statement,too-many-locals
+disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,no-self-use,missing-docstring,attribute-defined-outside-init,redefined-outer-name,import-error,redefined-variable-type,broad
-except,protected-access,global-statement,too-many-locals,abstract-method
[REPORTS]
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
new file mode 100644
index 0000000..1564292
--- /dev/null
+++ b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from aria.orchestrator.workflows import api
+from aria.orchestrator.workflows.core import engine
+from aria.orchestrator.workflows.executor import process
+from aria.orchestrator import workflow, operation
+from aria.orchestrator.workflows import exceptions
+
+import tests
+from tests import mock
+from tests import storage
+
+
+_TEST_RUNTIME_PROPERTIES = {
+ 'some': 'values', 'that': 'are', 'most': 'likely', 'only': 'set', 'here': 'yo'
+}
+
+
+def test_track_changes_of_successful_operation(context, executor):
+ _run_workflow(context=context, executor=executor, op_func=_mock_success_operation)
+ _assert_tracked_changes_are_applied(context)
+
+
+def test_track_changes_of_failed_operation(context, executor):
+ with pytest.raises(exceptions.ExecutorException):
+ _run_workflow(context=context, executor=executor, op_func=_mock_fail_operation)
+ _assert_tracked_changes_are_applied(context)
+
+
+def _assert_tracked_changes_are_applied(context):
+ instance = context.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+ assert instance.runtime_properties == _TEST_RUNTIME_PROPERTIES
+
+
+def _update_runtime_properties(context):
+ context.node_instance.runtime_properties.clear()
+ context.node_instance.runtime_properties.update(_TEST_RUNTIME_PROPERTIES)
+
+
+def _run_workflow(context, executor, op_func):
+ @workflow
+ def mock_workflow(ctx, graph):
+ node_instance = ctx.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+ node_instance.node.operations['test.op'] = {'operation': _operation_mapping(op_func)}
+ task = api.task.OperationTask.node_instance(instance=node_instance, name='test.op')
+ graph.add_tasks(task)
+ return graph
+ graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter
+ eng = engine.Engine(executor=executor, workflow_context=context, tasks_graph=graph)
+ eng.execute()
+
+
+@operation
+def _mock_success_operation(ctx):
+ _update_runtime_properties(ctx)
+
+
+@operation
+def _mock_fail_operation(ctx):
+ _update_runtime_properties(ctx)
+ raise RuntimeError
+
+
+def _operation_mapping(func):
+ return '{name}.{func.__name__}'.format(name=__name__, func=func)
+
+
+@pytest.fixture
+def executor():
+ result = process.ProcessExecutor(python_path=[tests.ROOT_DIR])
+ yield result
+ result.close()
+
+
+@pytest.fixture
+def context(tmpdir):
+ result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+ yield result
+ storage.release_sqlite_storage(result.model)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/tests/storage/test_instrumentation.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_instrumentation.py b/tests/storage/test_instrumentation.py
new file mode 100644
index 0000000..8e7f9aa
--- /dev/null
+++ b/tests/storage/test_instrumentation.py
@@ -0,0 +1,275 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from sqlalchemy import Column, Text, Integer, event
+
+from aria.storage import (
+ model,
+ structure,
+ type as aria_type,
+ ModelStorage,
+ sql_mapi,
+ instrumentation
+)
+from ..storage import get_sqlite_api_kwargs, release_sqlite_storage
+
+
+STUB = instrumentation._STUB
+Value = instrumentation._Value
+instruments_holder = []
+
+
+class TestInstrumentation(object):
+
+ def test_track_changes(self, storage):
+ model_kwargs = dict(
+ name='name',
+ dict1={'initial': 'value'},
+ dict2={'initial': 'value'},
+ list1=['initial'],
+ list2=['initial'],
+ int1=0,
+ int2=0,
+ string2='string')
+ model1_instance = MockModel1(**model_kwargs)
+ model2_instance = MockModel2(**model_kwargs)
+ storage.mock_model_1.put(model1_instance)
+ storage.mock_model_2.put(model2_instance)
+
+ instrument = self._track_changes({
+ MockModel1.dict1: dict,
+ MockModel1.list1: list,
+ MockModel1.int1: int,
+ MockModel1.string2: str,
+ MockModel2.dict2: dict,
+ MockModel2.list2: list,
+ MockModel2.int2: int,
+ MockModel2.name: str
+ })
+
+ assert not instrument.tracked_changes
+
+ storage_model1_instance = storage.mock_model_1.get(model1_instance.id)
+ storage_model2_instance = storage.mock_model_2.get(model2_instance.id)
+
+ storage_model1_instance.dict1 = {'hello': 'world'}
+ storage_model1_instance.dict2 = {'should': 'not track'}
+ storage_model1_instance.list1 = ['hello']
+ storage_model1_instance.list2 = ['should not track']
+ storage_model1_instance.int1 = 100
+ storage_model1_instance.int2 = 20000
+ storage_model1_instance.name = 'should not track'
+ storage_model1_instance.string2 = 'new_string'
+
+ storage_model2_instance.dict1.update({'should': 'not track'})
+ storage_model2_instance.dict2.update({'hello': 'world'})
+ storage_model2_instance.list1.append('should not track')
+ storage_model2_instance.list2.append('hello')
+ storage_model2_instance.int1 = 100
+ storage_model2_instance.int2 = 20000
+ storage_model2_instance.name = 'new_name'
+ storage_model2_instance.string2 = 'should not track'
+
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {
+ model1_instance.id: {
+ 'dict1': Value(STUB, {'hello': 'world'}),
+ 'list1': Value(STUB, ['hello']),
+ 'int1': Value(STUB, 100),
+ 'string2': Value(STUB, 'new_string')
+ }
+ },
+ 'mock_model_2': {
+ model2_instance.id: {
+ 'dict2': Value({'initial': 'value'}, {'hello': 'world', 'initial': 'value'}),
+ 'list2': Value(['initial'], ['initial', 'hello']),
+ 'int2': Value(STUB, 20000),
+ 'name': Value(STUB, 'new_name'),
+ }
+ }
+ }
+
+ def test_attribute_initial_none_value(self, storage):
+ instance1 = MockModel1(name='name1', dict1=None)
+ instance2 = MockModel1(name='name2', dict1=None)
+ storage.mock_model_1.put(instance1)
+ storage.mock_model_1.put(instance2)
+ instrument = self._track_changes({MockModel1.dict1: dict})
+ instance1 = storage.mock_model_1.get(instance1.id)
+ instance2 = storage.mock_model_1.get(instance2.id)
+ instance1.dict1 = {'new': 'value'}
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {
+ instance1.id: {'dict1': Value(STUB, {'new': 'value'})},
+ instance2.id: {'dict1': Value(None, None)},
+ }
+ }
+
+ def test_attribute_set_none_value(self, storage):
+ instance = MockModel1(name='name')
+ storage.mock_model_1.put(instance)
+ instrument = self._track_changes({
+ MockModel1.dict1: dict,
+ MockModel1.list1: list,
+ MockModel1.string2: str,
+ MockModel1.int1: int
+ })
+ instance = storage.mock_model_1.get(instance.id)
+ instance.dict1 = None
+ instance.list1 = None
+ instance.string2 = None
+ instance.int1 = None
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {
+ instance.id: {
+ 'dict1': Value(STUB, None),
+ 'list1': Value(STUB, None),
+ 'string2': Value(STUB, None),
+ 'int1': Value(STUB, None)
+ }
+ }
+ }
+
+ def test_restore(self):
+ instrument = self._track_changes({MockModel1.dict1: dict})
+ # set instance attribute, load instance, refresh instance and flush_refresh listeners
+ assert len(instrument.listeners) == 4
+ for listener_args in instrument.listeners:
+ assert event.contains(*listener_args)
+ instrument.restore()
+ assert len(instrument.listeners) == 4
+ for listener_args in instrument.listeners:
+ assert not event.contains(*listener_args)
+ return instrument
+
+ def test_restore_twice(self):
+ instrument = self.test_restore()
+ instrument.restore()
+
+ def test_instrumentation_context_manager(self, storage):
+ instance = MockModel1(name='name')
+ storage.mock_model_1.put(instance)
+ with self._track_changes({MockModel1.dict1: dict}) as instrument:
+ instance = storage.mock_model_1.get(instance.id)
+ instance.dict1 = {'new': 'value'}
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {instance.id: {'dict1': Value(STUB, {'new': 'value'})}}
+ }
+ assert len(instrument.listeners) == 4
+ for listener_args in instrument.listeners:
+ assert event.contains(*listener_args)
+ for listener_args in instrument.listeners:
+ assert not event.contains(*listener_args)
+
+ def test_apply_tracked_changes(self, storage):
+ initial_values = {'dict1': {'initial': 'value'}, 'list1': ['initial']}
+ instance1_1 = MockModel1(name='instance1_1', **initial_values)
+ instance1_2 = MockModel1(name='instance1_2', **initial_values)
+ instance2_1 = MockModel2(name='instance2_1', **initial_values)
+ instance2_2 = MockModel2(name='instance2_2', **initial_values)
+ storage.mock_model_1.put(instance1_1)
+ storage.mock_model_1.put(instance1_2)
+ storage.mock_model_2.put(instance2_1)
+ storage.mock_model_2.put(instance2_2)
+
+ instrument = self._track_changes({
+ MockModel1.dict1: dict,
+ MockModel1.list1: list,
+ MockModel2.dict1: dict,
+ MockModel2.list1: list
+ })
+
+ def get_instances():
+ return (storage.mock_model_1.get(instance1_1.id),
+ storage.mock_model_1.get(instance1_2.id),
+ storage.mock_model_2.get(instance2_1.id),
+ storage.mock_model_2.get(instance2_2.id))
+
+ instance1_1, instance1_2, instance2_1, instance2_2 = get_instances()
+ instance1_1.dict1 = {'new': 'value'}
+ instance1_2.list1 = ['new_value']
+ instance2_1.dict1.update({'new': 'value'})
+ instance2_2.list1.append('new_value')
+
+ instrument.restore()
+ storage.mock_model_1._session.expire_all()
+
+ instance1_1, instance1_2, instance2_1, instance2_2 = get_instances()
+ instance1_1.dict1 = {'overriding': 'value'}
+ instance1_2.list1 = ['overriding_value']
+ instance2_1.dict1 = {'overriding': 'value'}
+ instance2_2.list1 = ['overriding_value']
+ storage.mock_model_1.put(instance1_1)
+ storage.mock_model_1.put(instance1_2)
+ storage.mock_model_2.put(instance2_1)
+ storage.mock_model_2.put(instance2_2)
+ instance1_1, instance1_2, instance2_1, instance2_2 = get_instances()
+ assert instance1_1.dict1 == {'overriding': 'value'}
+ assert instance1_2.list1 == ['overriding_value']
+ assert instance2_1.dict1 == {'overriding': 'value'}
+ assert instance2_2.list1 == ['overriding_value']
+
+ instrumentation.apply_tracked_changes(
+ tracked_changes=instrument.tracked_changes,
+ model=storage)
+
+ instance1_1, instance1_2, instance2_1, instance2_2 = get_instances()
+ assert instance1_1.dict1 == {'new': 'value'}
+ assert instance1_2.list1 == ['new_value']
+ assert instance2_1.dict1 == {'initial': 'value', 'new': 'value'}
+ assert instance2_2.list1 == ['initial', 'new_value']
+
+ def _track_changes(self, instrumented):
+ instrument = instrumentation.track_changes(instrumented)
+ instruments_holder.append(instrument)
+ return instrument
+
+
+@pytest.fixture(autouse=True)
+def restore_instrumentation():
+ yield
+ for instrument in instruments_holder:
+ instrument.restore()
+ del instruments_holder[:]
+
+
+@pytest.fixture
+def storage():
+ result = ModelStorage(
+ api_cls=sql_mapi.SQLAlchemyModelAPI,
+ api_kwargs=get_sqlite_api_kwargs(),
+ items=(MockModel1, MockModel2))
+ yield result
+ release_sqlite_storage(result)
+
+
+class _MockModel(structure.ModelMixin):
+ name = Column(Text)
+ dict1 = Column(aria_type.Dict)
+ dict2 = Column(aria_type.Dict)
+ list1 = Column(aria_type.List)
+ list2 = Column(aria_type.List)
+ int1 = Column(Integer)
+ int2 = Column(Integer)
+ string2 = Column(Text)
+
+
+class MockModel1(model.DeclarativeBase, _MockModel):
+ __tablename__ = 'mock_model1'
+
+
+class MockModel2(model.DeclarativeBase, _MockModel):
+ __tablename__ = 'mock_model2'
[04/14] incubator-ariatosca git commit: ARIA-46 Execution plugin
Posted by mx...@apache.org.
ARIA-46 Execution plugin
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/c4429dd5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/c4429dd5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/c4429dd5
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: c4429dd59a80bd6e58ba3bec4a2528eeda27a897
Parents: 385b209
Author: Dan Kilman <da...@gigaspaces.com>
Authored: Tue Jan 3 17:00:46 2017 +0200
Committer: Dan Kilman <da...@gigaspaces.com>
Committed: Thu Jan 19 18:13:51 2017 +0200
----------------------------------------------------------------------
appveyor.yml | 2 +-
aria/orchestrator/context/common.py | 4 +-
aria/orchestrator/context/operation.py | 5 +-
aria/orchestrator/exceptions.py | 2 +-
aria/orchestrator/execution_plugin/__init__.py | 33 ++
aria/orchestrator/execution_plugin/common.py | 150 +++++
aria/orchestrator/execution_plugin/constants.py | 51 ++
.../execution_plugin/ctx_proxy/__init__.py | 16 +
.../execution_plugin/ctx_proxy/client.py | 105 ++++
.../execution_plugin/ctx_proxy/server.py | 239 ++++++++
.../execution_plugin/environment_globals.py | 60 ++
.../orchestrator/execution_plugin/exceptions.py | 38 ++
aria/orchestrator/execution_plugin/local.py | 125 ++++
.../orchestrator/execution_plugin/operations.py | 63 ++
.../execution_plugin/ssh/__init__.py | 14 +
.../execution_plugin/ssh/operations.py | 192 ++++++
.../orchestrator/execution_plugin/ssh/tunnel.py | 91 +++
aria/orchestrator/workflows/api/task.py | 12 +-
aria/orchestrator/workflows/core/engine.py | 1 -
aria/orchestrator/workflows/core/task.py | 3 +-
aria/orchestrator/workflows/executor/process.py | 3 +-
aria/storage/base_model.py | 26 +-
aria/storage/filesystem_rapi.py | 2 +-
aria/utils/exceptions.py | 19 +
requirements.txt | 5 +
setup.py | 9 +-
tests/orchestrator/execution_plugin/__init__.py | 14 +
.../execution_plugin/test_common.py | 193 ++++++
.../execution_plugin/test_ctx_proxy_server.py | 359 ++++++++++++
.../execution_plugin/test_global_ctx.py | 28 +
.../orchestrator/execution_plugin/test_local.py | 587 +++++++++++++++++++
tests/orchestrator/execution_plugin/test_ssh.py | 481 +++++++++++++++
tests/orchestrator/workflows/api/test_task.py | 5 +
tests/orchestrator/workflows/core/test_task.py | 38 +-
tests/orchestrator/workflows/helpers.py | 37 ++
tests/resources/scripts/test_ssh.sh | 81 +++
tests/test_logger.py | 5 +-
tests/utils/test_exceptions.py | 73 +++
tox.ini | 7 +
39 files changed, 3150 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/appveyor.yml
----------------------------------------------------------------------
diff --git a/appveyor.yml b/appveyor.yml
index 3ea8635..cb1ae42 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -3,7 +3,7 @@ environment:
TOX_ENV: pywin
matrix:
- - PYTHON: C:\Python27
+ - PYTHON: "C:\\Python27"
PYTHON_VERSION: 2.7.8
PYTHON_ARCH: 32
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/context/common.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/common.py b/aria/orchestrator/context/common.py
index 53844e8..691c17d 100644
--- a/aria/orchestrator/context/common.py
+++ b/aria/orchestrator/context/common.py
@@ -113,9 +113,7 @@ class BaseContext(logger.LoggerMixin):
template using the provided variables. ctx is available to the template without providing it
explicitly.
"""
- self.download_resource(destination=destination, path=path)
- with open(destination, 'rb') as f:
- resource_content = f.read()
+ resource_content = self.get_resource(path=path)
resource_content = self._render_resource(resource_content=resource_content,
variables=variables)
with open(destination, 'wb') as f:
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/context/operation.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/operation.py b/aria/orchestrator/context/operation.py
index 19bb73a..b33d107 100644
--- a/aria/orchestrator/context/operation.py
+++ b/aria/orchestrator/context/operation.py
@@ -42,6 +42,7 @@ class BaseOperationContext(BaseContext):
**kwargs)
self._task_id = task_id
self._actor_id = actor_id
+ self._task = None
def __repr__(self):
details = 'operation_mapping={task.operation_mapping}; ' \
@@ -55,7 +56,9 @@ class BaseOperationContext(BaseContext):
The task in the model storage
:return: Task model
"""
- return self.model.task.get(self._task_id)
+ if not self._task:
+ self._task = self.model.task.get(self._task_id)
+ return self._task
class NodeOperationContext(BaseOperationContext):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/exceptions.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/exceptions.py b/aria/orchestrator/exceptions.py
index bd5238e..c00b66b 100644
--- a/aria/orchestrator/exceptions.py
+++ b/aria/orchestrator/exceptions.py
@@ -36,7 +36,7 @@ class TaskRetryException(RuntimeError):
"""
Used internally when ctx.task.retry is called
"""
- def __init__(self, message, retry_interval):
+ def __init__(self, message, retry_interval=None):
super(TaskRetryException, self).__init__(message)
self.retry_interval = retry_interval
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/__init__.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/__init__.py b/aria/orchestrator/execution_plugin/__init__.py
new file mode 100644
index 0000000..372022f
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/__init__.py
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from contextlib import contextmanager
+
+# Populated during execution of python scripts
+ctx = None
+inputs = None
+
+
+@contextmanager
+def python_script_scope(operation_ctx, operation_inputs):
+ global ctx
+ global inputs
+ try:
+ ctx = operation_ctx
+ inputs = operation_inputs
+ yield
+ finally:
+ ctx = None
+ inputs = None
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/common.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/common.py b/aria/orchestrator/execution_plugin/common.py
new file mode 100644
index 0000000..47cb631
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/common.py
@@ -0,0 +1,150 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import tempfile
+
+import requests
+
+from . import constants
+from . import exceptions
+
+
+def is_windows():
+ return os.name == 'nt'
+
+
+def download_script(ctx, script_path):
+ split = script_path.split('://')
+ schema = split[0]
+ suffix = script_path.split('/')[-1]
+ file_descriptor, dest_script_path = tempfile.mkstemp(suffix='-{0}'.format(suffix))
+ os.close(file_descriptor)
+ try:
+ if schema in ['http', 'https']:
+ response = requests.get(script_path)
+ if response.status_code == 404:
+ ctx.task.abort('Failed to download script: {0} (status code: {1})'
+ .format(script_path, response.status_code))
+ content = response.text
+ with open(dest_script_path, 'wb') as f:
+ f.write(content)
+ else:
+ ctx.download_resource(destination=dest_script_path, path=script_path)
+ except:
+ os.remove(dest_script_path)
+ raise
+ return dest_script_path
+
+
+def create_process_config(script_path, process, operation_kwargs, quote_json_env_vars=False):
+ """
+ update a process with it's environment variables, and return it.
+
+ Get a dict representing a process and a dict representing the environment
+ variables. Convert each environment variable to a format of
+ <string representing the name of the variable> :
+ <json formatted string representing the value of the variable>.
+ Finally, update the process with the newly formatted environment variables,
+ and return the process.
+
+ :param process: a dict representing a process
+ :type process: dict
+ :param operation_kwargs: a dict representing environment variables that
+ should exist in the process' running environment.
+ :type operation_kwargs: dict
+ :return: the process updated with its environment variables.
+ :rtype: dict
+ """
+ process = process or {}
+ env_vars = operation_kwargs.copy()
+ if 'ctx' in env_vars:
+ del env_vars['ctx']
+ env_vars.update(process.get('env', {}))
+ for k, v in env_vars.items():
+ if isinstance(v, (dict, list, tuple, bool, int, float)):
+ v = json.dumps(v)
+ if quote_json_env_vars:
+ v = "'{0}'".format(v)
+ if is_windows():
+ # These <k,v> environment variables will subsequently
+ # be used in a subprocess.Popen() call, as the `env` parameter.
+ # In some windows python versions, if an environment variable
+ # name is not of type str (e.g. unicode), the Popen call will
+ # fail.
+ k = str(k)
+ # The windows shell removes all double quotes - escape them
+ # to still be able to pass JSON in env vars to the shell.
+ v = v.replace('"', '\\"')
+ del env_vars[k]
+ env_vars[k] = str(v)
+ process['env'] = env_vars
+ args = process.get('args')
+ command = script_path
+ command_prefix = process.get('command_prefix')
+ if command_prefix:
+ command = '{0} {1}'.format(command_prefix, command)
+ if args:
+ command = ' '.join([command] + [str(a) for a in args])
+ process['command'] = command
+ return process
+
+
+def patch_ctx(ctx):
+ ctx._error = None
+ task = ctx.task
+ task._original_abort = task.abort
+ task._original_retry = task.retry
+
+ def _validate_legal_action():
+ if ctx._error is not None:
+ ctx._error = RuntimeError(constants.ILLEGAL_CTX_OPERATION_MESSAGE)
+ raise ctx._error
+
+ def abort_operation(message=None):
+ _validate_legal_action()
+ ctx._error = exceptions.ScriptException(message=message, retry=False)
+ return ctx._error
+ task.abort = abort_operation
+
+ def retry_operation(message=None, retry_interval=None):
+ _validate_legal_action()
+ ctx._error = exceptions.ScriptException(message=message,
+ retry=True,
+ retry_interval=retry_interval)
+ return ctx._error
+ task.retry = retry_operation
+
+
+def check_error(ctx, error_check_func=None, reraise=False):
+ _error = ctx._error
+ # this happens when a script calls task.abort/task.retry more than once
+ if isinstance(_error, RuntimeError):
+ ctx.task._original_abort(str(_error))
+ # ScriptException is populated by the ctx proxy server when task.abort or task.retry
+ # are called
+ elif isinstance(_error, exceptions.ScriptException):
+ if _error.retry:
+ ctx.task._original_retry(_error.message, _error.retry_interval)
+ else:
+ ctx.task._original_abort(_error.message)
+ # local and ssh operations may pass an additional logic check for errors here
+ if error_check_func:
+ error_check_func()
+ # if this function is called from within an ``except`` clause, a re-raise maybe required
+ if reraise:
+ raise # pylint: disable=misplaced-bare-raise
+ return _error
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/constants.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/constants.py b/aria/orchestrator/execution_plugin/constants.py
new file mode 100644
index 0000000..0b54dbe
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/constants.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import exceptions
+
+# related to local
+PYTHON_SCRIPT_FILE_EXTENSION = '.py'
+POWERSHELL_SCRIPT_FILE_EXTENSION = '.ps1'
+DEFAULT_POWERSHELL_EXECUTABLE = 'powershell'
+
+# related to both local and ssh
+ILLEGAL_CTX_OPERATION_MESSAGE = 'ctx may only abort or retry once'
+
+# related to ssh
+DEFAULT_BASE_DIR = '/tmp/aria-ctx'
+FABRIC_ENV_DEFAULTS = {
+ 'connection_attempts': 5,
+ 'timeout': 10,
+ 'forward_agent': False,
+ 'abort_on_prompts': True,
+ 'keepalive': 0,
+ 'linewise': False,
+ 'pool_size': 0,
+ 'skip_bad_hosts': False,
+ 'status': False,
+ 'disable_known_hosts': True,
+ 'combine_stderr': True,
+ 'abort_exception': exceptions.TaskException,
+}
+VALID_FABRIC_GROUPS = set([
+ 'status',
+ 'aborts',
+ 'warnings',
+ 'running',
+ 'stdout',
+ 'stderr',
+ 'user',
+ 'everything'
+])
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/ctx_proxy/__init__.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ctx_proxy/__init__.py b/aria/orchestrator/execution_plugin/ctx_proxy/__init__.py
new file mode 100644
index 0000000..7571c15
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ctx_proxy/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import server, client
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/ctx_proxy/client.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ctx_proxy/client.py b/aria/orchestrator/execution_plugin/ctx_proxy/client.py
new file mode 100644
index 0000000..d965a5e
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ctx_proxy/client.py
@@ -0,0 +1,105 @@
+#! /usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+import sys
+import urllib2
+
+
+# Environment variable for the socket url (used by clients to locate the socket)
+CTX_SOCKET_URL = 'CTX_SOCKET_URL'
+
+
+class _RequestError(RuntimeError):
+
+ def __init__(self, ex_message, ex_type, ex_traceback):
+ super(_RequestError, self).__init__(self, '{0}: {1}'.format(ex_type, ex_message))
+ self.ex_type = ex_type
+ self.ex_message = ex_message
+ self.ex_traceback = ex_traceback
+
+
+def _http_request(socket_url, request, timeout):
+ response = urllib2.urlopen(
+ url=socket_url,
+ data=json.dumps(request),
+ timeout=timeout)
+ if response.code != 200:
+ raise RuntimeError('Request failed: {0}'.format(response))
+ return json.loads(response.read())
+
+
+def _client_request(socket_url, args, timeout):
+ response = _http_request(
+ socket_url=socket_url,
+ request={'args': args},
+ timeout=timeout)
+ payload = response['payload']
+ response_type = response.get('type')
+ if response_type == 'error':
+ ex_type = payload['type']
+ ex_message = payload['message']
+ ex_traceback = payload['traceback']
+ raise _RequestError(ex_message, ex_type, ex_traceback)
+ elif response_type == 'stop_operation':
+ raise SystemExit(payload['message'])
+ else:
+ return payload
+
+
+def _parse_args(args):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-t', '--timeout', type=int, default=30)
+ parser.add_argument('--socket-url', default=os.environ.get(CTX_SOCKET_URL))
+ parser.add_argument('--json-arg-prefix', default='@')
+ parser.add_argument('-j', '--json-output', action='store_true')
+ parser.add_argument('args', nargs='*')
+ args = parser.parse_args(args=args)
+ if not args.socket_url:
+ raise RuntimeError('Missing CTX_SOCKET_URL environment variable '
+ 'or socket_url command line argument. (ctx is supposed to be executed '
+ 'within an operation context)')
+ return args
+
+
+def _process_args(json_prefix, args):
+ processed_args = []
+ for arg in args:
+ if arg.startswith(json_prefix):
+ arg = json.loads(arg[1:])
+ processed_args.append(arg)
+ return processed_args
+
+
+def main(args=None):
+ args = _parse_args(args)
+ response = _client_request(
+ socket_url=args.socket_url,
+ args=_process_args(args.json_arg_prefix, args.args),
+ timeout=args.timeout)
+ if args.json_output:
+ response = json.dumps(response)
+ else:
+ if not response:
+ response = ''
+ response = str(response)
+ sys.stdout.write(response)
+
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/ctx_proxy/server.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ctx_proxy/server.py b/aria/orchestrator/execution_plugin/ctx_proxy/server.py
new file mode 100644
index 0000000..2782ae3
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ctx_proxy/server.py
@@ -0,0 +1,239 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import json
+import re
+import socket
+import threading
+import traceback
+import Queue
+import StringIO
+import wsgiref.simple_server
+
+import bottle
+
+from .. import exceptions
+
+
+class CtxProxy(object):
+
+ def __init__(self, ctx):
+ self.ctx = ctx
+ self.port = _get_unused_port()
+ self.socket_url = 'http://localhost:{0}'.format(self.port)
+ self.server = None
+ self._started = Queue.Queue(1)
+ self.thread = self._start_server()
+ self._started.get(timeout=5)
+
+ def _start_server(self):
+ proxy = self
+
+ class BottleServerAdapter(bottle.ServerAdapter):
+ def run(self, app):
+ class Server(wsgiref.simple_server.WSGIServer):
+ allow_reuse_address = True
+
+ def handle_error(self, request, client_address):
+ pass
+
+ class Handler(wsgiref.simple_server.WSGIRequestHandler):
+ def address_string(self):
+ return self.client_address[0]
+
+ def log_request(*args, **kwargs): # pylint: disable=no-method-argument
+ if not self.quiet:
+ return wsgiref.simple_server.WSGIRequestHandler.log_request(*args,
+ **kwargs)
+ server = wsgiref.simple_server.make_server(
+ host=self.host,
+ port=self.port,
+ app=app,
+ server_class=Server,
+ handler_class=Handler)
+ proxy.server = server
+ proxy._started.put(True)
+ server.serve_forever(poll_interval=0.1)
+
+ def serve():
+ bottle_app = bottle.Bottle()
+ bottle_app.post('/', callback=self._request_handler)
+ bottle.run(
+ app=bottle_app,
+ host='localhost',
+ port=self.port,
+ quiet=True,
+ server=BottleServerAdapter)
+ thread = threading.Thread(target=serve)
+ thread.daemon = True
+ thread.start()
+ return thread
+
+ def close(self):
+ if self.server:
+ self.server.shutdown()
+ self.server.server_close()
+
+ def _request_handler(self):
+ request = bottle.request.body.read() # pylint: disable=no-member
+ response = self._process(request)
+ return bottle.LocalResponse(
+ body=response,
+ status=200,
+ headers={'content-type': 'application/json'})
+
+ def _process(self, request):
+ try:
+ typed_request = json.loads(request)
+ args = typed_request['args']
+ payload = _process_ctx_request(self.ctx, args)
+ result_type = 'result'
+ if isinstance(payload, exceptions.ScriptException):
+ payload = dict(message=str(payload))
+ result_type = 'stop_operation'
+ result = json.dumps({
+ 'type': result_type,
+ 'payload': payload
+ })
+ except Exception as e:
+ traceback_out = StringIO.StringIO()
+ traceback.print_exc(file=traceback_out)
+ payload = {
+ 'type': type(e).__name__,
+ 'message': str(e),
+ 'traceback': traceback_out.getvalue()
+ }
+ result = json.dumps({
+ 'type': 'error',
+ 'payload': payload
+ })
+ return result
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args, **kwargs):
+ self.close()
+
+
+def _process_ctx_request(ctx, args):
+ current = ctx
+ num_args = len(args)
+ index = 0
+ while index < num_args:
+ arg = args[index]
+ attr = _desugar_attr(current, arg)
+ if attr:
+ current = getattr(current, attr)
+ elif isinstance(current, collections.MutableMapping):
+ key = arg
+ path_dict = _PathDictAccess(current)
+ if index + 1 == num_args:
+ # read dict prop by path
+ value = path_dict.get(key)
+ current = value
+ elif index + 2 == num_args:
+ # set dict prop by path
+ value = args[index + 1]
+ path_dict.set(key, value)
+ current = None
+ else:
+ raise RuntimeError('Illegal argument while accessing dict')
+ break
+ elif callable(current):
+ kwargs = {}
+ remaining_args = args[index:]
+ if isinstance(remaining_args[-1], collections.MutableMapping):
+ kwargs = remaining_args[-1]
+ remaining_args = remaining_args[:-1]
+ current = current(*remaining_args, **kwargs)
+ break
+ else:
+ raise RuntimeError('{0} cannot be processed in {1}'.format(arg, args))
+ index += 1
+ if callable(current):
+ current = current()
+ return current
+
+
+def _desugar_attr(obj, attr):
+ if not isinstance(attr, basestring):
+ return None
+ if hasattr(obj, attr):
+ return attr
+ attr = attr.replace('-', '_')
+ if hasattr(obj, attr):
+ return attr
+ return None
+
+
+class _PathDictAccess(object):
+ pattern = re.compile(r"(.+)\[(\d+)\]")
+
+ def __init__(self, obj):
+ self.obj = obj
+
+ def set(self, prop_path, value):
+ obj, prop_name = self._get_parent_obj_prop_name_by_path(prop_path)
+ obj[prop_name] = value
+
+ def get(self, prop_path):
+ value = self._get_object_by_path(prop_path)
+ return value
+
+ def _get_object_by_path(self, prop_path, fail_on_missing=True):
+ # when setting a nested object, make sure to also set all the
+ # intermediate path objects
+ current = self.obj
+ for prop_segment in prop_path.split('.'):
+ match = self.pattern.match(prop_segment)
+ if match:
+ index = int(match.group(2))
+ property_name = match.group(1)
+ if property_name not in current:
+ self._raise_illegal(prop_path)
+ if not isinstance(current[property_name], list):
+ self._raise_illegal(prop_path)
+ current = current[property_name][index]
+ else:
+ if prop_segment not in current:
+ if fail_on_missing:
+ self._raise_illegal(prop_path)
+ else:
+ current[prop_segment] = {}
+ current = current[prop_segment]
+ return current
+
+ def _get_parent_obj_prop_name_by_path(self, prop_path):
+ split = prop_path.split('.')
+ if len(split) == 1:
+ return self.obj, prop_path
+ parent_path = '.'.join(split[:-1])
+ parent_obj = self._get_object_by_path(parent_path, fail_on_missing=False)
+ prop_name = split[-1]
+ return parent_obj, prop_name
+
+ @staticmethod
+ def _raise_illegal(prop_path):
+ raise RuntimeError('illegal path: {0}'.format(prop_path))
+
+
+def _get_unused_port():
+ sock = socket.socket()
+ sock.bind(('127.0.0.1', 0))
+ _, port = sock.getsockname()
+ sock.close()
+ return port
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/environment_globals.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/environment_globals.py b/aria/orchestrator/execution_plugin/environment_globals.py
new file mode 100644
index 0000000..27311f0
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/environment_globals.py
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def create_initial_globals(path):
+ """ emulates a `globals()` call in a freshly loaded module
+
+ The implementation of this function is likely to raise a couple of
+ questions. If you read the implementation and nothing bothered you, feel
+ free to skip the rest of this docstring.
+
+ First, why is this function in its own module and not, say, in the same
+ module of the other environment-related functions?
+ Second, why is it implemented in such a way that copies the globals, then
+ deletes the item that represents this function, and then changes some
+ other entries?
+
+ Well, these two questions can be answered with one (elaborate) explanation.
+ If this function was in the same module with the other environment-related
+ functions, then we would have had to delete more items in globals than just
+ `create_initial_globals`. That is because all of the other function names
+ would also be in globals, and since there is no built-in mechanism that
+ return the name of the user-defined objects, this approach is quite an
+ overkill.
+
+ - But why do we rely on the copy-existing-globals-and-delete-entries
+ method, when it seems to force us to put `create_initial_globals` in its
+ own file?
+
+ Well, because there is no easier method of creating globals of a newly
+ loaded module.
+
+ - How about hard coding a 'global' dict? It seems that there are very few
+ entries: __doc__, __file__, __name__, __package__ (but don't forget
+ __builtins__).
+
+ That would be coupling our implementation to a specific `globals`
+ implementation. What if `globals` were to change?
+ """
+ copied_globals = globals().copy()
+ copied_globals.update({
+ '__doc__': 'Dynamically executed script',
+ '__file__': path,
+ '__name__': '__main__',
+ '__package__': None
+ })
+ del copied_globals[create_initial_globals.__name__]
+ return copied_globals
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/exceptions.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/exceptions.py b/aria/orchestrator/execution_plugin/exceptions.py
new file mode 100644
index 0000000..4641c80
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/exceptions.py
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class ProcessException(Exception):
+ """Raised when local scripts and remote ssh commands fail"""
+
+ def __init__(self, stderr=None, stdout=None, command=None, exit_code=None):
+ super(ProcessException, self).__init__(stderr)
+ self.command = command
+ self.exit_code = exit_code
+ self.stdout = stdout
+ self.stderr = stderr
+
+
+class TaskException(Exception):
+ """Raised when remote ssh scripts fail"""
+
+
+class ScriptException(Exception):
+ """Used by the ctx proxy server when task.retry or task.abort are called by scripts"""
+
+ def __init__(self, message=None, retry=None, retry_interval=None):
+ super(ScriptException, self).__init__(message)
+ self.retry = retry
+ self.retry_interval = retry_interval
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/local.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/local.py b/aria/orchestrator/execution_plugin/local.py
new file mode 100644
index 0000000..bc2d661
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/local.py
@@ -0,0 +1,125 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import threading
+import StringIO
+
+from . import ctx_proxy
+from . import exceptions
+from . import common
+from . import constants
+from . import environment_globals
+from . import python_script_scope
+
+
+def run_script(ctx, script_path, process, **kwargs):
+ if not script_path:
+ ctx.task.abort('Missing script_path')
+ process = process or {}
+ script_path = common.download_script(ctx, script_path)
+ script_func = _get_run_script_func(script_path, process)
+ return script_func(
+ ctx=ctx,
+ script_path=script_path,
+ process=process,
+ operation_kwargs=kwargs)
+
+
+def _get_run_script_func(script_path, process):
+ if _treat_script_as_python_script(script_path, process):
+ return _eval_script_func
+ else:
+ if _treat_script_as_powershell_script(script_path):
+ process.setdefault('command_prefix', constants.DEFAULT_POWERSHELL_EXECUTABLE)
+ return _execute_func
+
+
+def _treat_script_as_python_script(script_path, process):
+ eval_python = process.get('eval_python')
+ script_extension = os.path.splitext(script_path)[1].lower()
+ return (eval_python is True or (script_extension == constants.PYTHON_SCRIPT_FILE_EXTENSION and
+ eval_python is not False))
+
+
+def _treat_script_as_powershell_script(script_path):
+ script_extension = os.path.splitext(script_path)[1].lower()
+ return script_extension == constants.POWERSHELL_SCRIPT_FILE_EXTENSION
+
+
+def _eval_script_func(script_path, ctx, operation_kwargs, **_):
+ with python_script_scope(operation_ctx=ctx, operation_inputs=operation_kwargs):
+ execfile(script_path, environment_globals.create_initial_globals(script_path))
+
+
+def _execute_func(script_path, ctx, process, operation_kwargs):
+ os.chmod(script_path, 0755)
+ process = common.create_process_config(
+ script_path=script_path,
+ process=process,
+ operation_kwargs=operation_kwargs)
+ command = process['command']
+ env = os.environ.copy()
+ env.update(process['env'])
+ ctx.logger.info('Executing: {0}'.format(command))
+ common.patch_ctx(ctx)
+ with ctx_proxy.server.CtxProxy(ctx) as proxy:
+ env[ctx_proxy.client.CTX_SOCKET_URL] = proxy.socket_url
+ running_process = subprocess.Popen(
+ command,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env,
+ cwd=process.get('cwd'),
+ bufsize=1,
+ close_fds=not common.is_windows())
+ stdout_consumer = _OutputConsumer(running_process.stdout)
+ stderr_consumer = _OutputConsumer(running_process.stderr)
+ exit_code = running_process.wait()
+ stdout_consumer.join()
+ stderr_consumer.join()
+ ctx.logger.info('Execution done (exit_code={0}): {1}'.format(exit_code, command))
+
+ def error_check_func():
+ if exit_code:
+ raise exceptions.ProcessException(
+ command=command,
+ exit_code=exit_code,
+ stdout=stdout_consumer.read_output(),
+ stderr=stderr_consumer.read_output())
+ return common.check_error(ctx, error_check_func=error_check_func)
+
+
+class _OutputConsumer(object):
+
+ def __init__(self, out):
+ self._out = out
+ self._buffer = StringIO.StringIO()
+ self._consumer = threading.Thread(target=self._consume_output)
+ self._consumer.daemon = True
+ self._consumer.start()
+
+ def _consume_output(self):
+ for line in iter(self._out.readline, b''):
+ self._buffer.write(line)
+ self._out.close()
+
+ def read_output(self):
+ return self._buffer.getvalue()
+
+ def join(self):
+ self._consumer.join()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/operations.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/operations.py b/aria/orchestrator/execution_plugin/operations.py
new file mode 100644
index 0000000..5effa8a
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/operations.py
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from aria.orchestrator import operation
+from . import local as local_operations
+from .ssh import operations as ssh_operations
+
+
+@operation
+def run_script_locally(ctx,
+ script_path,
+ process=None,
+ **kwargs):
+ return local_operations.run_script(
+ ctx=ctx,
+ script_path=script_path,
+ process=process,
+ **kwargs)
+
+
+@operation
+def run_script_with_ssh(ctx,
+ script_path,
+ fabric_env=None,
+ process=None,
+ use_sudo=False,
+ hide_output=None,
+ **kwargs):
+ return ssh_operations.run_script(
+ ctx=ctx,
+ script_path=script_path,
+ fabric_env=fabric_env,
+ process=process,
+ use_sudo=use_sudo,
+ hide_output=hide_output,
+ **kwargs)
+
+
+@operation
+def run_commands_with_ssh(ctx,
+ commands,
+ fabric_env=None,
+ use_sudo=False,
+ hide_output=None,
+ **_):
+ return ssh_operations.run_commands(
+ ctx=ctx,
+ commands=commands,
+ fabric_env=fabric_env,
+ use_sudo=use_sudo,
+ hide_output=hide_output)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/ssh/__init__.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ssh/__init__.py b/aria/orchestrator/execution_plugin/ssh/__init__.py
new file mode 100644
index 0000000..ae1e83e
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ssh/__init__.py
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/ssh/operations.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ssh/operations.py b/aria/orchestrator/execution_plugin/ssh/operations.py
new file mode 100644
index 0000000..7589d42
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ssh/operations.py
@@ -0,0 +1,192 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import random
+import string
+import tempfile
+import StringIO
+
+import fabric.api
+import fabric.context_managers
+import fabric.contrib.files
+
+from .. import constants
+from .. import exceptions
+from .. import common
+from .. import ctx_proxy
+from . import tunnel
+
+
+_PROXY_CLIENT_PATH = ctx_proxy.client.__file__
+if _PROXY_CLIENT_PATH.endswith('.pyc'):
+ _PROXY_CLIENT_PATH = _PROXY_CLIENT_PATH[:-1]
+
+
+def run_commands(ctx, commands, fabric_env, use_sudo, hide_output, **_):
+ """Runs the provider 'commands' in sequence
+
+ :param commands: a list of commands to run
+ :param fabric_env: fabric configuration
+ """
+ with fabric.api.settings(_hide_output(ctx, groups=hide_output),
+ **_fabric_env(ctx, fabric_env, warn_only=True)):
+ for command in commands:
+ ctx.logger.info('Running command: {0}'.format(command))
+ run = fabric.api.sudo if use_sudo else fabric.api.run
+ result = run(command)
+ if result.failed:
+ raise exceptions.ProcessException(
+ command=result.command,
+ exit_code=result.return_code,
+ stdout=result.stdout,
+ stderr=result.stderr)
+
+
+def run_script(ctx, script_path, fabric_env, process, use_sudo, hide_output, **kwargs):
+ process = process or {}
+ paths = _Paths(base_dir=process.get('base_dir', constants.DEFAULT_BASE_DIR),
+ local_script_path=common.download_script(ctx, script_path))
+ with fabric.api.settings(_hide_output(ctx, groups=hide_output),
+ **_fabric_env(ctx, fabric_env, warn_only=False)):
+ # the remote host must have the ctx before running any fabric scripts
+ if not fabric.contrib.files.exists(paths.remote_ctx_path):
+ # there may be race conditions with other operations that
+ # may be running in parallel, so we pass -p to make sure
+ # we get 0 exit code if the directory already exists
+ fabric.api.run('mkdir -p {0} && mkdir -p {1}'.format(paths.remote_scripts_dir,
+ paths.remote_work_dir))
+ # this file has to be present before using ctx
+ fabric.api.put(_PROXY_CLIENT_PATH, paths.remote_ctx_path)
+ _patch_ctx(ctx)
+ process = common.create_process_config(
+ script_path=paths.remote_script_path,
+ process=process,
+ operation_kwargs=kwargs,
+ quote_json_env_vars=True)
+ fabric.api.put(paths.local_script_path, paths.remote_script_path)
+ with ctx_proxy.server.CtxProxy(ctx) as proxy:
+ local_port = proxy.port
+ with fabric.context_managers.cd(process.get('cwd', paths.remote_work_dir)): # pylint: disable=not-context-manager
+ with tunnel.remote(ctx, local_port=local_port) as remote_port:
+ local_socket_url = proxy.socket_url
+ remote_socket_url = local_socket_url.replace(str(local_port), str(remote_port))
+ env_script = _write_environment_script_file(
+ process=process,
+ paths=paths,
+ local_socket_url=local_socket_url,
+ remote_socket_url=remote_socket_url)
+ fabric.api.put(env_script, paths.remote_env_script_path)
+ try:
+ command = 'source {0} && {1}'.format(paths.remote_env_script_path,
+ process['command'])
+ run = fabric.api.sudo if use_sudo else fabric.api.run
+ run(command)
+ except exceptions.TaskException:
+ return common.check_error(ctx, reraise=True)
+ return common.check_error(ctx)
+
+
+def _patch_ctx(ctx):
+ common.patch_ctx(ctx)
+ original_download_resource = ctx.download_resource
+ original_download_resource_and_render = ctx.download_resource_and_render
+
+ def _download_resource(func, destination, **kwargs):
+ handle, temp_local_path = tempfile.mkstemp()
+ os.close(handle)
+ try:
+ func(destination=temp_local_path, **kwargs)
+ return fabric.api.put(temp_local_path, destination)
+ finally:
+ os.remove(temp_local_path)
+
+ def download_resource(destination, path=None):
+ _download_resource(
+ func=original_download_resource,
+ destination=destination,
+ path=path)
+ ctx.download_resource = download_resource
+
+ def download_resource_and_render(destination, path=None, variables=None):
+ _download_resource(
+ func=original_download_resource_and_render,
+ destination=destination,
+ path=path,
+ variables=variables)
+ ctx.download_resource_and_render = download_resource_and_render
+
+
+def _hide_output(ctx, groups):
+ """ Hides Fabric's output for every 'entity' in `groups` """
+ groups = set(groups or [])
+ if not groups.issubset(constants.VALID_FABRIC_GROUPS):
+ ctx.task.abort('`hide_output` must be a subset of {0} (Provided: {1})'
+ .format(', '.join(constants.VALID_FABRIC_GROUPS), ', '.join(groups)))
+ return fabric.api.hide(*groups)
+
+
+def _fabric_env(ctx, fabric_env, warn_only):
+ """Prepares fabric environment variables configuration"""
+ ctx.logger.debug('Preparing fabric environment...')
+ env = constants.FABRIC_ENV_DEFAULTS.copy()
+ env.update(fabric_env or {})
+ env.setdefault('warn_only', warn_only)
+ if 'host_string' not in env:
+ env['host_string'] = ctx.task.runs_on.ip
+ # validations
+ if not env.get('host_string'):
+ ctx.task.abort('`host_string` not supplied and ip cannot be deduced automatically')
+ if not (env.get('password') or env.get('key_filename') or env.get('key')):
+ ctx.task.abort(
+ 'Access credentials not supplied '
+ '(you must supply at least one of `key_filename`, `key` or `password`)')
+ if not env.get('user'):
+ ctx.task.abort('`user` not supplied')
+ ctx.logger.debug('Environment prepared successfully')
+ return env
+
+
+def _write_environment_script_file(process, paths, local_socket_url, remote_socket_url):
+ env_script = StringIO.StringIO()
+ env = process['env']
+ env['PATH'] = '{0}:$PATH'.format(paths.remote_ctx_dir)
+ env['PYTHONPATH'] = '{0}:$PYTHONPATH'.format(paths.remote_ctx_dir)
+ env_script.write('chmod +x {0}\n'.format(paths.remote_script_path))
+ env_script.write('chmod +x {0}\n'.format(paths.remote_ctx_path))
+ env.update({
+ ctx_proxy.client.CTX_SOCKET_URL: remote_socket_url,
+ 'LOCAL_{0}'.format(ctx_proxy.client.CTX_SOCKET_URL): local_socket_url
+ })
+ for key, value in env.iteritems():
+ env_script.write('export {0}={1}\n'.format(key, value))
+ return env_script
+
+
+class _Paths(object):
+
+ def __init__(self, base_dir, local_script_path):
+ self.local_script_path = local_script_path
+ self.remote_ctx_dir = base_dir
+ self.base_script_path = os.path.basename(self.local_script_path)
+ self.remote_ctx_path = '{0}/ctx'.format(self.remote_ctx_dir)
+ self.remote_scripts_dir = '{0}/scripts'.format(self.remote_ctx_dir)
+ self.remote_work_dir = '{0}/work'.format(self.remote_ctx_dir)
+ random_suffix = ''.join(random.choice(string.ascii_lowercase + string.digits)
+ for _ in range(8))
+ remote_path_suffix = '{0}-{1}'.format(self.base_script_path, random_suffix)
+ self.remote_env_script_path = '{0}/env-{1}'.format(self.remote_scripts_dir,
+ remote_path_suffix)
+ self.remote_script_path = '{0}/{1}'.format(self.remote_scripts_dir, remote_path_suffix)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/execution_plugin/ssh/tunnel.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ssh/tunnel.py b/aria/orchestrator/execution_plugin/ssh/tunnel.py
new file mode 100644
index 0000000..6fc8d54
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ssh/tunnel.py
@@ -0,0 +1,91 @@
+# This implementation was copied from the Fabric project directly:
+# https://github.com/fabric/fabric/blob/master/fabric/context_managers.py#L486
+# The purpose was to remove the rtunnel creation printouts here:
+# https://github.com/fabric/fabric/blob/master/fabric/context_managers.py#L547
+
+
+import contextlib
+import select
+import socket
+
+import fabric.api
+import fabric.state
+import fabric.thread_handling
+
+
+@contextlib.contextmanager
+def remote(ctx, local_port, remote_port=0, local_host='localhost', remote_bind_address='127.0.0.1'):
+ """Create a tunnel forwarding a locally-visible port to the remote target."""
+ sockets = []
+ channels = []
+ thread_handlers = []
+
+ def accept(channel, *args, **kwargs):
+ # This seemingly innocent statement seems to be doing nothing
+ # but the truth is far from it!
+ # calling fileno() on a paramiko channel the first time, creates
+ # the required plumbing to make the channel valid for select.
+ # While this would generally happen implicitly inside the _forwarder
+ # function when select is called, it may already be too late and may
+ # cause the select loop to hang.
+ # Specifically, when new data arrives to the channel, a flag is set
+ # on an "event" object which is what makes the select call work.
+ # problem is this will only happen if the event object is not None
+ # and it will be not-None only after channel.fileno() has been called
+ # for the first time. If we wait until _forwarder calls select for the
+ # first time it may be after initial data has reached the channel.
+ # calling it explicitly here in the paramiko transport main event loop
+ # guarantees this will not happen.
+ channel.fileno()
+
+ channels.append(channel)
+ sock = socket.socket()
+ sockets.append(sock)
+
+ try:
+ sock.connect((local_host, local_port))
+ except Exception as e:
+ try:
+ channel.close()
+ except Exception as ex2:
+ close_error = ' (While trying to close channel: {0})'.format(ex2)
+ else:
+ close_error = ''
+ ctx.task.abort('[{0}] rtunnel: cannot connect to {1}:{2} ({3}){4}'
+ .format(fabric.api.env.host_string, local_host, local_port, e,
+ close_error))
+
+ thread_handler = fabric.thread_handling.ThreadHandler('fwd', _forwarder, channel, sock)
+ thread_handlers.append(thread_handler)
+
+ transport = fabric.state.connections[fabric.api.env.host_string].get_transport()
+ remote_port = transport.request_port_forward(
+ remote_bind_address, remote_port, handler=accept)
+
+ try:
+ yield remote_port
+ finally:
+ for sock, chan, thread_handler in zip(sockets, channels, thread_handlers):
+ sock.close()
+ chan.close()
+ thread_handler.thread.join()
+ thread_handler.raise_if_needed()
+ transport.cancel_port_forward(remote_bind_address, remote_port)
+
+
+def _forwarder(chan, sock):
+ # Bidirectionally forward data between a socket and a Paramiko channel.
+ while True:
+ read = select.select([sock, chan], [], [])[0]
+ if sock in read:
+ data = sock.recv(1024)
+ if len(data) == 0:
+ break
+ chan.send(data)
+ if chan in read:
+ data = chan.recv(1024)
+ if len(data) == 0:
+ break
+ sock.send(data)
+ chan.close()
+ sock.close()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/workflows/api/task.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/api/task.py b/aria/orchestrator/workflows/api/task.py
index 70324a6..44715c1 100644
--- a/aria/orchestrator/workflows/api/task.py
+++ b/aria/orchestrator/workflows/api/task.py
@@ -68,7 +68,8 @@ class OperationTask(BaseTask):
retry_interval=None,
ignore_failure=None,
inputs=None,
- plugin=None):
+ plugin=None,
+ runs_on=None):
"""
Creates an operation task using the name, details, node instance and any additional kwargs.
:param name: the operation of the name.
@@ -89,6 +90,7 @@ class OperationTask(BaseTask):
if retry_interval is None else retry_interval)
self.ignore_failure = (self.workflow_context._task_ignore_failure
if ignore_failure is None else ignore_failure)
+ self.runs_on = runs_on
@classmethod
def node_instance(cls, instance, name, inputs=None, *args, **kwargs):
@@ -104,6 +106,7 @@ class OperationTask(BaseTask):
operation_details=instance.node.operations[name],
inputs=inputs,
plugins=instance.node.plugins or [],
+ runs_on=model.Task.RUNS_ON_NODE_INSTANCE,
*args,
**kwargs)
@@ -126,18 +129,22 @@ class OperationTask(BaseTask):
operation_details = getattr(instance.relationship, operation_end)[name]
if operation_end == cls.SOURCE_OPERATION:
plugins = instance.relationship.source_node.plugins
+ runs_on = model.Task.RUNS_ON_SOURCE
else:
plugins = instance.relationship.target_node.plugins
+ runs_on = model.Task.RUNS_ON_TARGET
return cls._instance(instance=instance,
name=name,
operation_details=operation_details,
inputs=inputs,
plugins=plugins or [],
+ runs_on=runs_on,
*args,
**kwargs)
@classmethod
- def _instance(cls, instance, name, operation_details, inputs, plugins, *args, **kwargs):
+ def _instance(cls, instance, name, operation_details, inputs, plugins, runs_on, *args,
+ **kwargs):
operation_mapping = operation_details.get('operation')
operation_inputs = operation_details.get('inputs', {})
operation_inputs.update(inputs or {})
@@ -151,6 +158,7 @@ class OperationTask(BaseTask):
operation_mapping=operation_mapping,
inputs=operation_inputs,
plugin=plugin,
+ runs_on=runs_on,
*args,
**kwargs)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/workflows/core/engine.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/core/engine.py b/aria/orchestrator/workflows/core/engine.py
index 47269a3..fd83614 100644
--- a/aria/orchestrator/workflows/core/engine.py
+++ b/aria/orchestrator/workflows/core/engine.py
@@ -52,7 +52,6 @@ class Engine(logger.LoggerMixin):
"""
try:
events.start_workflow_signal.send(self._workflow_context)
- cancel = False
while True:
cancel = self._is_cancel()
if cancel:
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/workflows/core/task.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/core/task.py b/aria/orchestrator/workflows/core/task.py
index 67be2ea..1deb66a 100644
--- a/aria/orchestrator/workflows/core/task.py
+++ b/aria/orchestrator/workflows/core/task.py
@@ -135,7 +135,8 @@ class OperationTask(BaseTask):
retry_interval=api_task.retry_interval,
ignore_failure=api_task.ignore_failure,
plugin=plugins[0] if plugins else None,
- execution=self._workflow_context.execution
+ execution=self._workflow_context.execution,
+ runs_on=api_task.runs_on
)
self._workflow_context.model.task.put(operation_task)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/orchestrator/workflows/executor/process.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py
index 51596c3..2cc9178 100644
--- a/aria/orchestrator/workflows/executor/process.py
+++ b/aria/orchestrator/workflows/executor/process.py
@@ -40,6 +40,7 @@ import Queue
import jsonpickle
from aria.utils import imports
+from aria.utils import exceptions
from aria.orchestrator.workflows.executor import base
from aria.orchestrator.context import serialization
from aria.storage import instrumentation
@@ -255,7 +256,7 @@ class _Messenger(object):
data = jsonpickle.dumps({
'type': type,
'task_id': self.task_id,
- 'exception': exception,
+ 'exception': exceptions.wrap_if_needed(exception),
'tracked_changes': tracked_changes
})
sock.send(struct.pack(_INT_FMT, len(data)))
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/storage/base_model.py
----------------------------------------------------------------------
diff --git a/aria/storage/base_model.py b/aria/storage/base_model.py
index a2bbb60..8566e66 100644
--- a/aria/storage/base_model.py
+++ b/aria/storage/base_model.py
@@ -695,8 +695,13 @@ class TaskBase(ModelMixin):
WAIT_STATES = [PENDING, RETRYING]
END_STATES = [SUCCESS, FAILED]
+ RUNS_ON_SOURCE = 'source'
+ RUNS_ON_TARGET = 'target'
+ RUNS_ON_NODE_INSTANCE = 'node_instance'
+ RUNS_ON = (RUNS_ON_NODE_INSTANCE, RUNS_ON_SOURCE, RUNS_ON_TARGET)
+
@orm.validates('max_attempts')
- def validate_max_attempts(self, _, value): # pylint: disable=no-self-use
+ def validate_max_attempts(self, _, value): # pylint: disable=no-self-use
"""Validates that max attempts is either -1 or a positive number"""
if value < 1 and value != TaskBase.INFINITE_RETRIES:
raise ValueError('Max attempts can be either -1 (infinite) or any positive number. '
@@ -718,6 +723,7 @@ class TaskBase(ModelMixin):
# Operation specific fields
operation_mapping = Column(String)
inputs = Column(Dict)
+ _runs_on = Column(Enum(*RUNS_ON, name='runs_on'), name='runs_on')
@property
def actor(self):
@@ -727,13 +733,23 @@ class TaskBase(ModelMixin):
"""
return self.node_instance or self.relationship_instance
+ @property
+ def runs_on(self):
+ if self._runs_on == self.RUNS_ON_NODE_INSTANCE:
+ return self.node_instance
+ elif self._runs_on == self.RUNS_ON_SOURCE:
+ return self.relationship_instance.source_node_instance # pylint: disable=no-member
+ elif self._runs_on == self.RUNS_ON_TARGET:
+ return self.relationship_instance.target_node_instance # pylint: disable=no-member
+ return None
+
@classmethod
- def as_node_instance(cls, instance, **kwargs):
- return cls(node_instance=instance, **kwargs)
+ def as_node_instance(cls, instance, runs_on, **kwargs):
+ return cls(node_instance=instance, _runs_on=runs_on, **kwargs)
@classmethod
- def as_relationship_instance(cls, instance, **kwargs):
- return cls(relationship_instance=instance, **kwargs)
+ def as_relationship_instance(cls, instance, runs_on, **kwargs):
+ return cls(relationship_instance=instance, _runs_on=runs_on, **kwargs)
@staticmethod
def abort(message=None):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/storage/filesystem_rapi.py
----------------------------------------------------------------------
diff --git a/aria/storage/filesystem_rapi.py b/aria/storage/filesystem_rapi.py
index c6b3a81..eb30e0b 100644
--- a/aria/storage/filesystem_rapi.py
+++ b/aria/storage/filesystem_rapi.py
@@ -132,7 +132,7 @@ class FileSystemResourceAPI(api.ResourceAPI):
if os.path.isfile(resource):
shutil.copy2(resource, destination)
else:
- dir_util.copy_tree(resource, destination) # pylint: disable=no-member
+ dir_util.copy_tree(resource, destination) # pylint: disable=no-member
def upload(self, entry_id, source, path=None, **_):
"""
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/aria/utils/exceptions.py
----------------------------------------------------------------------
diff --git a/aria/utils/exceptions.py b/aria/utils/exceptions.py
index 0370bb3..a19eb78 100644
--- a/aria/utils/exceptions.py
+++ b/aria/utils/exceptions.py
@@ -16,6 +16,8 @@
import sys
import linecache
+import jsonpickle
+
from clint.textui import indent
from .console import (puts, Colored)
@@ -39,6 +41,7 @@ def print_exception(e, full=True, cause=False, traceback=None):
traceback = e.cause_traceback if hasattr(e, 'cause_traceback') else None
print_exception(e.cause, full=full, cause=True, traceback=traceback)
+
def print_traceback(traceback=None):
"""
Prints the traceback with nice colors and such.
@@ -62,3 +65,19 @@ def print_traceback(traceback=None):
with indent(2):
puts(Colored.black(line.strip()))
traceback = traceback.tb_next
+
+
+class _WrappedException(Exception):
+
+ def __init__(self, exception_type, exception_str):
+ super(_WrappedException, self).__init__(exception_type, exception_str)
+ self.exception_type = exception_type
+ self.exception_str = exception_str
+
+
+def wrap_if_needed(exception):
+ try:
+ jsonpickle.loads(jsonpickle.dumps(exception))
+ return exception
+ except BaseException:
+ return _WrappedException(type(exception).__name__, str(exception))
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/requirements.txt
----------------------------------------------------------------------
diff --git a/requirements.txt b/requirements.txt
index 0005a5e..3f922e8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -26,3 +26,8 @@ CacheControl[filecache]==0.11.6
clint==0.5.1
SQLAlchemy==1.1.4
wagon==0.5.0
+bottle==0.12.11
+six==1.10.0
+Fabric==1.13.1
+# Fabric makes use of this library, but doesn't bring it :(
+pypiwin32==219 ; sys_platform == 'win32'
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/setup.py
----------------------------------------------------------------------
diff --git a/setup.py b/setup.py
index 112f13e..5a3d016 100644
--- a/setup.py
+++ b/setup.py
@@ -44,6 +44,11 @@ except IOError:
install_requires = []
+console_scripts = ['aria = aria.cli.cli:main']
+if os.environ.get('INSTALL_CTX'):
+ console_scripts.append('ctx = aria.orchestrator.execution_plugin.ctx_proxy.client:main')
+
+
setup(
name=_PACKAGE_NAME,
version=version,
@@ -77,8 +82,6 @@ setup(
zip_safe=False,
install_requires=install_requires,
entry_points={
- 'console_scripts': [
- 'aria = aria.cli.cli:main'
- ]
+ 'console_scripts': console_scripts
}
)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/orchestrator/execution_plugin/__init__.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/__init__.py b/tests/orchestrator/execution_plugin/__init__.py
new file mode 100644
index 0000000..ae1e83e
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/__init__.py
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/c4429dd5/tests/orchestrator/execution_plugin/test_common.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_common.py b/tests/orchestrator/execution_plugin/test_common.py
new file mode 100644
index 0000000..1048c27
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/test_common.py
@@ -0,0 +1,193 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+
+import requests
+import pytest
+
+from aria.storage import model
+from aria.orchestrator import exceptions
+from aria.orchestrator.execution_plugin import common
+
+
+class TestDownloadScript(object):
+
+ @pytest.fixture(autouse=True)
+ def patch_requests(self, mocker):
+ def _mock_requests_get(url):
+ response = namedtuple('Response', 'text status_code')
+ return response(url, self.status_code)
+ self.status_code = 200
+ mocker.patch.object(requests, 'get', _mock_requests_get)
+
+ def _test_url(self, url):
+ class Ctx(object):
+ task = model.Task
+
+ script_path = url
+ result = common.download_script(Ctx, script_path)
+ with open(result) as f:
+ assert script_path == f.read()
+ assert result.endswith('-some_script.py')
+
+ def test_http_url(self):
+ self._test_url('http://localhost/some_script.py')
+
+ def test_https_url(self):
+ self._test_url('https://localhost/some_script.py')
+
+ def test_url_status_code_404(self):
+ self.status_code = 404
+ with pytest.raises(exceptions.TaskAbortException) as exc_ctx:
+ self.test_http_url()
+ exception = exc_ctx.value
+ assert 'status code: 404' in str(exception)
+
+ def test_blueprint_resource(self):
+ test_script_path = 'my_script.py'
+
+ class Ctx(object):
+ @staticmethod
+ def download_resource(destination, path):
+ assert path == test_script_path
+ return destination
+ result = common.download_script(Ctx, test_script_path)
+ assert result.endswith(test_script_path)
+
+
+class TestCreateProcessConfig(object):
+
+ def test_plain_command(self):
+ script_path = 'path'
+ process = common.create_process_config(
+ script_path=script_path,
+ process={},
+ operation_kwargs={})
+ assert process['command'] == script_path
+
+ def test_command_with_args(self):
+ script_path = 'path'
+ process = {'args': [1, 2, 3]}
+ process = common.create_process_config(
+ script_path=script_path,
+ process=process,
+ operation_kwargs={})
+ assert process['command'] == '{0} 1 2 3'.format(script_path)
+
+ def test_command_prefix(self):
+ script_path = 'path'
+ command_prefix = 'prefix'
+ process = {'command_prefix': command_prefix}
+ process = common.create_process_config(
+ script_path=script_path,
+ process=process,
+ operation_kwargs={})
+ assert process['command'] == '{0} {1}'.format(command_prefix, script_path)
+
+ def test_command_with_args_and_prefix(self):
+ script_path = 'path'
+ command_prefix = 'prefix'
+ process = {'command_prefix': command_prefix,
+ 'args': [1, 2, 3]}
+ process = common.create_process_config(
+ script_path=script_path,
+ process=process,
+ operation_kwargs={})
+ assert process['command'] == '{0} {1} 1 2 3'.format(command_prefix, script_path)
+
+ def test_ctx_is_removed(self):
+ process = common.create_process_config(
+ script_path='',
+ process={},
+ operation_kwargs={'ctx': 1})
+ assert 'ctx' not in process['env']
+
+ def test_env_passed_explicitly(self):
+ env = {'one': '1', 'two': '2'}
+ process = common.create_process_config(
+ script_path='',
+ process={'env': env},
+ operation_kwargs={})
+ assert process['env'] == env
+
+ def test_env_populated_from_operation_kwargs(self):
+ operation_kwargs = {'one': '1', 'two': '2'}
+ process = common.create_process_config(
+ script_path='',
+ process={},
+ operation_kwargs=operation_kwargs)
+ assert process['env'] == operation_kwargs
+
+ def test_env_merged_from_operation_kwargs_and_process(self):
+ operation_kwargs = {'one': '1', 'two': '2'}
+ env = {'three': '3', 'four': '4'}
+ process = common.create_process_config(
+ script_path='',
+ process={'env': env},
+ operation_kwargs=operation_kwargs)
+ assert process['env'] == dict(operation_kwargs.items() + env.items())
+
+ def test_process_env_gets_precedence_over_operation_kwargs(self):
+ operation_kwargs = {'one': 'from_kwargs'}
+ env = {'one': 'from_env_process'}
+ process = common.create_process_config(
+ script_path='',
+ process={'env': env},
+ operation_kwargs=operation_kwargs)
+ assert process['env'] == env
+
+ def test_json_env_vars(self, mocker):
+ mocker.patch.object(common, 'is_windows', lambda: False)
+ operation_kwargs = {'a_dict': {'key': 'value'},
+ 'a_list': ['a', 'b', 'c'],
+ 'a_tuple': (4, 5, 6),
+ 'a_bool': True}
+ process = common.create_process_config(
+ script_path='',
+ process={},
+ operation_kwargs=operation_kwargs)
+ assert process['env'] == {'a_dict': '{"key": "value"}',
+ 'a_list': '["a", "b", "c"]',
+ 'a_tuple': '[4, 5, 6]',
+ 'a_bool': 'true'}
+
+ def test_quote_json_env_vars(self):
+ operation_kwargs = {'one': []}
+ process = common.create_process_config(
+ script_path='',
+ process={},
+ operation_kwargs=operation_kwargs,
+ quote_json_env_vars=True)
+ assert process['env']['one'] == "'[]'"
+
+ def test_env_keys_converted_to_string_on_windows(self, mocker):
+ mocker.patch.object(common, 'is_windows', lambda: True)
+ env = {u'one': '1'}
+ process = common.create_process_config(
+ script_path='',
+ process={'env': env},
+ operation_kwargs={})
+ print type(process['env'].keys()[0])
+ assert isinstance(process['env'].keys()[0], str)
+
+ def test_env_values_quotes_are_escaped_on_windows(self, mocker):
+ mocker.patch.object(common, 'is_windows', lambda: True)
+ env = {'one': '"hello"'}
+ process = common.create_process_config(
+ script_path='',
+ process={'env': env},
+ operation_kwargs={})
+ assert process['env']['one'] == '\\"hello\\"'
[05/14] incubator-ariatosca git commit: ARIA-65 Add validation for
relationship assignment type
Posted by mx...@apache.org.
ARIA-65 Add validation for relationship assignment type
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/8ecf5bba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/8ecf5bba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/8ecf5bba
Branch: refs/heads/ARIA-66-Convert-custom-parser-fields-into-sqla-based-fields
Commit: 8ecf5bba9a53135f993a43cf6e3b499857b19701
Parents: c4429dd
Author: Tal Liron <ta...@gmail.com>
Authored: Sun Jan 22 12:49:12 2017 +0200
Committer: Ran Ziv <ra...@gigaspaces.com>
Committed: Sun Jan 22 12:50:54 2017 +0200
----------------------------------------------------------------------
.../simple_v1_0/modeling/requirements.py | 17 +++++++++++++----
.../aria_extension_tosca/simple_v1_0/types.py | 8 ++++++++
2 files changed, 21 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8ecf5bba/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py
----------------------------------------------------------------------
diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py
index 8f6c38f..97e30ee 100644
--- a/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py
+++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py
@@ -159,12 +159,21 @@ def convert_requirement_from_definition_to_assignment(context, requirement_defin
relationship_template = relationship_type
relationship_type = relationship_template._get_type(context)
+ definition_relationship_type = None
+ relationship_definition = requirement_definition.relationship # RelationshipDefinition
+ if relationship_definition is not None:
+ definition_relationship_type = relationship_definition._get_type(context)
+
# If not exists, try at the node type
- relationship_definition = None
if relationship_type is None:
- relationship_definition = requirement_definition.relationship # RelationshipDefinition
- if relationship_definition is not None:
- relationship_type = relationship_definition._get_type(context)
+ relationship_type = definition_relationship_type
+ else:
+ # Make sure the type is derived
+ if not definition_relationship_type._is_descendant(context, relationship_type):
+ context.validation.report(
+ 'assigned relationship type "%s" is not a descendant of declared relationship type "%s"'
+ % (relationship_type._name, definition_relationship_type._name),
+ locator=container._locator, level=Issue.BETWEEN_TYPES)
if relationship_type is not None:
raw['relationship'] = OrderedDict()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/8ecf5bba/extensions/aria_extension_tosca/simple_v1_0/types.py
----------------------------------------------------------------------
diff --git a/extensions/aria_extension_tosca/simple_v1_0/types.py b/extensions/aria_extension_tosca/simple_v1_0/types.py
index 39843ac..2112f7f 100644
--- a/extensions/aria_extension_tosca/simple_v1_0/types.py
+++ b/extensions/aria_extension_tosca/simple_v1_0/types.py
@@ -484,6 +484,14 @@ class RelationshipType(ExtensiblePresentation):
'relationship_types')
@cachedmethod
+ def _is_descendant(self, context, the_type):
+ if the_type is None:
+ return False
+ elif the_type._name == self._name:
+ return True
+ return self._is_descendant(context, the_type._get_parent(context))
+
+ @cachedmethod
def _get_properties(self, context):
return FrozenDict(get_inherited_property_definitions(context, self, 'properties'))