You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by da...@apache.org on 2017/01/17 10:48:46 UTC
[1/6] incubator-ariatosca git commit:
ARIA-52-Support-order-management-in-relationships [Forced Update!]
Repository: incubator-ariatosca
Updated Branches:
refs/heads/ARIA-46-execution-plugin e203f6ce9 -> 06b2f4d47 (forced update)
ARIA-52-Support-order-management-in-relationships
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/17d07ed9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/17d07ed9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/17d07ed9
Branch: refs/heads/ARIA-46-execution-plugin
Commit: 17d07ed972341dd2780067316732f4d789e846ed
Parents: 3caf177
Author: mxmrlv <mx...@gmail.com>
Authored: Tue Dec 27 16:42:57 2016 +0200
Committer: mxmrlv <mx...@gmail.com>
Committed: Sun Jan 8 15:34:46 2017 +0200
----------------------------------------------------------------------
aria/storage/base_model.py | 73 ++++++++++++++----
aria/storage/sql_mapi.py | 2 +-
aria/storage/structure.py | 10 ++-
tests/storage/test_model_storage.py | 128 ++++++++++++++++++++++++++++++-
4 files changed, 194 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/17d07ed9/aria/storage/base_model.py
----------------------------------------------------------------------
diff --git a/aria/storage/base_model.py b/aria/storage/base_model.py
index 418d3b6..a2bbb60 100644
--- a/aria/storage/base_model.py
+++ b/aria/storage/base_model.py
@@ -51,6 +51,7 @@ from sqlalchemy import (
Float,
orm,
)
+from sqlalchemy.ext.orderinglist import ordering_list
from ..orchestrator.exceptions import TaskAbortException, TaskRetryException
from .structure import ModelMixin
@@ -415,7 +416,14 @@ class RelationshipBase(ModelMixin):
"""
__tablename__ = 'relationships'
- _private_fields = ['source_node_fk', 'target_node_fk']
+ _private_fields = ['source_node_fk', 'target_node_fk', 'source_position', 'target_position']
+
+ source_position = Column(Integer)
+ target_position = Column(Integer)
+
+ @declared_attr
+ def deployment_id(self):
+ return association_proxy('source_node', 'deployment_id')
@declared_attr
def source_node_fk(cls):
@@ -423,8 +431,14 @@ class RelationshipBase(ModelMixin):
@declared_attr
def source_node(cls):
- return cls.one_to_many_relationship('source_node_fk',
- backreference='outbound_relationships')
+ return cls.one_to_many_relationship(
+ 'source_node_fk',
+ backreference='outbound_relationships',
+ backref_kwargs=dict(
+ order_by=cls.source_position,
+ collection_class=ordering_list('source_position', count_from=0)
+ )
+ )
@declared_attr
def source_name(cls):
@@ -432,11 +446,18 @@ class RelationshipBase(ModelMixin):
@declared_attr
def target_node_fk(cls):
- return cls.foreign_key(NodeBase)
+ return cls.foreign_key(NodeBase, nullable=True)
@declared_attr
def target_node(cls):
- return cls.one_to_many_relationship('target_node_fk', backreference='inbound_relationships')
+ return cls.one_to_many_relationship(
+ 'target_node_fk',
+ backreference='inbound_relationships',
+ backref_kwargs=dict(
+ order_by=cls.target_position,
+ collection_class=ordering_list('target_position', count_from=0)
+ )
+ )
@declared_attr
def target_name(cls):
@@ -515,35 +536,60 @@ class RelationshipInstanceBase(ModelMixin):
__tablename__ = 'relationship_instances'
_private_fields = ['relationship_storage_fk',
'source_node_instance_fk',
- 'target_node_instance_fk']
+ 'target_node_instance_fk',
+ 'source_position',
+ 'target_position']
+
+ source_position = Column(Integer)
+ target_position = Column(Integer)
@declared_attr
def source_node_instance_fk(cls):
- return cls.foreign_key(NodeInstanceBase)
+ return cls.foreign_key(NodeInstanceBase, nullable=True)
@declared_attr
def source_node_instance(cls):
- return cls.one_to_many_relationship('source_node_instance_fk',
- backreference='outbound_relationship_instances')
+ return cls.one_to_many_relationship(
+ 'source_node_instance_fk',
+ backreference='outbound_relationship_instances',
+ backref_kwargs=dict(
+ order_by=cls.source_position,
+ collection_class=ordering_list('source_position', count_from=0)
+ )
+ )
@declared_attr
def source_node_instance_name(cls):
+ return association_proxy('source_node_instance', 'node_{0}'.format(cls.name_column_name()))
+
+ @declared_attr
+ def source_node_name(cls):
return association_proxy('source_node_instance', cls.name_column_name())
@declared_attr
def target_node_instance_fk(cls):
- return cls.foreign_key(NodeInstanceBase)
+ return cls.foreign_key(NodeInstanceBase, nullable=True)
@declared_attr
def target_node_instance(cls):
- return cls.one_to_many_relationship('target_node_instance_fk',
- backreference='inbound_relationship_instances')
+ return cls.one_to_many_relationship(
+ 'target_node_instance_fk',
+ backreference='inbound_relationship_instances',
+ backref_kwargs=dict(
+ order_by=cls.target_position,
+ collection_class=ordering_list('target_position', count_from=0)
+ )
+ )
@declared_attr
def target_node_instance_name(cls):
return association_proxy('target_node_instance', cls.name_column_name())
@declared_attr
+ def target_node_name(cls):
+ return association_proxy('target_node_instance', 'node_{0}'.format(cls.name_column_name()))
+
+ @declared_attr
def relationship_fk(cls):
return cls.foreign_key(RelationshipBase)
@@ -556,6 +602,7 @@ class RelationshipInstanceBase(ModelMixin):
return association_proxy('relationship', cls.name_column_name())
+
class PluginBase(ModelMixin):
"""
Plugin model representation.
@@ -658,7 +705,7 @@ class TaskBase(ModelMixin):
INFINITE_RETRIES = -1
- status = Column(Enum(*STATES), name='status', default=PENDING)
+ status = Column(Enum(*STATES, name='status'), default=PENDING)
due_at = Column(DateTime, default=datetime.utcnow)
started_at = Column(DateTime, default=None)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/17d07ed9/aria/storage/sql_mapi.py
----------------------------------------------------------------------
diff --git a/aria/storage/sql_mapi.py b/aria/storage/sql_mapi.py
index cde40c2..809f677 100644
--- a/aria/storage/sql_mapi.py
+++ b/aria/storage/sql_mapi.py
@@ -46,7 +46,7 @@ class SQLAlchemyModelAPI(api.ModelAPI):
if not result:
raise exceptions.StorageError(
- 'Requested {0} with ID `{1}` was not found'
+ 'Requested `{0}` with ID `{1}` was not found'
.format(self.model_cls.__name__, entry_id)
)
return result
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/17d07ed9/aria/storage/structure.py
----------------------------------------------------------------------
diff --git a/aria/storage/structure.py b/aria/storage/structure.py
index 431633b..fa592ac 100644
--- a/aria/storage/structure.py
+++ b/aria/storage/structure.py
@@ -80,7 +80,9 @@ class ModelMixin(object):
@classmethod
def one_to_many_relationship(cls,
foreign_key_column,
- backreference=None):
+ backreference=None,
+ backref_kwargs=None,
+ **kwargs):
"""Return a one-to-many SQL relationship object
Meant to be used from inside the *child* object
@@ -89,6 +91,7 @@ class ModelMixin(object):
:param foreign_key_column: The column of the foreign key (from the child table)
:param backreference: The name to give to the reference to the child (on the parent table)
"""
+ backref_kwargs = backref_kwargs or {}
parent_table = cls._get_cls_by_tablename(
getattr(cls, foreign_key_column).__remote_table_name)
primaryjoin_str = '{parent_class_name}.{parent_unique_id} == ' \
@@ -105,7 +108,8 @@ class ModelMixin(object):
foreign_keys=[getattr(cls, foreign_key_column)],
# The following line make sure that when the *parent* is
# deleted, all its connected children are deleted as well
- backref=backref(backreference or cls.__tablename__, cascade='all'),
+ backref=backref(backreference or cls.__tablename__, cascade='all', **backref_kwargs),
+ **kwargs
)
@classmethod
@@ -145,6 +149,8 @@ class ModelMixin(object):
field_value = list(field_value)
elif isinstance(field_value, dict):
field_value = dict(field_value)
+ elif isinstance(field_value, ModelMixin):
+ field_value = field_value.to_dict()
res[field] = field_value
return res
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/17d07ed9/tests/storage/test_model_storage.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py
index 6f9527e..0e8d1a0 100644
--- a/tests/storage/test_model_storage.py
+++ b/tests/storage/test_model_storage.py
@@ -27,7 +27,11 @@ from aria.storage import (
)
from aria import application_model_storage
from ..storage import get_sqlite_api_kwargs, release_sqlite_storage
-from ..mock import context as mock_context
+from ..mock import (
+ context as mock_context,
+ models,
+ operations
+)
class MockModel(model.DeclarativeBase, structure.ModelMixin): #pylint: disable=abstract-method
@@ -46,6 +50,11 @@ def storage():
release_sqlite_storage(base_storage)
+@pytest.fixture
+def context():
+ return mock_context.simple(get_sqlite_api_kwargs())
+
+
@pytest.fixture(scope='module', autouse=True)
def module_cleanup():
model.DeclarativeBase.metadata.remove(MockModel.__table__) #pylint: disable=no-member
@@ -104,8 +113,7 @@ def test_inner_list_update(storage):
assert storage_mm.model_list[0] == 'new_value'
-def test_model_to_dict():
- context = mock_context.simple(get_sqlite_api_kwargs())
+def test_model_to_dict(context):
deployment = context.deployment
deployment_dict = deployment.to_dict()
@@ -144,3 +152,117 @@ def test_application_storage_factory():
assert storage.execution
release_sqlite_storage(storage)
+
+
+def test_relationship_model_ordering(context):
+ deployment = context.model.deployment.get_by_name(models.DEPLOYMENT_NAME)
+ source_node = context.model.node.get_by_name(models.DEPENDENT_NODE_NAME)
+ source_node_instance = context.model.node_instance.get_by_name(
+ models.DEPENDENT_NODE_INSTANCE_NAME)
+ target_node = context.model.node.get_by_name(models.DEPENDENCY_NODE_NAME)
+ target_node_instance = context.model.node_instance.get_by_name(
+ models.DEPENDENCY_NODE_INSTANCE_NAME)
+ new_node = model.Node(
+ name='new_node',
+ type='test_node_type',
+ type_hierarchy=[],
+ number_of_instances=1,
+ planned_number_of_instances=1,
+ deploy_number_of_instances=1,
+ properties={},
+ operations=dict((key, {}) for key in operations.NODE_OPERATIONS),
+ min_number_of_instances=1,
+ max_number_of_instances=1,
+ deployment=deployment
+ )
+ source_to_new_relationship = model.Relationship(
+ source_node=source_node,
+ target_node=new_node,
+ source_interfaces={},
+ source_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
+ target_interfaces={},
+ target_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
+ type='rel_type',
+ type_hierarchy=[],
+ properties={},
+ )
+ new_node_instance = model.NodeInstance(
+ name='new_node_instance',
+ runtime_properties={},
+ version=None,
+ node=new_node,
+ state='',
+ scaling_groups=[]
+ )
+ source_to_new_relationship_instance = model.RelationshipInstance(
+ relationship=source_to_new_relationship,
+ source_node_instance=source_node_instance,
+ target_node_instance=new_node_instance,
+ )
+
+ new_to_target_relationship = model.Relationship(
+ source_node=new_node,
+ target_node=target_node,
+ source_interfaces={},
+ source_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
+ target_interfaces={},
+ target_operations=dict((key, {}) for key in operations.RELATIONSHIP_OPERATIONS),
+ type='rel_type',
+ type_hierarchy=[],
+ properties={},
+ )
+ new_to_target_relationship_instance = model.RelationshipInstance(
+ relationship=new_to_target_relationship,
+ source_node_instance=new_node_instance,
+ target_node_instance=target_node_instance,
+ )
+
+
+ context.model.node.put(new_node)
+ context.model.node_instance.put(new_node_instance)
+ context.model.relationship.put(source_to_new_relationship)
+ context.model.relationship.put(new_to_target_relationship)
+ context.model.relationship_instance.put(source_to_new_relationship_instance)
+ context.model.relationship_instance.put(new_to_target_relationship_instance)
+
+ def flip_and_assert(node_instance, direction):
+ """
+ Reversed the order of relationships and assert effects took place.
+ :param node_instance: the node instance to operatate on
+ :param direction: the type of relationships to flip (inbound/outbount)
+ :return:
+ """
+ assert direction in ('inbound', 'outbound')
+
+ relationships = getattr(node_instance.node, direction + '_relationships')
+ relationship_instances = getattr(node_instance, direction + '_relationship_instances')
+ assert len(relationships) == 2
+ assert len(relationship_instances) == 2
+
+ first_rel, second_rel = relationships
+ first_rel_instance, second_rel_instance = relationship_instances
+ assert getattr(first_rel, relationships.ordering_attr) == 0
+ assert getattr(second_rel, relationships.ordering_attr) == 1
+ assert getattr(first_rel_instance, relationship_instances.ordering_attr) == 0
+ assert getattr(second_rel_instance, relationship_instances.ordering_attr) == 1
+
+ reversed_relationships = list(reversed(relationships))
+ reversed_relationship_instances = list(reversed(relationship_instances))
+
+ assert relationships != reversed_relationships
+ assert relationship_instances != reversed_relationship_instances
+
+ relationships[:] = reversed_relationships
+ relationship_instances[:] = reversed_relationship_instances
+ context.model.node_instance.update(node_instance)
+
+ assert relationships == reversed_relationships
+ assert relationship_instances == reversed_relationship_instances
+
+ assert getattr(first_rel, relationships.ordering_attr) == 1
+ assert getattr(second_rel, relationships.ordering_attr) == 0
+ assert getattr(first_rel_instance, relationship_instances.ordering_attr) == 1
+ assert getattr(second_rel_instance, relationship_instances.ordering_attr) == 0
+
+ flip_and_assert(source_node_instance, 'outbound')
+ flip_and_assert(target_node_instance, 'inbound')
[2/6] incubator-ariatosca git commit: ARIA-59 Inherit default
interfaces in relationships
Posted by da...@apache.org.
ARIA-59 Inherit default interfaces in relationships
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/dac4da7f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/dac4da7f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/dac4da7f
Branch: refs/heads/ARIA-46-execution-plugin
Commit: dac4da7f616fa34d15e13e7162c706b9c3bd02a6
Parents: 17d07ed
Author: Tal Liron <tl...@gmail.com>
Authored: Sun Jan 8 23:12:12 2017 +0200
Committer: Ran Ziv <ra...@gigaspaces.com>
Committed: Sun Jan 8 23:12:35 2017 +0200
----------------------------------------------------------------------
extensions/aria_extension_tosca/__init__.py | 3 +--
.../simple_v1_0/modeling/requirements.py | 12 ++++++------
2 files changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/dac4da7f/extensions/aria_extension_tosca/__init__.py
----------------------------------------------------------------------
diff --git a/extensions/aria_extension_tosca/__init__.py b/extensions/aria_extension_tosca/__init__.py
index d93dce2..ff4fa7d 100644
--- a/extensions/aria_extension_tosca/__init__.py
+++ b/extensions/aria_extension_tosca/__init__.py
@@ -53,5 +53,4 @@ MODULES = (
'simple_nfv_v1_0')
__all__ = (
- 'MODULES',
- 'install_aria_extension')
+ 'MODULES',)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/dac4da7f/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py
----------------------------------------------------------------------
diff --git a/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py b/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py
index 9dea874..8f6c38f 100644
--- a/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py
+++ b/extensions/aria_extension_tosca/simple_v1_0/modeling/requirements.py
@@ -192,6 +192,12 @@ def convert_requirement_from_definition_to_assignment(context, requirement_defin
# InterfaceDefinition:
relationship_interface_definitions = OrderedDict(relationship_type._get_interfaces(context))
+ # Convert interface definitions to templates
+ convert_requirement_interface_definitions_from_type_to_raw_template(
+ context,
+ raw['relationship'],
+ relationship_interface_definitions)
+
if relationship_definition:
# Merge extra interface definitions
# InterfaceDefinition:
@@ -207,12 +213,6 @@ def convert_requirement_from_definition_to_assignment(context, requirement_defin
raw['relationship']['interfaces'] = OrderedDict()
for interface_name, interface in interfaces.iteritems():
raw['relationship']['interfaces'][interface_name] = interface._raw
- else:
- # Convert interface definitions to templates
- convert_requirement_interface_definitions_from_type_to_raw_template(
- context,
- raw['relationship'],
- relationship_interface_definitions)
return \
RequirementAssignment(name=requirement_definition._name, raw=raw, container=container), \
[3/6] incubator-ariatosca git commit: ARIA-60 Fix use of thread
locals in parser context
Posted by da...@apache.org.
ARIA-60 Fix use of thread locals in parser context
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/61b7094d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/61b7094d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/61b7094d
Branch: refs/heads/ARIA-46-execution-plugin
Commit: 61b7094d8fbd7cd34cebc14ab7cd537386143e57
Parents: dac4da7
Author: Tal Liron <ta...@gmail.com>
Authored: Sun Jan 15 16:45:35 2017 +0200
Committer: Ran Ziv <ra...@gigaspaces.com>
Committed: Sun Jan 15 16:46:05 2017 +0200
----------------------------------------------------------------------
aria/parser/consumption/context.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/61b7094d/aria/parser/consumption/context.py
----------------------------------------------------------------------
diff --git a/aria/parser/consumption/context.py b/aria/parser/consumption/context.py
index 8fb9bb6..a8b75df 100644
--- a/aria/parser/consumption/context.py
+++ b/aria/parser/consumption/context.py
@@ -24,6 +24,9 @@ from ..modeling import ModelingContext
from .style import Style
+_thread_locals = threading.local()
+
+
class ConsumptionContext(object):
"""
Properties:
@@ -44,8 +47,7 @@ class ConsumptionContext(object):
Gets the context attached to the current thread if there is one.
"""
- thread_locals = threading.local()
- return getattr(thread_locals, 'aria_consumption_context', None)
+ return getattr(_thread_locals, 'aria_consumption_context', None)
def __init__(self, set_thread_local=True):
self.args = []
@@ -65,8 +67,7 @@ class ConsumptionContext(object):
Attaches this context to the current thread.
"""
- thread_locals = threading.local()
- thread_locals.aria_consumption_context = self
+ _thread_locals.aria_consumption_context = self
def write(self, string):
"""
[6/6] incubator-ariatosca git commit: ARIA-46 Execution plugin
Posted by da...@apache.org.
ARIA-46 Execution plugin
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/06b2f4d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/06b2f4d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/06b2f4d4
Branch: refs/heads/ARIA-46-execution-plugin
Commit: 06b2f4d47a3a11f1557b70a89b7a03ab56d5a92f
Parents: 385b209
Author: Dan Kilman <da...@gigaspaces.com>
Authored: Tue Jan 3 17:00:46 2017 +0200
Committer: Dan Kilman <da...@gigaspaces.com>
Committed: Tue Jan 17 12:48:38 2017 +0200
----------------------------------------------------------------------
MANIFEST.in | 1 +
aria/orchestrator/execution_plugin/__init__.py | 33 ++
aria/orchestrator/execution_plugin/common.py | 145 +++++
aria/orchestrator/execution_plugin/constants.py | 51 ++
.../execution_plugin/ctx_proxy/__init__.py | 16 +
.../execution_plugin/ctx_proxy/client.py | 105 ++++
.../execution_plugin/ctx_proxy/server.py | 240 ++++++++
.../execution_plugin/environment_globals.py | 60 ++
.../orchestrator/execution_plugin/exceptions.py | 38 ++
aria/orchestrator/execution_plugin/local.py | 123 ++++
.../orchestrator/execution_plugin/operations.py | 59 ++
.../execution_plugin/ssh/__init__.py | 14 +
.../execution_plugin/ssh/operations.py | 190 ++++++
.../orchestrator/execution_plugin/ssh/tunnel.py | 91 +++
aria/orchestrator/workflows/api/task.py | 12 +-
aria/orchestrator/workflows/core/engine.py | 1 -
aria/orchestrator/workflows/core/task.py | 3 +-
aria/storage/base_model.py | 24 +-
aria/storage/filesystem_rapi.py | 2 +-
requirements.txt | 3 +
setup.py | 9 +-
tests/orchestrator/execution_plugin/__init__.py | 14 +
.../execution_plugin/test_common.py | 193 +++++++
.../execution_plugin/test_ctx_proxy_server.py | 355 ++++++++++++
.../orchestrator/execution_plugin/test_local.py | 572 +++++++++++++++++++
tests/orchestrator/execution_plugin/test_ssh.py | 378 ++++++++++++
tests/orchestrator/workflows/api/test_task.py | 5 +
tests/orchestrator/workflows/core/test_task.py | 38 +-
28 files changed, 2756 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/MANIFEST.in
----------------------------------------------------------------------
diff --git a/MANIFEST.in b/MANIFEST.in
index f9bd145..6b08eeb 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1 +1,2 @@
include requirements.txt
+include aria/orchestrator/execution_plugin/ctx_proxy/binaries/ctx
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/__init__.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/__init__.py b/aria/orchestrator/execution_plugin/__init__.py
new file mode 100644
index 0000000..372022f
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/__init__.py
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from contextlib import contextmanager
+
+# Populated during execution of python scripts
+ctx = None
+inputs = None
+
+
+@contextmanager
+def python_script_scope(operation_ctx, operation_inputs):
+ global ctx
+ global inputs
+ try:
+ ctx = operation_ctx
+ inputs = operation_inputs
+ yield
+ finally:
+ ctx = None
+ inputs = None
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/common.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/common.py b/aria/orchestrator/execution_plugin/common.py
new file mode 100644
index 0000000..b77474c
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/common.py
@@ -0,0 +1,145 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import tempfile
+
+import requests
+
+from . import constants
+from . import exceptions
+
+
+def is_windows():
+ return os.name == 'nt'
+
+
+def download_script(ctx, script_path):
+ split = script_path.split('://')
+ schema = split[0]
+ suffix = script_path.split('/')[-1]
+ file_descriptor, dest_script_path = tempfile.mkstemp(suffix='-{0}'.format(suffix))
+ os.close(file_descriptor)
+ try:
+ if schema in ['http', 'https']:
+ response = requests.get(script_path)
+ if response.status_code == 404:
+ ctx.task.abort('Failed to download script: {0} (status code: {1})'
+ .format(script_path, response.status_code))
+ content = response.text
+ with open(dest_script_path, 'wb') as f:
+ f.write(content)
+ else:
+ ctx.download_resource(destination=dest_script_path, path=script_path)
+ except:
+ os.remove(dest_script_path)
+ raise
+ return dest_script_path
+
+
+def create_process_config(script_path, process, operation_kwargs, quote_json_env_vars=False):
+ """
+ update a process with it's environment variables, and return it.
+
+ Get a dict representing a process and a dict representing the environment
+ variables. Convert each environment variable to a format of
+ <string representing the name of the variable> :
+ <json formatted string representing the value of the variable>.
+ Finally, update the process with the newly formatted environment variables,
+ and return the process.
+
+ :param process: a dict representing a process
+ :type process: dict
+ :param operation_kwargs: a dict representing environment variables that
+ should exist in the process' running environment.
+ :type operation_kwargs: dict
+ :return: the process updated with its environment variables.
+ :rtype: dict
+ """
+ process = process or {}
+ env_vars = operation_kwargs.copy()
+ if 'ctx' in env_vars:
+ del env_vars['ctx']
+ env_vars.update(process.get('env', {}))
+ for k, v in env_vars.items():
+ if isinstance(v, (dict, list, tuple, bool)):
+ env_var_value = json.dumps(v)
+ if is_windows():
+ # These <k,v> environment variables will subsequently
+ # be used in a subprocess.Popen() call, as the `env` parameter.
+ # In some windows python versions, if an environment variable
+ # name is not of type str (e.g. unicode), the Popen call will
+ # fail.
+ k = str(k)
+ # The windows shell removes all double quotes - escape them
+ # to still be able to pass JSON in env vars to the shell.
+ env_var_value = env_var_value.replace('"', '\\"')
+ if quote_json_env_vars:
+ env_var_value = "'{0}'".format(env_var_value)
+ env_vars[k] = env_var_value
+ process['env'] = env_vars
+ args = process.get('args')
+ command = script_path
+ command_prefix = process.get('command_prefix')
+ if command_prefix:
+ command = '{0} {1}'.format(command_prefix, command)
+ if args:
+ command = ' '.join([command] + [str(a) for a in args])
+ process['command'] = command
+ return process
+
+
+def patch_ctx(ctx):
+ ctx._error = None
+ task = ctx.task
+ task._original_abort = task.abort
+ task._original_retry = task.retry
+
+ def _validate_legal_action():
+ if ctx._error is not None:
+ ctx._error = RuntimeError(constants.ILLEGAL_CTX_OPERATION_MESSAGE)
+ raise ctx._error
+
+ def abort_operation(message=None):
+ _validate_legal_action()
+ ctx._error = exceptions.ScriptException(message=message, retry=False)
+ return ctx._error
+ task.abort = abort_operation
+
+ def retry_operation(message=None, retry_interval=None):
+ _validate_legal_action()
+ ctx._error = exceptions.ScriptException(message=message,
+ retry=True,
+ retry_interval=retry_interval)
+ return ctx._error
+ task.retry = retry_operation
+
+
+def check_error(ctx, error_check_func=None, reraise=False):
+ _error = ctx._error
+ # this happens when more than 1 ctx return action is invoked
+ if isinstance(_error, RuntimeError):
+ ctx.task._original_abort(str(_error))
+ elif isinstance(_error, exceptions.ScriptException):
+ if _error.retry:
+ ctx.task._original_retry(_error.message, _error.retry_interval)
+ else:
+ ctx.task._original_abort(_error.message)
+ if error_check_func:
+ error_check_func()
+ if reraise:
+ raise
+ return _error
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/constants.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/constants.py b/aria/orchestrator/execution_plugin/constants.py
new file mode 100644
index 0000000..76f3b94
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/constants.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import exceptions
+
+# related to local
+PYTHON_SCRIPT_FILE_EXTENSION = '.py'
+POWERSHELL_SCRIPT_FILE_EXTENSION = '.ps1'
+DEFAULT_POWERSHELL_EXECUTABLE = 'powershell'
+
+# related to both local and ssh
+ILLEGAL_CTX_OPERATION_MESSAGE = 'ctx may only abort or return once'
+
+# related to ssh
+DEFAULT_BASE_DIR = '/tmp/aria-ctx'
+FABRIC_ENV_DEFAULTS = {
+ 'connection_attempts': 5,
+ 'timeout': 10,
+ 'forward_agent': False,
+ 'abort_on_prompts': True,
+ 'keepalive': 0,
+ 'linewise': False,
+ 'pool_size': 0,
+ 'skip_bad_hosts': False,
+ 'status': False,
+ 'disable_known_hosts': True,
+ 'combine_stderr': True,
+ 'abort_exception': exceptions.TaskException,
+}
+VALID_FABRIC_GROUPS = set([
+ 'status',
+ 'aborts',
+ 'warnings',
+ 'running',
+ 'stdout',
+ 'stderr',
+ 'user',
+ 'everything'
+])
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/ctx_proxy/__init__.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ctx_proxy/__init__.py b/aria/orchestrator/execution_plugin/ctx_proxy/__init__.py
new file mode 100644
index 0000000..7571c15
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ctx_proxy/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import server, client
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/ctx_proxy/client.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ctx_proxy/client.py b/aria/orchestrator/execution_plugin/ctx_proxy/client.py
new file mode 100644
index 0000000..d965a5e
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ctx_proxy/client.py
@@ -0,0 +1,105 @@
+#! /usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+import sys
+import urllib2
+
+
+# Environment variable for the socket url (used by clients to locate the socket)
+CTX_SOCKET_URL = 'CTX_SOCKET_URL'
+
+
+class _RequestError(RuntimeError):
+
+ def __init__(self, ex_message, ex_type, ex_traceback):
+ super(_RequestError, self).__init__(self, '{0}: {1}'.format(ex_type, ex_message))
+ self.ex_type = ex_type
+ self.ex_message = ex_message
+ self.ex_traceback = ex_traceback
+
+
+def _http_request(socket_url, request, timeout):
+ response = urllib2.urlopen(
+ url=socket_url,
+ data=json.dumps(request),
+ timeout=timeout)
+ if response.code != 200:
+ raise RuntimeError('Request failed: {0}'.format(response))
+ return json.loads(response.read())
+
+
+def _client_request(socket_url, args, timeout):
+ response = _http_request(
+ socket_url=socket_url,
+ request={'args': args},
+ timeout=timeout)
+ payload = response['payload']
+ response_type = response.get('type')
+ if response_type == 'error':
+ ex_type = payload['type']
+ ex_message = payload['message']
+ ex_traceback = payload['traceback']
+ raise _RequestError(ex_message, ex_type, ex_traceback)
+ elif response_type == 'stop_operation':
+ raise SystemExit(payload['message'])
+ else:
+ return payload
+
+
+def _parse_args(args):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-t', '--timeout', type=int, default=30)
+ parser.add_argument('--socket-url', default=os.environ.get(CTX_SOCKET_URL))
+ parser.add_argument('--json-arg-prefix', default='@')
+ parser.add_argument('-j', '--json-output', action='store_true')
+ parser.add_argument('args', nargs='*')
+ args = parser.parse_args(args=args)
+ if not args.socket_url:
+ raise RuntimeError('Missing CTX_SOCKET_URL environment variable '
+ 'or socket_url command line argument. (ctx is supposed to be executed '
+ 'within an operation context)')
+ return args
+
+
+def _process_args(json_prefix, args):
+ processed_args = []
+ for arg in args:
+ if arg.startswith(json_prefix):
+ arg = json.loads(arg[1:])
+ processed_args.append(arg)
+ return processed_args
+
+
+def main(args=None):
+ args = _parse_args(args)
+ response = _client_request(
+ socket_url=args.socket_url,
+ args=_process_args(args.json_arg_prefix, args.args),
+ timeout=args.timeout)
+ if args.json_output:
+ response = json.dumps(response)
+ else:
+ if not response:
+ response = ''
+ response = str(response)
+ sys.stdout.write(response)
+
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/ctx_proxy/server.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ctx_proxy/server.py b/aria/orchestrator/execution_plugin/ctx_proxy/server.py
new file mode 100644
index 0000000..0732f12
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ctx_proxy/server.py
@@ -0,0 +1,240 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import json
+import re
+import socket
+import threading
+import traceback
+import Queue
+import StringIO
+import wsgiref.simple_server
+
+import bottle
+
+from .. import exceptions
+
+
+class CtxProxy(object):
+
+ def __init__(self, ctx):
+ self.ctx = ctx
+ self.port = _get_unused_port()
+ self.socket_url = 'http://localhost:{0}'.format(self.port)
+ self.server = None
+ self._started = Queue.Queue(1)
+ self.thread = self._start_server()
+ self._started.get(timeout=5)
+
+ def _start_server(self):
+ proxy = self
+
+ class BottleServerAdapter(bottle.ServerAdapter):
+ def run(self, app):
+ class Server(wsgiref.simple_server.WSGIServer):
+ allow_reuse_address = True
+
+ def handle_error(self, request, client_address):
+ pass
+
+ class Handler(wsgiref.simple_server.WSGIRequestHandler):
+ def address_string(self):
+ return self.client_address[0]
+
+ def log_request(*args, **kwargs): # pylint: disable=no-method-argument
+ if not self.quiet:
+ return wsgiref.simple_server.WSGIRequestHandler.log_request(*args,
+ **kwargs)
+ server = wsgiref.simple_server.make_server(
+ host=self.host,
+ port=self.port,
+ app=app,
+ server_class=Server,
+ handler_class=Handler)
+ proxy.server = server
+ self.port = server.server_port
+ proxy._started.put(True)
+ server.serve_forever(poll_interval=0.1)
+
+ def serve():
+ bottle_app = bottle.Bottle()
+ bottle_app.post('/', callback=self._request_handler)
+ bottle.run(
+ app=bottle_app,
+ host='localhost',
+ port=self.port,
+ quiet=True,
+ server=BottleServerAdapter)
+ thread = threading.Thread(target=serve)
+ thread.daemon = True
+ thread.start()
+ return thread
+
+ def close(self):
+ if self.server:
+ self.server.shutdown()
+ self.server.server_close()
+
+ def _request_handler(self):
+ request = bottle.request.body.read() # pylint: disable=no-member
+ response = self._process(request)
+ return bottle.LocalResponse(
+ body=response,
+ status=200,
+ headers={'content-type': 'application/json'})
+
+ def _process(self, request):
+ try:
+ typed_request = json.loads(request)
+ args = typed_request['args']
+ payload = _process_ctx_request(self.ctx, args)
+ result_type = 'result'
+ if isinstance(payload, exceptions.ScriptException):
+ payload = dict(message=str(payload))
+ result_type = 'stop_operation'
+ result = json.dumps({
+ 'type': result_type,
+ 'payload': payload
+ })
+ except Exception as e:
+ traceback_out = StringIO.StringIO()
+ traceback.print_exc(file=traceback_out)
+ payload = {
+ 'type': type(e).__name__,
+ 'message': str(e),
+ 'traceback': traceback_out.getvalue()
+ }
+ result = json.dumps({
+ 'type': 'error',
+ 'payload': payload
+ })
+ return result
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args, **kwargs):
+ self.close()
+
+
+def _process_ctx_request(ctx, args):
+ current = ctx
+ num_args = len(args)
+ index = 0
+ while index < num_args:
+ arg = args[index]
+ attr = _desugar_attr(current, arg)
+ if attr:
+ current = getattr(current, attr)
+ elif isinstance(current, collections.MutableMapping):
+ key = arg
+ path_dict = _PathDictAccess(current)
+ if index + 1 == num_args:
+ # read dict prop by path
+ value = path_dict.get(key)
+ current = value
+ elif index + 2 == num_args:
+ # set dict prop by path
+ value = args[index + 1]
+ path_dict.set(key, value)
+ current = None
+ else:
+ raise RuntimeError('Illegal argument while accessing dict')
+ break
+ elif callable(current):
+ kwargs = {}
+ remaining_args = args[index:]
+ if isinstance(remaining_args[-1], collections.MutableMapping):
+ kwargs = remaining_args[-1]
+ remaining_args = remaining_args[:-1]
+ current = current(*remaining_args, **kwargs)
+ break
+ else:
+ raise RuntimeError('{0} cannot be processed in {1}'.format(arg, args))
+ index += 1
+ if callable(current):
+ current = current()
+ return current
+
+
+def _desugar_attr(obj, attr):
+ if not isinstance(attr, basestring):
+ return None
+ if hasattr(obj, attr):
+ return attr
+ attr = attr.replace('-', '_')
+ if hasattr(obj, attr):
+ return attr
+ return None
+
+
+class _PathDictAccess(object):
+ pattern = re.compile(r"(.+)\[(\d+)\]")
+
+ def __init__(self, obj):
+ self.obj = obj
+
+ def set(self, prop_path, value):
+ obj, prop_name = self._get_parent_obj_prop_name_by_path(prop_path)
+ obj[prop_name] = value
+
+ def get(self, prop_path):
+ value = self._get_object_by_path(prop_path)
+ return value
+
+ def _get_object_by_path(self, prop_path, fail_on_missing=True):
+ # when setting a nested object, make sure to also set all the
+ # intermediate path objects
+ current = self.obj
+ for prop_segment in prop_path.split('.'):
+ match = self.pattern.match(prop_segment)
+ if match:
+ index = int(match.group(2))
+ property_name = match.group(1)
+ if property_name not in current:
+ self._raise_illegal(prop_path)
+ if not isinstance(current[property_name], list):
+ self._raise_illegal(prop_path)
+ current = current[property_name][index]
+ else:
+ if prop_segment not in current:
+ if fail_on_missing:
+ self._raise_illegal(prop_path)
+ else:
+ current[prop_segment] = {}
+ current = current[prop_segment]
+ return current
+
+ def _get_parent_obj_prop_name_by_path(self, prop_path):
+ split = prop_path.split('.')
+ if len(split) == 1:
+ return self.obj, prop_path
+ parent_path = '.'.join(split[:-1])
+ parent_obj = self._get_object_by_path(parent_path, fail_on_missing=False)
+ prop_name = split[-1]
+ return parent_obj, prop_name
+
+ @staticmethod
+ def _raise_illegal(prop_path):
+ raise RuntimeError('illegal path: {0}'.format(prop_path))
+
+
+def _get_unused_port():
+ sock = socket.socket()
+ sock.bind(('127.0.0.1', 0))
+ _, port = sock.getsockname()
+ sock.close()
+ return port
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/environment_globals.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/environment_globals.py b/aria/orchestrator/execution_plugin/environment_globals.py
new file mode 100644
index 0000000..27311f0
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/environment_globals.py
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def create_initial_globals(path):
+ """ emulates a `globals()` call in a freshly loaded module
+
+ The implementation of this function is likely to raise a couple of
+ questions. If you read the implementation and nothing bothered you, feel
+ free to skip the rest of this docstring.
+
+ First, why is this function in its own module and not, say, in the same
+ module of the other environment-related functions?
+ Second, why is it implemented in such a way that copies the globals, then
+ deletes the item that represents this function, and then changes some
+ other entries?
+
+ Well, these two questions can be answered with one (elaborate) explanation.
+ If this function was in the same module with the other environment-related
+ functions, then we would have had to delete more items in globals than just
+ `create_initial_globals`. That is because all of the other function names
+ would also be in globals, and since there is no built-in mechanism that
+ return the name of the user-defined objects, this approach is quite an
+ overkill.
+
+ - But why do we rely on the copy-existing-globals-and-delete-entries
+ method, when it seems to force us to put `create_initial_globals` in its
+ own file?
+
+ Well, because there is no easier method of creating globals of a newly
+ loaded module.
+
+ - How about hard coding a 'global' dict? It seems that there are very few
+ entries: __doc__, __file__, __name__, __package__ (but don't forget
+ __builtins__).
+
+ That would be coupling our implementation to a specific `globals`
+ implementation. What if `globals` were to change?
+ """
+ copied_globals = globals().copy()
+ copied_globals.update({
+ '__doc__': 'Dynamically executed script',
+ '__file__': path,
+ '__name__': '__main__',
+ '__package__': None
+ })
+ del copied_globals[create_initial_globals.__name__]
+ return copied_globals
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/exceptions.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/exceptions.py b/aria/orchestrator/execution_plugin/exceptions.py
new file mode 100644
index 0000000..2925638
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/exceptions.py
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class ProcessException(Exception):
+
+ # TODO: remove None's once process executor properly handles pickling *all* exceptions
+ def __init__(self, stderr=None, stdout=None, command=None, exit_code=None):
+ super(ProcessException, self).__init__(stderr)
+ self.command = command
+ self.exit_code = exit_code
+ self.stdout = stdout
+ self.stderr = stderr
+
+
+class TaskException(Exception):
+ pass
+
+
+class ScriptException(Exception):
+
+ # TODO: remove None's once process executor properly handles pickling *all* exceptions
+ def __init__(self, message=None, retry=None, retry_interval=None):
+ super(ScriptException, self).__init__(message)
+ self.retry = retry
+ self.retry_interval = retry_interval
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/local.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/local.py b/aria/orchestrator/execution_plugin/local.py
new file mode 100644
index 0000000..cc65f8c
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/local.py
@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import threading
+import StringIO
+
+from . import ctx_proxy
+from . import exceptions
+from . import common
+from . import constants
+from . import environment_globals
+from . import python_script_scope
+
+
+def run_script(ctx, script_path, process, **kwargs):
+ process = process or {}
+ script_path = common.download_script(ctx, script_path)
+ script_func = _get_run_script_func(script_path, process)
+ return script_func(
+ ctx=ctx,
+ script_path=script_path,
+ process=process,
+ operation_kwargs=kwargs)
+
+
+def _get_run_script_func(script_path, process):
+ if _treat_script_as_python_script(script_path, process):
+ return _eval_script_func
+ else:
+ if _treat_script_as_powershell_script(script_path):
+ process.setdefault('command_prefix', constants.DEFAULT_POWERSHELL_EXECUTABLE)
+ return _execute_func
+
+
+def _treat_script_as_python_script(script_path, process):
+ eval_python = process.get('eval_python')
+ script_extension = os.path.splitext(script_path)[1].lower()
+ return (eval_python is True or (script_extension == constants.PYTHON_SCRIPT_FILE_EXTENSION and
+ eval_python is not False))
+
+
+def _treat_script_as_powershell_script(script_path):
+ script_extension = os.path.splitext(script_path)[1].lower()
+ return script_extension == constants.POWERSHELL_SCRIPT_FILE_EXTENSION
+
+
+def _eval_script_func(script_path, ctx, _, operation_kwargs):
+ with python_script_scope(operation_ctx=ctx, operation_inputs=operation_kwargs):
+ execfile(script_path, environment_globals.create_initial_globals(script_path))
+
+
+def _execute_func(script_path, ctx, process, operation_kwargs):
+ os.chmod(script_path, 0755)
+ process = common.create_process_config(
+ script_path=script_path,
+ process=process,
+ operation_kwargs=operation_kwargs)
+ command = process['command']
+ env = os.environ.copy()
+ env.update(process['env'])
+ ctx.logger.info('Executing: {0}'.format(command))
+ common.patch_ctx(ctx)
+ with ctx_proxy.server.CtxProxy(ctx) as proxy:
+ env[ctx_proxy.client.CTX_SOCKET_URL] = proxy.socket_url
+ running_process = subprocess.Popen(
+ command,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env,
+ cwd=process.get('cwd'),
+ bufsize=1,
+ close_fds=not common.is_windows())
+ stdout_consumer = _OutputConsumer(running_process.stdout)
+ stderr_consumer = _OutputConsumer(running_process.stderr)
+ exit_code = running_process.wait()
+ stdout_consumer.join()
+ stderr_consumer.join()
+ ctx.logger.info('Execution done (exit_code={0}): {1}'.format(exit_code, command))
+
+ def error_check_func():
+ if exit_code:
+ raise exceptions.ProcessException(
+ command=command,
+ exit_code=exit_code,
+ stdout=stdout_consumer.read_output(),
+ stderr=stderr_consumer.read_output())
+ return common.check_error(ctx, error_check_func=error_check_func)
+
+
+class _OutputConsumer(object):
+
+ def __init__(self, out):
+ self._out = out
+ self._buffer = StringIO.StringIO()
+ self._consumer = threading.Thread(target=self._consume_output)
+ self._consumer.daemon = True
+ self._consumer.start()
+
+ def _consume_output(self):
+ for line in iter(self._out.readline, b''):
+ self._buffer.write(line)
+ self._out.close()
+
+ def read_output(self):
+ return self._buffer.getvalue()
+
+ def join(self):
+ self._consumer.join()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/operations.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/operations.py b/aria/orchestrator/execution_plugin/operations.py
new file mode 100644
index 0000000..3ad3c0e
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/operations.py
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from . import local as local_operations
+from .ssh import operations as ssh_operations
+
+
+def run_script_locally(ctx,
+ script_path,
+ process=None,
+ **kwargs):
+ return local_operations.run_script(
+ ctx=ctx,
+ script_path=script_path,
+ process=process,
+ **kwargs)
+
+
+def run_script_with_ssh(ctx,
+ script_path,
+ fabric_env=None,
+ process=None,
+ use_sudo=False,
+ hide_output=None,
+ **kwargs):
+ return ssh_operations.run_script(
+ ctx=ctx,
+ script_path=script_path,
+ fabric_env=fabric_env,
+ process=process,
+ use_use=use_sudo,
+ hide_output=hide_output,
+ **kwargs)
+
+
+def run_commands_with_ssh(ctx,
+ commands,
+ fabric_env=None,
+ use_sudo=False,
+ hide_output=None,
+ **_):
+ return ssh_operations.run_commands(
+ ctx=ctx,
+ commands=commands,
+ fabric_env=fabric_env,
+ use_sudo=use_sudo,
+ hide_output=hide_output)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/ssh/__init__.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ssh/__init__.py b/aria/orchestrator/execution_plugin/ssh/__init__.py
new file mode 100644
index 0000000..ae1e83e
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ssh/__init__.py
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/ssh/operations.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ssh/operations.py b/aria/orchestrator/execution_plugin/ssh/operations.py
new file mode 100644
index 0000000..7e81a6f
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ssh/operations.py
@@ -0,0 +1,190 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import random
+import string
+import StringIO
+
+import fabric.api
+import fabric.context_managers
+import fabric.contrib.files
+
+from .. import constants
+from .. import exceptions
+from .. import common
+from .. import ctx_proxy
+from . import tunnel
+
+
+_PROXY_CLIENT_PATH = ctx_proxy.client.__file__
+if _PROXY_CLIENT_PATH.endswith('.pyc'):
+ _PROXY_CLIENT_PATH = _PROXY_CLIENT_PATH[:-1]
+
+
+def run_commands(ctx, commands, fabric_env, use_sudo, hide_output, **_):
+ """Runs the provider 'commands' in sequence
+
+ :param commands: a list of commands to run
+ :param fabric_env: fabric configuration
+ """
+ with fabric.api.settings(_hide_output(ctx, groups=hide_output),
+ **_fabric_env(ctx, fabric_env, warn_only=True)):
+ for command in commands:
+ ctx.logger.info('Running command: {0}'.format(command))
+ run = fabric.api.sudo if use_sudo else fabric.api.run
+ result = run(command)
+ if result.failed:
+ raise exceptions.ProcessException(
+ command=result.command,
+ exit_code=result.return_code,
+ stdout=result.stdout,
+ stderr=result.stderr)
+
+
+def run_script(ctx, script_path, fabric_env, process, use_sudo, hide_output, **kwargs):
+ process = process or {}
+ paths = _Paths(base_dir=process.get('base_dir', constants.DEFAULT_BASE_DIR),
+ local_script_path=common.download_script(ctx, script_path))
+ with fabric.api.settings(_hide_output(ctx, groups=hide_output),
+ **_fabric_env(ctx, fabric_env, warn_only=False)):
+ # the remote host must have the ctx before running any fabric scripts
+ if not fabric.contrib.files.exists(paths.remote_ctx_path):
+ # there may be race conditions with other operations that
+ # may be running in parallel, so we pass -p to make sure
+ # we get 0 exit code if the directory already exists
+ fabric.api.run('mkdir -p {0} && mkdir -p {1}'.format(paths.remote_scripts_dir,
+ paths.remote_work_dir))
+ # this file has to be present before using ctx
+ fabric.api.put(_PROXY_CLIENT_PATH, paths.remote_ctx_path)
+ _patch_ctx(ctx, paths.remote_work_dir)
+ process = common.create_process_config(
+ script_path=paths.remote_script_path,
+ process=process,
+ operation_kwargs=kwargs,
+ quote_json_env_vars=True)
+ fabric.api.put(paths.local_script_path, paths.remote_script_path)
+ with ctx_proxy.server.CtxProxy(ctx) as proxy:
+ local_port = proxy.port
+ with fabric.context_managers.cd(process.get('cwd', paths.remote_work_dir)): # pylint: disable=not-context-manager
+ with tunnel.remote(ctx, local_port=local_port) as remote_port:
+ local_socket_url = proxy.socket_url
+ remote_socket_url = local_socket_url.replace(str(local_port), str(remote_port))
+ env_script = _write_environment_script_file(
+ process=process,
+ paths=paths,
+ local_socket_url=local_socket_url,
+ remote_socket_url=remote_socket_url)
+ fabric.api.put(env_script, paths.remote_env_script_path)
+ try:
+ command = 'source {0} && {1}'.format(paths.remote_env_script_path,
+ process['command'])
+ run = fabric.api.sudo if use_sudo else fabric.api.run
+ run(command)
+ except exceptions.TaskException:
+ return common.check_error(ctx, reraise=True)
+ return common.check_error(ctx)
+
+
+def _patch_ctx(ctx, remote_work_dir):
+ common.patch_ctx(ctx)
+ original_download_resource = ctx.download_resource
+ original_download_resource_and_render = ctx.download_resource_and_render
+
+ def fabric_put_in_remote_path(local_target_path, target_path):
+ if target_path:
+ remote_target_path = target_path
+ else:
+ remote_target_path = '{0}/{1}'.format(remote_work_dir,
+ os.path.basename(local_target_path))
+ fabric.api.put(local_target_path, remote_target_path)
+ return remote_target_path
+
+ def download_resource(resource_path, target_path=None):
+ local_target_path = original_download_resource(resource_path)
+ return fabric_put_in_remote_path(local_target_path, target_path)
+ ctx.download_resource = download_resource
+
+ def download_resource_and_render(resource_path,
+ target_path=None,
+ template_variables=None):
+ local_target_path = original_download_resource_and_render(
+ resource_path,
+ template_variables=template_variables)
+ return fabric_put_in_remote_path(local_target_path, target_path)
+ ctx.download_resource_and_render = download_resource_and_render
+
+
+def _hide_output(ctx, groups):
+ """ Hides Fabric's output for every 'entity' in `groups` """
+ groups = set(groups or [])
+ if not groups.issubset(constants.VALID_FABRIC_GROUPS):
+ ctx.task.abort('`hide_output` must be a subset of {0} (Provided: {1})'
+ .format(', '.join(constants.VALID_FABRIC_GROUPS), ', '.join(groups)))
+ return fabric.api.hide(*groups)
+
+
+def _fabric_env(ctx, fabric_env, warn_only):
+ """Prepares fabric environment variables configuration"""
+ ctx.logger.debug('Preparing fabric environment...')
+ env = constants.FABRIC_ENV_DEFAULTS.copy()
+ env.update(fabric_env or {})
+ env.setdefault('warn_only', warn_only)
+ if 'host_string' not in env:
+ env['host_string'] = ctx.task.runs_on.ip
+ # validations
+ if not env.get('host_string'):
+ ctx.task.abort('`host_string` not supplied and ip cannot be deduced automatically')
+ if not (env.get('password') or env.get('key_filename') or env.get('key')):
+ ctx.task.abort(
+ 'Access credentials not supplied '
+ '(you must supply at least one of `key_filename`, `key` or `password`)')
+ if not env.get('user'):
+ ctx.task.abort('`user` not supplied')
+ ctx.logger.debug('Environment prepared successfully')
+ return env
+
+
+def _write_environment_script_file(process, paths, local_socket_url, remote_socket_url):
+ env_script = StringIO.StringIO()
+ env = process['env']
+ env['PATH'] = '{0}:$PATH'.format(paths.remote_ctx_dir)
+ env['PYTHONPATH'] = '{0}:$PYTHONPATH'.format(paths.remote_ctx_dir)
+ env_script.write('chmod +x {0}\n'.format(paths.remote_script_path))
+ env_script.write('chmod +x {0}\n'.format(paths.remote_ctx_path))
+ env.update({
+ ctx_proxy.client.CTX_SOCKET_URL: remote_socket_url,
+ 'LOCAL_{0}'.format(ctx_proxy.client.CTX_SOCKET_URL): local_socket_url
+ })
+ for key, value in env.iteritems():
+ env_script.write('export {0}={1}\n'.format(key, value))
+ return env_script
+
+
+class _Paths(object):
+
+ def __init__(self, base_dir, local_script_path):
+ self.local_script_path = local_script_path
+ self.remote_ctx_dir = base_dir
+ self.base_script_path = os.path.basename(self.local_script_path)
+ self.remote_ctx_path = '{0}/ctx'.format(self.remote_ctx_dir)
+ self.remote_scripts_dir = '{0}/scripts'.format(self.remote_ctx_dir)
+ self.remote_work_dir = '{0}/work'.format(self.remote_ctx_dir)
+ random_suffix = ''.join(random.choice(string.ascii_lowercase + string.digits)
+ for _ in range(8))
+ remote_path_suffix = '{0}-{1}'.format(self.base_script_path, random_suffix)
+ self.remote_env_script_path = '{0}/env-{1}'.format(self.remote_scripts_dir,
+ remote_path_suffix)
+ self.remote_script_path = '{0}/{1}'.format(self.remote_scripts_dir, remote_path_suffix)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/execution_plugin/ssh/tunnel.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/execution_plugin/ssh/tunnel.py b/aria/orchestrator/execution_plugin/ssh/tunnel.py
new file mode 100644
index 0000000..6fc8d54
--- /dev/null
+++ b/aria/orchestrator/execution_plugin/ssh/tunnel.py
@@ -0,0 +1,91 @@
+# This implementation was copied from the Fabric project directly:
+# https://github.com/fabric/fabric/blob/master/fabric/context_managers.py#L486
+# The purpose was to remove the rtunnel creation printouts here:
+# https://github.com/fabric/fabric/blob/master/fabric/context_managers.py#L547
+
+
+import contextlib
+import select
+import socket
+
+import fabric.api
+import fabric.state
+import fabric.thread_handling
+
+
+@contextlib.contextmanager
+def remote(ctx, local_port, remote_port=0, local_host='localhost', remote_bind_address='127.0.0.1'):
+ """Create a tunnel forwarding a locally-visible port to the remote target."""
+ sockets = []
+ channels = []
+ thread_handlers = []
+
+ def accept(channel, *args, **kwargs):
+ # This seemingly innocent statement seems to be doing nothing
+ # but the truth is far from it!
+ # calling fileno() on a paramiko channel the first time, creates
+ # the required plumbing to make the channel valid for select.
+ # While this would generally happen implicitly inside the _forwarder
+ # function when select is called, it may already be too late and may
+ # cause the select loop to hang.
+ # Specifically, when new data arrives to the channel, a flag is set
+ # on an "event" object which is what makes the select call work.
+ # problem is this will only happen if the event object is not None
+ # and it will be not-None only after channel.fileno() has been called
+ # for the first time. If we wait until _forwarder calls select for the
+ # first time it may be after initial data has reached the channel.
+ # calling it explicitly here in the paramiko transport main event loop
+ # guarantees this will not happen.
+ channel.fileno()
+
+ channels.append(channel)
+ sock = socket.socket()
+ sockets.append(sock)
+
+ try:
+ sock.connect((local_host, local_port))
+ except Exception as e:
+ try:
+ channel.close()
+ except Exception as ex2:
+ close_error = ' (While trying to close channel: {0})'.format(ex2)
+ else:
+ close_error = ''
+ ctx.task.abort('[{0}] rtunnel: cannot connect to {1}:{2} ({3}){4}'
+ .format(fabric.api.env.host_string, local_host, local_port, e,
+ close_error))
+
+ thread_handler = fabric.thread_handling.ThreadHandler('fwd', _forwarder, channel, sock)
+ thread_handlers.append(thread_handler)
+
+ transport = fabric.state.connections[fabric.api.env.host_string].get_transport()
+ remote_port = transport.request_port_forward(
+ remote_bind_address, remote_port, handler=accept)
+
+ try:
+ yield remote_port
+ finally:
+ for sock, chan, thread_handler in zip(sockets, channels, thread_handlers):
+ sock.close()
+ chan.close()
+ thread_handler.thread.join()
+ thread_handler.raise_if_needed()
+ transport.cancel_port_forward(remote_bind_address, remote_port)
+
+
+def _forwarder(chan, sock):
+ # Bidirectionally forward data between a socket and a Paramiko channel.
+ while True:
+ read = select.select([sock, chan], [], [])[0]
+ if sock in read:
+ data = sock.recv(1024)
+ if len(data) == 0:
+ break
+ chan.send(data)
+ if chan in read:
+ data = chan.recv(1024)
+ if len(data) == 0:
+ break
+ sock.send(data)
+ chan.close()
+ sock.close()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/workflows/api/task.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/api/task.py b/aria/orchestrator/workflows/api/task.py
index 70324a6..8d93837 100644
--- a/aria/orchestrator/workflows/api/task.py
+++ b/aria/orchestrator/workflows/api/task.py
@@ -68,7 +68,8 @@ class OperationTask(BaseTask):
retry_interval=None,
ignore_failure=None,
inputs=None,
- plugin=None):
+ plugin=None,
+ runs_on=None):
"""
Creates an operation task using the name, details, node instance and any additional kwargs.
:param name: the operation of the name.
@@ -89,6 +90,7 @@ class OperationTask(BaseTask):
if retry_interval is None else retry_interval)
self.ignore_failure = (self.workflow_context._task_ignore_failure
if ignore_failure is None else ignore_failure)
+ self.runs_on = runs_on
@classmethod
def node_instance(cls, instance, name, inputs=None, *args, **kwargs):
@@ -104,6 +106,7 @@ class OperationTask(BaseTask):
operation_details=instance.node.operations[name],
inputs=inputs,
plugins=instance.node.plugins or [],
+ runs_on=model.Task.RUNS_ON_NODE,
*args,
**kwargs)
@@ -126,18 +129,22 @@ class OperationTask(BaseTask):
operation_details = getattr(instance.relationship, operation_end)[name]
if operation_end == cls.SOURCE_OPERATION:
plugins = instance.relationship.source_node.plugins
+ runs_on = model.Task.RUNS_ON_SOURCE
else:
plugins = instance.relationship.target_node.plugins
+ runs_on = model.Task.RUNS_ON_TARGET
return cls._instance(instance=instance,
name=name,
operation_details=operation_details,
inputs=inputs,
plugins=plugins or [],
+ runs_on=runs_on,
*args,
**kwargs)
@classmethod
- def _instance(cls, instance, name, operation_details, inputs, plugins, *args, **kwargs):
+ def _instance(cls, instance, name, operation_details, inputs, plugins, runs_on, *args,
+ **kwargs):
operation_mapping = operation_details.get('operation')
operation_inputs = operation_details.get('inputs', {})
operation_inputs.update(inputs or {})
@@ -151,6 +158,7 @@ class OperationTask(BaseTask):
operation_mapping=operation_mapping,
inputs=operation_inputs,
plugin=plugin,
+ runs_on=runs_on,
*args,
**kwargs)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/workflows/core/engine.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/core/engine.py b/aria/orchestrator/workflows/core/engine.py
index 47269a3..fd83614 100644
--- a/aria/orchestrator/workflows/core/engine.py
+++ b/aria/orchestrator/workflows/core/engine.py
@@ -52,7 +52,6 @@ class Engine(logger.LoggerMixin):
"""
try:
events.start_workflow_signal.send(self._workflow_context)
- cancel = False
while True:
cancel = self._is_cancel()
if cancel:
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/orchestrator/workflows/core/task.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/core/task.py b/aria/orchestrator/workflows/core/task.py
index 67be2ea..1deb66a 100644
--- a/aria/orchestrator/workflows/core/task.py
+++ b/aria/orchestrator/workflows/core/task.py
@@ -135,7 +135,8 @@ class OperationTask(BaseTask):
retry_interval=api_task.retry_interval,
ignore_failure=api_task.ignore_failure,
plugin=plugins[0] if plugins else None,
- execution=self._workflow_context.execution
+ execution=self._workflow_context.execution,
+ runs_on=api_task.runs_on
)
self._workflow_context.model.task.put(operation_task)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/storage/base_model.py
----------------------------------------------------------------------
diff --git a/aria/storage/base_model.py b/aria/storage/base_model.py
index a2bbb60..29973a2 100644
--- a/aria/storage/base_model.py
+++ b/aria/storage/base_model.py
@@ -695,6 +695,11 @@ class TaskBase(ModelMixin):
WAIT_STATES = [PENDING, RETRYING]
END_STATES = [SUCCESS, FAILED]
+ RUNS_ON_SOURCE = 'source'
+ RUNS_ON_TARGET = 'target'
+ RUNS_ON_NODE = 'node'
+ RUNS_ON = (RUNS_ON_NODE, RUNS_ON_SOURCE, RUNS_ON_TARGET)
+
@orm.validates('max_attempts')
def validate_max_attempts(self, _, value): # pylint: disable=no-self-use
"""Validates that max attempts is either -1 or a positive number"""
@@ -718,6 +723,7 @@ class TaskBase(ModelMixin):
# Operation specific fields
operation_mapping = Column(String)
inputs = Column(Dict)
+ _runs_on = Column(Enum(*RUNS_ON, name='runs_on'), name='runs_on')
@property
def actor(self):
@@ -727,13 +733,23 @@ class TaskBase(ModelMixin):
"""
return self.node_instance or self.relationship_instance
+ @property
+ def runs_on(self):
+ if self._runs_on == self.RUNS_ON_NODE:
+ return self.node_instance
+ elif self._runs_on == self.RUNS_ON_SOURCE:
+ return self.relationship_instance.source_node_instance # pylint: disable=no-member
+ elif self._runs_on == self.RUNS_ON_TARGET:
+ return self.relationship_instance.target_node_instance # pylint: disable=no-member
+ return None
+
@classmethod
- def as_node_instance(cls, instance, **kwargs):
- return cls(node_instance=instance, **kwargs)
+ def as_node_instance(cls, instance, runs_on, **kwargs):
+ return cls(node_instance=instance, _runs_on=runs_on, **kwargs)
@classmethod
- def as_relationship_instance(cls, instance, **kwargs):
- return cls(relationship_instance=instance, **kwargs)
+ def as_relationship_instance(cls, instance, runs_on, **kwargs):
+ return cls(relationship_instance=instance, _runs_on=runs_on, **kwargs)
@staticmethod
def abort(message=None):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/aria/storage/filesystem_rapi.py
----------------------------------------------------------------------
diff --git a/aria/storage/filesystem_rapi.py b/aria/storage/filesystem_rapi.py
index c6b3a81..eb30e0b 100644
--- a/aria/storage/filesystem_rapi.py
+++ b/aria/storage/filesystem_rapi.py
@@ -132,7 +132,7 @@ class FileSystemResourceAPI(api.ResourceAPI):
if os.path.isfile(resource):
shutil.copy2(resource, destination)
else:
- dir_util.copy_tree(resource, destination) # pylint: disable=no-member
+ dir_util.copy_tree(resource, destination) # pylint: disable=no-member
def upload(self, entry_id, source, path=None, **_):
"""
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/requirements.txt
----------------------------------------------------------------------
diff --git a/requirements.txt b/requirements.txt
index 0005a5e..055bafb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -26,3 +26,6 @@ CacheControl[filecache]==0.11.6
clint==0.5.1
SQLAlchemy==1.1.4
wagon==0.5.0
+bottle==0.12.11
+six==1.10.0
+Fabric==1.13.1
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/setup.py
----------------------------------------------------------------------
diff --git a/setup.py b/setup.py
index 112f13e..5a3d016 100644
--- a/setup.py
+++ b/setup.py
@@ -44,6 +44,11 @@ except IOError:
install_requires = []
+console_scripts = ['aria = aria.cli.cli:main']
+if os.environ.get('INSTALL_CTX'):
+ console_scripts.append('ctx = aria.orchestrator.execution_plugin.ctx_proxy.client:main')
+
+
setup(
name=_PACKAGE_NAME,
version=version,
@@ -77,8 +82,6 @@ setup(
zip_safe=False,
install_requires=install_requires,
entry_points={
- 'console_scripts': [
- 'aria = aria.cli.cli:main'
- ]
+ 'console_scripts': console_scripts
}
)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/tests/orchestrator/execution_plugin/__init__.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/__init__.py b/tests/orchestrator/execution_plugin/__init__.py
new file mode 100644
index 0000000..ae1e83e
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/__init__.py
@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/tests/orchestrator/execution_plugin/test_common.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_common.py b/tests/orchestrator/execution_plugin/test_common.py
new file mode 100644
index 0000000..7423ded
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/test_common.py
@@ -0,0 +1,193 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+
+import requests
+import pytest
+
+from aria.storage import model
+from aria.orchestrator import exceptions
+from aria.orchestrator.execution_plugin import common
+
+
+class TestDownloadScript(object):
+
+ @pytest.fixture(autouse=True)
+ def patch_requests(self, mocker):
+ def _mock_requests_get(url):
+ response = namedtuple('Response', 'text status_code')
+ return response(url, self.status_code)
+ self.status_code = 200
+ mocker.patch.object(requests, 'get', _mock_requests_get)
+
+ def _test_url(self, url):
+ class Ctx(object):
+ task = model.Task
+
+ script_path = url
+ result = common.download_script(Ctx, script_path)
+ with open(result) as f:
+ assert script_path == f.read()
+ assert result.endswith('-some_script.py')
+
+ def test_http_url(self):
+ self._test_url('http://localhost/some_script.py')
+
+ def test_https_url(self):
+ self._test_url('https://localhost/some_script.py')
+
+ def test_url_status_code_404(self):
+ self.status_code = 404
+ with pytest.raises(exceptions.TaskAbortException) as exc_ctx:
+ self.test_http_url()
+ exception = exc_ctx.value
+ assert 'status code: 404' in str(exception)
+
+ def test_blueprint_resource(self):
+ test_script_path = 'my_script.py'
+
+ class Ctx(object):
+ @staticmethod
+ def download_resource(destination, path):
+ assert path == test_script_path
+ return destination
+ result = common.download_script(Ctx, test_script_path)
+ assert result.endswith(test_script_path)
+
+
+class TestCreateProcessConfig(object):
+
+ def test_plain_command(self):
+ script_path = 'path'
+ process = common.create_process_config(
+ script_path=script_path,
+ process={},
+ operation_kwargs={})
+ assert process['command'] == script_path
+
+ def test_command_with_args(self):
+ script_path = 'path'
+ process = {'args': [1, 2, 3]}
+ process = common.create_process_config(
+ script_path=script_path,
+ process=process,
+ operation_kwargs={})
+ assert process['command'] == '{0} 1 2 3'.format(script_path)
+
+ def test_command_prefix(self):
+ script_path = 'path'
+ command_prefix = 'prefix'
+ process = {'command_prefix': command_prefix}
+ process = common.create_process_config(
+ script_path=script_path,
+ process=process,
+ operation_kwargs={})
+ assert process['command'] == '{0} {1}'.format(command_prefix, script_path)
+
+ def test_command_with_args_and_prefix(self):
+ script_path = 'path'
+ command_prefix = 'prefix'
+ process = {'command_prefix': command_prefix,
+ 'args': [1, 2, 3]}
+ process = common.create_process_config(
+ script_path=script_path,
+ process=process,
+ operation_kwargs={})
+ assert process['command'] == '{0} {1} 1 2 3'.format(command_prefix, script_path)
+
+ def test_ctx_is_removed(self):
+ process = common.create_process_config(
+ script_path='',
+ process={},
+ operation_kwargs={'ctx': 1})
+ assert 'ctx' not in process['env']
+
+ def test_env_passed_explicitly(self):
+ env = {'one': '1', 'two': '2'}
+ process = common.create_process_config(
+ script_path='',
+ process={'env': env},
+ operation_kwargs={})
+ assert process['env'] == env
+
+ def test_env_populated_from_operation_kwargs(self):
+ operation_kwargs = {'one': '1', 'two': '2'}
+ process = common.create_process_config(
+ script_path='',
+ process={},
+ operation_kwargs=operation_kwargs)
+ assert process['env'] == operation_kwargs
+
+ def test_env_merged_from_operation_kwargs_and_process(self):
+ operation_kwargs = {'one': '1', 'two': '2'}
+ env = {'three': '3', 'four': 4}
+ process = common.create_process_config(
+ script_path='',
+ process={'env': env},
+ operation_kwargs=operation_kwargs)
+ assert process['env'] == dict(operation_kwargs.items() + env.items())
+
+ def test_process_env_gets_precedence_over_operation_kwargs(self):
+ operation_kwargs = {'one': 'from_kwargs'}
+ env = {'one': 'from_env_process'}
+ process = common.create_process_config(
+ script_path='',
+ process={'env': env},
+ operation_kwargs=operation_kwargs)
+ assert process['env'] == env
+
+ def test_json_env_vars(self):
+ operation_kwargs = {'a_dict': {'key': 'value'},
+ 'a_list': ['a', 'b', 'c'],
+ 'a_tuple': (4, 5, 6),
+ 'a_bool': True}
+ process = common.create_process_config(
+ script_path='',
+ process={},
+ operation_kwargs=operation_kwargs)
+ assert process['env'] == {'a_dict': '{"key": "value"}',
+ 'a_list': '["a", "b", "c"]',
+ 'a_tuple': '[4, 5, 6]',
+ 'a_bool': 'true'}
+
+ def test_quote_json_env_vars(self):
+ operation_kwargs = {'one': []}
+ process = common.create_process_config(
+ script_path='',
+ process={},
+ operation_kwargs=operation_kwargs,
+ quote_json_env_vars=True)
+ assert process['env']['one'] == "'[]'"
+
+ @pytest.mark.skipif(not common.is_windows(), reason='windows')
+ def test_env_keys_converted_to_string_on_windows(self):
+ env = {u'one': '1'}
+ process = common.create_process_config(
+ script_path='',
+ process={'env': env},
+ operation_kwargs={})
+ print process
+ assert isinstance(process['env'].keys()[0], str)
+
+ @pytest.mark.skipif(not common.is_windows(), reason='windows')
+ def test_env_values_quotes_are_escaped_on_windows(self):
+ env = {'one': '"hello"'}
+ process = common.create_process_config(
+ script_path='',
+ process={'env': env},
+ operation_kwargs={})
+ print process
+ assert process['env']['one'] == '\\"hello\\"'
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/tests/orchestrator/execution_plugin/test_ctx_proxy_server.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_ctx_proxy_server.py b/tests/orchestrator/execution_plugin/test_ctx_proxy_server.py
new file mode 100644
index 0000000..f133e15
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/test_ctx_proxy_server.py
@@ -0,0 +1,355 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import time
+import sys
+import subprocess
+import StringIO
+
+import pytest
+
+from aria.orchestrator.execution_plugin import ctx_proxy
+
+
+class TestCtxProxy(object):
+
+ def test_attribute_access(self, server):
+ response = self.request(server, 'stub_attr', 'some_property')
+ assert response == 'some_value'
+
+ def test_sugared_attribute_access(self, server):
+ response = self.request(server, 'stub-attr', 'some-property')
+ assert response == 'some_value'
+
+ def test_dict_prop_access_get_key(self, server):
+ response = self.request(server, 'node', 'properties', 'prop1')
+ assert response == 'value1'
+
+ def test_dict_prop_access_get_key_nested(self, server):
+ response = self.request(server, 'node', 'properties', 'prop2.nested_prop1')
+ assert response == 'nested_value1'
+
+ def test_dict_prop_access_get_with_list_index(self, server):
+ response = self.request(server, 'node', 'properties', 'prop3[2].value')
+ assert response == 'value_2'
+
+ def test_dict_prop_access_set(self, server, ctx):
+ self.request(server, 'node', 'properties', 'prop4.key', 'new_value')
+ self.request(server, 'node', 'properties', 'prop3[2].value', 'new_value_2')
+ self.request(server, 'node', 'properties', 'prop4.some.new.path',
+ 'some_new_value')
+ assert ctx.node.properties['prop4']['key'] == 'new_value'
+ assert ctx.node.properties['prop3'][2]['value'] == 'new_value_2'
+ assert ctx.node.properties['prop4']['some']['new']['path'] == 'some_new_value'
+
+ def test_illegal_dict_access(self, server):
+ self.request(server, 'node', 'properties', 'prop4.key', 'new_value')
+ with pytest.raises(RuntimeError):
+ self.request(server, 'node', 'properties', 'prop4.key', 'new_value', 'what')
+
+ def test_method_invocation(self, server):
+ args = ['arg1', 'arg2', 'arg3']
+ response_args = self.request(server, 'stub-method', *args)
+ assert response_args == args
+
+ def test_method_invocation_no_args(self, server):
+ response = self.request(server, 'stub-method')
+ assert response == []
+
+ def test_method_invocation_kwargs(self, server):
+ arg1 = 'arg1'
+ arg2 = 'arg2'
+ arg4 = 'arg4_override'
+ arg5 = 'arg5'
+ kwargs = dict(
+ arg4=arg4,
+ arg5=arg5)
+ response = self.request(server, 'stub_args', arg1, arg2, kwargs)
+ assert response == dict(
+ arg1=arg1,
+ arg2=arg2,
+ arg3='arg3',
+ arg4=arg4,
+ args=[],
+ kwargs=dict(
+ arg5=arg5))
+
+ def test_empty_return_value(self, server):
+ response = self.request(server, 'stub_none')
+ assert response is None
+
+ def test_client_request_timeout(self, server):
+ with pytest.raises(IOError):
+ ctx_proxy.client._client_request(server.socket_url,
+ args=['stub-sleep', '0.5'],
+ timeout=0.1)
+
+ def test_processing_exception(self, server):
+ with pytest.raises(ctx_proxy.client._RequestError):
+ self.request(server, 'property_that_does_not_exist')
+
+ def test_not_json_serializable(self, server):
+ with pytest.raises(ctx_proxy.client._RequestError):
+ self.request(server, 'logger')
+
+ def test_no_string_arg(self, server):
+ args = ['stub_method', 1, 2]
+ response = self.request(server, *args)
+ assert response == args[1:]
+
+ class StubAttribute(object):
+ some_property = 'some_value'
+
+ class NodeAttribute(object):
+ def __init__(self, properties):
+ self.properties = properties
+
+ @staticmethod
+ def stub_method(*args):
+ return args
+
+ @staticmethod
+ def stub_sleep(seconds):
+ time.sleep(float(seconds))
+
+ @staticmethod
+ def stub_args(arg1, arg2, arg3='arg3', arg4='arg4', *args, **kwargs):
+ return dict(
+ arg1=arg1,
+ arg2=arg2,
+ arg3=arg3,
+ arg4=arg4,
+ args=args,
+ kwargs=kwargs)
+
+ @pytest.fixture
+ def ctx(self):
+ class MockCtx(object):
+ pass
+ ctx = MockCtx()
+ properties = {
+ 'prop1': 'value1',
+ 'prop2': {
+ 'nested_prop1': 'nested_value1'
+ },
+ 'prop3': [
+ {'index': 0, 'value': 'value_0'},
+ {'index': 1, 'value': 'value_1'},
+ {'index': 2, 'value': 'value_2'}
+ ],
+ 'prop4': {
+ 'key': 'value'
+ }
+ }
+ ctx.stub_none = None
+ ctx.stub_method = self.stub_method
+ ctx.stub_sleep = self.stub_sleep
+ ctx.stub_args = self.stub_args
+ ctx.stub_attr = self.StubAttribute()
+ ctx.node = self.NodeAttribute(properties)
+ return ctx
+
+ @pytest.fixture
+ def server(self, ctx):
+ result = ctx_proxy.server.CtxProxy(ctx)
+ yield result
+ result.close()
+
+ def request(self, server, *args):
+ return ctx_proxy.client._client_request(server.socket_url, args, timeout=5)
+
+
+class TestArgumentParsing(object):
+
+ def test_socket_url_arg(self):
+ self.expected.update(dict(socket_url='sock_url'))
+ ctx_proxy.client.main(['--socket-url', self.expected.get('socket_url')])
+
+ def test_socket_url_env(self):
+ expected_socket_url = 'env_sock_url'
+ os.environ['CTX_SOCKET_URL'] = expected_socket_url
+ self.expected.update(dict(socket_url=expected_socket_url))
+ ctx_proxy.client.main([])
+
+ def test_socket_url_missing(self):
+ del os.environ['CTX_SOCKET_URL']
+ with pytest.raises(RuntimeError):
+ ctx_proxy.client.main([])
+
+ def test_args(self):
+ self.expected.update(dict(args=['1', '2', '3']))
+ ctx_proxy.client.main(self.expected.get('args'))
+
+ def test_timeout(self):
+ self.expected.update(dict(timeout='10'))
+ ctx_proxy.client.main(['--timeout', self.expected.get('timeout')])
+ self.expected.update(dict(timeout='15'))
+ ctx_proxy.client.main(['-t', self.expected.get('timeout')])
+
+ def test_mixed_order(self):
+ self.expected.update(dict(
+ args=['1', '2', '3'], timeout='20', socket_url='mixed_socket_url'))
+ ctx_proxy.client.main(
+ ['-t', self.expected.get('timeout')] +
+ ['--socket-url', self.expected.get('socket_url')] +
+ self.expected.get('args'))
+ ctx_proxy.client.main(
+ ['-t', self.expected.get('timeout')] +
+ self.expected.get('args') +
+ ['--socket-url', self.expected.get('socket_url')])
+ ctx_proxy.client.main(
+ self.expected.get('args') +
+ ['-t', self.expected.get('timeout')] +
+ ['--socket-url', self.expected.get('socket_url')])
+
+ def test_json_args(self):
+ args = ['@1', '@[1,2,3]', '@{"key":"value"}']
+ expected_args = [1, [1, 2, 3], {'key': 'value'}]
+ self.expected.update(dict(args=expected_args))
+ ctx_proxy.client.main(args)
+
+ def test_json_arg_prefix(self):
+ args = ['_1', '@1']
+ expected_args = [1, '@1']
+ self.expected.update(dict(args=expected_args))
+ ctx_proxy.client.main(args + ['--json-arg-prefix', '_'])
+
+ def test_json_output(self):
+ self.assert_valid_output('string', 'string', '"string"')
+ self.assert_valid_output(1, '1', '1')
+ self.assert_valid_output([1, '2'], "[1, '2']", '[1, "2"]')
+ self.assert_valid_output({'key': 1},
+ "{'key': 1}",
+ '{"key": 1}')
+ self.assert_valid_output(False, '', 'false')
+ self.assert_valid_output(True, 'True', 'true')
+ self.assert_valid_output([], '', '[]')
+ self.assert_valid_output({}, '', '{}')
+
+ def assert_valid_output(self, response, ex_typed_output, ex_json_output):
+ self.mock_response = response
+ current_stdout = sys.stdout
+
+ def run(args, expected):
+ output = StringIO.StringIO()
+ sys.stdout = output
+ ctx_proxy.client.main(args)
+ assert output.getvalue() == expected
+
+ try:
+ run([], ex_typed_output)
+ run(['-j'], ex_json_output)
+ run(['--json-output'], ex_json_output)
+ finally:
+ sys.stdout = current_stdout
+
+ def mock_client_request(self, socket_url, args, timeout):
+ assert socket_url == self.expected.get('socket_url')
+ assert args == self.expected.get('args')
+ assert timeout == int(self.expected.get('timeout'))
+ return self.mock_response
+
+ @pytest.fixture(autouse=True)
+ def patch_client_request(self, mocker):
+ mocker.patch.object(ctx_proxy.client,
+ ctx_proxy.client._client_request.__name__,
+ self.mock_client_request)
+ mocker.patch.dict('os.environ', {'CTX_SOCKET_URL': 'stub'})
+
+ @pytest.fixture(autouse=True)
+ def defaults(self):
+ self.expected = dict(args=[], timeout=30, socket_url='stub')
+ self.mock_response = None
+
+
+class TestCtxEntryPoint(object):
+
+ def test_ctx_in_path(self):
+ subprocess.check_output(['ctx', '--help'])
+
+
+class TestPathDictAccess(object):
+ def test_simple_set(self):
+ obj = {}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ path_dict.set('foo', 42)
+ assert obj == {'foo': 42}
+
+ def test_nested_set(self):
+ obj = {'foo': {}}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ path_dict.set('foo.bar', 42)
+ assert obj == {'foo': {'bar': 42}}
+
+ def test_set_index(self):
+ obj = {'foo': [None, {'bar': 0}]}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ path_dict.set('foo[1].bar', 42)
+ assert obj == {'foo': [None, {'bar': 42}]}
+
+ def test_set_nonexistent_parent(self):
+ obj = {}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ path_dict.set('foo.bar', 42)
+ assert obj == {'foo': {'bar': 42}}
+
+ def test_set_nonexistent_parent_nested(self):
+ obj = {}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ path_dict.set('foo.bar.baz', 42)
+ assert obj == {'foo': {'bar': {'baz': 42}}}
+
+ def test_simple_get(self):
+ obj = {'foo': 42}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ result = path_dict.get('foo')
+ assert result == 42
+
+ def test_nested_get(self):
+ obj = {'foo': {'bar': 42}}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ result = path_dict.get('foo.bar')
+ assert result == 42
+
+ def test_nested_get_shadows_dotted_name(self):
+ obj = {'foo': {'bar': 42}, 'foo.bar': 58}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ result = path_dict.get('foo.bar')
+ assert result == 42
+
+ def test_index_get(self):
+ obj = {'foo': [0, 1]}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ result = path_dict.get('foo[1]')
+ assert result == 1
+
+ def test_get_nonexistent(self):
+ obj = {}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ with pytest.raises(RuntimeError):
+ path_dict.get('foo')
+
+ def test_get_by_index_not_list(self):
+ obj = {'foo': {0: 'not-list'}}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ with pytest.raises(RuntimeError):
+ path_dict.get('foo[0]')
+
+ def test_get_by_index_nonexistent_parent(self):
+ obj = {}
+ path_dict = ctx_proxy.server._PathDictAccess(obj)
+ with pytest.raises(RuntimeError):
+ path_dict.get('foo[1]')
[4/6] incubator-ariatosca git commit: ARIA-63 Implement attribute
tracking for subprocesses
Posted by da...@apache.org.
ARIA-63 Implement attribute tracking for subprocesses
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/385b209d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/385b209d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/385b209d
Branch: refs/heads/ARIA-46-execution-plugin
Commit: 385b209db2bf797cdd9877c3d031f3b258b8f79f
Parents: 61b7094
Author: Dan Kilman <da...@gigaspaces.com>
Authored: Sun Jan 15 17:42:23 2017 +0200
Committer: Dan Kilman <da...@gigaspaces.com>
Committed: Tue Jan 17 12:46:04 2017 +0200
----------------------------------------------------------------------
aria/orchestrator/workflows/executor/process.py | 48 ++--
aria/storage/instrumentation.py | 157 +++++++++++
aria/storage/type.py | 38 ++-
tests/.pylintrc | 2 +-
.../test_process_executor_tracked_changes.py | 95 +++++++
tests/storage/test_instrumentation.py | 275 +++++++++++++++++++
6 files changed, 595 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/aria/orchestrator/workflows/executor/process.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py
index e0a8aeb..51596c3 100644
--- a/aria/orchestrator/workflows/executor/process.py
+++ b/aria/orchestrator/workflows/executor/process.py
@@ -42,6 +42,8 @@ import jsonpickle
from aria.utils import imports
from aria.orchestrator.workflows.executor import base
from aria.orchestrator.context import serialization
+from aria.storage import instrumentation
+from aria.storage import type as storage_type
_IS_WIN = os.name == 'nt'
@@ -139,10 +141,17 @@ class ProcessExecutor(base.BaseExecutor):
if message_type == 'started':
self._task_started(self._tasks[task_id])
elif message_type == 'succeeded':
- self._task_succeeded(self._remove_task(task_id))
+ task = self._remove_task(task_id)
+ instrumentation.apply_tracked_changes(
+ tracked_changes=message['tracked_changes'],
+ model=task.context.model)
+ self._task_succeeded(task)
elif message_type == 'failed':
- self._task_failed(self._remove_task(task_id),
- exception=message['exception'])
+ task = self._remove_task(task_id)
+ instrumentation.apply_tracked_changes(
+ tracked_changes=message['tracked_changes'],
+ model=task.context.model)
+ self._task_failed(task, exception=message['exception'])
else:
raise RuntimeError('Invalid state')
except BaseException as e:
@@ -227,26 +236,27 @@ class _Messenger(object):
"""Task started message"""
self._send_message(type='started')
- def succeeded(self):
+ def succeeded(self, tracked_changes):
"""Task succeeded message"""
- self._send_message(type='succeeded')
+ self._send_message(type='succeeded', tracked_changes=tracked_changes)
- def failed(self, exception):
+ def failed(self, tracked_changes, exception):
"""Task failed message"""
- self._send_message(type='failed', exception=exception)
+ self._send_message(type='failed', tracked_changes=tracked_changes, exception=exception)
def closed(self):
"""Executor closed message"""
self._send_message(type='closed')
- def _send_message(self, type, exception=None):
+ def _send_message(self, type, tracked_changes=None, exception=None):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', self.port))
try:
data = jsonpickle.dumps({
'type': type,
'task_id': self.task_id,
- 'exception': exception
+ 'exception': exception,
+ 'tracked_changes': tracked_changes
})
sock.send(struct.pack(_INT_FMT, len(data)))
sock.sendall(data)
@@ -271,13 +281,19 @@ def _main():
operation_mapping = arguments['operation_mapping']
operation_inputs = arguments['operation_inputs']
context_dict = arguments['context']
- try:
- ctx = serialization.operation_context_from_dict(context_dict)
- task_func = imports.load_attribute(operation_mapping)
- task_func(ctx=ctx, **operation_inputs)
- messenger.succeeded()
- except BaseException as e:
- messenger.failed(exception=e)
+
+ # This is required for the instrumentation work properly.
+ # See docstring of `remove_mutable_association_listener` for further details
+ storage_type.remove_mutable_association_listener()
+
+ with instrumentation.track_changes() as instrument:
+ try:
+ ctx = serialization.operation_context_from_dict(context_dict)
+ task_func = imports.load_attribute(operation_mapping)
+ task_func(ctx=ctx, **operation_inputs)
+ messenger.succeeded(tracked_changes=instrument.tracked_changes)
+ except BaseException as e:
+ messenger.failed(exception=e, tracked_changes=instrument.tracked_changes)
if __name__ == '__main__':
_main()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/aria/storage/instrumentation.py
----------------------------------------------------------------------
diff --git a/aria/storage/instrumentation.py b/aria/storage/instrumentation.py
new file mode 100644
index 0000000..d71190d
--- /dev/null
+++ b/aria/storage/instrumentation.py
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+
+import sqlalchemy.event
+
+from . import api
+from . import model as _model
+
+_STUB = object()
+_INSTRUMENTED = {
+ _model.NodeInstance.runtime_properties: dict
+}
+
+
+def track_changes(instrumented=None):
+ """Track changes in the specified model columns
+
+ This call will register event listeners using sqlalchemy's event mechanism. The listeners
+ instrument all returned objects such that the attributes specified in ``instrumented``, will
+ be replaced with a value that is stored in the returned instrumentation context
+ ``tracked_changes`` property.
+
+ Why should this be implemented when sqlalchemy already does a fantastic job at tracking changes
+ you ask? Well, when sqlalchemy is used with sqlite, due to how sqlite works, only one process
+ can hold a write lock to the database. This does not work well when ARIA runs tasks in
+ subprocesses (by the process executor) and these tasks wish to change some state as well. These
+ tasks certainly deserve a chance to do so!
+
+ To enable this, the subprocess calls ``track_changes()`` before any state changes are made.
+ At the end of the subprocess execution, it should return the ``tracked_changes`` attribute of
+ the instrumentation context returned from this call, to the parent process. The parent process
+ will then call ``apply_tracked_changes()`` that resides in this module as well.
+ At that point, the changes will actually be written back to the database.
+
+ :param instrumented: A dict from model columns to their python native type
+ :return: The instrumentation context
+ """
+ return _Instrumentation(instrumented or _INSTRUMENTED)
+
+
+class _Instrumentation(object):
+
+ def __init__(self, instrumented):
+ self.tracked_changes = {}
+ self.listeners = []
+ self._track_changes(instrumented)
+
+ def _track_changes(self, instrumented):
+ instrumented_classes = {}
+ for instrumented_attribute, attribute_type in instrumented.items():
+ self._register_set_attribute_listener(
+ instrumented_attribute=instrumented_attribute,
+ attribute_type=attribute_type)
+ instrumented_class = instrumented_attribute.parent.entity
+ instrumented_class_attributes = instrumented_classes.setdefault(instrumented_class, {})
+ instrumented_class_attributes[instrumented_attribute.key] = attribute_type
+ for instrumented_class, instrumented_attributes in instrumented_classes.items():
+ self._register_instance_listeners(
+ instrumented_class=instrumented_class,
+ instrumented_attributes=instrumented_attributes)
+
+ def _register_set_attribute_listener(self, instrumented_attribute, attribute_type):
+ def listener(target, value, *_):
+ mapi_name = api.generate_lower_name(target.__class__)
+ tracked_instances = self.tracked_changes.setdefault(mapi_name, {})
+ tracked_attributes = tracked_instances.setdefault(target.id, {})
+ if value is None:
+ current = None
+ else:
+ current = copy.deepcopy(attribute_type(value))
+ tracked_attributes[instrumented_attribute.key] = _Value(_STUB, current)
+ return current
+ listener_args = (instrumented_attribute, 'set', listener)
+ sqlalchemy.event.listen(*listener_args, retval=True)
+ self.listeners.append(listener_args)
+
+ def _register_instance_listeners(self, instrumented_class, instrumented_attributes):
+ def listener(target, *_):
+ mapi_name = api.generate_lower_name(instrumented_class)
+ tracked_instances = self.tracked_changes.setdefault(mapi_name, {})
+ tracked_attributes = tracked_instances.setdefault(target.id, {})
+ for attribute_name, attribute_type in instrumented_attributes.items():
+ if attribute_name not in tracked_attributes:
+ initial = getattr(target, attribute_name)
+ if initial is None:
+ current = None
+ else:
+ current = copy.deepcopy(attribute_type(initial))
+ tracked_attributes[attribute_name] = _Value(initial, current)
+ target.__dict__[attribute_name] = tracked_attributes[attribute_name].current
+ for listener_args in [(instrumented_class, 'load', listener),
+ (instrumented_class, 'refresh', listener),
+ (instrumented_class, 'refresh_flush', listener)]:
+ sqlalchemy.event.listen(*listener_args)
+ self.listeners.append(listener_args)
+
+ def restore(self):
+ """Remove all listeners registered by this instrumentation"""
+ for listener_args in self.listeners:
+ if sqlalchemy.event.contains(*listener_args):
+ sqlalchemy.event.remove(*listener_args)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.restore()
+
+
+class _Value(object):
+ # You may wonder why is this a full blown class and not a named tuple. The reason is that
+ # jsonpickle that is used to serialize the tracked_changes, does not handle named tuples very
+ # well. At the very least, I could not get it to behave.
+
+ def __init__(self, initial, current):
+ self.initial = initial
+ self.current = current
+
+ def __eq__(self, other):
+ if not isinstance(other, _Value):
+ return False
+ return self.initial == other.initial and self.current == other.current
+
+ def __hash__(self):
+ return hash(self.initial) ^ hash(self.current)
+
+
+def apply_tracked_changes(tracked_changes, model):
+ """Write tracked changes back to the database using provided model storage
+
+ :param tracked_changes: The ``tracked_changes`` attribute of the instrumentation context
+ returned by calling ``track_changes()``
+ :param model: The model storage used to actually apply the changes
+ """
+ for mapi_name, tracked_instances in tracked_changes.items():
+ mapi = getattr(model, mapi_name)
+ for instance_id, tracked_attributes in tracked_instances.items():
+ instance = None
+ for attribute_name, value in tracked_attributes.items():
+ if value.initial != value.current:
+ if not instance:
+ instance = mapi.get(instance_id)
+ setattr(instance, attribute_name, value.current)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/aria/storage/type.py
----------------------------------------------------------------------
diff --git a/aria/storage/type.py b/aria/storage/type.py
index ab50b0f..dd2a5ce 100644
--- a/aria/storage/type.py
+++ b/aria/storage/type.py
@@ -16,7 +16,8 @@ import json
from sqlalchemy import (
TypeDecorator,
- VARCHAR
+ VARCHAR,
+ event
)
from sqlalchemy.ext import mutable
@@ -84,5 +85,36 @@ class _MutableList(mutable.MutableList):
raise exceptions.StorageError('SQL Storage error: {0}'.format(str(e)))
-_MutableList.associate_with(List)
-_MutableDict.associate_with(Dict)
+def _mutable_association_listener(mapper, cls):
+ for prop in mapper.column_attrs:
+ column_type = prop.columns[0].type
+ if isinstance(column_type, Dict):
+ _MutableDict.associate_with_attribute(getattr(cls, prop.key))
+ if isinstance(column_type, List):
+ _MutableList.associate_with_attribute(getattr(cls, prop.key))
+_LISTENER_ARGS = (mutable.mapper, 'mapper_configured', _mutable_association_listener)
+
+
+def _register_mutable_association_listener():
+ event.listen(*_LISTENER_ARGS)
+
+
+def remove_mutable_association_listener():
+ """
+ Remove the event listener that associates ``Dict`` and ``List`` column types with
+ ``MutableDict`` and ``MutableList``, respectively.
+
+ This call must happen before any model instance is instantiated.
+ This is because once it does, that would trigger the listener we are trying to remove.
+ Once it is triggered, many other listeners will then be registered.
+ At that point, it is too late.
+
+ The reason this function exists is that the association listener, interferes with ARIA change
+ tracking instrumentation, so a way to disable it is required.
+
+ Note that the event listener this call removes is registered by default.
+ """
+ if event.contains(*_LISTENER_ARGS):
+ event.remove(*_LISTENER_ARGS)
+
+_register_mutable_association_listener()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/tests/.pylintrc
----------------------------------------------------------------------
diff --git a/tests/.pylintrc b/tests/.pylintrc
index 23251af..5de0691 100644
--- a/tests/.pylintrc
+++ b/tests/.pylintrc
@@ -77,7 +77,7 @@ confidence=
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,no-self-use,missing-docstring,attribute-defined-outside-init,redefined-outer-name,import-error,redefined-variable-type,broad
-except,protected-access,global-statement,too-many-locals
+disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,redefined-builtin,no-self-use,missing-docstring,attribute-defined-outside-init,redefined-outer-name,import-error,redefined-variable-type,broad
-except,protected-access,global-statement,too-many-locals,abstract-method
[REPORTS]
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
new file mode 100644
index 0000000..1564292
--- /dev/null
+++ b/tests/orchestrator/workflows/executor/test_process_executor_tracked_changes.py
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from aria.orchestrator.workflows import api
+from aria.orchestrator.workflows.core import engine
+from aria.orchestrator.workflows.executor import process
+from aria.orchestrator import workflow, operation
+from aria.orchestrator.workflows import exceptions
+
+import tests
+from tests import mock
+from tests import storage
+
+
+_TEST_RUNTIME_PROPERTIES = {
+ 'some': 'values', 'that': 'are', 'most': 'likely', 'only': 'set', 'here': 'yo'
+}
+
+
+def test_track_changes_of_successful_operation(context, executor):
+ _run_workflow(context=context, executor=executor, op_func=_mock_success_operation)
+ _assert_tracked_changes_are_applied(context)
+
+
+def test_track_changes_of_failed_operation(context, executor):
+ with pytest.raises(exceptions.ExecutorException):
+ _run_workflow(context=context, executor=executor, op_func=_mock_fail_operation)
+ _assert_tracked_changes_are_applied(context)
+
+
+def _assert_tracked_changes_are_applied(context):
+ instance = context.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+ assert instance.runtime_properties == _TEST_RUNTIME_PROPERTIES
+
+
+def _update_runtime_properties(context):
+ context.node_instance.runtime_properties.clear()
+ context.node_instance.runtime_properties.update(_TEST_RUNTIME_PROPERTIES)
+
+
+def _run_workflow(context, executor, op_func):
+ @workflow
+ def mock_workflow(ctx, graph):
+ node_instance = ctx.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+ node_instance.node.operations['test.op'] = {'operation': _operation_mapping(op_func)}
+ task = api.task.OperationTask.node_instance(instance=node_instance, name='test.op')
+ graph.add_tasks(task)
+ return graph
+ graph = mock_workflow(ctx=context) # pylint: disable=no-value-for-parameter
+ eng = engine.Engine(executor=executor, workflow_context=context, tasks_graph=graph)
+ eng.execute()
+
+
+@operation
+def _mock_success_operation(ctx):
+ _update_runtime_properties(ctx)
+
+
+@operation
+def _mock_fail_operation(ctx):
+ _update_runtime_properties(ctx)
+ raise RuntimeError
+
+
+def _operation_mapping(func):
+ return '{name}.{func.__name__}'.format(name=__name__, func=func)
+
+
+@pytest.fixture
+def executor():
+ result = process.ProcessExecutor(python_path=[tests.ROOT_DIR])
+ yield result
+ result.close()
+
+
+@pytest.fixture
+def context(tmpdir):
+ result = mock.context.simple(storage.get_sqlite_api_kwargs(str(tmpdir)))
+ yield result
+ storage.release_sqlite_storage(result.model)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/385b209d/tests/storage/test_instrumentation.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_instrumentation.py b/tests/storage/test_instrumentation.py
new file mode 100644
index 0000000..8e7f9aa
--- /dev/null
+++ b/tests/storage/test_instrumentation.py
@@ -0,0 +1,275 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from sqlalchemy import Column, Text, Integer, event
+
+from aria.storage import (
+ model,
+ structure,
+ type as aria_type,
+ ModelStorage,
+ sql_mapi,
+ instrumentation
+)
+from ..storage import get_sqlite_api_kwargs, release_sqlite_storage
+
+
+STUB = instrumentation._STUB
+Value = instrumentation._Value
+instruments_holder = []
+
+
+class TestInstrumentation(object):
+
+ def test_track_changes(self, storage):
+ model_kwargs = dict(
+ name='name',
+ dict1={'initial': 'value'},
+ dict2={'initial': 'value'},
+ list1=['initial'],
+ list2=['initial'],
+ int1=0,
+ int2=0,
+ string2='string')
+ model1_instance = MockModel1(**model_kwargs)
+ model2_instance = MockModel2(**model_kwargs)
+ storage.mock_model_1.put(model1_instance)
+ storage.mock_model_2.put(model2_instance)
+
+ instrument = self._track_changes({
+ MockModel1.dict1: dict,
+ MockModel1.list1: list,
+ MockModel1.int1: int,
+ MockModel1.string2: str,
+ MockModel2.dict2: dict,
+ MockModel2.list2: list,
+ MockModel2.int2: int,
+ MockModel2.name: str
+ })
+
+ assert not instrument.tracked_changes
+
+ storage_model1_instance = storage.mock_model_1.get(model1_instance.id)
+ storage_model2_instance = storage.mock_model_2.get(model2_instance.id)
+
+ storage_model1_instance.dict1 = {'hello': 'world'}
+ storage_model1_instance.dict2 = {'should': 'not track'}
+ storage_model1_instance.list1 = ['hello']
+ storage_model1_instance.list2 = ['should not track']
+ storage_model1_instance.int1 = 100
+ storage_model1_instance.int2 = 20000
+ storage_model1_instance.name = 'should not track'
+ storage_model1_instance.string2 = 'new_string'
+
+ storage_model2_instance.dict1.update({'should': 'not track'})
+ storage_model2_instance.dict2.update({'hello': 'world'})
+ storage_model2_instance.list1.append('should not track')
+ storage_model2_instance.list2.append('hello')
+ storage_model2_instance.int1 = 100
+ storage_model2_instance.int2 = 20000
+ storage_model2_instance.name = 'new_name'
+ storage_model2_instance.string2 = 'should not track'
+
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {
+ model1_instance.id: {
+ 'dict1': Value(STUB, {'hello': 'world'}),
+ 'list1': Value(STUB, ['hello']),
+ 'int1': Value(STUB, 100),
+ 'string2': Value(STUB, 'new_string')
+ }
+ },
+ 'mock_model_2': {
+ model2_instance.id: {
+ 'dict2': Value({'initial': 'value'}, {'hello': 'world', 'initial': 'value'}),
+ 'list2': Value(['initial'], ['initial', 'hello']),
+ 'int2': Value(STUB, 20000),
+ 'name': Value(STUB, 'new_name'),
+ }
+ }
+ }
+
+ def test_attribute_initial_none_value(self, storage):
+ instance1 = MockModel1(name='name1', dict1=None)
+ instance2 = MockModel1(name='name2', dict1=None)
+ storage.mock_model_1.put(instance1)
+ storage.mock_model_1.put(instance2)
+ instrument = self._track_changes({MockModel1.dict1: dict})
+ instance1 = storage.mock_model_1.get(instance1.id)
+ instance2 = storage.mock_model_1.get(instance2.id)
+ instance1.dict1 = {'new': 'value'}
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {
+ instance1.id: {'dict1': Value(STUB, {'new': 'value'})},
+ instance2.id: {'dict1': Value(None, None)},
+ }
+ }
+
+ def test_attribute_set_none_value(self, storage):
+ instance = MockModel1(name='name')
+ storage.mock_model_1.put(instance)
+ instrument = self._track_changes({
+ MockModel1.dict1: dict,
+ MockModel1.list1: list,
+ MockModel1.string2: str,
+ MockModel1.int1: int
+ })
+ instance = storage.mock_model_1.get(instance.id)
+ instance.dict1 = None
+ instance.list1 = None
+ instance.string2 = None
+ instance.int1 = None
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {
+ instance.id: {
+ 'dict1': Value(STUB, None),
+ 'list1': Value(STUB, None),
+ 'string2': Value(STUB, None),
+ 'int1': Value(STUB, None)
+ }
+ }
+ }
+
+ def test_restore(self):
+ instrument = self._track_changes({MockModel1.dict1: dict})
+ # set instance attribute, load instance, refresh instance and flush_refresh listeners
+ assert len(instrument.listeners) == 4
+ for listener_args in instrument.listeners:
+ assert event.contains(*listener_args)
+ instrument.restore()
+ assert len(instrument.listeners) == 4
+ for listener_args in instrument.listeners:
+ assert not event.contains(*listener_args)
+ return instrument
+
+ def test_restore_twice(self):
+ instrument = self.test_restore()
+ instrument.restore()
+
+ def test_instrumentation_context_manager(self, storage):
+ instance = MockModel1(name='name')
+ storage.mock_model_1.put(instance)
+ with self._track_changes({MockModel1.dict1: dict}) as instrument:
+ instance = storage.mock_model_1.get(instance.id)
+ instance.dict1 = {'new': 'value'}
+ assert instrument.tracked_changes == {
+ 'mock_model_1': {instance.id: {'dict1': Value(STUB, {'new': 'value'})}}
+ }
+ assert len(instrument.listeners) == 4
+ for listener_args in instrument.listeners:
+ assert event.contains(*listener_args)
+ for listener_args in instrument.listeners:
+ assert not event.contains(*listener_args)
+
+ def test_apply_tracked_changes(self, storage):
+ initial_values = {'dict1': {'initial': 'value'}, 'list1': ['initial']}
+ instance1_1 = MockModel1(name='instance1_1', **initial_values)
+ instance1_2 = MockModel1(name='instance1_2', **initial_values)
+ instance2_1 = MockModel2(name='instance2_1', **initial_values)
+ instance2_2 = MockModel2(name='instance2_2', **initial_values)
+ storage.mock_model_1.put(instance1_1)
+ storage.mock_model_1.put(instance1_2)
+ storage.mock_model_2.put(instance2_1)
+ storage.mock_model_2.put(instance2_2)
+
+ instrument = self._track_changes({
+ MockModel1.dict1: dict,
+ MockModel1.list1: list,
+ MockModel2.dict1: dict,
+ MockModel2.list1: list
+ })
+
+ def get_instances():
+ return (storage.mock_model_1.get(instance1_1.id),
+ storage.mock_model_1.get(instance1_2.id),
+ storage.mock_model_2.get(instance2_1.id),
+ storage.mock_model_2.get(instance2_2.id))
+
+ instance1_1, instance1_2, instance2_1, instance2_2 = get_instances()
+ instance1_1.dict1 = {'new': 'value'}
+ instance1_2.list1 = ['new_value']
+ instance2_1.dict1.update({'new': 'value'})
+ instance2_2.list1.append('new_value')
+
+ instrument.restore()
+ storage.mock_model_1._session.expire_all()
+
+ instance1_1, instance1_2, instance2_1, instance2_2 = get_instances()
+ instance1_1.dict1 = {'overriding': 'value'}
+ instance1_2.list1 = ['overriding_value']
+ instance2_1.dict1 = {'overriding': 'value'}
+ instance2_2.list1 = ['overriding_value']
+ storage.mock_model_1.put(instance1_1)
+ storage.mock_model_1.put(instance1_2)
+ storage.mock_model_2.put(instance2_1)
+ storage.mock_model_2.put(instance2_2)
+ instance1_1, instance1_2, instance2_1, instance2_2 = get_instances()
+ assert instance1_1.dict1 == {'overriding': 'value'}
+ assert instance1_2.list1 == ['overriding_value']
+ assert instance2_1.dict1 == {'overriding': 'value'}
+ assert instance2_2.list1 == ['overriding_value']
+
+ instrumentation.apply_tracked_changes(
+ tracked_changes=instrument.tracked_changes,
+ model=storage)
+
+ instance1_1, instance1_2, instance2_1, instance2_2 = get_instances()
+ assert instance1_1.dict1 == {'new': 'value'}
+ assert instance1_2.list1 == ['new_value']
+ assert instance2_1.dict1 == {'initial': 'value', 'new': 'value'}
+ assert instance2_2.list1 == ['initial', 'new_value']
+
+ def _track_changes(self, instrumented):
+ instrument = instrumentation.track_changes(instrumented)
+ instruments_holder.append(instrument)
+ return instrument
+
+
+@pytest.fixture(autouse=True)
+def restore_instrumentation():
+ yield
+ for instrument in instruments_holder:
+ instrument.restore()
+ del instruments_holder[:]
+
+
+@pytest.fixture
+def storage():
+ result = ModelStorage(
+ api_cls=sql_mapi.SQLAlchemyModelAPI,
+ api_kwargs=get_sqlite_api_kwargs(),
+ items=(MockModel1, MockModel2))
+ yield result
+ release_sqlite_storage(result)
+
+
+class _MockModel(structure.ModelMixin):
+ name = Column(Text)
+ dict1 = Column(aria_type.Dict)
+ dict2 = Column(aria_type.Dict)
+ list1 = Column(aria_type.List)
+ list2 = Column(aria_type.List)
+ int1 = Column(Integer)
+ int2 = Column(Integer)
+ string2 = Column(Text)
+
+
+class MockModel1(model.DeclarativeBase, _MockModel):
+ __tablename__ = 'mock_model1'
+
+
+class MockModel2(model.DeclarativeBase, _MockModel):
+ __tablename__ = 'mock_model2'
[5/6] incubator-ariatosca git commit: ARIA-46 Execution plugin
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/tests/orchestrator/execution_plugin/test_local.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_local.py b/tests/orchestrator/execution_plugin/test_local.py
new file mode 100644
index 0000000..8123ae8
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/test_local.py
@@ -0,0 +1,572 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+import pytest
+
+from aria import workflow
+from aria.orchestrator.workflows import api
+from aria.orchestrator.exceptions import TaskAbortException
+from aria.orchestrator.execution_plugin import operations
+from aria.orchestrator.execution_plugin.exceptions import ProcessException
+from aria.orchestrator.execution_plugin import local
+from aria.orchestrator.execution_plugin import constants
+from aria.orchestrator.workflows.executor import process
+from aria.orchestrator.workflows.core import engine
+
+from tests import mock, storage
+
+IS_WINDOWS = os.name == 'nt'
+
+import logging
+logging.basicConfig()
+
+
+class TestLocalRunScript(object):
+
+ def test_script_path_parameter(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties map.key value
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties map.key value
+ ''')
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ assert props['map']['key'] == 'value'
+
+ def test_process_env(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties map.key1 $key1
+ ctx node-instance runtime-properties map.key2 $key2
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties map.key1 %key1%
+ ctx node-instance runtime-properties map.key2 %key2%
+ ''')
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ process={
+ 'env': {
+ 'key1': 'value1',
+ 'key2': 'value2'
+ }
+ })
+ p_map = props['map']
+ assert p_map['key1'] == 'value1'
+ assert p_map['key2'] == 'value2'
+
+ def test_process_cwd(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties map.cwd $PWD
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties map.cwd %CD%
+ ''')
+ tmpdir = str(tmpdir)
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ process={
+ 'cwd': tmpdir
+ })
+ p_map = props['map']
+ assert p_map['cwd'] == tmpdir
+
+ def test_process_command_prefix(self, executor, workflow_context, tmpdir):
+ use_ctx = 'ctx node-instance runtime-properties map.key value'
+ python_script = ['import subprocess',
+ 'subprocess.Popen("{0}").split(' ')).communicate()[0]'.format(use_ctx)]
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='\n'.join(python_script),
+ windows_script='''
+ ctx node-instance runtime-properties map.key $env:TEST_KEY
+ ''',
+ windows_suffix='.ps1')
+ # TODO: this (powershell) happens implicitly
+ command_prefix = 'powershell' if IS_WINDOWS else 'python'
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ process={
+ 'env': {'TEST_KEY': 'value'},
+ 'command_prefix': command_prefix
+ })
+ p_map = props['map']
+ assert p_map['key'] == 'value'
+
+ def test_process_args(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties map.arg1 "$1"
+ ctx node-instance runtime-properties map.arg2 $2
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties map.arg1 %1
+ ctx node-instance runtime-properties map.arg2 %2
+ ''')
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ process={
+ 'args': ['"arg with spaces"', 'arg2']
+ })
+ assert 'arg with spaces' == props['map']['arg1']
+ assert 'arg2' == props['map']['arg2']
+
+ def test_no_script_path(self, executor, workflow_context):
+ with pytest.raises(TaskAbortException):
+ self._run(
+ executor, workflow_context,
+ script_path=None)
+
+ def test_script_error(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ echo 123123
+ command_that_does_not_exist
+ ''',
+ windows_script='''
+ @echo off
+ echo 123123
+ command_that_does_not_exist
+ ''')
+ with pytest.raises(ProcessException) as ex_ctx:
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ exception = ex_ctx.value
+ assert os.path.basename(script_path) in exception.command
+ assert exception.exit_code == 1
+ assert exception.stdout.strip() == '123123'
+ assert 'command_that_does_not_exist' in exception.stderr
+
+ def test_script_error_from_bad_ctx_request(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx property_that_does_not_exist
+ ''',
+ windows_script='''
+ ctx property_that_does_not_exist
+ ''')
+ with pytest.raises(ProcessException) as ex_ctx:
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ exception = ex_ctx.value
+ assert os.path.basename(script_path) in exception.command
+ assert exception.exit_code == 1
+ assert 'RequestError' in exception.stderr
+ assert 'property_that_does_not_exist' in exception.stderr
+
+ def test_python_script(self, executor, workflow_context, tmpdir):
+ script = '''
+from cloudify import ctx
+if __name__ == '__main__':
+ ctx.node_instance.runtime_properties['key'] = 'value'
+'''
+ suffix = '.py'
+ script_path = self._create_script(
+ tmpdir,
+ linux_script=script,
+ windows_script=script,
+ linux_suffix=suffix,
+ windows_suffix=suffix)
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ assert props['key'] == 'value'
+
+ def test_inputs_as_environment_variables(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties key "${input_as_env_var}"
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties key "%input_as_env_var%"
+ ''')
+
+ def test(value):
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ env_var=value)
+ expected = props['key'] if isinstance(value, basestring) else json.loads(props['key'])
+ assert expected == value
+ test('string-value')
+ test([1, 2, 3])
+ test(999)
+ test(3.14)
+ test(False)
+ test({
+ 'complex1': {
+ 'complex2': {
+ 'key': 'value'
+ },
+ 'list': [1, 2, 3]
+ }
+ })
+
+ def test_explicit_env_variables_inputs_override(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties key "${input_as_env_var}"
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties key "%input_as_env_var%"
+ ''')
+
+ def test(value):
+ props = self._run(
+ executor, workflow_context,
+ script_path=script_path,
+ env_var='test-value',
+ process={
+ 'env': {
+ 'input_as_env_var': value
+ }
+ })
+ expected = props['key'] if isinstance(value, basestring) else json.loads(props['key'])
+ assert expected == value
+ test('override')
+ test({'key': 'value'})
+
+ def test_get_nonexistent_runtime_property(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx node-instance runtime-properties nonexistent
+ ''',
+ windows_script='''
+ ctx node-instance runtime-properties nonexistent
+ ''')
+ with pytest.raises(ProcessException) as exc_ctx:
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ exception = exc_ctx.value
+ assert os.path.basename(script_path) in exception.command
+ assert exception.exit_code == 1
+ assert 'RequestError' in exception.stderr
+ assert 'nonexistent' in exception.stderr
+
+ def test_get_nonexistent_runtime_property_json(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx -j instance runtime-properties nonexistent
+ ''',
+ windows_script='''
+ ctx -j instance runtime-properties nonexistent
+ ''')
+ with pytest.raises(ProcessException) as exc_ctx:
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ exception = exc_ctx.value
+ assert os.path.basename(script_path) in exception.command
+ assert exception.exit_code == 1
+ assert 'RequestError' in exception.stderr
+ assert 'nonexistent' in exception.stderr
+
+ def test_abort(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task abort abort-message
+ ''',
+ windows_script='''
+ ctx task abort abort-message
+ ''')
+ with pytest.raises(TaskAbortException):
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+
+ def test_retry(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task retry retry-message
+ ''',
+ windows_script='''
+ ctx task retry retry-message
+ ''')
+ with pytest.raises(TaskAbortException):
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+
+ def test_crash_abort_after_retry(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task retry retry-message
+ ctx task abort should-raise-a-runtime-error
+ ''',
+ windows_script='''
+ ctx task retry retry-message
+ ctx task abort should-raise-a-runtime-error
+ ''')
+ with pytest.raises(TaskAbortException):
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+
+ def test_crash_retry_after_abort(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task abort abort-message
+ ctx task retry should-raise-a-runtime-error
+ ''',
+ windows_script='''
+ ctx task abort abort-message
+ ctx task retry should-raise-a-runtime-error
+ ''')
+ with pytest.raises(TaskAbortException):
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+
+ def test_crash_abort_after_abort(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash
+ ctx task abort abort-message
+ ctx task abort should-raise-a-runtime-error
+ ''',
+ windows_script='''
+ ctx task abort abort-message
+ ctx task abort should-raise-a-runtime-error
+ ''')
+ with pytest.raises(TaskAbortException) as exc_ctx:
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ exception = exc_ctx.value
+ assert str(exception) == constants.ILLEGAL_CTX_OPERATION_MESSAGE
+
+ def test_crash_retry_after_retry(self, executor, workflow_context, tmpdir):
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash
+ ctx task retry retry-message
+ ctx task retry should-raise-a-runtime-error
+ ''',
+ windows_script='''
+ ctx task retry retry-message
+ ctx task retry should-raise-a-runtime-error
+ ''')
+ with pytest.raises(TaskAbortException) as exc_ctx:
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ exception = exc_ctx.value
+ assert str(exception) == constants.ILLEGAL_CTX_OPERATION_MESSAGE
+
+ def test_retry_returns_a_nonzero_exit_code(self, executor, workflow_context, tmpdir):
+ log_path = tmpdir.join('temp.log')
+ message = 'message'
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task retry '{0}' 2> {1}
+ echo should-not-run > {1}
+ '''.format(message, log_path),
+ windows_script='''
+ ctx task retry "{0}" 2> {1}
+ if %errorlevel% neq 0 exit /b %errorlevel%
+ echo should-not-run > {1}
+ '''.format(message, log_path))
+ with pytest.raises(TaskAbortException):
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ assert log_path.read().strip() == message
+
+ def test_abort_returns_a_nonzero_exit_code(self, executor, workflow_context, tmpdir):
+ log_path = tmpdir.join('temp.log')
+ message = 'message'
+ script_path = self._create_script(
+ tmpdir,
+ linux_script='''#! /bin/bash -e
+ ctx task abort '{0}' 2> {1}
+ echo should-not-run > {1}
+ '''.format(message, log_path),
+ windows_script='''
+ ctx task abort "{0}" 2> {1}
+ if %errorlevel% neq 0 exit /b %errorlevel%
+ echo should-not-run > {1}
+ '''.format(message, log_path))
+ with pytest.raises(TaskAbortException):
+ self._run(
+ executor, workflow_context,
+ script_path=script_path)
+ assert log_path.read().strip() == message
+
+ def _create_script(self,
+ tmpdir,
+ linux_script,
+ windows_script,
+ windows_suffix='.bat',
+ linux_suffix=''):
+ suffix = windows_suffix if IS_WINDOWS else linux_suffix
+ script = windows_script if IS_WINDOWS else linux_script
+ script_path = tmpdir.join('script{0}'.format(suffix))
+ script_path.write(script)
+ return str(script_path)
+
+ def _run(self,
+ executor,
+ workflow_context,
+ script_path,
+ process=None,
+ env_var='value'):
+ @workflow
+ def mock_workflow(ctx, graph):
+ op = 'test.op'
+ node_instance = ctx.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+ node_instance.node.operations[op] = {
+ 'operation': '{0}.{1}'.format(operations.__name__,
+ operations.run_script_locally.__name__)}
+ graph.add_tasks(api.task.OperationTask.node_instance(
+ instance=node_instance,
+ name=op,
+ inputs={
+ 'script_path': script_path,
+ 'process': process,
+ 'input_as_env_var': env_var
+ }))
+ workflow_context.resource.deployment.upload(
+ entry_id=str(workflow_context.deployment.id),
+ source=script_path,
+ path=os.path.basename(script_path))
+ tasks_graph = mock_workflow(ctx=workflow_context)
+ eng = engine.Engine(
+ executor=executor,
+ workflow_context=workflow_context,
+ tasks_graph=tasks_graph)
+ eng.execute()
+ return workflow_context.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME).runtime_properties
+
+ @pytest.fixture
+ def executor(self):
+ result = process.ProcessExecutor()
+ yield result
+ result.close()
+
+ @pytest.fixture
+ def workflow_context(self, tmpdir):
+ workflow_context = mock.context.simple(
+ storage.get_sqlite_api_kwargs(str(tmpdir)),
+ resources_dir=str(tmpdir.join('resources')))
+ workflow_context.states = []
+ workflow_context.exception = None
+ yield workflow_context
+ storage.release_sqlite_storage(workflow_context.model)
+
+
+class BaseTestConfiguration(object):
+
+ @pytest.fixture(autouse=True)
+ def mock_execute(self, mocker):
+ def eval_func(**_):
+ self.called = 'eval'
+
+ def execute_func(process, **_):
+ self.process = process
+ self.called = 'execute'
+ self.process = {}
+ self.called = None
+ mocker.patch.object(local, '_execute_func', execute_func)
+ mocker.patch.object(local, '_eval_script_func', eval_func)
+
+ class Ctx(object):
+ @staticmethod
+ def download_resource(destination, path):
+ return destination
+
+ def _run(self, script_path, process=None):
+ local.run_script(
+ script_path=script_path,
+ process=process,
+ ctx=self.Ctx)
+
+
+class TestPowerShellConfiguration(BaseTestConfiguration):
+
+ def test_implicit_powershell_call_with_ps1_extension(self):
+ self._run(script_path='script_path.ps1')
+ assert self.process['command_prefix'] == 'powershell'
+
+ def test_command_prefix_is_overridden_for_ps1_extension(self):
+ self._run(script_path='script_path.ps1',
+ process={'command_prefix': 'bash'})
+ assert self.process['command_prefix'] == 'bash'
+
+ def test_explicit_powershell_call(self):
+ self._run(script_path='script_path.ps1',
+ process={'command_prefix': 'powershell'})
+ assert self.process['command_prefix'] == 'powershell'
+
+
+class TestEvalPythonConfiguration(BaseTestConfiguration):
+
+ def test_explicit_eval_without_py_extension(self):
+ self._run(script_path='script_path',
+ process={'eval_python': True})
+ assert self.called == 'eval'
+
+ def test_explicit_eval_with_py_extension(self):
+ self._run(script_path='script_path.py',
+ process={'eval_python': True})
+ assert self.called == 'eval'
+
+ def test_implicit_eval(self):
+ self._run(script_path='script_path.py')
+ assert self.called == 'eval'
+
+ def test_explicit_execute_without_py_extension(self):
+ self._run(script_path='script_path',
+ process={'eval_python': False})
+ assert self.called == 'execute'
+
+ def test_explicit_execute_with_py_extension(self):
+ self._run(script_path='script_path.py',
+ process={'eval_python': False})
+ assert self.called == 'execute'
+
+ def test_implicit_execute(self):
+ self._run(script_path='script_path')
+ assert self.called == 'execute'
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/tests/orchestrator/execution_plugin/test_ssh.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/execution_plugin/test_ssh.py b/tests/orchestrator/execution_plugin/test_ssh.py
new file mode 100644
index 0000000..05c2cb0
--- /dev/null
+++ b/tests/orchestrator/execution_plugin/test_ssh.py
@@ -0,0 +1,378 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import contextlib
+import json
+import getpass
+import logging
+import StringIO
+
+import pytest
+
+from fabric import api
+from fabric.contrib import files
+from fabric import context_managers
+
+from aria.storage import model
+from aria.orchestrator.exceptions import TaskAbortException
+from aria.orchestrator.execution_plugin.exceptions import ProcessException
+from aria.orchestrator.execution_plugin.ssh import operations as ssh_operations
+from aria.orchestrator.execution_plugin import constants
+
+
+_SSH_USER = 'travis'
+_SSH_PASSWORD = 'password'
+
+
+@pytest.mark.skipif(getpass.getuser() != _SSH_USER, reason='actual ssh server required')
+class TestWithActualSSHServer(object):
+
+ @pytest.fixture(autouse=True)
+ def setup(self, request):
+ self.test_name = request.node.name
+ self.custom_base_dir = '/tmp/new-aria-ctx'
+ self.default_fabric_env = {
+ 'host_string': 'localhost',
+ 'user': _SSH_USER,
+ 'password': _SSH_PASSWORD
+ }
+ with context_managers.settings(**self.default_fabric_env):
+ if files.exists(constants.DEFAULT_BASE_DIR):
+ api.run('rm -rf {0}'.format(constants.DEFAULT_BASE_DIR))
+ if files.exists(self.custom_base_dir):
+ api.run('rm -rf {0}'.format(self.custom_base_dir))
+
+ def test_run_script(self):
+ expected_runtime_property_value = 'some_value'
+ self._execute(
+ env={'test_value': expected_runtime_property_value})
+ instance = None
+ assert instance.runtime_properties['test_value'] == expected_runtime_property_value
+
+ def test_run_script_as_sudo(self):
+ self._execute(use_sudo=True)
+ with context_managers.settings(**self.default_fabric_env):
+ assert files.exists('/opt/test_dir')
+ api.sudo('rm -rf /opt/test_dir')
+
+ def test_run_script_default_base_dir(self):
+ self._execute()
+ instance = None
+ assert instance.runtime_properties['work_dir'] == constants.DEFAULT_BASE_DIR
+
+ def test_run_script_with_hide(self, mocker):
+ def execute_script(hide_groups):
+ stdout = StringIO.StringIO()
+ mocker.patch('sys.stdout', stdout)
+ try:
+ self._execute(hide_output=hide_groups)
+ finally:
+ mocker.resetall()
+ return stdout.getvalue().strip()
+ expected_log_message = '[localhost] run: source {0}/scripts/'.format(self.custom_base_dir)
+ output = execute_script(hide_groups=[])
+ assert expected_log_message in output
+ output = execute_script(hide_groups=['everything'])
+ assert expected_log_message not in output
+
+ def test_run_script_process_config(self):
+ expected_env_value = 'test_value_env'
+ expected_arg1_value = 'test_value_arg1'
+ expected_arg2_value = 'test_value_arg2'
+ expected_cwd = '/tmp'
+ expected_base_dir = self.custom_base_dir
+ self._execute(
+ process={
+ 'env': {
+ 'test_value_env': expected_env_value
+ },
+ 'args': [expected_arg1_value, expected_arg2_value],
+ 'cwd': expected_cwd,
+ 'base_dir': expected_base_dir
+ })
+ instance = None
+ assert instance.runtime_properties['env_value'] == expected_env_value
+ assert len(instance.runtime_properties['bash_version']) > 0
+ assert instance.runtime_properties['arg1_value'] == expected_arg1_value
+ assert instance.runtime_properties['arg2_value'] == expected_arg2_value
+ assert instance.runtime_properties['cwd'] == expected_cwd
+ assert instance.runtime_properties['ctx_path'] == '{0}/ctx'.format(expected_base_dir)
+
+ def test_run_script_command_prefix(self):
+ self._execute(
+ process={
+ 'command_prefix': 'dash'
+ })
+ instance = None
+ assert len(instance.runtime_properties['bash_version']) == 0
+ assert instance.runtime_properties['sanity'] == 'sanity'
+
+ def test_run_script_reuse_existing_ctx(self):
+ expected_test_value_1 = 'test_value_1'
+ expected_test_value_2 = 'test_value_2'
+ self._execute(
+ env={
+ 'test_operation': '{0}_1'.format(self.test_name),
+ 'test_value': expected_test_value_1
+ })
+ instance = None
+ assert instance.runtime_properties['test_value'] == expected_test_value_1
+ self._execute(
+ env={
+ 'test_operation': '{0}_2'.format(self.test_name),
+ 'test_value': expected_test_value_2
+ })
+ instance = None
+ assert instance.runtime_properties['test_value'] == expected_test_value_2
+
+ def test_run_script_return_value(self):
+ expected_return_value = 'expected_return_value'
+ return_value = self._execute(
+ env={'return_value': expected_return_value})
+ assert return_value == expected_return_value
+
+ def test_run_script_download_resource(self):
+ assert self._execute() == 'content'
+
+ def test_run_script_download_resource_and_render(self):
+ assert self._execute() == 'test'
+
+ def test_run_script_inputs_as_env_variables(self):
+ def test(value):
+ return_value = self._execute(
+ custom_input=value)
+ expected = return_value if isinstance(value, basestring) else json.loads(return_value)
+ assert value == expected
+ test('string-value')
+ test([1, 2, 3])
+ test({'key': 'value'})
+
+ def test_run_script_inputs_as_env_variables_process_env_override(self):
+ def test_override(value):
+ return_value = self._execute(
+ custom_input='custom-input-value',
+ env={'custom_env_var': value})
+ expected = return_value if isinstance(value, basestring) else json.loads(return_value)
+ assert value == expected
+ test_override('string-value')
+ test_override([1, 2, 3])
+ test_override({'key': 'value'})
+
+
+class TestFabricEnvHideGroupsAndRunCommands(object):
+
+ def test_fabric_env_default_override(self):
+ # first sanity for no override
+ self._run()
+ assert self.mock.settings_merged['timeout'] == constants.FABRIC_ENV_DEFAULTS['timeout']
+ # now override
+ invocation_fabric_env = self.default_fabric_env.copy()
+ timeout = 1000000
+ invocation_fabric_env['timeout'] = timeout
+ self._run(fabric_env=invocation_fabric_env)
+ assert self.mock.settings_merged['timeout'] == timeout
+
+ def test_implicit_host_string(self, mocker):
+ expected_ip = '1.1.1.1'
+ mocker.patch.object(self._Ctx.task.runs_on, 'ip', expected_ip)
+ fabric_env = self.default_fabric_env.copy()
+ del fabric_env['host_string']
+ self._run(fabric_env=fabric_env)
+ assert self.mock.settings_merged['host_string'] == expected_ip
+
+ def test_explicit_host_string(self):
+ fabric_env = self.default_fabric_env.copy()
+ host_string = 'explicit_host_string'
+ fabric_env['host_string'] = host_string
+ self._run(fabric_env=fabric_env)
+ assert self.mock.settings_merged['host_string'] == host_string
+
+ def test_override_warn_only(self):
+ fabric_env = self.default_fabric_env.copy()
+ self._run(fabric_env=fabric_env)
+ assert self.mock.settings_merged['warn_only'] is True
+ fabric_env = self.default_fabric_env.copy()
+ fabric_env['warn_only'] = False
+ self._run(fabric_env=fabric_env)
+ assert self.mock.settings_merged['warn_only'] is False
+
+ def test_missing_host_string(self):
+ with pytest.raises(TaskAbortException) as exc_ctx:
+ fabric_env = self.default_fabric_env.copy()
+ del fabric_env['host_string']
+ self._run(fabric_env=fabric_env)
+ assert '`host_string` not supplied' in str(exc_ctx.value)
+
+ def test_missing_user(self):
+ with pytest.raises(TaskAbortException) as exc_ctx:
+ fabric_env = self.default_fabric_env.copy()
+ del fabric_env['user']
+ self._run(fabric_env=fabric_env)
+ assert '`user` not supplied' in str(exc_ctx.value)
+
+ def test_missing_key_or_password(self):
+ with pytest.raises(TaskAbortException) as exc_ctx:
+ fabric_env = self.default_fabric_env.copy()
+ del fabric_env['key_filename']
+ self._run(fabric_env=fabric_env)
+ assert 'Access credentials not supplied' in str(exc_ctx.value)
+
+ def test_hide_in_settings_and_non_viable_groups(self):
+ groups = ('running', 'stdout')
+ self._run(hide_output=groups)
+ assert self.mock.settings_merged['hide_output'] == groups
+ with pytest.raises(TaskAbortException) as exc_ctx:
+ self._run(hide_output=('running', 'bla'))
+ assert '`hide_output` must be a subset of' in str(exc_ctx.value)
+
+ def test_run_commands(self):
+ def test(use_sudo):
+ commands = ['command1', 'command2']
+ self._run(
+ commands=commands,
+ use_sudo=use_sudo)
+ assert all(item in self.mock.settings_merged.items() for
+ item in self.default_fabric_env.items())
+ assert self.mock.settings_merged['warn_only'] is True
+ assert self.mock.settings_merged['use_sudo'] == use_sudo
+ assert self.mock.commands == commands
+ self.mock.settings_merged = {}
+ self.mock.commands = []
+ test(use_sudo=False)
+ test(use_sudo=True)
+
+ def test_failed_command(self):
+ with pytest.raises(ProcessException) as exc_ctx:
+ self._run(commands=['fail'])
+ exception = exc_ctx.value
+ assert exception.stdout == self.MockCommandResult.stdout
+ assert exception.stderr == self.MockCommandResult.stderr
+ assert exception.command == self.MockCommandResult.command
+ assert exception.exit_code == self.MockCommandResult.return_code
+
+ class MockCommandResult(object):
+ stdout = 'mock_stdout'
+ stderr = 'mock_stderr'
+ command = 'mock_command'
+ return_code = 1
+
+ def __init__(self, failed):
+ self.failed = failed
+
+ class MockFabricApi(object):
+
+ def __init__(self):
+ self.commands = []
+ self.settings_merged = {}
+
+ @contextlib.contextmanager
+ def settings(self, *args, **kwargs):
+ self.settings_merged.update(kwargs)
+ if args:
+ groups = args[0]
+ self.settings_merged.update({'hide_output': groups})
+ yield
+
+ def run(self, command):
+ self.commands.append(command)
+ self.settings_merged['use_sudo'] = False
+ return TestFabricEnvHideGroupsAndRunCommands.MockCommandResult(command == 'fail')
+
+ def sudo(self, command):
+ self.commands.append(command)
+ self.settings_merged['use_sudo'] = True
+ return TestFabricEnvHideGroupsAndRunCommands.MockCommandResult(command == 'fail')
+
+ def hide(self, *groups):
+ return groups
+
+ def exists(self, path):
+ raise RuntimeError
+
+ class _Ctx(object):
+ class Stub(object):
+ @staticmethod
+ def abort(message=None):
+ model.Task.abort(message)
+ ip = None
+ task = Stub
+ task.runs_on = Stub
+ logger = logging.getLogger()
+
+ @pytest.fixture(autouse=True)
+ def setup(self, mocker):
+ self.default_fabric_env = {
+ 'host_string': 'test',
+ 'user': 'test',
+ 'key_filename': 'test',
+ }
+ self.mock = self.MockFabricApi()
+ mocker.patch('fabric.api', self.mock)
+
+ def _run(self,
+ commands=(),
+ fabric_env=None,
+ process=None,
+ use_sudo=False,
+ hide_output=None):
+ ssh_operations.run_commands(
+ ctx=self._Ctx,
+ commands=commands,
+ process=process,
+ fabric_env=fabric_env or self.default_fabric_env,
+ use_sudo=use_sudo,
+ hide_output=hide_output)
+
+
+class TestUtilityFunctions(object):
+
+ def test_paths(self):
+ base_dir = '/path'
+ local_script_path = '/local/script/path.py'
+ paths = ssh_operations._Paths(base_dir=base_dir,
+ local_script_path=local_script_path)
+ assert paths.local_script_path == local_script_path
+ assert paths.remote_ctx_dir == base_dir
+ assert paths.base_script_path == 'path.py'
+ assert paths.remote_ctx_path == '/path/ctx'
+ assert paths.remote_scripts_dir == '/path/scripts'
+ assert paths.remote_work_dir == '/path/work'
+ assert paths.remote_env_script_path.startswith('/path/scripts/env-path.py-')
+ assert paths.remote_script_path.startswith('/path/scripts/path.py-')
+
+ def test_write_environment_script_file(self):
+ base_dir = '/path'
+ local_script_path = '/local/script/path.py'
+ paths = ssh_operations._Paths(base_dir=base_dir,
+ local_script_path=local_script_path)
+ env = {'one': "'1'"}
+ local_socket_url = 'local_socket_url'
+ remote_socket_url = 'remote_socket_url'
+ env_script_lines = set([l for l in ssh_operations._write_environment_script_file(
+ process={'env': env},
+ paths=paths,
+ local_socket_url=local_socket_url,
+ remote_socket_url=remote_socket_url
+ ).getvalue().split('\n') if l])
+ expected_env_script_lines = set([
+ 'export PATH=/path:$PATH',
+ 'export PYTHONPATH=/path:$PYTHONPATH',
+ 'chmod +x /path/ctx',
+ 'chmod +x {0}'.format(paths.remote_script_path),
+ 'export CTX_SOCKET_URL={0}'.format(remote_socket_url),
+ 'export LOCAL_CTX_SOCKET_URL={0}'.format(local_socket_url),
+ 'export one=\'1\''
+ ])
+ assert env_script_lines == expected_env_script_lines
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/tests/orchestrator/workflows/api/test_task.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/api/test_task.py b/tests/orchestrator/workflows/api/test_task.py
index 58e387f..51ff66d 100644
--- a/tests/orchestrator/workflows/api/test_task.py
+++ b/tests/orchestrator/workflows/api/test_task.py
@@ -16,6 +16,7 @@
import pytest
+from aria.storage import model
from aria.orchestrator import context
from aria.orchestrator.workflows import api
@@ -72,6 +73,7 @@ class TestOperationTask(object):
assert api_task.plugin == {'name': 'plugin',
'package_name': 'package',
'package_version': '0.1'}
+ assert api_task.runs_on == model.Task.RUNS_ON_NODE
def test_source_relationship_operation_task_creation(self, ctx):
operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
@@ -104,6 +106,7 @@ class TestOperationTask(object):
assert api_task.plugin == {'name': 'plugin',
'package_name': 'package',
'package_version': '0.1'}
+ assert api_task.runs_on == model.Task.RUNS_ON_SOURCE
def test_target_relationship_operation_task_creation(self, ctx):
operation_name = 'aria.interfaces.relationship_lifecycle.preconfigure'
@@ -136,6 +139,7 @@ class TestOperationTask(object):
assert api_task.plugin == {'name': 'plugin',
'package_name': 'package',
'package_version': '0.1'}
+ assert api_task.runs_on == model.Task.RUNS_ON_TARGET
def test_operation_task_default_values(self, ctx):
dependency_node_instance = ctx.model.node_instance.get_by_name(
@@ -151,6 +155,7 @@ class TestOperationTask(object):
assert task.max_attempts == ctx._task_max_attempts
assert task.ignore_failure == ctx._task_ignore_failure
assert task.plugin == {}
+ assert task.runs_on is None
class TestWorkflowTask(object):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/06b2f4d4/tests/orchestrator/workflows/core/test_task.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/core/test_task.py b/tests/orchestrator/workflows/core/test_task.py
index 5381f5d..f145400 100644
--- a/tests/orchestrator/workflows/core/test_task.py
+++ b/tests/orchestrator/workflows/core/test_task.py
@@ -38,7 +38,7 @@ def ctx(tmpdir):
class TestOperationTask(object):
- def _create_operation_task(self, ctx, node_instance):
+ def _create_node_operation_task(self, ctx, node_instance):
with workflow_context.current.push(ctx):
api_task = api.task.OperationTask.node_instance(
instance=node_instance,
@@ -46,21 +46,31 @@ class TestOperationTask(object):
core_task = core.task.OperationTask(api_task=api_task)
return api_task, core_task
- def test_operation_task_creation(self, ctx):
+ def _create_relationship_operation_task(self, ctx, relationship_instance, operation_end):
+ with workflow_context.current.push(ctx):
+ api_task = api.task.OperationTask.relationship_instance(
+ instance=relationship_instance,
+ name='aria.interfaces.relationship_lifecycle.preconfigure',
+ operation_end=operation_end)
+ core_task = core.task.OperationTask(api_task=api_task)
+ return api_task, core_task
+
+ def test_node_operation_task_creation(self, ctx):
storage_plugin = mock.models.get_plugin(package_name='p1', package_version='0.1')
storage_plugin_other = mock.models.get_plugin(package_name='p0', package_version='0.0')
ctx.model.plugin.put(storage_plugin_other)
ctx.model.plugin.put(storage_plugin)
- node_instance = \
- ctx.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
+ node_instance = ctx.model.node_instance.get_by_name(
+ mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
node = node_instance.node
node.plugins = [{'name': 'plugin1',
'package_name': 'p1',
'package_version': '0.1'}]
node.operations['aria.interfaces.lifecycle.create'] = {'plugin': 'plugin1'}
- api_task, core_task = self._create_operation_task(ctx, node_instance)
+ api_task, core_task = self._create_node_operation_task(ctx, node_instance)
storage_task = ctx.model.task.get_by_name(core_task.name)
assert storage_task.execution_name == ctx.execution.name
+ assert storage_task.runs_on.id == core_task.context.node_instance.id
assert core_task.model_task == storage_task
assert core_task.name == api_task.name
assert core_task.operation_mapping == api_task.operation_mapping
@@ -68,11 +78,25 @@ class TestOperationTask(object):
assert core_task.inputs == api_task.inputs == storage_task.inputs
assert core_task.plugin == storage_plugin
+ def test_source_relationship_operation_task_creation(self, ctx):
+ relationship_instance = ctx.model.relationship_instance.list()[0]
+ api_task, core_task = self._create_relationship_operation_task(
+ ctx, relationship_instance,
+ api.task.OperationTask.SOURCE_OPERATION)
+ assert core_task.model_task.runs_on.id == relationship_instance.source_node_instance.id
+
+ def test_target_relationship_operation_task_creation(self, ctx):
+ relationship_instance = ctx.model.relationship_instance.list()[0]
+ api_task, core_task = self._create_relationship_operation_task(
+ ctx, relationship_instance,
+ api.task.OperationTask.TARGET_OPERATION)
+ assert core_task.model_task.runs_on.id == relationship_instance.target_node_instance.id
+
def test_operation_task_edit_locked_attribute(self, ctx):
node_instance = \
ctx.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
- _, core_task = self._create_operation_task(ctx, node_instance)
+ _, core_task = self._create_node_operation_task(ctx, node_instance)
now = datetime.utcnow()
with pytest.raises(exceptions.TaskException):
core_task.status = core_task.STARTED
@@ -89,7 +113,7 @@ class TestOperationTask(object):
node_instance = \
ctx.model.node_instance.get_by_name(mock.models.DEPENDENCY_NODE_INSTANCE_NAME)
- _, core_task = self._create_operation_task(ctx, node_instance)
+ _, core_task = self._create_node_operation_task(ctx, node_instance)
future_time = datetime.utcnow() + timedelta(seconds=3)
with core_task._update():