You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by ra...@apache.org on 2017/04/26 17:24:44 UTC

[1/2] incubator-ariatosca git commit: Improve execution cancelling [Forced Update!]

Repository: incubator-ariatosca
Updated Branches:
  refs/heads/ARIA-153-end-to-end-tests 8c8bb5f2e -> 7cc71bbbc (forced update)


Improve execution cancelling

Unhandled execution status transitions resulting from cancelling an
execution via the CLI, that we indentified and tried to address:

1. TERMINATED -> CANCELLING
You cancel the execution, but by the time we try to set the status to
CANCELLING, the execution thread had already finished, and therefore,
in SUCCEEDED status.

2. FAILED -> CANCELLING
You cancel the execution, but by the time we try to set the status to
CANCELLING, the execution thread had already encountered an error,
and therefore, in FAILED state.

3. TERMINATED -> CANCELLED
Similar to #1, but with CANCELLED instead of CANCELLING.

4. FAILED -> CANCELLED
Similar to #1, but with CANCELLED instead of CANCELLING.

In all of the above cases (#1-#4), we skip updating the execution
status, and log that the execution already succeeded/failed before we
were able to cancel it.

5. CANCELLING -> STARTED
You cancel the execution while it is still in pending state. Meanwhile,
while the execution status was already set to CANCELLING, we try to set
the execution status

6. CANCELLED -> STARTED
Similar to #5, but after the status is set to CANCELLING, it also gets
set to CANCELLED before attempting to set it to STARTED.

In cases #5-#6, we skip updtating the execution status, and nothing
is logged.


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/5cd7aec5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/5cd7aec5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/5cd7aec5

Branch: refs/heads/ARIA-153-end-to-end-tests
Commit: 5cd7aec550a4720fb63fa8b36c27ff52921d0b14
Parents: 7a4a1dd
Author: Avia Efrat <av...@gigaspaces.com>
Authored: Thu Apr 20 13:23:32 2017 +0300
Committer: Avia Efrat <av...@gigaspaces.com>
Committed: Wed Apr 26 15:37:06 2017 +0300

----------------------------------------------------------------------
 aria/cli/commands/executions.py                 | 12 +++-----
 aria/modeling/orchestration.py                  |  9 +++---
 .../workflows/core/events_handler.py            | 32 +++++++++++++++-----
 aria/orchestrator/workflows/events_logging.py   |  1 +
 tests/modeling/test_models.py                   | 22 +++++++-------
 tests/orchestrator/test_workflow_runner.py      |  2 +-
 .../orchestrator/workflows/core/test_engine.py  |  2 +-
 7 files changed, 47 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/5cd7aec5/aria/cli/commands/executions.py
----------------------------------------------------------------------
diff --git a/aria/cli/commands/executions.py b/aria/cli/commands/executions.py
index 6a1f02a..cb86376 100644
--- a/aria/cli/commands/executions.py
+++ b/aria/cli/commands/executions.py
@@ -139,7 +139,6 @@ def start(workflow_name,
     execution_thread_name = '{0}_{1}'.format(service_name, workflow_name)
     execution_thread = threading.ExceptionThread(target=workflow_runner.execute,
                                                  name=execution_thread_name)
-    execution_thread.daemon = True  # allows force-cancel to exit immediately
 
     logger.info('Starting {0}execution. Press Ctrl+C cancel'.format('dry ' if dry else ''))
     execution_thread.start()
@@ -172,11 +171,10 @@ def start(workflow_name,
 
 def _cancel_execution(workflow_runner, execution_thread, logger, log_iterator):
     logger.info('Cancelling execution. Press Ctrl+C again to force-cancel')
-    try:
-        workflow_runner.cancel()
-        while execution_thread.is_alive():
+    workflow_runner.cancel()
+    while execution_thread.is_alive():
+        try:
             execution_logging.log_list(log_iterator)
             execution_thread.join(1)
-    except KeyboardInterrupt:
-        logger.info('Force-cancelling execution')
-        # TODO handle execution (update status etc.) and exit process
+        except KeyboardInterrupt:
+            pass

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/5cd7aec5/aria/modeling/orchestration.py
----------------------------------------------------------------------
diff --git a/aria/modeling/orchestration.py b/aria/modeling/orchestration.py
index b9a75e9..f163903 100644
--- a/aria/modeling/orchestration.py
+++ b/aria/modeling/orchestration.py
@@ -56,21 +56,20 @@ class ExecutionBase(ModelMixin):
     __private_fields__ = ['service_fk',
                           'service_template']
 
-    TERMINATED = 'terminated'
+    SUCCEEDED = 'succeeded'
     FAILED = 'failed'
     CANCELLED = 'cancelled'
     PENDING = 'pending'
     STARTED = 'started'
     CANCELLING = 'cancelling'
-    FORCE_CANCELLING = 'force_cancelling'
 
-    STATES = (TERMINATED, FAILED, CANCELLED, PENDING, STARTED, CANCELLING, FORCE_CANCELLING)
-    END_STATES = (TERMINATED, FAILED, CANCELLED)
+    STATES = (SUCCEEDED, FAILED, CANCELLED, PENDING, STARTED, CANCELLING)
+    END_STATES = (SUCCEEDED, FAILED, CANCELLED)
 
     VALID_TRANSITIONS = {
         PENDING: (STARTED, CANCELLED),
         STARTED: END_STATES + (CANCELLING,),
-        CANCELLING: END_STATES + (FORCE_CANCELLING,)
+        CANCELLING: END_STATES
     }
 
     @orm.validates('status')

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/5cd7aec5/aria/orchestrator/workflows/core/events_handler.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/core/events_handler.py b/aria/orchestrator/workflows/core/events_handler.py
index 7f61bfa..b43082b 100644
--- a/aria/orchestrator/workflows/core/events_handler.py
+++ b/aria/orchestrator/workflows/core/events_handler.py
@@ -81,6 +81,9 @@ def _task_succeeded(task, *args, **kwargs):
 @events.start_workflow_signal.connect
 def _workflow_started(workflow_context, *args, **kwargs):
     execution = workflow_context.execution
+    # the execution may already be in the process of cancelling
+    if execution.status in (execution.CANCELLING, execution.CANCELLED):
+        return
     execution.status = execution.STARTED
     execution.started_at = datetime.utcnow()
     workflow_context.execution = execution
@@ -98,7 +101,7 @@ def _workflow_failed(workflow_context, exception, *args, **kwargs):
 @events.on_success_workflow_signal.connect
 def _workflow_succeeded(workflow_context, *args, **kwargs):
     execution = workflow_context.execution
-    execution.status = execution.TERMINATED
+    execution.status = execution.SUCCEEDED
     execution.ended_at = datetime.utcnow()
     workflow_context.execution = execution
 
@@ -106,13 +109,16 @@ def _workflow_succeeded(workflow_context, *args, **kwargs):
 @events.on_cancelled_workflow_signal.connect
 def _workflow_cancelled(workflow_context, *args, **kwargs):
     execution = workflow_context.execution
-    # _workflow_cancelling function may have called this function
-    # already
+    # _workflow_cancelling function may have called this function already
     if execution.status == execution.CANCELLED:
         return
-    execution.status = execution.CANCELLED
-    execution.ended_at = datetime.utcnow()
-    workflow_context.execution = execution
+    # the execution may have already been finished
+    elif execution.status in (execution.SUCCEEDED, execution.FAILED):
+        _log_tried_to_cancel_execution_but_it_already_ended(workflow_context, execution.status)
+    else:
+        execution.status = execution.CANCELLED
+        execution.ended_at = datetime.utcnow()
+        workflow_context.execution = execution
 
 
 @events.on_cancelling_workflow_signal.connect
@@ -120,8 +126,12 @@ def _workflow_cancelling(workflow_context, *args, **kwargs):
     execution = workflow_context.execution
     if execution.status == execution.PENDING:
         return _workflow_cancelled(workflow_context=workflow_context)
-    execution.status = execution.CANCELLING
-    workflow_context.execution = execution
+    # the execution may have already been finished
+    elif execution.status in (execution.SUCCEEDED, execution.FAILED):
+        _log_tried_to_cancel_execution_but_it_already_ended(workflow_context, execution.status)
+    else:
+        execution.status = execution.CANCELLING
+        workflow_context.execution = execution
 
 
 def _update_node_state_if_necessary(task, is_transitional=False):
@@ -135,3 +145,9 @@ def _update_node_state_if_necessary(task, is_transitional=False):
         if state:
             node.state = state
             task.context.model.node.update(node)
+
+
+def _log_tried_to_cancel_execution_but_it_already_ended(workflow_context, status):
+    workflow_context.logger.info(
+        "'{workflow_name}' workflow execution {status} before the cancel request"
+        "was fully processed".format(workflow_name=workflow_context.workflow_name, status=status))

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/5cd7aec5/aria/orchestrator/workflows/events_logging.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/workflows/events_logging.py b/aria/orchestrator/workflows/events_logging.py
index 7d15c81..3ffe18b 100644
--- a/aria/orchestrator/workflows/events_logging.py
+++ b/aria/orchestrator/workflows/events_logging.py
@@ -52,6 +52,7 @@ def _failure_operation_handler(task, traceback, **kwargs):
         .format(name=_get_task_name(task), task=task), extra=dict(traceback=traceback)
     )
 
+
 @events.start_workflow_signal.connect
 def _start_workflow_handler(context, **kwargs):
     context.logger.info("Starting '{ctx.workflow_name}' workflow execution".format(ctx=context))

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/5cd7aec5/tests/modeling/test_models.py
----------------------------------------------------------------------
diff --git a/tests/modeling/test_models.py b/tests/modeling/test_models.py
index d64cdba..2da2154 100644
--- a/tests/modeling/test_models.py
+++ b/tests/modeling/test_models.py
@@ -310,40 +310,40 @@ class TestExecution(object):
                                 Execution.CANCELLED,
                                 Execution.PENDING],
             Execution.STARTED: [Execution.FAILED,
-                                Execution.TERMINATED,
+                                Execution.SUCCEEDED,
                                 Execution.CANCELLED,
                                 Execution.CANCELLING,
                                 Execution.STARTED],
             Execution.CANCELLING: [Execution.FAILED,
-                                   Execution.TERMINATED,
+                                   Execution.SUCCEEDED,
                                    Execution.CANCELLED,
                                    Execution.CANCELLING],
             Execution.FAILED: [Execution.FAILED],
-            Execution.TERMINATED: [Execution.TERMINATED],
+            Execution.SUCCEEDED: [Execution.SUCCEEDED],
             Execution.CANCELLED: [Execution.CANCELLED]
         }
 
         invalid_transitions = {
             Execution.PENDING: [Execution.FAILED,
-                                Execution.TERMINATED,
+                                Execution.SUCCEEDED,
                                 Execution.CANCELLING],
             Execution.STARTED: [Execution.PENDING],
             Execution.CANCELLING: [Execution.PENDING,
                                    Execution.STARTED],
             Execution.FAILED: [Execution.PENDING,
                                Execution.STARTED,
-                               Execution.TERMINATED,
+                               Execution.SUCCEEDED,
                                Execution.CANCELLED,
                                Execution.CANCELLING],
-            Execution.TERMINATED: [Execution.PENDING,
-                                   Execution.STARTED,
-                                   Execution.FAILED,
-                                   Execution.CANCELLED,
-                                   Execution.CANCELLING],
+            Execution.SUCCEEDED: [Execution.PENDING,
+                                  Execution.STARTED,
+                                  Execution.FAILED,
+                                  Execution.CANCELLED,
+                                  Execution.CANCELLING],
             Execution.CANCELLED: [Execution.PENDING,
                                   Execution.STARTED,
                                   Execution.FAILED,
-                                  Execution.TERMINATED,
+                                  Execution.SUCCEEDED,
                                   Execution.CANCELLING],
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/5cd7aec5/tests/orchestrator/test_workflow_runner.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/test_workflow_runner.py b/tests/orchestrator/test_workflow_runner.py
index 54e940f..7374e50 100644
--- a/tests/orchestrator/test_workflow_runner.py
+++ b/tests/orchestrator/test_workflow_runner.py
@@ -86,7 +86,7 @@ def test_existing_active_executions(request, service, model):
 def test_existing_executions_but_no_active_ones(request, service, model):
     existing_terminated_execution = models.Execution(
         service=service,
-        status=models.Execution.TERMINATED,
+        status=models.Execution.SUCCEEDED,
         workflow_name='uninstall')
     model.execution.put(existing_terminated_execution)
     # no active executions exist, so no error should be raised

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/5cd7aec5/tests/orchestrator/workflows/core/test_engine.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/core/test_engine.py b/tests/orchestrator/workflows/core/test_engine.py
index 1a88f13..af9af17 100644
--- a/tests/orchestrator/workflows/core/test_engine.py
+++ b/tests/orchestrator/workflows/core/test_engine.py
@@ -157,7 +157,7 @@ class TestEngine(BaseTest):
         execution = workflow_context.execution
         assert execution.started_at <= execution.ended_at <= datetime.utcnow()
         assert execution.error is None
-        assert execution.status == models.Execution.TERMINATED
+        assert execution.status == models.Execution.SUCCEEDED
 
     def test_single_task_successful_execution(self, workflow_context, executor):
         @workflow


[2/2] incubator-ariatosca git commit: ARIA-153 Write end-to-end tests for ARIA

Posted by ra...@apache.org.
ARIA-153 Write end-to-end tests for ARIA

Created infrastructure for end-to-end tests,
plus a test for the hello-world example.


Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/7cc71bbb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/7cc71bbb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/7cc71bbb

Branch: refs/heads/ARIA-153-end-to-end-tests
Commit: 7cc71bbbc3a8ffc69b1ea4a168b4985679d4e444
Parents: 5cd7aec
Author: Ran Ziv <ra...@gigaspaces.com>
Authored: Wed Apr 26 20:21:19 2017 +0300
Committer: Ran Ziv <ra...@gigaspaces.com>
Committed: Wed Apr 26 20:24:31 2017 +0300

----------------------------------------------------------------------
 tests/end2end/__init__.py         |  0
 tests/end2end/test_hello_world.py | 52 ++++++++++++++++++
 tests/end2end/testenv.py          | 96 ++++++++++++++++++++++++++++++++++
 tests/helpers.py                  | 31 +++++++++++
 tests/parser/service_templates.py |  9 ++--
 tests/parser/utils.py             | 16 ------
 tests/requirements.txt            |  1 +
 7 files changed, 185 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7cc71bbb/tests/end2end/__init__.py
----------------------------------------------------------------------
diff --git a/tests/end2end/__init__.py b/tests/end2end/__init__.py
new file mode 100644
index 0000000..e69de29

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7cc71bbb/tests/end2end/test_hello_world.py
----------------------------------------------------------------------
diff --git a/tests/end2end/test_hello_world.py b/tests/end2end/test_hello_world.py
new file mode 100644
index 0000000..0b6a09a
--- /dev/null
+++ b/tests/end2end/test_hello_world.py
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import requests
+
+from .testenv import testenv  # pylint: disable=unused-import
+from .. import helpers
+
+
+def test_hello_world(testenv):
+    hello_world_template_uri = helpers.get_example_uri('hello-world', 'helloworld.yaml')
+    service_name = testenv.install_service(hello_world_template_uri)
+
+    try:
+        _verify_deployed_service_in_storage(service_name, testenv.model_storage)
+        _verify_webserver_running('http://localhost:9090')
+    finally:
+        # Even if some assertions failed, attempt to execute uninstall so the
+        # webserver process doesn't stay up once the test is finished
+        # TODO: remove force_service_delete=True
+        testenv.uninstall_service(force_service_delete=True)
+
+    testenv.verify_clean_storage()
+
+
+def _verify_webserver_running(http_endpoint):
+    server_response = requests.get(http_endpoint, timeout=10)
+    assert server_response.status_code == 200
+
+
+def _verify_deployed_service_in_storage(service_name, model_storage):
+    service_templates = model_storage.service_template.list()
+    assert len(service_templates) == 1
+    assert len(service_templates[0].services) == 1
+    service = service_templates[0].services[0]
+    assert service.name == service_name
+    assert len(service.executions) == 1
+    assert len(service.nodes) == 2
+    # TODO: validate node states
+    assert len(service.executions[0].logs) > 0

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7cc71bbb/tests/end2end/testenv.py
----------------------------------------------------------------------
diff --git a/tests/end2end/testenv.py b/tests/end2end/testenv.py
new file mode 100644
index 0000000..ffc65be
--- /dev/null
+++ b/tests/end2end/testenv.py
@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+import pytest
+import sh
+
+
+@pytest.fixture
+def testenv(tmpdir, request):
+    test_name = request.node.name
+    return TestEnvironment(str(tmpdir), test_name)
+
+
+class TestEnvironment(object):
+
+    def __init__(self, workdir, test_name):
+        self.workdir = workdir
+        self.test_name = test_name
+
+        # setting the workdir for the CLI to work with
+        os.environ['ARIA_WORKDIR'] = self.workdir
+        self.cli = self._get_cli()
+
+        env = self._get_aria_env()
+        self.model_storage = env.model_storage
+        self.resource_storage = env.resource_storage
+        self.plugin_manager = env.plugin_manager
+
+    def install_service(self, service_template_path, dry=False, service_template_name=None,
+                        service_name=None):
+        service_template_name = service_template_name or self.test_name
+        service_name = service_name or self.test_name
+
+        self.cli.service_templates.store(service_template_path, service_template_name)
+        self.cli.services.create(service_name, service_template_name=service_template_name)
+        self.execute_workflow(service_name, 'install', dry=dry)
+        return service_name
+
+    def uninstall_service(self, service_name=None, service_template_name=None, dry=False,
+                          force_service_delete=False):
+        service_name = service_name or self.test_name
+        self.execute_workflow(service_name, 'uninstall', dry=dry)
+        self.cli.services.delete(service_name, force=force_service_delete)
+        self.cli.service_templates.delete(service_template_name or self.test_name)
+
+    def execute_workflow(self, service_name, workflow_name, dry=False):
+        self.cli.executions.start(workflow_name, service_name=service_name, dry=dry)
+
+    def verify_clean_storage(self):
+        assert len(self.model_storage.service_template.list()) == 0
+        assert len(self.model_storage.service.list()) == 0
+        assert len(self.model_storage.execution.list()) == 0
+        assert len(self.model_storage.node_template.list()) == 0
+        assert len(self.model_storage.node.list()) == 0
+        assert len(self.model_storage.log.list()) == 0
+
+    def _get_cli(self):
+        cli = sh.aria.bake(_out=sys.stdout.write, _err=sys.stderr.write)
+
+        # the `sh` library supports underscore-dash auto-replacement for commands and option flags
+        # yet not for subcommands (e.g. `aria service-templates`); The following class fixes this.
+        class PatchedCli(object):
+            def __getattr__(self, attr):
+                if '_' in attr:
+                    return cli.bake(attr.replace('_', '-'))
+                return getattr(cli, attr)
+
+            def __call__(self, *args, **kwargs):
+                # this is to support the `aria` command itself (e.g. `aria --version` calls)
+                return cli(*args, **kwargs)
+
+        return PatchedCli()
+
+    def _get_aria_env(self):
+        # a somewhat hackish but most simple way of acquiring environment context such as
+        # the model storage, resource storage etc.
+        # note that the `ARIA_WORKDIR` environment variable must be exported before the import
+        # below is used, as the import itself will initialize the `.aria` directory.
+        from aria.cli import env as cli_env
+        reload(cli_env)  # reloading the module in-between tests
+        return cli_env.env

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7cc71bbb/tests/helpers.py
----------------------------------------------------------------------
diff --git a/tests/helpers.py b/tests/helpers.py
new file mode 100644
index 0000000..472d696
--- /dev/null
+++ b/tests/helpers.py
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from . import ROOT_DIR
+from .resources import DIR as RESOURCES_DIR
+
+
+def get_example_uri(*args):
+    return os.path.join(ROOT_DIR, 'examples', *args)
+
+
+def get_resource_uri(*args):
+    return os.path.join(RESOURCES_DIR, *args)
+
+
+def get_service_template_uri(*args):
+    return os.path.join(RESOURCES_DIR, 'service-templates', *args)

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7cc71bbb/tests/parser/service_templates.py
----------------------------------------------------------------------
diff --git a/tests/parser/service_templates.py b/tests/parser/service_templates.py
index a8fde14..56f75ab 100644
--- a/tests/parser/service_templates.py
+++ b/tests/parser/service_templates.py
@@ -17,7 +17,8 @@ import os
 
 from aria.utils.caching import cachedmethod
 
-from .utils import (get_example_uri, get_test_uri, create_context, create_consumer)
+from .utils import (create_context, create_consumer)
+from ..helpers import (get_example_uri, get_service_template_uri)
 
 
 def consume_use_case(use_case_name, consumer_class_name='instance', cache=True):
@@ -37,10 +38,10 @@ def consume_use_case(use_case_name, consumer_class_name='instance', cache=True):
 
 def consume_node_cellar(consumer_class_name='instance', cache=True):
     cachedmethod.ENABLED = cache
-    uri = get_test_uri('tosca-simple-1.0', 'node-cellar', 'node-cellar.yaml')
+    uri = get_service_template_uri('tosca-simple-1.0', 'node-cellar', 'node-cellar.yaml')
     context = create_context(uri)
-    context.args.append('--inputs=' + get_test_uri('tosca-simple-1.0', 'node-cellar',
-                                                   'inputs.yaml'))
+    context.args.append('--inputs=' + get_service_template_uri('tosca-simple-1.0', 'node-cellar',
+                                                               'inputs.yaml'))
     consumer, dumper = create_consumer(context, consumer_class_name)
     consumer.consume()
     context.validation.dump_issues()

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7cc71bbb/tests/parser/utils.py
----------------------------------------------------------------------
diff --git a/tests/parser/utils.py b/tests/parser/utils.py
index 8460de8..f0e890f 100644
--- a/tests/parser/utils.py
+++ b/tests/parser/utils.py
@@ -13,8 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-
 from aria.parser.loading import UriLocation
 from aria.parser.consumption import (
     ConsumptionContext,
@@ -28,20 +26,6 @@ from aria.parser.consumption import (
 )
 from aria.utils.imports import import_fullname
 
-from tests import ROOT_DIR
-from tests.resources import DIR
-
-
-SERVICE_TEMPLATES_DIR = os.path.join(DIR, 'service-templates')
-
-
-def get_example_uri(*args):
-    return os.path.join(ROOT_DIR, 'examples', *args)
-
-
-def get_test_uri(*args):
-    return os.path.join(SERVICE_TEMPLATES_DIR, *args)
-
 
 def create_context(uri,
                    loader_source='aria.parser.loading.DefaultLoaderSource',

http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/7cc71bbb/tests/requirements.txt
----------------------------------------------------------------------
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 2f0245a..71a227a 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -12,6 +12,7 @@
 
 testtools
 fasteners==0.13.0
+sh==1.12.13
 mock==1.0.1
 pylint==1.6.4
 pytest==3.0.2