You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by mx...@apache.org on 2016/10/20 13:17:05 UTC
[2/8] incubator-ariatosca git commit: Pylint tools package
Pylint tools package
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/2e14ee40
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/2e14ee40
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/2e14ee40
Branch: refs/heads/pylint-aria-workflow-builtin
Commit: 2e14ee40747b4502182dcf006abba5620e6a1687
Parents: 66ed02a
Author: Dan Kilman <da...@gmail.com>
Authored: Wed Oct 19 16:02:21 2016 +0300
Committer: Dan Kilman <da...@gmail.com>
Committed: Thu Oct 20 12:43:25 2016 +0300
----------------------------------------------------------------------
aria/tools/application.py | 79 +++++++++++++++++++++++++-----------------
aria/tools/lru_cache.py | 7 +++-
aria/tools/module.py | 9 +++++
aria/tools/plugin.py | 13 +++----
aria/tools/process.py | 59 ++++++++++++++++++++++++-------
aria/tools/validation.py | 32 +++++++++++++++--
6 files changed, 145 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/application.py
----------------------------------------------------------------------
diff --git a/aria/tools/application.py b/aria/tools/application.py
index 0f36ca2..32feeff 100644
--- a/aria/tools/application.py
+++ b/aria/tools/application.py
@@ -13,6 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+Convenience storage related tools.
+# TODO rename module name
+"""
+
import os
import json
import shutil
@@ -25,6 +30,10 @@ from aria.exceptions import StorageError
class StorageManager(LoggerMixin):
+ """
+ Convenience wrapper to simplify work with the lower level storage mechanism
+ """
+
def __init__(
self,
model_storage,
@@ -51,11 +60,14 @@ class StorageManager(LoggerMixin):
resource_storage,
deployment_id,
deployment_plan):
+ """
+ Create a StorageManager from a deployment
+ """
return cls(
- model_storage,
- resource_storage,
- deployment_id,
- deployment_plan,
+ model_storage=model_storage,
+ resource_storage=resource_storage,
+ deployment_id=deployment_id,
+ deployment_plan=deployment_plan,
blueprint_path=None,
blueprint_plan=None,
blueprint_id=None
@@ -69,6 +81,9 @@ class StorageManager(LoggerMixin):
blueprint_path,
blueprint_id,
blueprint_plan):
+ """
+ Create a StorageManager from a blueprint
+ """
return cls(
model_storage,
resource_storage,
@@ -248,32 +263,32 @@ class StorageManager(LoggerMixin):
def _load_plugin_from_archive(tar_source):
- if not tarfile.is_tarfile(tar_source):
- # TODO: go over the exceptions
- raise StorageError(
- 'the provided tar archive can not be read.')
+ if not tarfile.is_tarfile(tar_source):
+ # TODO: go over the exceptions
+ raise StorageError(
+ 'the provided tar archive can not be read.')
- with tarfile.open(tar_source) as tar:
- tar_members = tar.getmembers()
- # a wheel plugin will contain exactly one sub directory
- if not tar_members:
- raise StorageError(
- 'archive file structure malformed. expecting exactly one '
- 'sub directory; got none.')
- package_json_path = os.path.join(tar_members[0].name,
- 'package.json')
- try:
- package_member = tar.getmember(package_json_path)
- except KeyError:
- raise StorageError("'package.json' was not found under {0}"
- .format(package_json_path))
- try:
- package_json = tar.extractfile(package_member)
- except tarfile.ExtractError as e:
- raise StorageError(str(e))
- try:
- return json.load(package_json)
- except ValueError as e:
- raise StorageError("'package.json' is not a valid json: "
- "{json_str}. error is {error}"
- .format(json_str=package_json.read(), error=str(e)))
+ with tarfile.open(tar_source) as tar:
+ tar_members = tar.getmembers()
+ # a wheel plugin will contain exactly one sub directory
+ if not tar_members:
+ raise StorageError(
+ 'archive file structure malformed. expecting exactly one '
+ 'sub directory; got none.')
+ package_json_path = os.path.join(tar_members[0].name,
+ 'package.json')
+ try:
+ package_member = tar.getmember(package_json_path)
+ except KeyError:
+ raise StorageError("'package.json' was not found under {0}"
+ .format(package_json_path))
+ try:
+ package_json = tar.extractfile(package_member)
+ except tarfile.ExtractError as e:
+ raise StorageError(str(e))
+ try:
+ return json.load(package_json)
+ except ValueError as e:
+ raise StorageError("'package.json' is not a valid json: "
+ "{json_str}. error is {error}"
+ .format(json_str=package_json.read(), error=str(e)))
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/lru_cache.py
----------------------------------------------------------------------
diff --git a/aria/tools/lru_cache.py b/aria/tools/lru_cache.py
index 2cd2864..5863376 100755
--- a/aria/tools/lru_cache.py
+++ b/aria/tools/lru_cache.py
@@ -25,6 +25,7 @@ from collections import OrderedDict
class _LRUCache(object):
+
def __init__(self, input_func, max_size, timeout):
self._input_func = input_func
self._max_size = max_size
@@ -38,13 +39,17 @@ class _LRUCache(object):
# in case called from a regular function - the caller is None.
self._caches_dict = {}
- def _prepare_key(self, *args, **kwargs):
+ @staticmethod
+ def _prepare_key(*args, **kwargs):
kwargs_key = "".join(
imap(lambda x: str(x) + str(type(kwargs[x])) + str(kwargs[x]),
sorted(kwargs)))
return "".join(imap(lambda x: str(type(x)) + str(x), args)) + kwargs_key
def cache_clear(self, caller=None):
+ """
+ Clears the cache, optionally, only for a specific caller
+ """
# Remove the cache for the caller, only if exists:
if caller in self._caches_dict:
del self._caches_dict[caller]
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/module.py
----------------------------------------------------------------------
diff --git a/aria/tools/module.py b/aria/tools/module.py
index 535f7aa..3afc0ff 100644
--- a/aria/tools/module.py
+++ b/aria/tools/module.py
@@ -13,10 +13,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+Utility methods for dynamically loading python code
+"""
+
import importlib
def load_attribute(attribute_path):
+ """
+ Dynamically load an attribute based on the path to it.
+ e.g. some_package.some_module.some_attribute, will load the some_attribute from the
+ some_package.some_module module
+ """
module_name, attribute_name = attribute_path.rsplit('.', 1)
try:
module = importlib.import_module(module_name)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/plugin.py
----------------------------------------------------------------------
diff --git a/aria/tools/plugin.py b/aria/tools/plugin.py
index 976ac93..bb2b974 100644
--- a/aria/tools/plugin.py
+++ b/aria/tools/plugin.py
@@ -13,18 +13,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+Contains utility methods that enable dynamic python code loading
+# TODO: merge with tools.module
+"""
+
import os
from importlib import import_module
def plugin_installer(path, plugin_suffix, package=None, callback=None):
"""
-
- :param path:
- :param plugin_suffix:
- :param package:
- :param callback:
- :return:
+ Load each module under ``path`` that ends with ``plugin_suffix``. If ``callback`` is supplied,
+ call it with each loaded module.
"""
assert callback is None or callable(callback)
plugin_suffix = '{0}.py'.format(plugin_suffix)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/process.py
----------------------------------------------------------------------
diff --git a/aria/tools/process.py b/aria/tools/process.py
index 7df0026..5a3d8a0 100644
--- a/aria/tools/process.py
+++ b/aria/tools/process.py
@@ -13,6 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+Subprocess utility methods
+"""
+
import os
import subprocess
from signal import SIGKILL
@@ -23,6 +27,10 @@ from aria.exceptions import ExecutorException, ProcessException
class Process(LoggerMixin):
+ """
+ Subprocess wrapper
+ """
+
def __init__(
self,
args,
@@ -32,12 +40,7 @@ class Process(LoggerMixin):
env=None,
**kwargs):
"""
- Process class - subprocess wrapper
- :param args:
- :param stdout:
- :param stderr:
- :param cwd:
- :param env:
+ Subprocess wrapper
"""
super(Process, self).__init__(**kwargs)
self.args = args
@@ -48,7 +51,7 @@ class Process(LoggerMixin):
self._stderr = stderr
def __repr__(self):
- return '{cls.__name__}(args=self.args, cwd=self.cwd)'.format(
+ return '{cls.__name__}(args={self.args}, cwd={self.cwd})'.format(
cls=self.__class__, self=self)
def __getattr__(self, item):
@@ -56,25 +59,40 @@ class Process(LoggerMixin):
@property
def name(self):
+ """
+ The process name
+ """
return self.args[0]
@property
def pid(self):
+ """
+ The process pid (if running)
+ """
if self.is_running():
return self.process.pid
@property
def stdout(self):
- assert self.process, 'Need to run before calling thie method'
+ """
+ The process stdout
+ """
+ assert self.process, 'Need to run before calling this method'
return self.process.stdout
@property
def stderr(self):
- assert self.process, 'Need to run before calling thie method'
+ """
+ The process stderr
+ """
+ assert self.process, 'Need to run before calling this method'
return self.process.stderr
@property
def return_code(self):
+ """
+ The process return code. Will wait for process to end if it hasn't already
+ """
if self.process is None:
return None
if self.is_running():
@@ -85,6 +103,10 @@ class Process(LoggerMixin):
return self.process.returncode
def terminate(self):
+ """
+ Terminates the process by sending a SIGTERM to it. If the process did not stop after that,
+ sends a SIGKILL with 1 second interval for a maximum of 10 times.
+ """
if self.process is not None and self.process.poll() is None:
self.logger.debug('terminating process {0:d} ({1})'.format(self.process.pid, self.name))
self.process.terminate()
@@ -96,16 +118,22 @@ class Process(LoggerMixin):
sleep(1)
kill_attempts += 1
- def killpg(self):
+ def kill(self):
+ """
+ Kill the process by sending a SIGKILL to it
+ """
if self.is_running():
os.killpg(os.getpgid(self.pid), SIGKILL)
def is_running(self):
+ """
+ Returns ``True`` if the process is currently running
+ """
return self.process.poll() is None if self.process else False
def wait(self):
"""
- Block till child process finishes
+ Block until process finishes
"""
assert self.process, 'Need to run before calling thie method'
self.process.wait()
@@ -113,8 +141,6 @@ class Process(LoggerMixin):
def run(self, nice=None, universal_newlines=True):
"""
Run the child process. This call does not block.
- :param int nice: nice on the child process run
- :param bool universal_newlines:
"""
self.logger.debug('Running child process: {0}'.format(' '.join(self.args)))
self.process = subprocess.Popen(
@@ -128,6 +154,9 @@ class Process(LoggerMixin):
universal_newlines=universal_newlines)
def run_in_shell(self, nice=None, universal_newlines=True):
+ """
+ Run the child process in a shell. This call does not block.
+ """
command = ' '.join(self.args)
self.logger.debug('Running child process in shell: {0}'.format(command))
self.process = subprocess.Popen(
@@ -142,6 +171,10 @@ class Process(LoggerMixin):
universal_newlines=universal_newlines)
def raise_failure(self):
+ """
+ Raise a ProcessException if the process terminated with a non zero return code. Will wait
+ for the process to finish if it hasn't already
+ """
if self.is_running():
self.wait()
if self.return_code == 0:
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/2e14ee40/aria/tools/validation.py
----------------------------------------------------------------------
diff --git a/aria/tools/validation.py b/aria/tools/validation.py
index 1de995b..ea1dae7 100644
--- a/aria/tools/validation.py
+++ b/aria/tools/validation.py
@@ -13,14 +13,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""
+Contains validation related utilities
+"""
+
class ValidatorMixin(object):
+ """
+ A mixin that should be added to classes that require validating user input
+ """
+
_ARGUMENT_TYPE_MESSAGE = '{name} argument must be {type} based, got {arg!r}'
_ACTION_MESSAGE = 'action arg options: {actions}, got {action}'
_ARGUMENT_CHOICE_MESSAGE = '{name} argument must be in {choices}, got {arg!r}'
@classmethod
def validate_actions(cls, action):
+ """
+ Validate action is defined in the class ``ACTIONS`` attribute
+ """
# todo: remove this and use validate choice
if action not in cls.ACTIONS:
raise TypeError(cls._ACTION_MESSAGE.format(
@@ -28,33 +39,50 @@ class ValidatorMixin(object):
@classmethod
def validate_in_choice(cls, name, argument, choices):
+ """
+ Validate ``argument`` is in ``choices``
+ """
if argument not in choices:
raise TypeError(cls._ARGUMENT_CHOICE_MESSAGE.format(
name=name, choices=choices, arg=argument))
@classmethod
def validate_type(cls, argument_name, argument, expected_type):
+ """
+ Validate ``argument`` is a subclass of ``expected_type``
+ """
if not issubclass(argument, expected_type):
raise TypeError(cls._ARGUMENT_TYPE_MESSAGE.format(
name=argument_name, type=expected_type, arg=argument))
@classmethod
def validate_instance(cls, argument_name, argument, expected_type):
+ """
+ Validate ``argument`` is a instance of ``expected_type``
+ """
if not isinstance(argument, expected_type):
raise TypeError(cls._ARGUMENT_TYPE_MESSAGE.format(
name=argument_name, type=expected_type, arg=argument))
@classmethod
def validate_callable(cls, argument_name, argument):
+ """
+ Validate ``argument`` is callable
+ """
if not callable(argument):
raise TypeError(cls._ARGUMENT_TYPE_MESSAGE.format(
name=argument_name, type='callable', arg=argument))
def validate_function_arguments(func, func_kwargs):
- _KWARGS_FLAG = 8
+ """
+ Validates all required arguments are supplied to ``func`` and that no additional arguments are
+ supplied
+ """
+
+ _kwargs_flags = 8
- has_kwargs = func.func_code.co_flags & _KWARGS_FLAG != 0
+ has_kwargs = func.func_code.co_flags & _kwargs_flags != 0
args_count = func.func_code.co_argcount
# all args without the ones with default values