You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by mx...@apache.org on 2016/11/27 16:21:31 UTC
[1/2] incubator-ariatosca git commit: Moved to SQLAlchemy based
models and changed architecture [Forced Update!]
Repository: incubator-ariatosca
Updated Branches:
refs/heads/SQLAlchemy-based-models 6feb4cb31 -> 1bb1f6bad (forced update)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/storage/states.py
----------------------------------------------------------------------
diff --git a/aria/storage/states.py b/aria/storage/states.py
new file mode 100644
index 0000000..b3f6828
--- /dev/null
+++ b/aria/storage/states.py
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class DeploymentModificationState(object):
+ STARTED = 'started'
+ FINISHED = 'finished'
+ ROLLEDBACK = 'rolledback'
+
+ STATES = [STARTED, FINISHED, ROLLEDBACK]
+ END_STATES = [FINISHED, ROLLEDBACK]
+
+
+class SnapshotState(object):
+ CREATED = 'created'
+ FAILED = 'failed'
+ CREATING = 'creating'
+ UPLOADED = 'uploaded'
+
+ STATES = [CREATED, FAILED, CREATING, UPLOADED]
+ END_STATES = [CREATED, FAILED, UPLOADED]
+
+
+class ExecutionState(object):
+ TERMINATED = 'terminated'
+ FAILED = 'failed'
+ CANCELLED = 'cancelled'
+ PENDING = 'pending'
+ STARTED = 'started'
+ CANCELLING = 'cancelling'
+ FORCE_CANCELLING = 'force_cancelling'
+
+ STATES = [TERMINATED, FAILED, CANCELLED, PENDING, STARTED,
+ CANCELLING, FORCE_CANCELLING]
+ END_STATES = [TERMINATED, FAILED, CANCELLED]
+ ACTIVE_STATES = [state for state in STATES if state not in END_STATES]
+
+
+class TaskState(object):
+ PENDING = 'pending'
+ RETRYING = 'retrying'
+ SENT = 'sent'
+ STARTED = 'started'
+ SUCCESS = 'success'
+ FAILED = 'failed'
+ STATES = (
+ PENDING,
+ RETRYING,
+ SENT,
+ STARTED,
+ SUCCESS,
+ FAILED,
+ )
+
+ WAIT_STATES = [PENDING, RETRYING]
+ END_STATES = [SUCCESS, FAILED]
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/storage/storage_api.py
----------------------------------------------------------------------
diff --git a/aria/storage/storage_api.py b/aria/storage/storage_api.py
new file mode 100644
index 0000000..8fa6e7f
--- /dev/null
+++ b/aria/storage/storage_api.py
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from contextlib import contextmanager
+
+from . import exceptions
+
+
+class StorageAPI(object):
+ def __init__(self, name, *args, **kwargs):
+ super(StorageAPI, self).__init__(*args, **kwargs)
+ self._name = name
+
+ @property
+ def name(self):
+ return self._name
+
+ def create(self, model_name, **kwargs):
+ raise NotImplementedError('Subclass must implement abstract create method')
+
+ @contextmanager
+ def connect(self):
+ try:
+ self._establish_connection()
+ yield self
+ except BaseException as e:
+ raise exceptions.StorageError(str(e))
+ finally:
+ self._destroy_connection()
+
+ def _establish_connection(self):
+ pass
+
+ def _destroy_connection(self):
+ pass
+
+
+class ModelAPI(StorageAPI):
+ def __init__(self, model_cls, **kwargs):
+ super(ModelAPI, self).__init__(**kwargs)
+ self._model_cls = model_cls
+
+ @property
+ def model_cls(self):
+ return self._model_cls
+
+ def get(self, model_name, entry_id, **kwargs):
+ raise NotImplementedError('Subclass must implement abstract get method')
+
+ def store(self, model_name, entry, **kwargs):
+ raise NotImplementedError('Subclass must implement abstract store method')
+
+ def delete(self, model_name, entry_id, **kwargs):
+ raise NotImplementedError('Subclass must implement abstract delete method')
+
+ def iter(self, model_name, **kwargs):
+ raise NotImplementedError('Subclass must implement abstract iter method')
+
+ def update(self, model_name, entry_id, **kwargs):
+ raise NotImplementedError('Subclass must implement abstract update method')
+
+
+class ResourceAPI(StorageAPI):
+ def data(self, entry_id, path=None, **kwargs):
+ raise NotImplementedError('Subclass must implement abstract data method')
+
+ def download(self, entry_id, destination, path=None, **kwargs):
+ raise NotImplementedError('Subclass must implement abstract download method')
+
+ def upload(self, entry_id, source, path=None, **kwargs):
+ raise NotImplementedError('Subclass must implement abstract upload method')
+
+
+def generate_lower_name(model_cls):
+ """
+ Generates the name of the class from the class object. e.g. SomeClass -> some_class
+ :param model_cls: the class to evaluate.
+ :return: lower name
+ :rtype: basestring
+ """
+ return ''.join(
+ character if character.islower() else '_{0}'.format(character.lower())
+ for character in model_cls.__name__)[1:]
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/storage/structures.py
----------------------------------------------------------------------
diff --git a/aria/storage/structures.py b/aria/storage/structures.py
index b02366e..922025c 100644
--- a/aria/storage/structures.py
+++ b/aria/storage/structures.py
@@ -26,282 +26,198 @@ classes:
* IterPointerField - represents an iterable pointers field.
* Model - abstract model implementation.
"""
-import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
-)
-
-
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+import jsonpickle
+from flask_sqlalchemy import SQLAlchemy
+db = SQLAlchemy()
-class Field(ValidatorMixin):
- """
- A single field implementation
- """
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
+class classproperty(object):
+ """A class that acts a a decorator for class-level properties
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
+ class A(object):
+ _prop1 = 1
+ _prop2 = 2
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ @classproperty
+ def foo(cls):
+ return cls._prop1 + cls._prop2
+ And use it like this:
+ print A.foo # 3
-class IterField(Field):
"""
- Represents an iterable field.
+ def __init__(self, get_func):
+ self.get_func = get_func
+
+ def __get__(self, owner_self, owner_cls):
+ return self.get_func(owner_cls)
+
+
+def foreign_key(
+ parent_table,
+ id_col_name='storage_id',
+ nullable=False,
+ column_type=db.Integer
+):
+ """Return a ForeignKey object with the relevant
+
+ :param parent_table: SQL name of the parent table
+ :param id_col_name: Name of the parent table's ID column [default: `id`]
+ :param nullable: Should the column be allowed to remain empty
+ :param column_type: The type (integer/text/etc.) of the column
+ :return:
"""
- def __init__(self, **kwargs):
- """
- Simple iterable field manager.
- This field type don't have choices option.
-
- :param kwargs: kwargs to be passed to next in line classes.
- """
- super(IterField, self).__init__(choices=(), **kwargs)
-
- def validate_value(self, name, values, *args):
- """
- Validates the value of each iterable value.
-
- :param name: the name of the field.
- :param values: the values of the field.
- """
- for value in values:
- self.validate_instance(name, value, self.type)
-
-
-class PointerField(Field):
+ return db.Column(
+ column_type,
+ db.ForeignKey(
+ '{0}.{1}'.format(parent_table.__tablename__, id_col_name),
+ ondelete='CASCADE'
+ ),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(
+ child_class_name,
+ column_name,
+ parent_class_name,
+ back_reference_name,
+ parent_id_name='storage_id'
+):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param child_class_name: Class name of the child table
+ :param column_name: Name of the column pointing to the parent table
+ :param parent_class_name: Class name of the parent table
+ :param back_reference_name: The name to give to the reference to the child
+ :param parent_id_name: Name of the parent table's ID column [default: `id`]
+ :return:
"""
- A single pointer field implementation.
-
- Any PointerField points via id to another document.
+ return db.relationship(
+ parent_class_name,
+ primaryjoin='{0}.{1} == {2}.{3}'.format(
+ child_class_name,
+ column_name,
+ parent_class_name,
+ parent_id_name
+ ),
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=db.backref(back_reference_name, cascade='all')
+ )
+
+
+def many_to_many_relationship(
+ other_table_class_name,
+ connecting_table,
+ back_reference_name
+):
+ """Return a many-to-many SQL relationship object
+
+ :param other_table_class_name: The name of the table we're connecting to
+ :param connecting_table: The secondary table used in the relationship
+ :param back_reference_name: The name to give to the reference to the
+ current table from the other table
+ :return:
"""
-
- def __init__(self, type, **kwargs):
- assert issubclass(type, Model)
- super(PointerField, self).__init__(type=type, **kwargs)
-
-
-class IterPointerField(IterField, PointerField):
+ return db.relationship(
+ other_table_class_name,
+ secondary=connecting_table,
+ backref=db.backref(back_reference_name, lazy='dynamic')
+ )
+
+
+class UTCDateTime(db.TypeDecorator):
+
+ impl = db.DateTime
+
+ def process_result_value(self, value, engine):
+ # Adhering to the same norms used in the rest of the code
+ if value is not None:
+ # When the date has a microsecond value equal to 0,
+ # isoformat returns the time as 17:22:11 instead of
+ # 17:22:11.000, so we need to adjust the returned value
+ if value.microsecond:
+ return '{0}Z'.format(value.isoformat()[:-3])
+ else:
+ return '{0}.000Z'.format(value.isoformat())
+
+ def process_bind_param(self, value, dialect):
+ if isinstance(value, basestring):
+ # SQLite only accepts datetime objects
+ return date_parser.parse(value)
+ else:
+ return value
+
+
+class SQLModelBase(db.Model):
+ """Abstract base class for all SQL models that allows [de]serialization
"""
- An iterable pointers field.
+ # SQLAlchemy syntax
+ __abstract__ = True
- Any IterPointerField points via id to other documents.
- """
- pass
+ # Indicates to the storage manager whether the table is a resource or not
+ is_resource = False
+ _private_fields = []
-class Model(object):
- """
- Base class for all of the storage models.
- """
- id = None
+ # Indicates whether the `id` column in this class should be unique
+ is_id_unique = True
- def __init__(self, **fields):
- """
- Abstract class for any model in the storage.
- The Initializer creates attributes according to the (keyword arguments) that given
- Each value is validated according to the Field.
- Each model has to have and ID Field.
+ def to_dict(self, suppress_error=False):
+ """Return a dict representation of the model
- :param fields: each item is validated and transformed into instance attributes.
+ :param suppress_error: If set to True, sets `None` to attributes that
+ it's unable to retrieve (e.g., if a relationship wasn't established
+ yet, and so it's impossible to access a property through it)
"""
- self._assert_model_have_id_field(**fields)
- missing_fields, unexpected_fields = self._setup_fields(fields)
+ if suppress_error:
+ res = dict()
+ for field in self.fields:
+ try:
+ field_value = getattr(self, field)
+ except AttributeError:
+ field_value = None
+ res[field] = field_value
+ else:
+ res = {field: getattr(self, field) for field in self.fields}
+ return res
- if missing_fields:
- raise StorageError(
- 'Model {name} got missing keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=missing_fields))
+ def to_json(self):
+ return jsonpickle.encode(self.to_dict(), unpicklable=False)
- if unexpected_fields:
- raise StorageError(
- 'Model {name} got unexpected keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=unexpected_fields))
+ @classproperty
+ def fields(cls):
+ """Return the list of field names for this table
- def __repr__(self):
- return '{name}(fields={0})'.format(sorted(self.fields), name=self.__class__.__name__)
-
- def __eq__(self, other):
- return (
- isinstance(other, self.__class__) and
- self.fields_dict == other.fields_dict)
-
- @property
- def fields(self):
+ Mostly for backwards compatibility in the code (that uses `fields`)
"""
- Iterates over the fields of the model.
- :yields: the class's field name
- """
- for name, field in vars(self.__class__).items():
- if isinstance(field, Field):
- yield name
+ return cls.__table__.columns.keys()
@property
def fields_dict(self):
- """
- Transforms the instance attributes into a dict.
+ return dict((field, getattr(self, field)) for field in self.fields)
- :return: all fields in dict format.
- :rtype dict
+ def _get_unique_id(self):
+ """A method to allow classes to override the default representation
"""
- return dict((name, getattr(self, name)) for name in self.fields)
+ return 'id', self.id
- @property
- def json(self):
- """
- Transform the dict of attributes into json
- :return:
- """
- return json.dumps(self.fields_dict)
-
- @classmethod
- def _assert_model_have_id_field(cls, **fields_initializer_values):
- if not getattr(cls, 'id', None):
- raise StorageError('Model {cls.__name__} must have id field'.format(cls=cls))
-
- if cls.id.default == cls.id.NO_DEFAULT and 'id' not in fields_initializer_values:
- raise StorageError(
- 'Model {cls.__name__} is missing required '
- 'keyword-only argument: "id"'.format(cls=cls))
-
- def _setup_fields(self, input_fields):
- missing = []
- for field_name in self.fields:
- try:
- field_obj = input_fields.pop(field_name)
- setattr(self, field_name, field_obj)
- except KeyError:
- field = getattr(self.__class__, field_name)
- if field.default == field.NO_DEFAULT:
- missing.append(field_name)
-
- unexpected_fields = input_fields.keys()
- return missing, unexpected_fields
-
-
-class Storage(LoggerMixin):
- """
- Represents the storage
- """
- def __init__(self, driver, items=(), **kwargs):
- super(Storage, self).__init__(**kwargs)
- self.driver = driver
- self.registered = {}
- for item in items:
- self.register(item)
- self.logger.debug('{name} object is ready: {0!r}'.format(
- self, name=self.__class__.__name__))
+ def __str__(self):
+ id_name, id_value = self._get_unique_id()
+ return '<{0} {1}=`{2}`>'.format(
+ self.__class__.__name__,
+ id_name,
+ id_value
+ )
def __repr__(self):
- return '{name}(driver={self.driver})'.format(
- name=self.__class__.__name__, self=self)
+ return str(self)
- def __getattr__(self, item):
- try:
- return self.registered[item]
- except KeyError:
- return super(Storage, self).__getattribute__(item)
+ def __unicode__(self):
+ return str(self)
- def setup(self):
- """
- Setup and create all storage items
- """
- for name, api in self.registered.iteritems():
- try:
- api.create()
- self.logger.debug(
- 'setup {name} in storage {self!r}'.format(name=name, self=self))
- except StorageError:
- pass
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self.fields_dict == other.fields_dict
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/mock/context.py
----------------------------------------------------------------------
diff --git a/tests/mock/context.py b/tests/mock/context.py
index 5fda07e..0790ba6 100644
--- a/tests/mock/context.py
+++ b/tests/mock/context.py
@@ -17,11 +17,11 @@ from aria import application_model_storage
from aria.orchestrator import context
from . import models
-from ..storage import InMemoryModelDriver
+from aria.storage.api_driver.in_memory import InMemoryModelAPI
def simple(**kwargs):
- storage = application_model_storage(InMemoryModelDriver())
+ storage = application_model_storage(InMemoryModelAPI())
storage.setup()
storage.blueprint.store(models.get_blueprint())
storage.deployment.store(models.get_deployment())
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/mock/models.py
----------------------------------------------------------------------
diff --git a/tests/mock/models.py b/tests/mock/models.py
index 327b0b9..585bffa 100644
--- a/tests/mock/models.py
+++ b/tests/mock/models.py
@@ -36,7 +36,7 @@ def get_dependency_node():
return models.Node(
id=DEPENDENCY_NODE_ID,
host_id=DEPENDENCY_NODE_ID,
- blueprint_id=BLUEPRINT_ID,
+ # blueprint_id=BLUEPRINT_ID,
type='test_node_type',
type_hierarchy=[],
number_of_instances=1,
@@ -57,8 +57,8 @@ def get_dependency_node_instance(dependency_node=None):
deployment_id=DEPLOYMENT_ID,
runtime_properties={'ip': '1.1.1.1'},
version=None,
- relationship_instances=[],
- node=dependency_node or get_dependency_node()
+ # relationship_instances=[],
+ node_storage_id=dependency_node.id or DEPENDENCY_NODE_ID
)
@@ -146,9 +146,15 @@ def get_deployment():
now = datetime.utcnow()
return models.Deployment(
id=DEPLOYMENT_ID,
- description=None,
+ description='',
created_at=now,
updated_at=now,
- blueprint_id=BLUEPRINT_ID,
- workflows={}
+ workflows={},
+ inputs={},
+ groups={},
+ permalink='',
+ policy_triggers={},
+ policy_types={},
+ outputs={},
+ scaling_groups={},
)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/orchestrator/context/test_workflow.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/context/test_workflow.py b/tests/orchestrator/context/test_workflow.py
index 258f0c5..51342e9 100644
--- a/tests/orchestrator/context/test_workflow.py
+++ b/tests/orchestrator/context/test_workflow.py
@@ -19,9 +19,9 @@ import pytest
from aria import application_model_storage
from aria.orchestrator import context
+from aria.storage.api_driver.in_memory import InMemoryModelAPI
from tests.mock import models
-from tests.storage import InMemoryModelDriver
class TestWorkflowContext(object):
@@ -57,7 +57,7 @@ class TestWorkflowContext(object):
@pytest.fixture(scope='function')
def storage():
- result = application_model_storage(InMemoryModelDriver())
+ result = application_model_storage(InMemoryModelAPI())
result.setup()
result.blueprint.store(models.get_blueprint())
result.deployment.store(models.get_deployment())
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/orchestrator/workflows/core/test_engine.py
----------------------------------------------------------------------
diff --git a/tests/orchestrator/workflows/core/test_engine.py b/tests/orchestrator/workflows/core/test_engine.py
index 1b00bf6..1bcc1e3 100644
--- a/tests/orchestrator/workflows/core/test_engine.py
+++ b/tests/orchestrator/workflows/core/test_engine.py
@@ -33,6 +33,7 @@ from aria.orchestrator.workflows import (
)
from aria.orchestrator.workflows.core import engine
from aria.orchestrator.workflows.executor import thread
+from aria.storage.api_driver.in_memory import InMemoryModelAPI
import tests.storage
@@ -129,7 +130,7 @@ class BaseTest(object):
@pytest.fixture(scope='function')
def workflow_context(self):
- model_storage = aria.application_model_storage(tests.storage.InMemoryModelDriver())
+ model_storage = aria.application_model_storage(InMemoryModelAPI())
model_storage.setup()
blueprint = mock.models.get_blueprint()
deployment = mock.models.get_deployment()
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/storage/__init__.py
----------------------------------------------------------------------
diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py
index 3408f2b..d5720e8 100644
--- a/tests/storage/__init__.py
+++ b/tests/storage/__init__.py
@@ -16,33 +16,6 @@
from tempfile import mkdtemp
from shutil import rmtree
-from aria.storage import ModelDriver
-
-
-class InMemoryModelDriver(ModelDriver):
- def __init__(self, **kwargs):
- super(InMemoryModelDriver, self).__init__(**kwargs)
- self.storage = {}
-
- def create(self, name, *args, **kwargs):
- self.storage[name] = {}
-
- def get(self, name, entry_id, **kwargs):
- return self.storage[name][entry_id].copy()
-
- def store(self, name, entry_id, entry, **kwargs):
- self.storage[name][entry_id] = entry
-
- def delete(self, name, entry_id, **kwargs):
- self.storage[name].pop(entry_id)
-
- def iter(self, name, **kwargs):
- for item in self.storage[name].itervalues():
- yield item.copy()
-
- def update(self, name, entry_id, **kwargs):
- self.storage[name][entry_id].update(**kwargs)
-
class TestFileSystem(object):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/storage/test_drivers.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_drivers.py b/tests/storage/test_drivers.py
deleted file mode 100644
index dccbe98..0000000
--- a/tests/storage/test_drivers.py
+++ /dev/null
@@ -1,135 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import pytest
-
-from aria.storage.drivers import FileSystemModelDriver, Driver, ModelDriver, ResourceDriver
-from aria.storage.exceptions import StorageError
-
-from . import InMemoryModelDriver, TestFileSystem
-
-
-def test_base_storage_driver():
- driver = Driver()
- driver.connect()
- driver.disconnect()
- driver.create('name')
- with driver as connection:
- assert driver is connection
- with pytest.raises(StorageError):
- with driver:
- raise StorageError()
-
-
-def test_model_base_driver():
- driver = ModelDriver()
- with pytest.raises(NotImplementedError):
- driver.get('name', 'id')
- with pytest.raises(NotImplementedError):
- driver.store('name', entry={}, entry_id=None)
- with pytest.raises(NotImplementedError):
- driver.update('name', 'id', update_field=1)
- with pytest.raises(NotImplementedError):
- driver.delete('name', 'id')
- with pytest.raises(NotImplementedError):
- driver.iter('name')
-
-
-def test_resource_base_driver():
- driver = ResourceDriver()
- with pytest.raises(NotImplementedError):
- driver.download('name', 'id', destination='dest')
- with pytest.raises(NotImplementedError):
- driver.upload('name', 'id', source='')
- with pytest.raises(NotImplementedError):
- driver.data('name', 'id')
-
-
-def test_custom_driver():
- entry_dict = {
- 'id': 'entry_id',
- 'entry_value': 'entry_value'
- }
-
- with InMemoryModelDriver() as driver:
- driver.create('entry')
- assert driver.storage['entry'] == {}
-
- driver.store(name='entry', entry=entry_dict, entry_id=entry_dict['id'])
- assert driver.get(name='entry', entry_id='entry_id') == entry_dict
-
- assert list(node for node in driver.iter('entry')) == [entry_dict]
-
- driver.update(name='entry', entry_id=entry_dict['id'], entry_value='new_value')
- assert driver.get(name='entry', entry_id='entry_id') == entry_dict
-
- driver.delete(name='entry', entry_id='entry_id')
-
- with pytest.raises(KeyError):
- driver.get(name='entry', entry_id='entry_id')
-
-
-class TestFileSystemDriver(TestFileSystem):
-
- def setup_method(self):
- super(TestFileSystemDriver, self).setup_method()
- self.driver = FileSystemModelDriver(directory=self.path)
-
- def test_name(self):
- assert repr(self.driver) == (
- 'FileSystemModelDriver(directory={self.path})'.format(self=self))
-
- def test_create(self):
- self.driver.create(name='node')
- assert os.path.exists(os.path.join(self.path, 'node'))
-
- def test_store(self):
- self.test_create()
- self.driver.store(name='node', entry_id='test_id', entry={'test': 'test'})
- assert os.path.exists(os.path.join(self.path, 'node', 'test_id'))
-
- def test_update(self):
- self.test_store()
- self.driver.update(name='node', entry_id='test_id', test='updated_test')
- entry = self.driver.get(name='node', entry_id='test_id')
- assert entry == {'test': 'updated_test'}
-
- def test_get(self):
- self.test_store()
- entry = self.driver.get(name='node', entry_id='test_id')
- assert entry == {'test': 'test'}
-
- def test_delete(self):
- self.test_store()
- self.driver.delete(name='node', entry_id='test_id')
- assert not os.path.exists(os.path.join(self.path, 'node', 'test_id'))
-
- def test_iter(self):
- self.test_create()
- entries = [
- {'test': 'test0'},
- {'test': 'test1'},
- {'test': 'test2'},
- {'test': 'test3'},
- {'test': 'test4'},
- ]
- for entry_id, entry in enumerate(entries):
- self.driver.store('node', str(entry_id), entry)
-
- for entry in self.driver.iter('node'):
- entries.pop(entries.index(entry))
-
- assert not entries
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/storage/test_field.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_field.py b/tests/storage/test_field.py
deleted file mode 100644
index cab218f..0000000
--- a/tests/storage/test_field.py
+++ /dev/null
@@ -1,124 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import pytest
-
-from aria.storage.structures import (
- Field,
- IterField,
- PointerField,
- IterPointerField,
- Model,
-)
-
-
-def model_factory():
- class TestModel(Model):
- id = Field(default='test_id')
- return TestModel()
-
-
-def test_base_field():
- field = Field()
- assert vars(field) == vars(Field(type=None, choices=(), default=Field.NO_DEFAULT))
-
-
-def test_type_check():
- field = Field(type=int)
- assert vars(field) == vars(Field(type=int, choices=(), default=Field.NO_DEFAULT))
- with pytest.raises(TypeError):
- field.validate_instance('field', 'any_value', int)
- field.validate_instance('field', 1, int)
-
-
-def test_field_choices():
- field = Field(choices=[1, 2])
- assert vars(field) == vars(Field(type=None, choices=[1, 2], default=Field.NO_DEFAULT))
- field.validate_in_choice('field', 1, field.choices)
-
- with pytest.raises(TypeError):
- field.validate_in_choice('field', 'value', field.choices)
-
-
-def test_field_without_default():
- class Test(object):
- field = Field()
- test = Test()
- with pytest.raises(AttributeError, message="'Test' object has no attribute 'field'"):
- assert test.field
-
-
-def test_field_default_func():
- def true_func():
- return True
-
- field = Field(default=true_func)
- assert vars(field) == vars(Field(type=None, choices=(), default=true_func))
- assert field.default
-
-
-def test_field_default():
- field = Field(default='value')
- assert vars(field) == vars(Field(type=None, choices=(), default='value'))
-
-
-def test_iterable_field():
- iter_field = IterField(type=int)
- assert vars(iter_field) == vars(Field(type=int, default=Field.NO_DEFAULT))
- iter_field.validate_value('iter_field', [1, 2])
- with pytest.raises(TypeError):
- iter_field.validate_value('iter_field', ['a', 1])
-
-
-def test_pointer_field():
- test_model = model_factory()
-
- pointer_field = PointerField(type=Model)
- assert vars(pointer_field) == \
- vars(PointerField(type=Model, choices=(), default=Field.NO_DEFAULT))
- with pytest.raises(AssertionError):
- PointerField(type=list)
- pointer_field.validate_value('pointer_field', test_model, None)
- with pytest.raises(TypeError):
- pointer_field.validate_value('pointer_field', int, None)
-
-
-def test_iterable_pointer_field():
- test_model = model_factory()
- iter_pointer_field = IterPointerField(type=Model)
- assert vars(iter_pointer_field) == \
- vars(IterPointerField(type=Model, default=Field.NO_DEFAULT))
- with pytest.raises(AssertionError):
- IterPointerField(type=list)
-
- iter_pointer_field.validate_value('iter_pointer_field', [test_model, test_model], None)
- with pytest.raises(TypeError):
- iter_pointer_field.validate_value('iter_pointer_field', [int, test_model], None)
-
-
-def test_custom_field_validation():
- def validation_func(name, value, instance):
- assert name == 'id'
- assert value == 'value'
- assert isinstance(instance, TestModel)
-
- class TestModel(Model):
- id = Field(default='_', validation_func=validation_func)
-
- obj = TestModel()
- obj.id = 'value'
-
- with pytest.raises(AssertionError):
- obj.id = 'not_value'
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/storage/test_model_storage.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_model_storage.py b/tests/storage/test_model_storage.py
index 17e11ae..3288e93 100644
--- a/tests/storage/test_model_storage.py
+++ b/tests/storage/test_model_storage.py
@@ -15,38 +15,35 @@
import pytest
+from aria import application_model_storage
+
from aria.storage import (
- Storage,
ModelStorage,
+ ResourceStorage,
models,
+ exceptions,
+ api_driver as storage_api,
)
-from aria.storage import structures
-from aria.storage.exceptions import StorageError
-from aria.storage.structures import Model, Field, PointerField
-from aria import application_model_storage
-
-from . import InMemoryModelDriver
def test_storage_base():
- driver = InMemoryModelDriver()
- storage = Storage(driver)
+ api = storage_api.InMemoryModelAPI
+ storage = ModelStorage(api)
- assert storage.driver == driver
+ assert storage.api == api
with pytest.raises(AttributeError):
storage.non_existent_attribute()
def test_model_storage():
- storage = ModelStorage(InMemoryModelDriver())
+ storage = ModelStorage(storage_api.InMemoryModelAPI)
storage.register(models.ProviderContext)
- storage.setup()
pc = models.ProviderContext(context={}, name='context_name', id='id1')
storage.provider_context.store(pc)
- assert storage.provider_context.get('id1') == pc
+ assert storage.provider_context.get('id1').fields_dict == pc.fields_dict
assert [pc_from_storage for pc_from_storage in storage.provider_context.iter()] == [pc]
assert [pc_from_storage for pc_from_storage in storage.provider_context] == [pc]
@@ -55,34 +52,30 @@ def test_model_storage():
assert storage.provider_context.get('id1').context == {'update_key': 0}
storage.provider_context.delete('id1')
- with pytest.raises(StorageError):
+ with pytest.raises(exceptions.StorageError):
storage.provider_context.get('id1')
def test_storage_driver():
- storage = ModelStorage(InMemoryModelDriver())
+ storage = ModelStorage(storage_api.InMemoryModelAPI)
storage.register(models.ProviderContext)
- storage.setup()
+
pc = models.ProviderContext(context={}, name='context_name', id='id2')
- storage.driver.store(name='provider_context', entry=pc.fields_dict, entry_id=pc.id)
+ storage.registered['provider_context'].store(entry=pc)
- assert storage.driver.get(
- name='provider_context',
- entry_id='id2',
- model_cls=models.ProviderContext) == pc.fields_dict
+ assert storage.registered['provider_context'].get(entry_id='id2').fields_dict == pc.fields_dict
- assert [i for i in storage.driver.iter(name='provider_context')] == [pc.fields_dict]
+ assert next(i.fields_dict for i in storage.registered['provider_context'].iter()) == pc.fields_dict
assert [i for i in storage.provider_context] == [pc]
- storage.provider_context.delete('id2')
+ storage.registered['provider_context'].delete('id2')
- with pytest.raises(StorageError):
- storage.provider_context.get('id2')
+ with pytest.raises(exceptions.StorageError):
+ storage.registered['provider_context'].get('id2')
def test_application_storage_factory():
- driver = InMemoryModelDriver()
- storage = application_model_storage(driver)
+ storage = application_model_storage(storage_api.InMemoryModelAPI)
assert storage.node
assert storage.node_instance
assert storage.plugin
@@ -95,68 +88,68 @@ def test_application_storage_factory():
assert storage.execution
assert storage.provider_context
- reused_storage = application_model_storage(driver)
+ reused_storage = application_model_storage(storage_api.InMemoryModelAPI)
assert reused_storage == storage
-def test_storage_pointers():
- class PointedModel(Model):
- id = Field()
-
- class PointingModel(Model):
- id = Field()
- pointing_field = PointerField(type=PointedModel)
-
- storage = ModelStorage(InMemoryModelDriver(), model_classes=[PointingModel])
- storage.setup()
-
- assert storage.pointed_model
- assert storage.pointing_model
-
- pointed_model = PointedModel(id='pointed_id')
-
- pointing_model = PointingModel(id='pointing_id', pointing_field=pointed_model)
- storage.pointing_model.store(pointing_model)
-
- assert storage.pointed_model.get('pointed_id') == pointed_model
- assert storage.pointing_model.get('pointing_id') == pointing_model
-
- storage.pointing_model.delete('pointing_id')
-
- with pytest.raises(StorageError):
- assert storage.pointed_model.get('pointed_id')
- assert storage.pointing_model.get('pointing_id')
-
-
-def test_storage_iter_pointers():
- class PointedIterModel(models.Model):
- id = structures.Field()
-
- class PointingIterModel(models.Model):
- id = models.Field()
- pointing_field = structures.IterPointerField(type=PointedIterModel)
-
- storage = ModelStorage(InMemoryModelDriver(), model_classes=[PointingIterModel])
- storage.setup()
-
- assert storage.pointed_iter_model
- assert storage.pointing_iter_model
-
- pointed_iter_model1 = PointedIterModel(id='pointed_id1')
- pointed_iter_model2 = PointedIterModel(id='pointed_id2')
-
- pointing_iter_model = PointingIterModel(
- id='pointing_id',
- pointing_field=[pointed_iter_model1, pointed_iter_model2])
- storage.pointing_iter_model.store(pointing_iter_model)
-
- assert storage.pointed_iter_model.get('pointed_id1') == pointed_iter_model1
- assert storage.pointed_iter_model.get('pointed_id2') == pointed_iter_model2
- assert storage.pointing_iter_model.get('pointing_id') == pointing_iter_model
+# def test_storage_pointers():
+# class PointedModel(Model):
+# id = Field()
+#
+# class PointingModel(Model):
+# id = Field()
+# pointing_field = PointerField(type=PointedModel)
+#
+# storage = ModelStorage(InMemoryModelDriver(), model_classes=[PointingModel])
+# storage.setup()
+#
+# assert storage.pointed_model
+# assert storage.pointing_model
+#
+# pointed_model = PointedModel(id='pointed_id')
+#
+# pointing_model = PointingModel(id='pointing_id', pointing_field=pointed_model)
+# storage.pointing_model.store(pointing_model)
+#
+# assert storage.pointed_model.get('pointed_id') == pointed_model
+# assert storage.pointing_model.get('pointing_id') == pointing_model
+#
+# storage.pointing_model.delete('pointing_id')
+#
+# with pytest.raises(StorageError):
+# assert storage.pointed_model.get('pointed_id')
+# assert storage.pointing_model.get('pointing_id')
- storage.pointing_iter_model.delete('pointing_id')
- with pytest.raises(StorageError):
- assert storage.pointed_iter_model.get('pointed_id1')
- assert storage.pointed_iter_model.get('pointed_id2')
- assert storage.pointing_iter_model.get('pointing_id')
+# def test_storage_iter_pointers():
+# class PointedIterModel(models.Model):
+# id = structures.Field()
+#
+# class PointingIterModel(models.Model):
+# id = models.Field()
+# pointing_field = structures.IterPointerField(type=PointedIterModel)
+#
+# storage = ModelStorage(InMemoryModelDriver(), model_classes=[PointingIterModel])
+# storage.setup()
+#
+# assert storage.pointed_iter_model
+# assert storage.pointing_iter_model
+#
+# pointed_iter_model1 = PointedIterModel(id='pointed_id1')
+# pointed_iter_model2 = PointedIterModel(id='pointed_id2')
+#
+# pointing_iter_model = PointingIterModel(
+# id='pointing_id',
+# pointing_field=[pointed_iter_model1, pointed_iter_model2])
+# storage.pointing_iter_model.store(pointing_iter_model)
+#
+# assert storage.pointed_iter_model.get('pointed_id1') == pointed_iter_model1
+# assert storage.pointed_iter_model.get('pointed_id2') == pointed_iter_model2
+# assert storage.pointing_iter_model.get('pointing_id') == pointing_iter_model
+#
+# storage.pointing_iter_model.delete('pointing_id')
+#
+# with pytest.raises(StorageError):
+# assert storage.pointed_iter_model.get('pointed_id1')
+# assert storage.pointed_iter_model.get('pointed_id2')
+# assert storage.pointing_iter_model.get('pointing_id')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/storage/test_models.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_models.py b/tests/storage/test_models.py
deleted file mode 100644
index 7e289e6..0000000
--- a/tests/storage/test_models.py
+++ /dev/null
@@ -1,364 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-from datetime import datetime
-
-import pytest
-
-from aria.storage import Model, Field
-from aria.storage.exceptions import StorageError
-from aria.storage.models import (
- DeploymentUpdateStep,
- Relationship,
- RelationshipInstance,
- Node,
- NodeInstance,
- Blueprint,
- Execution,
- Task
-)
-from tests.mock import models
-
-# TODO: add tests per model
-
-
-def test_base_model_without_fields():
- with pytest.raises(StorageError, message="Id field has to be in model fields"):
- Model()
-
-
-def test_base_model_members():
- _test_field = Field()
-
- class TestModel1(Model):
- test_field = _test_field
- id = Field(default='test_id')
-
- assert _test_field is TestModel1.test_field
-
- test_model = TestModel1(test_field='test_field_value', id='test_id')
-
- assert repr(test_model) == "TestModel1(fields=['id', 'test_field'])"
- expected = {'test_field': 'test_field_value', 'id': 'test_id'}
- assert json.loads(test_model.json) == expected
- assert test_model.fields_dict == expected
-
- with pytest.raises(StorageError):
- TestModel1()
-
- with pytest.raises(StorageError):
- TestModel1(test_field='test_field_value', id='test_id', unsupported_field='value')
-
- class TestModel2(Model):
- test_field = Field()
- id = Field()
-
- with pytest.raises(StorageError):
- TestModel2()
-
-
-def test_blueprint_model():
- Blueprint(
- plan={},
- id='id',
- description='description',
- created_at=datetime.utcnow(),
- updated_at=datetime.utcnow(),
- main_file_name='/path',
- )
- with pytest.raises(TypeError):
- Blueprint(
- plan=None,
- id='id',
- description='description',
- created_at=datetime.utcnow(),
- updated_at=datetime.utcnow(),
- main_file_name='/path',
- )
- with pytest.raises(TypeError):
- Blueprint(
- plan={},
- id=999,
- description='description',
- created_at=datetime.utcnow(),
- updated_at=datetime.utcnow(),
- main_file_name='/path',
- )
- with pytest.raises(TypeError):
- Blueprint(
- plan={},
- id='id',
- description=999,
- created_at=datetime.utcnow(),
- updated_at=datetime.utcnow(),
- main_file_name='/path',
- )
- with pytest.raises(TypeError):
- Blueprint(
- plan={},
- id='id',
- description='description',
- created_at='error',
- updated_at=datetime.utcnow(),
- main_file_name='/path',
- )
- with pytest.raises(TypeError):
- Blueprint(
- plan={},
- id='id',
- description='description',
- created_at=datetime.utcnow(),
- updated_at=None,
- main_file_name='/path',
- )
- with pytest.raises(TypeError):
- Blueprint(
- plan={},
- id='id',
- description='description',
- created_at=datetime.utcnow(),
- updated_at=None,
- main_file_name=88,
- )
- Blueprint(
- plan={},
- description='description',
- created_at=datetime.utcnow(),
- updated_at=datetime.utcnow(),
- main_file_name='/path',
- )
-
-
-def test_deployment_update_step_model():
- add_node = DeploymentUpdateStep(
- id='add_step',
- action='add',
- entity_type='node',
- entity_id='node_id')
-
- modify_node = DeploymentUpdateStep(
- id='modify_step',
- action='modify',
- entity_type='node',
- entity_id='node_id')
-
- remove_node = DeploymentUpdateStep(
- id='remove_step',
- action='remove',
- entity_type='node',
- entity_id='node_id')
-
- for step in (add_node, modify_node, remove_node):
- assert hash((step.id, step.entity_id)) == hash(step)
-
- assert remove_node < modify_node < add_node
- assert not remove_node > modify_node > add_node
-
- add_rel = DeploymentUpdateStep(
- id='add_step',
- action='add',
- entity_type='relationship',
- entity_id='relationship_id')
-
- # modify_rel = DeploymentUpdateStep(
- # id='modify_step',
- # action='modify',
- # entity_type='relationship',
- # entity_id='relationship_id')
-
- remove_rel = DeploymentUpdateStep(
- id='remove_step',
- action='remove',
- entity_type='relationship',
- entity_id='relationship_id')
-
- assert remove_rel < remove_node < add_node < add_rel
- assert not add_node < None
- # TODO fix logic here so that pylint is happy
- # assert not modify_node < modify_rel and not modify_rel < modify_node
-
-
-def _relationship(id=''):
- return Relationship(
- id='rel{0}'.format(id),
- target_id='target{0}'.format(id),
- source_id='source{0}'.format(id),
- source_interfaces={},
- source_operations={},
- target_interfaces={},
- target_operations={},
- type='type{0}'.format(id),
- type_hierarchy=[],
- properties={})
-
-
-def test_relationships():
- relationships = [_relationship(index) for index in xrange(3)]
-
- node = Node(
- blueprint_id='blueprint_id',
- type='type',
- type_hierarchy=None,
- number_of_instances=1,
- planned_number_of_instances=1,
- deploy_number_of_instances=1,
- properties={},
- operations={},
- relationships=relationships,
- min_number_of_instances=1,
- max_number_of_instances=1)
-
- for index in xrange(3):
- assert relationships[index] is \
- next(node.relationships_by_target('target{0}'.format(index)))
-
- relationship = _relationship()
-
- node = Node(
- blueprint_id='blueprint_id',
- type='type',
- type_hierarchy=None,
- number_of_instances=1,
- planned_number_of_instances=1,
- deploy_number_of_instances=1,
- properties={},
- operations={},
- relationships=[relationship, relationship, relationship],
- min_number_of_instances=1,
- max_number_of_instances=1)
-
- for node_relationship in node.relationships_by_target('target'):
- assert relationship is node_relationship
-
-
-def test_relationship_instance():
- relationship = _relationship()
- relationship_instances = [RelationshipInstance(
- id='rel{0}'.format(index),
- target_id='target_{0}'.format(index % 2),
- source_id='source_{0}'.format(index % 2),
- source_name='',
- target_name='',
- relationship=relationship,
- type='type{0}'.format(index)) for index in xrange(3)]
-
- node_instance = NodeInstance(
- deployment_id='deployment_id',
- runtime_properties={},
- version='1',
- relationship_instances=relationship_instances,
- node=Node(
- blueprint_id='blueprint_id',
- type='type',
- type_hierarchy=None,
- number_of_instances=1,
- planned_number_of_instances=1,
- deploy_number_of_instances=1,
- properties={},
- operations={},
- relationships=[],
- min_number_of_instances=1,
- max_number_of_instances=1),
- scaling_groups=()
- )
-
- from itertools import chain
-
- assert set(relationship_instances) == set(chain(
- node_instance.relationships_by_target('target_0'),
- node_instance.relationships_by_target('target_1')))
-
-
-def test_execution_status_transition():
- def create_execution(status):
- return Execution(
- id='e_id',
- deployment_id='d_id',
- workflow_id='w_id',
- blueprint_id='b_id',
- status=status,
- parameters={}
- )
-
- valid_transitions = {
- Execution.PENDING: [Execution.STARTED,
- Execution.CANCELLED,
- Execution.PENDING],
- Execution.STARTED: [Execution.FAILED,
- Execution.TERMINATED,
- Execution.CANCELLED,
- Execution.CANCELLING,
- Execution.STARTED],
- Execution.CANCELLING: [Execution.FAILED,
- Execution.TERMINATED,
- Execution.CANCELLED,
- Execution.CANCELLING],
- Execution.FAILED: [Execution.FAILED],
- Execution.TERMINATED: [Execution.TERMINATED],
- Execution.CANCELLED: [Execution.CANCELLED]
- }
-
- invalid_transitions = {
- Execution.PENDING: [Execution.FAILED,
- Execution.TERMINATED,
- Execution.CANCELLING],
- Execution.STARTED: [Execution.PENDING],
- Execution.CANCELLING: [Execution.PENDING,
- Execution.STARTED],
- Execution.FAILED: [Execution.PENDING,
- Execution.STARTED,
- Execution.TERMINATED,
- Execution.CANCELLED,
- Execution.CANCELLING],
- Execution.TERMINATED: [Execution.PENDING,
- Execution.STARTED,
- Execution.FAILED,
- Execution.CANCELLED,
- Execution.CANCELLING],
- Execution.CANCELLED: [Execution.PENDING,
- Execution.STARTED,
- Execution.FAILED,
- Execution.TERMINATED,
- Execution.CANCELLING],
- }
-
- for current_status, valid_transitioned_statues in valid_transitions.items():
- for transitioned_status in valid_transitioned_statues:
- execution = create_execution(current_status)
- execution.status = transitioned_status
-
- for current_status, invalid_transitioned_statues in invalid_transitions.items():
- for transitioned_status in invalid_transitioned_statues:
- execution = create_execution(current_status)
- with pytest.raises(ValueError):
- execution.status = transitioned_status
-
-
-def test_task_max_attempts_validation():
- def create_task(max_attempts):
- Task(execution_id='eid',
- name='name',
- operation_mapping='',
- inputs={},
- actor=models.get_dependency_node_instance(),
- max_attempts=max_attempts)
- create_task(max_attempts=1)
- create_task(max_attempts=2)
- create_task(max_attempts=Task.INFINITE_RETRIES)
- with pytest.raises(ValueError):
- create_task(max_attempts=0)
- with pytest.raises(ValueError):
- create_task(max_attempts=-2)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/storage/test_models_api.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_models_api.py b/tests/storage/test_models_api.py
deleted file mode 100644
index 2b92820..0000000
--- a/tests/storage/test_models_api.py
+++ /dev/null
@@ -1,70 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import pytest
-
-from aria.storage import _ModelApi, models
-from aria.storage.exceptions import StorageError
-
-from . import InMemoryModelDriver
-
-
-def test_models_api_base():
- driver = InMemoryModelDriver()
- driver.create('provider_context')
- table = _ModelApi('provider_context', driver, models.ProviderContext)
- assert repr(table) == (
- '{table.name}(driver={table.driver}, '
- 'model={table.model_cls})'.format(table=table))
- provider_context = models.ProviderContext(context={}, name='context_name', id='id')
-
- table.store(provider_context)
- assert table.get('id') == provider_context
-
- assert [i for i in table.iter()] == [provider_context]
- assert [i for i in table] == [provider_context]
-
- table.delete('id')
-
- with pytest.raises(StorageError):
- table.get('id')
-
-
-def test_iterable_model_api():
- driver = InMemoryModelDriver()
- driver.create('deployment_update')
- driver.create('deployment_update_step')
- model_api = _ModelApi('deployment_update', driver, models.DeploymentUpdate)
- deployment_update = models.DeploymentUpdate(
- id='id',
- deployment_id='deployment_id',
- deployment_plan={},
- execution_id='execution_id',
- steps=[models.DeploymentUpdateStep(
- id='step_id',
- action='add',
- entity_type='node',
- entity_id='node_id'
- )]
- )
-
- model_api.store(deployment_update)
- assert [i for i in model_api.iter()] == [deployment_update]
- assert [i for i in model_api] == [deployment_update]
-
- model_api.delete('id')
-
- with pytest.raises(StorageError):
- model_api.get('id')
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/tests/storage/test_resource_storage.py
----------------------------------------------------------------------
diff --git a/tests/storage/test_resource_storage.py b/tests/storage/test_resource_storage.py
index 918b270..42e69f5 100644
--- a/tests/storage/test_resource_storage.py
+++ b/tests/storage/test_resource_storage.py
@@ -18,15 +18,17 @@ import tempfile
import pytest
-from aria.storage.exceptions import StorageError
-from aria.storage import ResourceStorage, FileSystemResourceDriver
+from aria.storage import (
+ api_driver,
+ exceptions,
+ ResourceStorage
+)
from . import TestFileSystem
class TestResourceStorage(TestFileSystem):
def _create(self, storage):
storage.register('blueprint')
- storage.setup()
def _upload(self, storage, tmp_path, id):
with open(tmp_path, 'w') as f:
@@ -41,24 +43,27 @@ class TestResourceStorage(TestFileSystem):
storage.blueprint.upload(entry_id=id, source=tmp_dir)
+ def _create_storage(self):
+ return ResourceStorage(api_driver.FileSystemResourceAPI,
+ api_params=dict(directory=self.path))
+
def test_name(self):
- driver = FileSystemResourceDriver(directory=self.path)
- storage = ResourceStorage(driver, resources=['blueprint'])
- assert repr(storage) == 'ResourceStorage(driver={driver})'.format(
- driver=driver
- )
- assert repr(storage.registered['blueprint']) == (
- 'ResourceApi(driver={driver}, resource={resource_name})'.format(
- driver=driver,
- resource_name='blueprint'))
+ api = api_driver.FileSystemResourceAPI
+ storage = ResourceStorage(api_driver.FileSystemResourceAPI,
+ items=['blueprint'],
+ api_params=dict(directory=self.path))
+ assert repr(storage) == 'ResourceStorage(api={api})'.format(api=api)
+ assert 'directory={resource_dir}'.format(api=api, resource_dir=self.path) in \
+ repr(storage.registered['blueprint'])
def test_create(self):
- storage = ResourceStorage(FileSystemResourceDriver(directory=self.path))
+ storage = self._create_storage()
self._create(storage)
assert os.path.exists(os.path.join(self.path, 'blueprint'))
def test_upload_file(self):
- storage = ResourceStorage(FileSystemResourceDriver(directory=self.path))
+ storage = ResourceStorage(api_driver.FileSystemResourceAPI,
+ api_params=dict(directory=self.path))
self._create(storage)
tmpfile_path = tempfile.mkstemp(suffix=self.__class__.__name__, dir=self.path)[1]
self._upload(storage, tmpfile_path, id='blueprint_id')
@@ -74,7 +79,7 @@ class TestResourceStorage(TestFileSystem):
assert f.read() == 'fake context'
def test_download_file(self):
- storage = ResourceStorage(FileSystemResourceDriver(directory=self.path))
+ storage = self._create_storage()
self._create(storage)
tmpfile_path = tempfile.mkstemp(suffix=self.__class__.__name__, dir=self.path)[1]
tmpfile_name = os.path.basename(tmpfile_path)
@@ -90,19 +95,19 @@ class TestResourceStorage(TestFileSystem):
assert f.read() == 'fake context'
def test_download_non_existing_file(self):
- storage = ResourceStorage(FileSystemResourceDriver(directory=self.path))
+ storage = self._create_storage()
self._create(storage)
- with pytest.raises(StorageError):
+ with pytest.raises(exceptions.StorageError):
storage.blueprint.download(entry_id='blueprint_id', destination='', path='fake_path')
def test_data_non_existing_file(self):
- storage = ResourceStorage(FileSystemResourceDriver(directory=self.path))
+ storage = self._create_storage()
self._create(storage)
- with pytest.raises(StorageError):
+ with pytest.raises(exceptions.StorageError):
storage.blueprint.data(entry_id='blueprint_id', path='fake_path')
def test_data_file(self):
- storage = ResourceStorage(FileSystemResourceDriver(directory=self.path))
+ storage = self._create_storage()
self._create(storage)
tmpfile_path = tempfile.mkstemp(suffix=self.__class__.__name__, dir=self.path)[1]
self._upload(storage, tmpfile_path, 'blueprint_id')
@@ -110,7 +115,7 @@ class TestResourceStorage(TestFileSystem):
assert storage.blueprint.data(entry_id='blueprint_id') == 'fake context'
def test_upload_dir(self):
- storage = ResourceStorage(FileSystemResourceDriver(directory=self.path))
+ storage = self._create_storage()
self._create(storage)
tmp_dir = tempfile.mkdtemp(suffix=self.__class__.__name__, dir=self.path)
second_level_tmp_dir = tempfile.mkdtemp(dir=tmp_dir)
@@ -127,7 +132,7 @@ class TestResourceStorage(TestFileSystem):
assert os.path.isfile(destination)
def test_upload_path_in_dir(self):
- storage = ResourceStorage(FileSystemResourceDriver(directory=self.path))
+ storage = self._create_storage()
self._create(storage)
tmp_dir = tempfile.mkdtemp(suffix=self.__class__.__name__, dir=self.path)
second_level_tmp_dir = tempfile.mkdtemp(dir=tmp_dir)
@@ -151,7 +156,7 @@ class TestResourceStorage(TestFileSystem):
os.path.basename(second_update_file)))
def test_download_dir(self):
- storage = ResourceStorage(FileSystemResourceDriver(directory=self.path))
+ storage = self._create_storage()
self._create(storage)
tmp_dir = tempfile.mkdtemp(suffix=self.__class__.__name__, dir=self.path)
second_level_tmp_dir = tempfile.mkdtemp(dir=tmp_dir)
@@ -174,7 +179,7 @@ class TestResourceStorage(TestFileSystem):
assert f.read() == 'fake context'
def test_data_dir(self):
- storage = ResourceStorage(FileSystemResourceDriver(directory=self.path))
+ storage = self._create_storage()
self._create(storage)
tmp_dir = tempfile.mkdtemp(suffix=self.__class__.__name__, dir=self.path)
@@ -183,5 +188,5 @@ class TestResourceStorage(TestFileSystem):
storage.blueprint.upload(entry_id='blueprint_id', source=tmp_dir)
- with pytest.raises(StorageError):
+ with pytest.raises(exceptions.StorageError):
storage.blueprint.data(entry_id='blueprint_id')
[2/2] incubator-ariatosca git commit: Moved to SQLAlchemy based
models and changed architecture
Posted by mx...@apache.org.
Moved to SQLAlchemy based models and changed architecture
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/1bb1f6ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/1bb1f6ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/1bb1f6ba
Branch: refs/heads/SQLAlchemy-based-models
Commit: 1bb1f6bad90e5fcf8699faa672d2bc53f16998e1
Parents: b33c70e
Author: mxmrlv <mx...@gmail.com>
Authored: Sun Nov 27 13:20:46 2016 +0200
Committer: mxmrlv <mx...@gmail.com>
Committed: Sun Nov 27 18:20:54 2016 +0200
----------------------------------------------------------------------
aria/__init__.py | 39 +-
aria/orchestrator/context/workflow.py | 2 +-
aria/storage/__init__.py | 372 +-------
aria/storage/api_driver/__init__.py | 18 +
aria/storage/api_driver/file_system.py | 229 +++++
aria/storage/api_driver/in_memory.py | 146 ++++
aria/storage/api_driver/sql.py | 44 +
aria/storage/drivers.py | 416 ---------
aria/storage/models.py | 853 +++++++++++--------
aria/storage/states.py | 68 ++
aria/storage/storage_api.py | 94 ++
aria/storage/structures.py | 410 ++++-----
tests/mock/context.py | 4 +-
tests/mock/models.py | 18 +-
tests/orchestrator/context/test_workflow.py | 4 +-
.../orchestrator/workflows/core/test_engine.py | 3 +-
tests/storage/__init__.py | 27 -
tests/storage/test_drivers.py | 135 ---
tests/storage/test_field.py | 124 ---
tests/storage/test_model_storage.py | 167 ++--
tests/storage/test_models.py | 364 --------
tests/storage/test_models_api.py | 70 --
tests/storage/test_resource_storage.py | 55 +-
23 files changed, 1456 insertions(+), 2206 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/__init__.py
----------------------------------------------------------------------
diff --git a/aria/__init__.py b/aria/__init__.py
index 3f81f98..4dd23c9 100644
--- a/aria/__init__.py
+++ b/aria/__init__.py
@@ -23,7 +23,7 @@ import pkgutil
from .VERSION import version as __version__
from .orchestrator.decorators import workflow, operation
-from .storage import ModelStorage, ResourceStorage, models, ModelDriver, ResourceDriver
+from . import storage
from . import (
utils,
parser,
@@ -58,37 +58,36 @@ def install_aria_extensions():
del sys.modules[module_name]
-def application_model_storage(driver):
+def application_model_storage(api):
"""
Initiate model storage for the supplied storage driver
"""
- assert isinstance(driver, ModelDriver)
- if driver not in _model_storage:
- _model_storage[driver] = ModelStorage(
- driver, model_classes=[
- models.Node,
- models.NodeInstance,
- models.Plugin,
- models.Blueprint,
- models.Snapshot,
- models.Deployment,
- models.DeploymentUpdate,
- models.DeploymentModification,
- models.Execution,
- models.ProviderContext,
- models.Task,
+ if api not in _model_storage:
+ _model_storage[api] = storage.ModelStorage(
+ api, items=[
+ storage.models.Node,
+ storage.models.NodeInstance,
+ storage.models.Plugin,
+ storage.models.Blueprint,
+ storage.models.Snapshot,
+ storage.models.Deployment,
+ storage.models.DeploymentUpdate,
+ storage.models.DeploymentUpdateStep,
+ storage.models.DeploymentModification,
+ storage.models.Execution,
+ storage.models.ProviderContext,
+ storage.models.Task,
])
- return _model_storage[driver]
+ return _model_storage[api]
def application_resource_storage(driver):
"""
Initiate resource storage for the supplied storage driver
"""
- assert isinstance(driver, ResourceDriver)
if driver not in _resource_storage:
- _resource_storage[driver] = ResourceStorage(
+ _resource_storage[driver] = storage.ResourceStorage(
driver,
resources=[
'blueprint',
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/orchestrator/context/workflow.py
----------------------------------------------------------------------
diff --git a/aria/orchestrator/context/workflow.py b/aria/orchestrator/context/workflow.py
index 3dc222b..280390c 100644
--- a/aria/orchestrator/context/workflow.py
+++ b/aria/orchestrator/context/workflow.py
@@ -76,7 +76,7 @@ class WorkflowContext(BaseContext):
class _CurrentContext(threading.local):
"""
- Provides thread-level context, which sugarcoats the task api.
+ Provides thread-level context, which sugarcoats the task api_driver.
"""
def __init__(self):
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/storage/__init__.py
----------------------------------------------------------------------
diff --git a/aria/storage/__init__.py b/aria/storage/__init__.py
index 2d142a5..9cb6c77 100644
--- a/aria/storage/__init__.py
+++ b/aria/storage/__init__.py
@@ -20,14 +20,14 @@ Path: aria.storage
Storage package is a generic abstraction over different storage types.
We define this abstraction with the following components:
-1. storage: simple api to use
-2. driver: implementation of the database client api.
+1. storage: simple api_driver to use
+2. driver: implementation of the database client api_driver.
3. model: defines the structure of the table/document.
4. field: defines a field/item in the model.
API:
* application_storage_factory - function, default Aria storage factory.
- * Storage - class, simple storage api.
+ * Storage - class, simple storage api_driver.
* models - module, default Aria standard models.
* structures - module, default Aria structures - holds the base model,
and different fields types.
@@ -37,354 +37,60 @@ API:
* drivers - module, a pool of Aria standard drivers.
* StorageDriver - class, abstract model implementation.
"""
-# todo: rewrite the above package documentation
-# (something like explaning the two types of storage - models and resources)
-from collections import namedtuple
+from aria.logger import LoggerMixin
+from aria.storage import storage_api
+from aria.storage.exceptions import StorageError
+from . import models, exceptions, storage_api, structures
-from .structures import Storage, Field, Model, IterField, PointerField
-from .drivers import (
- ModelDriver,
- ResourceDriver,
- FileSystemResourceDriver,
- FileSystemModelDriver,
-)
-from . import models, exceptions
__all__ = (
'ModelStorage',
- 'ResourceStorage',
- 'FileSystemModelDriver',
'models',
'structures',
- 'Field',
- 'IterField',
- 'PointerField',
- 'Model',
- 'drivers',
- 'ModelDriver',
- 'ResourceDriver',
- 'FileSystemResourceDriver',
)
-# todo: think about package output api's...
-# todo: in all drivers name => entry_type
-# todo: change in documentation str => basestring
-
-
-class ModelStorage(Storage):
- """
- Managing the models storage.
- """
- def __init__(self, driver, model_classes=(), **kwargs):
- """
- Simple storage client api for Aria applications.
- The storage instance defines the tables/documents/code api.
-
- :param ModelDriver driver: model storage driver.
- :param model_classes: the models to register.
- """
- assert isinstance(driver, ModelDriver)
- super(ModelStorage, self).__init__(driver, model_classes, **kwargs)
-
- def __getattr__(self, table):
- """
- getattr is a shortcut to simple api
-
- for Example:
- >> storage = ModelStorage(driver=FileSystemModelDriver('/tmp'))
- >> node_table = storage.node
- >> for node in node_table:
- >> print node
-
- :param str table: table name to get
- :return: a storage object that mapped to the table name
- """
- return super(ModelStorage, self).__getattr__(table)
-
- def register(self, model_cls):
- """
- Registers the model type in the resource storage manager.
- :param model_cls: the model to register.
- """
- model_name = generate_lower_name(model_cls)
- model_api = _ModelApi(model_name, self.driver, model_cls)
- self.registered[model_name] = model_api
-
- for pointer_schema_register in model_api.pointer_mapping.values():
- model_cls = pointer_schema_register.model_cls
- self.register(model_cls)
-
-_Pointer = namedtuple('_Pointer', 'name, is_iter')
-
-
-class _ModelApi(object):
- def __init__(self, name, driver, model_cls):
- """
- Managing the model in the storage, using the driver.
-
- :param basestring name: the name of the model.
- :param ModelDriver driver: the driver which supports this model in the storage.
- :param Model model_cls: table/document class model.
- """
- assert isinstance(driver, ModelDriver)
- assert issubclass(model_cls, Model)
- self.name = name
- self.driver = driver
- self.model_cls = model_cls
- self.pointer_mapping = {}
- self._setup_pointers_mapping()
-
- def _setup_pointers_mapping(self):
- for field_name, field_cls in vars(self.model_cls).items():
- if not(isinstance(field_cls, PointerField) and field_cls.type):
- continue
- pointer_key = _Pointer(field_name, is_iter=isinstance(field_cls, IterField))
- self.pointer_mapping[pointer_key] = self.__class__(
- name=generate_lower_name(field_cls.type),
- driver=self.driver,
- model_cls=field_cls.type)
-
- def __iter__(self):
- return self.iter()
-
- def __repr__(self):
- return '{self.name}(driver={self.driver}, model={self.model_cls})'.format(self=self)
-
- def create(self):
- """
- Creates the model in the storage.
- """
- with self.driver as connection:
- connection.create(self.name)
-
- def get(self, entry_id, **kwargs):
- """
- Getter for the model from the storage.
-
- :param basestring entry_id: the id of the table/document.
- :return: model instance
- :rtype: Model
- """
- with self.driver as connection:
- data = connection.get(
- name=self.name,
- entry_id=entry_id,
- **kwargs)
- data.update(self._get_pointers(data, **kwargs))
- return self.model_cls(**data)
-
- def store(self, entry, **kwargs):
- """
- Setter for the model in the storage.
-
- :param Model entry: the table/document to store.
- """
- assert isinstance(entry, self.model_cls)
- with self.driver as connection:
- data = entry.fields_dict
- data.update(self._store_pointers(data, **kwargs))
- connection.store(
- name=self.name,
- entry_id=entry.id,
- entry=data,
- **kwargs)
- def delete(self, entry_id, **kwargs):
- """
- Delete the model from storage.
- :param basestring entry_id: id of the entity to delete from storage.
- """
- entry = self.get(entry_id)
- with self.driver as connection:
- self._delete_pointers(entry, **kwargs)
- connection.delete(
- name=self.name,
- entry_id=entry_id,
- **kwargs)
-
- def iter(self, **kwargs):
- """
- Generator over the entries of model in storage.
- """
- with self.driver as connection:
- for data in connection.iter(name=self.name, **kwargs):
- data.update(self._get_pointers(data, **kwargs))
- yield self.model_cls(**data)
-
- def update(self, entry_id, **kwargs):
- """
- Updates and entry in storage.
-
- :param str entry_id: the id of the table/document.
- :param kwargs: the fields to update.
- :return:
- """
- with self.driver as connection:
- connection.update(
- name=self.name,
- entry_id=entry_id,
- **kwargs
- )
-
- def _get_pointers(self, data, **kwargs):
- pointers = {}
- for field, schema in self.pointer_mapping.items():
- if field.is_iter:
- pointers[field.name] = [
- schema.get(entry_id=pointer_id, **kwargs)
- for pointer_id in data[field.name]
- if pointer_id]
- elif data[field.name]:
- pointers[field.name] = schema.get(entry_id=data[field.name], **kwargs)
- return pointers
-
- def _store_pointers(self, data, **kwargs):
- pointers = {}
- for field, model_api in self.pointer_mapping.items():
- if field.is_iter:
- pointers[field.name] = []
- for iter_entity in data[field.name]:
- pointers[field.name].append(iter_entity.id)
- model_api.store(iter_entity, **kwargs)
- else:
- pointers[field.name] = data[field.name].id
- model_api.store(data[field.name], **kwargs)
- return pointers
-
- def _delete_pointers(self, entry, **kwargs):
- for field, schema in self.pointer_mapping.items():
- if field.is_iter:
- for iter_entry in getattr(entry, field.name):
- schema.delete(iter_entry.id, **kwargs)
- else:
- schema.delete(getattr(entry, field.name).id, **kwargs)
-
-
-class ResourceApi(object):
+class Storage(LoggerMixin):
"""
- Managing the resource in the storage, using the driver.
-
- :param basestring name: the name of the resource.
- :param ResourceDriver driver: the driver which supports this resource in the storage.
+ Represents the storage
"""
- def __init__(self, driver, resource_name):
- """
- Managing the resources in the storage, using the driver.
-
- :param ResourceDriver driver: the driver which supports this model in the storage.
- :param basestring resource_name: the type of the entry this resourceAPI manages.
- """
- assert isinstance(driver, ResourceDriver)
- self.driver = driver
- self.resource_name = resource_name
+ def __init__(self, api, items=(), api_params=None, **kwargs):
+ self._api_params = api_params or {}
+ super(Storage, self).__init__(**kwargs)
+ self.api = api
+ self.registered = {}
+ for item in items:
+ self.register(item)
+ self.logger.debug('{name} object is ready: {0!r}'.format(
+ self, name=self.__class__.__name__))
def __repr__(self):
- return '{name}(driver={self.driver}, resource={self.resource_name})'.format(
- name=self.__class__.__name__, self=self)
-
- def create(self):
- """
- Create the resource dir in the storage.
- """
- with self.driver as connection:
- connection.create(self.resource_name)
-
- def data(self, entry_id, path=None, **kwargs):
- """
- Retrieve the content of a storage resource.
-
- :param basestring entry_id: the id of the entry.
- :param basestring path: path of the resource on the storage.
- :param kwargs: resources to be passed to the driver..
- :return the content of a single file:
- """
- with self.driver as connection:
- return connection.data(
- entry_type=self.resource_name,
- entry_id=entry_id,
- path=path,
- **kwargs)
+ return '{name}(api={self.api})'.format(name=self.__class__.__name__, self=self)
- def download(self, entry_id, destination, path=None, **kwargs):
- """
- Download a file/dir from the resource storage.
+ def __getattr__(self, item):
+ try:
+ return self.registered[item]
+ except KeyError:
+ return super(Storage, self).__getattribute__(item)
- :param basestring entry_id: the id of the entry.
- :param basestring destination: the destination of the file/dir.
- :param basestring path: path of the resource on the storage.
- """
- with self.driver as connection:
- connection.download(
- entry_type=self.resource_name,
- entry_id=entry_id,
- destination=destination,
- path=path,
- **kwargs)
-
- def upload(self, entry_id, source, path=None, **kwargs):
- """
- Upload a file/dir from the resource storage.
-
- :param basestring entry_id: the id of the entry.
- :param basestring source: the source path of the file to upload.
- :param basestring path: the destination of the file, relative to the root dir
- of the resource
- """
- with self.driver as connection:
- connection.upload(
- entry_type=self.resource_name,
- entry_id=entry_id,
- source=source,
- path=path,
- **kwargs)
-
-
-def generate_lower_name(model_cls):
- """
- Generates the name of the class from the class object. e.g. SomeClass -> some_class
- :param model_cls: the class to evaluate.
- :return: lower name
- :rtype: basestring
- """
- return ''.join(
- character if character.islower() else '_{0}'.format(character.lower())
- for character in model_cls.__name__)[1:]
+ def register(self, name):
+ raise NotImplementedError("BBBBBBB")
class ResourceStorage(Storage):
- """
- Managing the resource storage.
- """
- def __init__(self, driver, resources=(), **kwargs):
- """
- Simple storage client api for Aria applications.
- The storage instance defines the tables/documents/code api.
-
- :param ResourceDriver driver: resource storage driver
- :param resources: the resources to register.
- """
- assert isinstance(driver, ResourceDriver)
- super(ResourceStorage, self).__init__(driver, resources, **kwargs)
+ def register(self, name):
+ self.registered[name] = self.api(name=name,
+ **self._api_params)
+ self.registered[name].create()
+ self.logger.debug('setup {name} in storage {self!r}'.format(name=name, self=self))
- def register(self, resource):
- """
- Registers the resource type in the resource storage manager.
- :param resource: the resource to register.
- """
- self.registered[resource] = ResourceApi(self.driver, resource_name=resource)
- def __getattr__(self, resource):
- """
- getattr is a shortcut to simple api
-
- for Example:
- >> storage = ResourceStorage(driver=FileSystemResourceDriver('/tmp'))
- >> blueprint_resources = storage.blueprint
- >> blueprint_resources.download(blueprint_id, destination='~/blueprint/')
-
- :param str resource: resource name to download
- :return: a storage object that mapped to the resource name
- :rtype: ResourceApi
- """
- return super(ResourceStorage, self).__getattr__(resource)
+class ModelStorage(Storage):
+ def register(self, model):
+ model_name = storage_api.generate_lower_name(model)
+ self.registered[model_name] = self.api(name=model_name,
+ model_cls=model,
+ **self._api_params)
+ self.registered[model_name].create()
+ self.logger.debug('setup {name} in storage {self!r}'.format(name=model_name, self=self))
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/storage/api_driver/__init__.py
----------------------------------------------------------------------
diff --git a/aria/storage/api_driver/__init__.py b/aria/storage/api_driver/__init__.py
new file mode 100644
index 0000000..9d0095b
--- /dev/null
+++ b/aria/storage/api_driver/__init__.py
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .file_system import FileSystemModelAPI, FileSystemResourceAPI
+from .in_memory import InMemoryModelAPI
+from .sql import SQLAlchemyModelAPI
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/storage/api_driver/file_system.py
----------------------------------------------------------------------
diff --git a/aria/storage/api_driver/file_system.py b/aria/storage/api_driver/file_system.py
new file mode 100644
index 0000000..45ecbe8
--- /dev/null
+++ b/aria/storage/api_driver/file_system.py
@@ -0,0 +1,229 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import shutil
+from functools import partial
+from multiprocessing import RLock
+from distutils import dir_util
+
+import jsonpickle
+
+from .. import (
+ storage_api,
+ StorageError
+)
+
+
+# TODO: fix docs
+
+
+class FileSystemAPI(storage_api.StorageAPI):
+ """
+ Base class which handles storage on the file system.
+ """
+ def __init__(self, *args, **kwargs):
+ super(FileSystemAPI, self).__init__(*args, **kwargs)
+ self._lock = RLock()
+
+ def _establish_connection(self):
+ self._lock.acquire()
+
+ def _destroy_connection(self):
+ self._lock.release()
+
+ def __getstate__(self):
+ obj_dict = super(FileSystemAPI, self).__getstate__()
+ del obj_dict['_lock']
+ return obj_dict
+
+ def __setstate__(self, obj_dict):
+ super(FileSystemAPI, self).__setstate__(obj_dict)
+ vars(self).update(_lock=RLock(), **obj_dict)
+
+
+class FileSystemModelAPI(storage_api.ModelAPI, FileSystemAPI):
+ """
+ File system model storage.
+ """
+
+ def __init__(self, directory, **kwargs):
+ """
+ File system implementation for storage api.
+ :param str directory: root dir for storage.
+ """
+ super(FileSystemModelAPI, self).__init__(**kwargs)
+ self.directory = directory
+
+ self._join_path = partial(os.path.join, self.directory)
+
+ def __repr__(self):
+ return '{cls.__name__}(directory={self.directory})'.format(
+ cls=self.__class__, self=self)
+
+ def create(self, **kwargs):
+ """
+ Create directory in storage by path.
+ tries to create the root directory as well.
+ :param str name: path of file in storage.
+ """
+ try:
+ os.makedirs(self.directory)
+ except (OSError, IOError):
+ pass
+ os.makedirs(self._join_path(self.name))
+
+ def get(self, entry_id, **kwargs):
+ """
+ Getter from storage.
+ :param str name: name of directory in storage.
+ :param str entry_id: id of the file to get from storage.
+ :return: value of file from storage.
+ :rtype: dict
+ """
+ with open(self._join_path(self.name, entry_id)) as file_obj:
+ return jsonpickle.loads(file_obj.read())
+
+ def store(self, entry, **kwargs):
+ """
+ Delete from storage.
+ :param Model entry: name of directory in storage.
+ """
+ with open(self._join_path(self.name, entry.id), 'w') as file_obj:
+ file_obj.write(jsonpickle.dumps(entry))
+
+ def delete(self, entry_id, **kwargs):
+ """
+ Delete from storage.
+ :param str name: name of directory in storage.
+ :param str entry_id: id of the file to delete from storage.
+ """
+ os.remove(self._join_path(self.name, entry_id))
+
+ def iter(self, filters=None, **kwargs):
+ """
+ Generator over the entries of directory in storage.
+ :param dict filters: filters for query
+ """
+ filters = filters or {}
+
+ for entry_id in os.listdir(self._join_path(self.name)):
+ value = self.get(self.name, entry_id=entry_id)
+ for filter_name, filter_value in filters.items():
+ if value.get(filter_name) != filter_value:
+ break
+ else:
+ yield value
+
+ def update(self, entry_id, **kwargs):
+ """
+ Updates and entry in storage.
+
+ :param str name: name of table/document in storage.
+ :param str entry_id: id of the document to get from storage.
+ :param kwargs: the fields to update.
+ :return:
+ """
+ entry_dict = self.get(entry_id)
+ entry_dict.update(**kwargs)
+ self.store(entry_dict)
+
+
+class FileSystemResourceAPI(storage_api.ResourceAPI, FileSystemAPI):
+ """
+ File system resource storage.
+ """
+
+ def __init__(self, directory, **kwargs):
+ """
+ File system implementation for storage api.
+ :param str directory: root dir for storage.
+ """
+ super(FileSystemResourceAPI, self).__init__(**kwargs)
+ self.directory = directory
+ self._join_path = partial(os.path.join, self.directory)
+
+ def __repr__(self):
+ return '{cls.__name__}(directory={self.directory})'.format(
+ cls=self.__class__, self=self)
+
+ def create(self, **kwargs):
+ """
+ Create directory in storage by path.
+ tries to create the root directory as well.
+ :param str name: path of file in storage.
+ """
+ try:
+ os.makedirs(self.directory)
+ except (OSError, IOError):
+ pass
+ os.makedirs(self._join_path(self.name))
+
+ def data(self, entry_id, path=None, **_):
+ """
+ Retrieve the content of a file system storage resource.
+
+ :param str entry_type: the type of the entry.
+ :param str entry_id: the id of the entry.
+ :param str path: a path to a specific resource.
+ :return: the content of the file
+ :rtype: bytes
+ """
+ resource_relative_path = os.path.join(self.name, entry_id, path or '')
+ resource = os.path.join(self.directory, resource_relative_path)
+ if not os.path.exists(resource):
+ raise StorageError("Resource {0} does not exist".format(resource_relative_path))
+ if not os.path.isfile(resource):
+ resources = os.listdir(resource)
+ if len(resources) != 1:
+ raise StorageError('No resource in path: {0}'.format(resource))
+ resource = os.path.join(resource, resources[0])
+ with open(resource, 'rb') as resource_file:
+ return resource_file.read()
+
+ def download(self, entry_id, destination, path=None, **_):
+ """
+ Download a specific file or dir from the file system resource storage.
+
+ :param str entry_type: the name of the entry.
+ :param str entry_id: the id of the entry
+ :param str destination: the destination of the files.
+ :param str path: a path on the remote machine relative to the root of the entry.
+ """
+ resource_relative_path = os.path.join(self.name, entry_id, path or '')
+ resource = os.path.join(self.directory, resource_relative_path)
+ if not os.path.exists(resource):
+ raise StorageError("Resource {0} does not exist".format(resource_relative_path))
+ if os.path.isfile(resource):
+ shutil.copy2(resource, destination)
+ else:
+ dir_util.copy_tree(resource, destination) # pylint: disable=no-member
+
+ def upload(self, entry_id, source, path=None, **_):
+ """
+ Uploads a specific file or dir to the file system resource storage.
+
+ :param str entry_type: the name of the entry.
+ :param str entry_id: the id of the entry
+ :param source: the source of the files to upload.
+ :param path: the destination of the file/s relative to the entry root dir.
+ """
+ resource_directory = os.path.join(self.directory, self.name, entry_id)
+ if not os.path.exists(resource_directory):
+ os.makedirs(resource_directory)
+ destination = os.path.join(resource_directory, path or '')
+ if os.path.isfile(source):
+ shutil.copy2(source, destination)
+ else:
+ dir_util.copy_tree(source, destination) # pylint: disable=no-member
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/storage/api_driver/in_memory.py
----------------------------------------------------------------------
diff --git a/aria/storage/api_driver/in_memory.py b/aria/storage/api_driver/in_memory.py
new file mode 100644
index 0000000..b2d76f1
--- /dev/null
+++ b/aria/storage/api_driver/in_memory.py
@@ -0,0 +1,146 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+
+from aria.storage.structures import db
+
+from .. import storage_api
+
+
+_Pointer = namedtuple('_Pointer', 'name, is_iter')
+
+storage = {}
+
+
+class InMemoryModelAPI(storage_api.ModelAPI):
+ def __init__(self, *args, **kwargs):
+ """
+ Managing the model in the storage, using the driver.
+
+ :param basestring name: the name of the model.
+ :param ModelDriver driver: the driver which supports this model in the storage.
+ :param Model model_cls: table/document class model.
+ """
+ super(InMemoryModelAPI, self).__init__(*args, **kwargs)
+ self.pointer_mapping = {}
+
+ def __iter__(self):
+ return self.iter()
+
+ def create(self):
+ """
+ Creates the model in the storage.
+ """
+ with self.connect():
+ storage[self.name] = {}
+ self._setup_pointers_mapping()
+
+ def _setup_pointers_mapping(self,):
+ for field_name, field_cls in vars(self.model_cls).items():
+ if not(isinstance(field_cls, db.RelationshipProperty) and field_cls.type):
+ continue
+ pointer_key = _Pointer(field_name, is_iter=False)
+ self.pointer_mapping[pointer_key] = self.__class__(
+ name=storage_api.generate_lower_name(field_cls.type),
+ model_cls=field_cls.type)
+
+ def get(self, entry_id, **kwargs):
+ """
+ Getter for the model from the storage.
+
+ :param basestring entry_id: the id of the table/document.
+ :return: model instance
+ :rtype: Model
+ """
+ with self.connect():
+ data = storage[self.name][entry_id]
+ data.update(self._get_pointers(data, **kwargs))
+ return self.model_cls(**data)
+
+ def store(self, entry, **kwargs):
+ """
+ Setter for the model in the storage.
+
+ :param Model entry: the table/document to store.
+ """
+ with self.connect():
+ data = entry.fields_dict
+ data.update(self._store_pointers(data, **kwargs))
+ storage[self.name][entry.id] = data
+
+ def delete(self, entry_id, **kwargs):
+ """
+ Delete the model from storage.
+
+ :param basestring entry_id: id of the entity to delete from storage.
+ """
+ entry = self.get(entry_id)
+ with self.connect():
+ self._delete_pointers(entry, **kwargs)
+ storage[self.name].pop(entry_id)
+
+ def iter(self, **kwargs):
+ """
+ Generator over the entries of model in storage.
+ """
+ with self.connect():
+ for data in storage[self.name].values():
+ data.update(self._get_pointers(data, **kwargs))
+ yield self.model_cls(**data)
+
+ def update(self, entry_id, **kwargs):
+ """
+ Updates and entry in storage.
+
+ :param str entry_id: the id of the table/document.
+ :param kwargs: the fields to update.
+ :return:
+ """
+ with self.connect():
+ storage[self.name][entry_id].update(**kwargs)
+
+ def _get_pointers(self, data, **kwargs):
+ pointers = {}
+ for field, schema in self.pointer_mapping.items():
+ if field.is_iter:
+ pointers[field.name] = [
+ schema.get(entry_id=pointer_id, **kwargs)
+ for pointer_id in data[field.name]
+ if pointer_id]
+ elif data[field.name]:
+ pointers[field.name] = schema.get(entry_id=data[field.name], **kwargs)
+ return pointers
+
+ def _store_pointers(self, data, **kwargs):
+ pointers = {}
+ for field, model_api in self.pointer_mapping.items():
+ if field.is_iter:
+ pointers[field.name] = []
+ for iter_entity in data[field.name]:
+ pointers[field.name].append(iter_entity.id)
+ model_api.store(iter_entity, **kwargs)
+ else:
+ pointers[field.name] = data[field.name].id
+ model_api.store(data[field.name], **kwargs)
+ return pointers
+
+ def _delete_pointers(self, entry, **kwargs):
+ for field, schema in self.pointer_mapping.items():
+ if field.is_iter:
+ for iter_entry in getattr(entry, field.name):
+ schema.delete(iter_entry.id, **kwargs)
+ else:
+ schema.delete(getattr(entry, field.name).id, **kwargs)
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/storage/api_driver/sql.py
----------------------------------------------------------------------
diff --git a/aria/storage/api_driver/sql.py b/aria/storage/api_driver/sql.py
new file mode 100644
index 0000000..f9bc4fe
--- /dev/null
+++ b/aria/storage/api_driver/sql.py
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .. import storage_api
+
+
+class SQLAlchemyModelAPI(storage_api.ModelAPI):
+ def _destroy_connection(self):
+ pass
+
+ def _establish_connection(self):
+ pass
+
+ def store(self, entry, **kwargs):
+ pass
+
+ def create(self):
+ pass
+
+ def update(self, entry_id, **kwargs):
+ pass
+
+ def delete(self, entry_id, **kwargs):
+ pass
+
+ def iter(self, **kwargs):
+ pass
+
+ def get(self, entry_id, **kwargs):
+ pass
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/storage/drivers.py
----------------------------------------------------------------------
diff --git a/aria/storage/drivers.py b/aria/storage/drivers.py
deleted file mode 100644
index 1f96956..0000000
--- a/aria/storage/drivers.py
+++ /dev/null
@@ -1,416 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Aria's storage.drivers module
-Path: aria.storage.driver
-
-drivers module holds a generic abstract implementation of drivers.
-
-classes:
- * Driver - abstract storage driver implementation.
- * ModelDriver - abstract model base storage driver.
- * ResourceDriver - abstract resource base storage driver.
- * FileSystemModelDriver - file system implementation for model storage driver.
- * FileSystemResourceDriver - file system implementation for resource storage driver.
-"""
-
-import distutils.dir_util # pylint: disable=no-name-in-module, import-error
-import os
-import shutil
-from functools import partial
-from multiprocessing import RLock
-
-import jsonpickle
-
-from ..logger import LoggerMixin
-from .exceptions import StorageError
-
-__all__ = (
- 'ModelDriver',
- 'FileSystemModelDriver',
- 'ResourceDriver',
- 'FileSystemResourceDriver',
-)
-
-
-class Driver(LoggerMixin):
- """
- Driver: storage driver context manager - abstract driver implementation.
- In the implementation level, It is a good practice to raise StorageError on Errors.
- """
-
- def __enter__(self):
- """
- Context manager entry method, executes connect.
- :return: context manager instance
- :rtype: Driver
- """
- self.connect()
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- """
- Context manager exit method, executes disconnect.
- """
- self.disconnect()
- if not exc_type:
- return
- # self.logger.debug(
- # '{name} had an error'.format(name=self.__class__.__name__),
- # exc_info=(exc_type, exc_val, exc_tb))
- if StorageError in exc_type.mro():
- return
- raise StorageError('Exception had occurred, {type}: {message}'.format(
- type=exc_type, message=str(exc_val)))
-
- def connect(self):
- """
- Open storage connection.
- In some cases, This method can get the connection from a connection pool.
- """
- pass
-
- def disconnect(self):
- """
- Close storage connection.
- In some cases, This method can release the connection to the connection pool.
- """
- pass
-
- def create(self, name, *args, **kwargs):
- """
- Create table/document in storage by name.
- :param str name: name of table/document in storage.
- """
- pass
-
-
-class ModelDriver(Driver):
- """
- ModelDriver context manager.
- Base Driver for Model based storage.
- """
-
- def get(self, name, entry_id, **kwargs):
- """
- Getter from storage.
- :param str name: name of table/document in storage.
- :param str entry_id: id of the document to get from storage.
- :return: value of entity from the storage.
- """
- raise NotImplementedError('Subclass must implement abstract get method')
-
- def delete(self, name, entry_id, **kwargs):
- """
- Delete from storage.
- :param str name: name of table/document in storage.
- :param str entry_id: id of the entity to delete from storage.
- :param dict kwargs: extra kwargs if needed.
- """
- raise NotImplementedError('Subclass must implement abstract delete method')
-
- def store(self, name, entry_id, entry, **kwargs):
- """
- Setter to storage.
- :param str name: name of table/document in storage.
- :param str entry_id: id of the entity to store in the storage.
- :param dict entry: content to store.
- """
- raise NotImplementedError('Subclass must implement abstract store method')
-
- def iter(self, name, **kwargs):
- """
- Generator over the entries of table/document in storage.
- :param str name: name of table/document/file in storage to iter over.
- """
- raise NotImplementedError('Subclass must implement abstract iter method')
-
- def update(self, name, entry_id, **kwargs):
- """
- Updates and entry in storage.
-
- :param str name: name of table/document in storage.
- :param str entry_id: id of the document to get from storage.
- :param kwargs: the fields to update.
- :return:
- """
- raise NotImplementedError('Subclass must implement abstract store method')
-
-
-class ResourceDriver(Driver):
- """
- ResourceDriver context manager.
- Base Driver for Resource based storage.
-
- Resource storage structure is a file system base.
- <resource root directory>/<resource_name>/<entry_id>/<entry>
- entry: can be one single file or multiple files and directories.
- """
-
- def data(self, entry_type, entry_id, path=None, **kwargs):
- """
- Get the binary data from a file in a resource entry.
- If the entry is a single file no path needed,
- If the entry contain number of files the path will gide to the relevant file.
-
- resource path:
- <resource root directory>/<name>/<entry_id>/<path>
-
- :param basestring entry_type: resource name.
- :param basestring entry_id: id of the entity to resource in the storage.
- :param basestring path: path to resource relative to entry_id folder in the storage.
- :return: entry file object.
- :rtype: bytes
- """
- raise NotImplementedError('Subclass must implement abstract get method')
-
- def download(self, entry_type, entry_id, destination, path=None, **kwargs):
- """
- Download the resource to a destination.
- Like data method bat this method isn't returning data,
- Instead it create a new file in local file system.
-
- resource path:
- <resource root directory>/<name>/<entry_id>/<path>
- copy to:
- /<destination>
- destination can be file or directory
-
- :param basestring entry_type: resource name.
- :param basestring entry_id: id of the entity to resource in the storage.
- :param basestring destination: path in local file system to download to.
- :param basestring path: path to resource relative to entry_id folder in the storage.
- """
- raise NotImplementedError('Subclass must implement abstract get method')
-
- def upload(self, entry_type, entry_id, source, path=None, **kwargs):
- """
- Upload the resource from source.
- source can be file or directory with files.
-
- copy from:
- /<source>
- to resource path:
- <resource root directory>/<name>/<entry_id>/<path>
-
- :param basestring entry_type: resource name.
- :param basestring entry_id: id of the entity to resource in the storage.
- :param basestring source: source can be file or directory with files.
- :param basestring path: path to resource relative to entry_id folder in the storage.
- """
- raise NotImplementedError('Subclass must implement abstract get method')
-
-
-class BaseFileSystemDriver(Driver):
- """
- Base class which handles storage on the file system.
- """
- def __init__(self, *args, **kwargs):
- super(BaseFileSystemDriver, self).__init__(*args, **kwargs)
- self._lock = RLock()
-
- def connect(self):
- self._lock.acquire()
-
- def disconnect(self):
- self._lock.release()
-
- def __getstate__(self):
- obj_dict = super(BaseFileSystemDriver, self).__getstate__()
- del obj_dict['_lock']
- return obj_dict
-
- def __setstate__(self, obj_dict):
- super(BaseFileSystemDriver, self).__setstate__(obj_dict)
- vars(self).update(_lock=RLock(), **obj_dict)
-
-
-class FileSystemModelDriver(ModelDriver, BaseFileSystemDriver):
- """
- FileSystemModelDriver context manager.
- """
-
- def __init__(self, directory, **kwargs):
- """
- File system implementation for storage driver.
- :param str directory: root dir for storage.
- """
- super(FileSystemModelDriver, self).__init__(**kwargs)
- self.directory = directory
-
- self._join_path = partial(os.path.join, self.directory)
-
- def __repr__(self):
- return '{cls.__name__}(directory={self.directory})'.format(
- cls=self.__class__, self=self)
-
- def create(self, name):
- """
- Create directory in storage by path.
- tries to create the root directory as well.
- :param str name: path of file in storage.
- """
- try:
- os.makedirs(self.directory)
- except (OSError, IOError):
- pass
- os.makedirs(self._join_path(name))
-
- def get(self, name, entry_id, **kwargs):
- """
- Getter from storage.
- :param str name: name of directory in storage.
- :param str entry_id: id of the file to get from storage.
- :return: value of file from storage.
- :rtype: dict
- """
- with open(self._join_path(name, entry_id)) as file_obj:
- return jsonpickle.loads(file_obj.read())
-
- def store(self, name, entry_id, entry, **kwargs):
- """
- Delete from storage.
- :param str name: name of directory in storage.
- :param str entry_id: id of the file to delete from storage.
- """
- with open(self._join_path(name, entry_id), 'w') as file_obj:
- file_obj.write(jsonpickle.dumps(entry))
-
- def delete(self, name, entry_id, **kwargs):
- """
- Delete from storage.
- :param str name: name of directory in storage.
- :param str entry_id: id of the file to delete from storage.
- """
- os.remove(self._join_path(name, entry_id))
-
- def iter(self, name, filters=None, **kwargs):
- """
- Generator over the entries of directory in storage.
- :param str name: name of directory in storage to iter over.
- :param dict filters: filters for query
- """
- filters = filters or {}
-
- for entry_id in os.listdir(self._join_path(name)):
- value = self.get(name, entry_id=entry_id)
- for filter_name, filter_value in filters.items():
- if value.get(filter_name) != filter_value:
- break
- else:
- yield value
-
- def update(self, name, entry_id, **kwargs):
- """
- Updates and entry in storage.
-
- :param str name: name of table/document in storage.
- :param str entry_id: id of the document to get from storage.
- :param kwargs: the fields to update.
- :return:
- """
- entry_dict = self.get(name, entry_id)
- entry_dict.update(**kwargs)
- self.store(name, entry_id, entry_dict)
-
-
-class FileSystemResourceDriver(ResourceDriver, BaseFileSystemDriver):
- """
- FileSystemResourceDriver context manager.
- """
-
- def __init__(self, directory, **kwargs):
- """
- File system implementation for storage driver.
- :param str directory: root dir for storage.
- """
- super(FileSystemResourceDriver, self).__init__(**kwargs)
- self.directory = directory
- self._join_path = partial(os.path.join, self.directory)
-
- def __repr__(self):
- return '{cls.__name__}(directory={self.directory})'.format(
- cls=self.__class__, self=self)
-
- def create(self, name):
- """
- Create directory in storage by path.
- tries to create the root directory as well.
- :param basestring name: path of file in storage.
- """
- try:
- os.makedirs(self.directory)
- except (OSError, IOError):
- pass
- os.makedirs(self._join_path(name))
-
- def data(self, entry_type, entry_id, path=None):
- """
- Retrieve the content of a file system storage resource.
-
- :param basestring entry_type: the type of the entry.
- :param basestring entry_id: the id of the entry.
- :param basestring path: a path to a specific resource.
- :return: the content of the file
- :rtype: bytes
- """
- resource_relative_path = os.path.join(entry_type, entry_id, path or '')
- resource = os.path.join(self.directory, resource_relative_path)
- if not os.path.exists(resource):
- raise StorageError("Resource {0} does not exist".format(resource_relative_path))
- if not os.path.isfile(resource):
- resources = os.listdir(resource)
- if len(resources) != 1:
- raise StorageError('No resource in path: {0}'.format(resource))
- resource = os.path.join(resource, resources[0])
- with open(resource, 'rb') as resource_file:
- return resource_file.read()
-
- def download(self, entry_type, entry_id, destination, path=None):
- """
- Download a specific file or dir from the file system resource storage.
-
- :param basestring entry_type: the name of the entry.
- :param basestring entry_id: the id of the entry
- :param basestring destination: the destination of the files.
- :param basestring path: a path on the remote machine relative to the root of the entry.
- """
- resource_relative_path = os.path.join(entry_type, entry_id, path or '')
- resource = os.path.join(self.directory, resource_relative_path)
- if not os.path.exists(resource):
- raise StorageError("Resource {0} does not exist".format(resource_relative_path))
- if os.path.isfile(resource):
- shutil.copy2(resource, destination)
- else:
- distutils.dir_util.copy_tree(resource, destination) # pylint: disable=no-member
-
- def upload(self, entry_type, entry_id, source, path=None):
- """
- Uploads a specific file or dir to the file system resource storage.
-
- :param basestring entry_type: the name of the entry.
- :param basestring entry_id: the id of the entry
- :param source: the source of the files to upload.
- :param path: the destination of the file/s relative to the entry root dir.
- """
- resource_directory = os.path.join(self.directory, entry_type, entry_id)
- if not os.path.exists(resource_directory):
- os.makedirs(resource_directory)
- destination = os.path.join(resource_directory, path or '')
- if os.path.isfile(source):
- shutil.copy2(source, destination)
- else:
- distutils.dir_util.copy_tree(source, destination) # pylint: disable=no-member
http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/1bb1f6ba/aria/storage/models.py
----------------------------------------------------------------------
diff --git a/aria/storage/models.py b/aria/storage/models.py
index d24ad75..17b0608 100644
--- a/aria/storage/models.py
+++ b/aria/storage/models.py
@@ -36,11 +36,13 @@ classes:
* ProviderContext - provider context implementation model.
* Plugin - plugin implementation model.
"""
-
from datetime import datetime
-from types import NoneType
+from uuid import uuid4
-from .structures import Field, IterPointerField, Model, uuid_generator, PointerField
+from aria.storage import (
+ structures,
+ states,
+)
__all__ = (
'Model',
@@ -60,344 +62,516 @@ __all__ = (
)
# todo: sort this, maybe move from mgr or move from aria???
-ACTION_TYPES = ()
-ENTITY_TYPES = ()
+# TODO: this must change
+ACTION_TYPES = ('a')
+ENTITY_TYPES = ('b')
-class Blueprint(Model):
+def uuid_generator():
"""
- A Model which represents a blueprint
+ wrapper function which generates ids
"""
- plan = Field(type=dict)
- id = Field(type=basestring, default=uuid_generator)
- description = Field(type=(basestring, NoneType))
- created_at = Field(type=datetime)
- updated_at = Field(type=datetime)
- main_file_name = Field(type=basestring)
+ return str(uuid4())
-class Snapshot(Model):
- """
- A Model which represents a snapshot
- """
- CREATED = 'created'
- FAILED = 'failed'
- CREATING = 'creating'
- UPLOADED = 'uploaded'
- END_STATES = [CREATED, FAILED, UPLOADED]
+class Blueprint(structures.SQLModelBase):
+ __tablename__ = 'blueprints'
- id = Field(type=basestring, default=uuid_generator)
- created_at = Field(type=datetime)
- status = Field(type=basestring)
- error = Field(type=basestring, default=None)
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
+ created_at = structures.db.Column(structures.UTCDateTime, nullable=False, index=True)
+ main_file_name = structures.db.Column(structures.db.Text, nullable=False)
+ plan = structures.db.Column(structures.db.PickleType, nullable=False)
+ updated_at = structures.db.Column(structures.UTCDateTime)
+ description = structures.db.Column(structures.db.Text)
-class Deployment(Model):
- """
- A Model which represents a deployment
- """
- id = Field(type=basestring, default=uuid_generator)
- description = Field(type=(basestring, NoneType))
- created_at = Field(type=datetime)
- updated_at = Field(type=datetime)
- blueprint_id = Field(type=basestring)
- workflows = Field(type=dict)
- inputs = Field(type=dict, default=lambda: {})
- policy_types = Field(type=dict, default=lambda: {})
- policy_triggers = Field(type=dict, default=lambda: {})
- groups = Field(type=dict, default=lambda: {})
- outputs = Field(type=dict, default=lambda: {})
- scaling_groups = Field(type=dict, default=lambda: {})
-
-
-class DeploymentUpdateStep(Model):
- """
- A Model which represents a deployment update step
- """
- id = Field(type=basestring, default=uuid_generator)
- action = Field(type=basestring, choices=ACTION_TYPES)
- entity_type = Field(type=basestring, choices=ENTITY_TYPES)
- entity_id = Field(type=basestring)
- supported = Field(type=bool, default=True)
-
- def __hash__(self):
- return hash((self.id, self.entity_id))
-
- def __lt__(self, other):
- """
- the order is 'remove' < 'modify' < 'add'
- :param other:
- :return:
- """
- if not isinstance(other, self.__class__):
- return not self >= other
-
- if self.action != other.action:
- if self.action == 'remove':
- return_value = True
- elif self.action == 'add':
- return_value = False
- else:
- return_value = other.action == 'add'
- return return_value
-
- if self.action == 'add':
- return self.entity_type == 'node' and other.entity_type == 'relationship'
- if self.action == 'remove':
- return self.entity_type == 'relationship' and other.entity_type == 'node'
- return False
-
-
-class DeploymentUpdate(Model):
- """
- A Model which represents a deployment update
- """
- INITIALIZING = 'initializing'
- SUCCESSFUL = 'successful'
- UPDATING = 'updating'
- FINALIZING = 'finalizing'
- EXECUTING_WORKFLOW = 'executing_workflow'
- FAILED = 'failed'
-
- STATES = [
- INITIALIZING,
- SUCCESSFUL,
- UPDATING,
- FINALIZING,
- EXECUTING_WORKFLOW,
- FAILED,
- ]
-
- # '{0}-{1}'.format(kwargs['deployment_id'], uuid4())
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- state = Field(type=basestring, choices=STATES, default=INITIALIZING)
- deployment_plan = Field()
- deployment_update_nodes = Field(default=None)
- deployment_update_node_instances = Field(default=None)
- deployment_update_deployment = Field(default=None)
- modified_entity_ids = Field(default=None)
- execution_id = Field(type=basestring)
- steps = IterPointerField(type=DeploymentUpdateStep, default=())
-
-
-class Execution(Model):
- """
- A Model which represents an execution
- """
- class _Validation(object):
+class Snapshot(structures.SQLModelBase):
+ __tablename__ = 'snapshots'
- @staticmethod
- def execution_status_transition_validation(_, value, instance):
- """Validation function that verifies execution status transitions are OK"""
- try:
- current_status = instance.status
- except AttributeError:
- return
- valid_transitions = Execution.VALID_TRANSITIONS.get(current_status, [])
- if current_status != value and value not in valid_transitions:
- raise ValueError('Cannot change execution status from {current} to {new}'.format(
- current=current_status,
- new=value))
-
- TERMINATED = 'terminated'
- FAILED = 'failed'
- CANCELLED = 'cancelled'
- PENDING = 'pending'
- STARTED = 'started'
- CANCELLING = 'cancelling'
- STATES = (
- TERMINATED,
- FAILED,
- CANCELLED,
- PENDING,
- STARTED,
- CANCELLING,
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
+
+ created_at = structures.db.Column(structures.UTCDateTime, nullable=False, index=True)
+ status = structures.db.Column(structures.db.Enum(*states.SnapshotState.STATES, name='snapshot_status'))
+ error = structures.db.Column(structures.db.Text)
+
+
+class Deployment(structures.SQLModelBase):
+ __tablename__ = 'deployments'
+
+ # See base class for an explanation on these properties
+ join_properties = {
+ 'blueprint_id': {
+ # No need to provide the Blueprint table, as it's already joined
+ 'models': [Blueprint],
+ 'column': Blueprint.id.label('blueprint_id')
+ },
+ }
+ join_order = 2
+
+ _private_fields = ['blueprint_storage_id']
+
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
+
+ created_at = structures.db.Column(structures.UTCDateTime, nullable=False, index=True)
+ description = structures.db.Column(structures.db.Text)
+ inputs = structures.db.Column(structures.db.PickleType)
+ groups = structures.db.Column(structures.db.PickleType)
+ permalink = structures.db.Column(structures.db.Text)
+ policy_triggers = structures.db.Column(structures.db.PickleType)
+ policy_types = structures.db.Column(structures.db.PickleType)
+ outputs = structures.db.Column(structures.db.PickleType(comparator=lambda *a: False))
+ scaling_groups = structures.db.Column(structures.db.PickleType)
+ updated_at = structures.db.Column(structures.UTCDateTime)
+ workflows = structures.db.Column(structures.db.PickleType(comparator=lambda *a: False))
+
+ blueprint_storage_id = structures.foreign_key(Blueprint)
+ blueprint = structures.one_to_many_relationship(
+ child_class_name='Deployment',
+ column_name='blueprint_storage_id',
+ parent_class_name='Blueprint',
+ back_reference_name='deployments'
)
- END_STATES = [TERMINATED, FAILED, CANCELLED]
- ACTIVE_STATES = [state for state in STATES if state not in END_STATES]
- VALID_TRANSITIONS = {
- PENDING: [STARTED, CANCELLED],
- STARTED: END_STATES + [CANCELLING],
- CANCELLING: END_STATES
+
+ @property
+ def blueprint_id(self):
+ return self.blueprint.id
+
+
+class Execution(structures.SQLModelBase):
+ __tablename__ = 'executions'
+
+ # See base class for an explanation on these properties
+ join_properties = {
+ 'blueprint_id': {
+ 'models': [Deployment, Blueprint],
+ 'column': Blueprint.id.label('blueprint_id')
+ },
+ 'deployment_id': {
+ 'models': [Deployment],
+ 'column': Deployment.id.label('deployment_id')
+ }
}
+ join_order = 3
- id = Field(type=basestring, default=uuid_generator)
- status = Field(type=basestring, choices=STATES,
- validation_func=_Validation.execution_status_transition_validation)
- deployment_id = Field(type=basestring)
- workflow_id = Field(type=basestring)
- blueprint_id = Field(type=basestring)
- created_at = Field(type=datetime, default=datetime.utcnow)
- started_at = Field(type=datetime, default=None)
- ended_at = Field(type=datetime, default=None)
- error = Field(type=basestring, default=None)
- parameters = Field()
+ _private_fields = ['deployment_storage_id']
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
-class Relationship(Model):
- """
- A Model which represents a relationship
- """
- id = Field(type=basestring, default=uuid_generator)
- source_id = Field(type=basestring)
- target_id = Field(type=basestring)
- source_interfaces = Field(type=dict)
- source_operations = Field(type=dict)
- target_interfaces = Field(type=dict)
- target_operations = Field(type=dict)
- type = Field(type=basestring)
- type_hierarchy = Field(type=list)
- properties = Field(type=dict)
-
-
-class Node(Model):
- """
- A Model which represents a node
- """
- id = Field(type=basestring, default=uuid_generator)
- blueprint_id = Field(type=basestring)
- type = Field(type=basestring)
- type_hierarchy = Field()
- number_of_instances = Field(type=int)
- planned_number_of_instances = Field(type=int)
- deploy_number_of_instances = Field(type=int)
- host_id = Field(type=basestring, default=None)
- properties = Field(type=dict)
- operations = Field(type=dict)
- plugins = Field(type=list, default=())
- relationships = IterPointerField(type=Relationship)
- plugins_to_install = Field(type=list, default=())
- min_number_of_instances = Field(type=int)
- max_number_of_instances = Field(type=int)
-
- def relationships_by_target(self, target_id):
- """
- Retreives all of the relationship by target.
- :param target_id: the node id of the target of the relationship
- :yields: a relationship which target and node with the specified target_id
- """
- for relationship in self.relationships:
- if relationship.target_id == target_id:
- yield relationship
- # todo: maybe add here Exception if isn't exists (didn't yield one's)
-
-
-class RelationshipInstance(Model):
- """
- A Model which represents a relationship instance
- """
- id = Field(type=basestring, default=uuid_generator)
- target_id = Field(type=basestring)
- target_name = Field(type=basestring)
- source_id = Field(type=basestring)
- source_name = Field(type=basestring)
- type = Field(type=basestring)
- relationship = PointerField(type=Relationship)
+ created_at = structures.db.Column(structures.UTCDateTime, nullable=False, index=True)
+ error = structures.db.Column(structures.db.Text)
+ is_system_workflow = structures.db.Column(structures.db.Boolean, nullable=False)
+ parameters = structures.db.Column(structures.db.PickleType)
+ status = structures.db.Column(
+ structures.db.Enum(*states.ExecutionState.STATES, name='execution_status')
+ )
+ workflow_id = structures.db.Column(structures.db.Text, nullable=False)
+
+ deployment_storage_id = structures.foreign_key(Deployment, nullable=True)
+ deployment = structures.one_to_many_relationship(
+ child_class_name='Execution',
+ column_name='deployment_storage_id',
+ parent_class_name='Deployment',
+ back_reference_name='executions'
+ )
+ @property
+ def deployment_id(self):
+ return self.deployment.id if self.deployment else None
+
+ @property
+ def blueprint_id(self):
+ return self.deployment.blueprint_id if self.deployment else None
+
+ def __str__(self):
+ id_name, id_value = self._get_unique_id()
+ return '<{0} {1}=`{2}` (status={3})>'.format(
+ self.__class__.__name__,
+ id_name,
+ id_value,
+ self.status
+ )
+
+
+class DeploymentUpdate(structures.SQLModelBase):
+ __tablename__ = 'deployment_updates'
+
+ # See base class for an explanation on these properties
+ join_properties = {
+ 'execution_id': {
+ 'models': [Execution],
+ 'column': Execution.id.label('execution_id')
+ },
+ 'deployment_id': {
+ 'models': [Deployment],
+ 'column': Deployment.id.label('deployment_id')
+ },
+ }
+ join_order = 4
+
+ _private_fields = ['execution_storage_id']
+
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
+
+ created_at = structures.db.Column(structures.UTCDateTime, nullable=False, index=True)
+ deployment_plan = structures.db.Column(structures.db.PickleType)
+ deployment_update_node_instances = structures.db.Column(structures.db.PickleType)
+ deployment_update_deployment = structures.db.Column(structures.db.PickleType)
+ deployment_update_nodes = structures.db.Column(structures.db.PickleType)
+ modified_entity_ids = structures.db.Column(structures.db.PickleType)
+ state = structures.db.Column(structures.db.Text)
+
+ execution_storage_id = structures.foreign_key(Execution, nullable=True)
+ execution = structures.one_to_many_relationship(
+ child_class_name='DeploymentUpdate',
+ column_name='execution_storage_id',
+ parent_class_name='Execution',
+ back_reference_name='deployment_updates'
+ )
-class NodeInstance(Model):
- """
- A Model which represents a node instance
- """
- # todo: add statuses
- UNINITIALIZED = 'uninitialized'
- INITIALIZING = 'initializing'
- CREATING = 'creating'
- CONFIGURING = 'configuring'
- STARTING = 'starting'
- DELETED = 'deleted'
- STOPPING = 'stopping'
- DELETING = 'deleting'
- STATES = (
- UNINITIALIZED,
- INITIALIZING,
- CREATING,
- CONFIGURING,
- STARTING,
- DELETED,
- STOPPING,
- DELETING
+ deployment_storage_id = structures.foreign_key(Deployment)
+ deployment = structures.one_to_many_relationship(
+ child_class_name='DeploymentUpdate',
+ column_name='deployment_storage_id',
+ parent_class_name='Deployment',
+ back_reference_name='deployment_updates'
)
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- runtime_properties = Field(type=dict)
- state = Field(type=basestring, choices=STATES, default=UNINITIALIZED)
- version = Field(type=(basestring, NoneType))
- relationship_instances = IterPointerField(type=RelationshipInstance)
- node = PointerField(type=Node)
- host_id = Field(type=basestring, default=None)
- scaling_groups = Field(default=())
-
- def relationships_by_target(self, target_id):
- """
- Retreives all of the relationship by target.
- :param target_id: the instance id of the target of the relationship
- :yields: a relationship instance which target and node with the specified target_id
- """
- for relationship_instance in self.relationship_instances:
- if relationship_instance.target_id == target_id:
- yield relationship_instance
- # todo: maybe add here Exception if isn't exists (didn't yield one's)
-
-
-class DeploymentModification(Model):
- """
- A Model which represents a deployment modification
- """
- STARTED = 'started'
- FINISHED = 'finished'
- ROLLEDBACK = 'rolledback'
- END_STATES = [FINISHED, ROLLEDBACK]
-
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- modified_nodes = Field(type=(dict, NoneType))
- added_and_related = IterPointerField(type=NodeInstance)
- removed_and_related = IterPointerField(type=NodeInstance)
- extended_and_related = IterPointerField(type=NodeInstance)
- reduced_and_related = IterPointerField(type=NodeInstance)
- # before_modification = IterPointerField(type=NodeInstance)
- status = Field(type=basestring, choices=(STARTED, FINISHED, ROLLEDBACK))
- created_at = Field(type=datetime)
- ended_at = Field(type=(datetime, NoneType))
- context = Field()
-
-
-class ProviderContext(Model):
- """
- A Model which represents a provider context
- """
- id = Field(type=basestring, default=uuid_generator)
- context = Field(type=dict)
- name = Field(type=basestring)
+ @property
+ def execution_id(self):
+ return self.execution.id if self.execution else None
+ @property
+ def deployment_id(self):
+ return self.deployment.id
-class Plugin(Model):
- """
- A Model which represents a plugin
- """
- id = Field(type=basestring, default=uuid_generator)
- package_name = Field(type=basestring)
- archive_name = Field(type=basestring)
- package_source = Field(type=dict)
- package_version = Field(type=basestring)
- supported_platform = Field(type=basestring)
- distribution = Field(type=basestring)
- distribution_version = Field(type=basestring)
- distribution_release = Field(type=basestring)
- wheels = Field()
- excluded_wheels = Field()
- supported_py_versions = Field(type=list)
- uploaded_at = Field(type=datetime)
-
-
-class Task(Model):
+ def to_dict(self, suppress_error=False):
+ dep_update_dict = super(DeploymentUpdate, self).to_dict(suppress_error)
+ # Taking care of the fact the DeploymentSteps are objects
+ dep_update_dict['steps'] = [step.to_dict() for step in self.steps]
+ return dep_update_dict
+
+
+class DeploymentUpdateStep(structures.SQLModelBase):
+ __tablename__ = 'deployment_update_steps'
+
+ # See base class for an explanation on these properties
+ join_properties = {
+ 'deployment_update_id': {
+ 'models': [DeploymentUpdate],
+ 'column': DeploymentUpdate.id.label('deployment_update_id')
+ },
+ }
+ join_order = 5
+
+ _private_fields = ['deployment_update_storage_id']
+
+ id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+
+ action = structures.db.Column(structures.db.Enum(*ACTION_TYPES, name='action_type'))
+ entity_id = structures.db.Column(structures.db.Text, nullable=False)
+ entity_type = structures.db.Column(structures.db.Enum(*ENTITY_TYPES, name='entity_type'))
+
+ deployment_update_storage_id = structures.foreign_key(DeploymentUpdate)
+ deployment_update = structures.one_to_many_relationship(
+ child_class_name='DeploymentUpdateStep',
+ column_name='deployment_update_storage_id',
+ parent_class_name='DeploymentUpdate',
+ back_reference_name='steps'
+ )
+
+ @property
+ def tenant(self):
+ return self.deployment_update.tenant
+
+ @property
+ def deployment_update_id(self):
+ return self.deployment_update.id
+
+
+class DeploymentModification(structures.SQLModelBase):
+ __tablename__ = 'deployment_modifications'
+
+ # See base class for an explanation on these properties
+ join_properties = {
+ 'deployment_id': {
+ 'models': [Deployment],
+ 'column': Deployment.id.label('deployment_id')
+ },
+ }
+ join_order = 3
+
+ _private_fields = ['deployment_storage_id']
+
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
+
+ context = structures.db.Column(structures.db.PickleType)
+ created_at = structures.db.Column(structures.UTCDateTime, nullable=False, index=True)
+ ended_at = structures.db.Column(structures.UTCDateTime, index=True)
+ modified_nodes = structures.db.Column(structures.db.PickleType)
+ node_instances = structures.db.Column(structures.db.PickleType)
+ status = structures.db.Column(structures.db.Enum(
+ *states.DeploymentModificationState.STATES,
+ name='deployment_modification_status'
+ ))
+
+ deployment_storage_id = structures.foreign_key(Deployment)
+ deployment = structures.one_to_many_relationship(
+ child_class_name='DeploymentModification',
+ column_name='deployment_storage_id',
+ parent_class_name='Deployment',
+ back_reference_name='modifications'
+ )
+
+ @property
+ def tenant(self):
+ return self.deployment.tenant
+
+ @property
+ def deployment_id(self):
+ return self.deployment.id
+
+
+class Node(structures.SQLModelBase):
+ __tablename__ = 'nodes'
+
+ # See base class for an explanation on these properties
+ is_id_unique = False
+ join_properties = {
+ 'blueprint_id': {
+ 'models': [Deployment, Blueprint],
+ 'column': Blueprint.id.label('blueprint_id')
+ },
+ 'deployment_id': {
+ 'models': [Deployment],
+ 'column': Deployment.id.label('deployment_id')
+ },
+ }
+ join_order = 3
+
+ _private_fields = ['deployment_storage_id']
+
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
+
+ deploy_number_of_instances = structures.db.Column(structures.db.Integer, nullable=False)
+ # TODO: This probably should be a foreign key, but there's no guarantee
+ # in the code, currently, that the host will be created beforehand
+ host_id = structures.db.Column(structures.db.Text)
+ max_number_of_instances = structures.db.Column(structures.db.Integer, nullable=False)
+ min_number_of_instances = structures.db.Column(structures.db.Integer, nullable=False)
+ number_of_instances = structures.db.Column(structures.db.Integer, nullable=False)
+ planned_number_of_instances = structures.db.Column(structures.db.Integer, nullable=False)
+ plugins = structures.db.Column(structures.db.PickleType)
+ plugins_to_install = structures.db.Column(structures.db.PickleType)
+ properties = structures.db.Column(structures.db.PickleType)
+ # relationships = structures.db.Column(structures.db.PickleType)
+ operations = structures.db.Column(structures.db.PickleType)
+ type = structures.db.Column(structures.db.Text, nullable=False, index=True)
+ type_hierarchy = structures.db.Column(structures.db.PickleType)
+
+ deployment_storage_id = structures.foreign_key(Deployment)
+ deployment = structures.one_to_many_relationship(
+ child_class_name='Node',
+ column_name='deployment_storage_id',
+ parent_class_name='Deployment',
+ back_reference_name='nodes'
+ )
+
+ @property
+ def tenant(self):
+ return self.deployment.tenant
+
+ @property
+ def deployment_id(self):
+ return self.deployment.id
+
+ @property
+ def blueprint_id(self):
+ return self.deployment.blueprint_id
+
+
+class Relationship(structures.SQLModelBase):
+ __tablename__ = 'relationships'
+
+ join_properties = {
+ 'blueprint_id': {
+ 'models': [Node, Deployment, Blueprint],
+ 'column': Blueprint.id.label('blueprint_id')
+ },
+ 'deployment_id': {
+ 'models': [Node, Deployment],
+ 'column': Deployment.id.label('deployment_id')
+ }
+ }
+ join_order = 4
+ _private_fields = ['relationship_storage_source_node_id',
+ 'relationship_storage_target_node_id']
+
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
+
+ source_interfaces = structures.db.Column(structures.db.PickleType)
+ source_operations = structures.db.Column(structures.db.PickleType)
+ target_interfaces = structures.db.Column(structures.db.PickleType)
+ target_operations = structures.db.Column(structures.db.PickleType)
+ type = structures.db.Column(structures.db.String)
+ type_hierarchy = structures.db.Column(structures.db.PickleType) # TODO: this should be list
+ properties = structures.db.Column(structures.db.PickleType)
+
+ source_node_storage_id = structures.foreign_key(Node)
+ target_node_storage_id = structures.foreign_key(Node)
+
+ source_node = structures.one_to_many_relationship(
+ child_class_name='Relationship',
+ column_name='source_node_storage_id',
+ parent_class_name='Node',
+ back_reference_name='relationship_source'
+ )
+ target_node = structures.one_to_many_relationship(
+ child_class_name='Relationship',
+ column_name='target_node_storage_id',
+ parent_class_name='Node',
+ back_reference_name='relationship_target'
+ )
+
+
+class NodeInstance(structures.SQLModelBase):
+ __tablename__ = 'node_instances'
+
+ # See base class for an explanation on these properties
+ join_properties = {
+ 'node_id': {
+ 'models': [Node],
+ 'column': Node.id.label('node_id')
+ },
+ 'deployment_id': {
+ 'models': [Node, Deployment],
+ 'column': Deployment.id.label('deployment_id')
+ },
+ }
+ join_order = 4
+
+ _private_fields = ['node_storage_id']
+
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
+
+ # TODO: This probably should be a foreign key, but there's no guarantee
+ # in the code, currently, that the host will be created beforehand
+ host_id = structures.db.Column(structures.db.Text)
+ runtime_properties = structures.db.Column(structures.db.PickleType)
+ scaling_groups = structures.db.Column(structures.db.PickleType)
+ state = structures.db.Column(structures.db.Text, nullable=False)
+ version = structures.db.Column(structures.db.Integer, default=1)
+
+ node_storage_id = structures.foreign_key(Node)
+ node = structures.one_to_many_relationship(
+ child_class_name='NodeInstance',
+ column_name='node_storage_id',
+ parent_class_name='Node',
+ back_reference_name='node_instances'
+ )
+
+ @property
+ def tenant(self):
+ return self.deployment.tenant
+
+ @property
+ def node_id(self):
+ return self.node.id
+
+ @property
+ def deployment_id(self):
+ return self.node.deployment_id
+
+
+class RelationshipInstance(structures.SQLModelBase):
+ __tablename__ = 'relationship_instances'
+
+ join_properties = {
+ 'blueprint_id': {
+ 'models': [Relationship, Node, Deployment, Blueprint],
+ 'column': Blueprint.id.label('blueprint_id')
+ },
+ 'deployment_id': {
+ 'models': [Relationship, Node, Deployment],
+ 'column': Deployment.id.label('deployment_id')
+ }
+ }
+ join_order = 5
+
+ _private_fields = ['relationship_storage_id',
+ 'source_node_instance_id',
+ 'target_node_instance_id']
+
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
+
+ type = structures.db.Column(structures.db.String)
+
+ source_node_instance_id = structures.foreign_key(NodeInstance)
+ source_node_instance = structures.one_to_many_relationship(
+ child_class_name='RelationshipInstance',
+ column_name='source_node_instance_id',
+ parent_class_name='NodeInstance',
+ back_reference_name='relationship_instance_source'
+ )
+ target_node_instance_id = structures.foreign_key(NodeInstance)
+ target_node_instance = structures.one_to_many_relationship(
+ child_class_name='RelationshipInstance',
+ column_name='target_node_instance_id',
+ parent_class_name='NodeInstance',
+ back_reference_name='relationship_instance_target'
+ )
+ relationship_storage_id = structures.foreign_key(Relationship)
+ relationship = structures.one_to_many_relationship(
+ child_class_name='RelationshipInstance',
+ column_name='relationship_storage_id',
+ parent_class_name='Relationship',
+ back_reference_name='relationship_instances'
+ )
+
+
+class ProviderContext(structures.SQLModelBase):
+ __tablename__ = 'provider_context'
+
+ id = structures.db.Column(structures.db.Text, primary_key=True)
+ name = structures.db.Column(structures.db.Text, nullable=False)
+ context = structures.db.Column(structures.db.PickleType, nullable=False)
+
+
+class Plugin(structures.SQLModelBase):
+ __tablename__ = 'plugins'
+
+ storage_id = structures.db.Column(structures.db.Integer, primary_key=True, autoincrement=True)
+ id = structures.db.Column(structures.db.Text, index=True)
+
+ archive_name = structures.db.Column(structures.db.Text, nullable=False, index=True)
+ distribution = structures.db.Column(structures.db.Text)
+ distribution_release = structures.db.Column(structures.db.Text)
+ distribution_version = structures.db.Column(structures.db.Text)
+ excluded_wheels = structures.db.Column(structures.db.PickleType)
+ package_name = structures.db.Column(structures.db.Text, nullable=False, index=True)
+ package_source = structures.db.Column(structures.db.Text)
+ package_version = structures.db.Column(structures.db.Text)
+ supported_platform = structures.db.Column(structures.db.PickleType)
+ supported_py_versions = structures.db.Column(structures.db.PickleType)
+ uploaded_at = structures.db.Column(structures.UTCDateTime, nullable=False, index=True)
+ wheels = structures.db.Column(structures.db.PickleType, nullable=False)
+
+
+class Task(structures.SQLModelBase):
"""
A Model which represents an task
"""
+ __tablename__ = 'task'
class _Validation(object):
@@ -408,37 +582,24 @@ class Task(Model):
raise ValueError('Max attempts can be either -1 (infinite) or any positive number. '
'Got {value}'.format(value=value))
- PENDING = 'pending'
- RETRYING = 'retrying'
- SENT = 'sent'
- STARTED = 'started'
- SUCCESS = 'success'
- FAILED = 'failed'
- STATES = (
- PENDING,
- RETRYING,
- SENT,
- STARTED,
- SUCCESS,
- FAILED,
- )
- WAIT_STATES = [PENDING, RETRYING]
- END_STATES = [SUCCESS, FAILED]
INFINITE_RETRIES = -1
- id = Field(type=basestring, default=uuid_generator)
- status = Field(type=basestring, choices=STATES, default=PENDING)
- execution_id = Field(type=basestring)
- due_at = Field(type=datetime, default=datetime.utcnow)
- started_at = Field(type=datetime, default=None)
- ended_at = Field(type=datetime, default=None)
- max_attempts = Field(type=int, default=1, validation_func=_Validation.validate_max_attempts)
- retry_count = Field(type=int, default=0)
- retry_interval = Field(type=(int, float), default=0)
- ignore_failure = Field(type=bool, default=False)
+ id = structures.db.Column(structures.db.String, primary_key=True, default=uuid_generator)
+ status = structures.db.Column(structures.db.Enum(*states.TaskState.STATES),
+ name='task_status',
+ default=states.TaskState.PENDING)
+
+ execution_id = structures.db.Column(structures.db.String)
+ due_at = structures.db.Column(structures.UTCDateTime, default=datetime.utcnow)
+ started_at = structures.db.Column(structures.UTCDateTime, default=None)
+ ended_at = structures.db.Column(structures.UTCDateTime, default=None)
+ max_attempts = structures.db.Column(structures.db.Integer, default=1) # , validation_func=_Validation.validate_max_attempts)
+ retry_count = structures.db.Column(structures.db.Integer, default=0)
+ retry_interval = structures.db.Column(structures.db.Float, default=0)
+ ignore_failure = structures.db.Column(structures.db.Boolean, default=False)
# Operation specific fields
- name = Field(type=basestring)
- operation_mapping = Field(type=basestring)
- actor = Field()
- inputs = Field(type=dict, default=lambda: {})
+ name = structures.db.Column(structures.db.String)
+ operation_mapping = structures.db.Column(structures.db.String)
+ actor = structures.db.Column()
+ inputs = structures.db.column(structures.db.PickleType(comparator=lambda *a: False))