You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ariatosca.apache.org by mxmrlv <gi...@git.apache.org> on 2016/12/01 10:58:35 UTC
[GitHub] incubator-ariatosca pull request #31: Aria 30 sql based storage implementati...
GitHub user mxmrlv opened a pull request:
https://github.com/apache/incubator-ariatosca/pull/31
Aria 30 sql based storage implementation
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/apache/incubator-ariatosca ARIA-30-SQL-based-storage-implementation
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/incubator-ariatosca/pull/31.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #31
----
commit 2d6b9375d0665f4fabbd2acac8d5f10a0d20adcb
Author: mxmrlv <mx...@gmail.com>
Date: 2016-11-27T11:20:46Z
Storage is now sql based with SQLAlchemy based models
commit 91f9de43b0042d5619f9d9539350fc5847287be6
Author: mxmrlv <mx...@gmail.com>
Date: 2016-12-01T10:56:11Z
more lynting
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90778657
--- Diff: aria/storage/models.py ---
@@ -148,265 +302,220 @@ def __lt__(self, other):
return False
-class DeploymentUpdate(Model):
+class DeploymentModification(SQLModelBase):
"""
- A Model which represents a deployment update
+ Deployment modification model representation.
"""
- INITIALIZING = 'initializing'
- SUCCESSFUL = 'successful'
- UPDATING = 'updating'
- FINALIZING = 'finalizing'
- EXECUTING_WORKFLOW = 'executing_workflow'
- FAILED = 'failed'
+ __tablename__ = 'deployment_modifications'
- STATES = [
- INITIALIZING,
- SUCCESSFUL,
- UPDATING,
- FINALIZING,
- EXECUTING_WORKFLOW,
- FAILED,
- ]
-
- # '{0}-{1}'.format(kwargs['deployment_id'], uuid4())
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- state = Field(type=basestring, choices=STATES, default=INITIALIZING)
- deployment_plan = Field()
- deployment_update_nodes = Field(default=None)
- deployment_update_node_instances = Field(default=None)
- deployment_update_deployment = Field(default=None)
- modified_entity_ids = Field(default=None)
- execution_id = Field(type=basestring)
- steps = IterPointerField(type=DeploymentUpdateStep, default=())
-
-
-class Execution(Model):
- """
- A Model which represents an execution
- """
+ STARTED = 'started'
+ FINISHED = 'finished'
+ ROLLEDBACK = 'rolledback'
- class _Validation(object):
-
- @staticmethod
- def execution_status_transition_validation(_, value, instance):
- """Validation function that verifies execution status transitions are OK"""
- try:
- current_status = instance.status
- except AttributeError:
- return
- valid_transitions = Execution.VALID_TRANSITIONS.get(current_status, [])
- if current_status != value and value not in valid_transitions:
- raise ValueError('Cannot change execution status from {current} to {new}'.format(
- current=current_status,
- new=value))
+ STATES = [STARTED, FINISHED, ROLLEDBACK]
+ END_STATES = [FINISHED, ROLLEDBACK]
- TERMINATED = 'terminated'
- FAILED = 'failed'
- CANCELLED = 'cancelled'
- PENDING = 'pending'
- STARTED = 'started'
- CANCELLING = 'cancelling'
- STATES = (
- TERMINATED,
- FAILED,
- CANCELLED,
- PENDING,
- STARTED,
- CANCELLING,
- )
- END_STATES = [TERMINATED, FAILED, CANCELLED]
- ACTIVE_STATES = [state for state in STATES if state not in END_STATES]
- VALID_TRANSITIONS = {
- PENDING: [STARTED, CANCELLED],
- STARTED: END_STATES + [CANCELLING],
- CANCELLING: END_STATES
- }
+ deployment_fk = foreign_key(Deployment.storage_id)
+ _private_fields = ['deployment_fk']
+ deployment_id = association_proxy('deployment', 'id')
- id = Field(type=basestring, default=uuid_generator)
- status = Field(type=basestring, choices=STATES,
- validation_func=_Validation.execution_status_transition_validation)
- deployment_id = Field(type=basestring)
- workflow_id = Field(type=basestring)
- blueprint_id = Field(type=basestring)
- created_at = Field(type=datetime, default=datetime.utcnow)
- started_at = Field(type=datetime, default=None)
- ended_at = Field(type=datetime, default=None)
- error = Field(type=basestring, default=None)
- parameters = Field()
+ context = Column(MutableDict.as_mutable(Dict))
+ created_at = Column(DateTime, nullable=False, index=True)
+ ended_at = Column(DateTime, index=True)
+ modified_nodes = Column(MutableDict.as_mutable(Dict))
+ node_instances = Column(MutableDict.as_mutable(Dict))
+ status = Column(Enum(*STATES, name='deployment_modification_status'))
+ @declared_attr
+ def deployment(cls):
+ return one_to_many_relationship(cls,
+ Deployment,
+ cls.deployment_fk,
+ backreference='modifications')
-class Relationship(Model):
+
+class Node(SQLModelBase):
"""
- A Model which represents a relationship
+ Node model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- source_id = Field(type=basestring)
- target_id = Field(type=basestring)
- source_interfaces = Field(type=dict)
- source_operations = Field(type=dict)
- target_interfaces = Field(type=dict)
- target_operations = Field(type=dict)
- type = Field(type=basestring)
- type_hierarchy = Field(type=list)
- properties = Field(type=dict)
-
-
-class Node(Model):
+ __tablename__ = 'nodes'
+
+ # See base class for an explanation on these properties
+ is_id_unique = False
+
+ _private_fields = ['deployment_fk']
+ deployment_fk = foreign_key(Deployment.storage_id)
+
+ deployment_id = association_proxy('deployment', 'id')
+ blueprint_id = association_proxy('blueprint', 'id')
+
+ @declared_attr
+ def deployment(cls):
+ return one_to_many_relationship(cls, Deployment, cls.deployment_fk)
+
+ deploy_number_of_instances = Column(Integer, nullable=False)
+ # TODO: This probably should be a foreign key, but there's no guarantee
+ # in the code, currently, that the host will be created beforehand
+ host_id = Column(Text)
+ max_number_of_instances = Column(Integer, nullable=False)
+ min_number_of_instances = Column(Integer, nullable=False)
+ number_of_instances = Column(Integer, nullable=False)
+ planned_number_of_instances = Column(Integer, nullable=False)
+ plugins = Column(MutableDict.as_mutable(Dict))
+ plugins_to_install = Column(MutableDict.as_mutable(Dict))
+ properties = Column(MutableDict.as_mutable(Dict))
+ operations = Column(MutableDict.as_mutable(Dict))
+ type = Column(Text, nullable=False, index=True)
+ type_hierarchy = Column(PickleType)
+
+
+class Relationship(SQLModelBase):
"""
- A Model which represents a node
+ Relationship model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- blueprint_id = Field(type=basestring)
- type = Field(type=basestring)
- type_hierarchy = Field()
- number_of_instances = Field(type=int)
- planned_number_of_instances = Field(type=int)
- deploy_number_of_instances = Field(type=int)
- host_id = Field(type=basestring, default=None)
- properties = Field(type=dict)
- operations = Field(type=dict)
- plugins = Field(type=list, default=())
- relationships = IterPointerField(type=Relationship)
- plugins_to_install = Field(type=list, default=())
- min_number_of_instances = Field(type=int)
- max_number_of_instances = Field(type=int)
-
- def relationships_by_target(self, target_id):
- """
- Retreives all of the relationship by target.
- :param target_id: the node id of the target of the relationship
- :yields: a relationship which target and node with the specified target_id
- """
- for relationship in self.relationships:
- if relationship.target_id == target_id:
- yield relationship
- # todo: maybe add here Exception if isn't exists (didn't yield one's)
+ __tablename__ = 'relationships'
+ blueprint_id = association_proxy('blueprint', 'id')
+ deployment_id = association_proxy('deployment', 'id')
-class RelationshipInstance(Model):
- """
- A Model which represents a relationship instance
- """
- id = Field(type=basestring, default=uuid_generator)
- target_id = Field(type=basestring)
- target_name = Field(type=basestring)
- source_id = Field(type=basestring)
- source_name = Field(type=basestring)
- type = Field(type=basestring)
- relationship = PointerField(type=Relationship)
+ _private_fields = ['source_node_fk', 'target_node_fk']
+
+ source_node_fk = foreign_key(Node.storage_id)
+ target_node_fk = foreign_key(Node.storage_id)
+
+ @declared_attr
+ def source_node(cls):
+ return one_to_many_relationship(cls, Node, cls.source_node_fk, 'outbound_relationships')
+ @declared_attr
+ def target_node(cls):
+ return one_to_many_relationship(cls, Node, cls.target_node_fk, 'inbound_relationships')
-class NodeInstance(Model):
+ source_interfaces = Column(MutableDict.as_mutable(Dict))
+ source_operations = Column(MutableDict.as_mutable(Dict))
+ target_interfaces = Column(MutableDict.as_mutable(Dict))
+ target_operations = Column(MutableDict.as_mutable(Dict))
+ type = Column(String)
+ type_hierarchy = Column(PickleType)
+ properties = Column(MutableDict.as_mutable(Dict))
+
+
+class NodeInstance(SQLModelBase):
"""
- A Model which represents a node instance
+ Node instance model representation.
"""
- # todo: add statuses
- UNINITIALIZED = 'uninitialized'
- INITIALIZING = 'initializing'
- CREATING = 'creating'
- CONFIGURING = 'configuring'
- STARTING = 'starting'
- DELETED = 'deleted'
- STOPPING = 'stopping'
- DELETING = 'deleting'
- STATES = (
- UNINITIALIZED,
- INITIALIZING,
- CREATING,
- CONFIGURING,
- STARTING,
- DELETED,
- STOPPING,
- DELETING
- )
+ __tablename__ = 'node_instances'
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- runtime_properties = Field(type=dict)
- state = Field(type=basestring, choices=STATES, default=UNINITIALIZED)
- version = Field(type=(basestring, NoneType))
- relationship_instances = IterPointerField(type=RelationshipInstance)
- node = PointerField(type=Node)
- host_id = Field(type=basestring, default=None)
- scaling_groups = Field(default=())
-
- def relationships_by_target(self, target_id):
- """
- Retreives all of the relationship by target.
- :param target_id: the instance id of the target of the relationship
- :yields: a relationship instance which target and node with the specified target_id
- """
- for relationship_instance in self.relationship_instances:
- if relationship_instance.target_id == target_id:
- yield relationship_instance
- # todo: maybe add here Exception if isn't exists (didn't yield one's)
+ node_fk = foreign_key(Node.storage_id)
+ deployment_fk = foreign_key(Deployment.storage_id)
+ _private_fields = ['node_fk', 'deployment_fk']
+
+ node_id = association_proxy('node', 'id')
+ deployment_id = association_proxy('node', 'deployment_id')
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r91277691
--- Diff: tests/storage/__init__.py ---
@@ -33,10 +35,30 @@ def teardown_method(self):
rmtree(self.path, ignore_errors=True)
-def get_sqlite_api_kwargs():
- engine = create_engine('sqlite:///:memory:',
- connect_args={'check_same_thread': False},
- poolclass=StaticPool)
- session = orm.sessionmaker(bind=engine)()
+def get_sqlite_api_kwargs(base_dir=None, filename='memory'):
--- End diff --
memory->db.sqlit
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90503205
--- Diff: aria/storage/mapi/sql.py ---
@@ -0,0 +1,368 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+try:
+ from psycopg2 import DatabaseError as Psycopg2DBError
+ sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
--- End diff --
remove psycopg2
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by ran-z <gi...@git.apache.org>.
Github user ran-z commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90774109
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
--- End diff --
context manager for locking
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by ran-z <gi...@git.apache.org>.
Github user ran-z commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90774188
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
+ query = self._get_query(include, filters)
+ if locking:
+ query = query.with_for_update()
+ result = query.first()
+
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with ID `{1}` was not found'
+ .format(self.model_cls.__name__, entry_id)
+ )
+ return result
+
+ def iter(self,
--- End diff --
1) list
2) use paginate etc.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90778672
--- Diff: aria/storage/models.py ---
@@ -148,265 +302,220 @@ def __lt__(self, other):
return False
-class DeploymentUpdate(Model):
+class DeploymentModification(SQLModelBase):
"""
- A Model which represents a deployment update
+ Deployment modification model representation.
"""
- INITIALIZING = 'initializing'
- SUCCESSFUL = 'successful'
- UPDATING = 'updating'
- FINALIZING = 'finalizing'
- EXECUTING_WORKFLOW = 'executing_workflow'
- FAILED = 'failed'
+ __tablename__ = 'deployment_modifications'
- STATES = [
- INITIALIZING,
- SUCCESSFUL,
- UPDATING,
- FINALIZING,
- EXECUTING_WORKFLOW,
- FAILED,
- ]
-
- # '{0}-{1}'.format(kwargs['deployment_id'], uuid4())
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- state = Field(type=basestring, choices=STATES, default=INITIALIZING)
- deployment_plan = Field()
- deployment_update_nodes = Field(default=None)
- deployment_update_node_instances = Field(default=None)
- deployment_update_deployment = Field(default=None)
- modified_entity_ids = Field(default=None)
- execution_id = Field(type=basestring)
- steps = IterPointerField(type=DeploymentUpdateStep, default=())
-
-
-class Execution(Model):
- """
- A Model which represents an execution
- """
+ STARTED = 'started'
+ FINISHED = 'finished'
+ ROLLEDBACK = 'rolledback'
- class _Validation(object):
-
- @staticmethod
- def execution_status_transition_validation(_, value, instance):
- """Validation function that verifies execution status transitions are OK"""
- try:
- current_status = instance.status
- except AttributeError:
- return
- valid_transitions = Execution.VALID_TRANSITIONS.get(current_status, [])
- if current_status != value and value not in valid_transitions:
- raise ValueError('Cannot change execution status from {current} to {new}'.format(
- current=current_status,
- new=value))
+ STATES = [STARTED, FINISHED, ROLLEDBACK]
+ END_STATES = [FINISHED, ROLLEDBACK]
- TERMINATED = 'terminated'
- FAILED = 'failed'
- CANCELLED = 'cancelled'
- PENDING = 'pending'
- STARTED = 'started'
- CANCELLING = 'cancelling'
- STATES = (
- TERMINATED,
- FAILED,
- CANCELLED,
- PENDING,
- STARTED,
- CANCELLING,
- )
- END_STATES = [TERMINATED, FAILED, CANCELLED]
- ACTIVE_STATES = [state for state in STATES if state not in END_STATES]
- VALID_TRANSITIONS = {
- PENDING: [STARTED, CANCELLED],
- STARTED: END_STATES + [CANCELLING],
- CANCELLING: END_STATES
- }
+ deployment_fk = foreign_key(Deployment.storage_id)
+ _private_fields = ['deployment_fk']
+ deployment_id = association_proxy('deployment', 'id')
- id = Field(type=basestring, default=uuid_generator)
- status = Field(type=basestring, choices=STATES,
- validation_func=_Validation.execution_status_transition_validation)
- deployment_id = Field(type=basestring)
- workflow_id = Field(type=basestring)
- blueprint_id = Field(type=basestring)
- created_at = Field(type=datetime, default=datetime.utcnow)
- started_at = Field(type=datetime, default=None)
- ended_at = Field(type=datetime, default=None)
- error = Field(type=basestring, default=None)
- parameters = Field()
+ context = Column(MutableDict.as_mutable(Dict))
+ created_at = Column(DateTime, nullable=False, index=True)
+ ended_at = Column(DateTime, index=True)
+ modified_nodes = Column(MutableDict.as_mutable(Dict))
+ node_instances = Column(MutableDict.as_mutable(Dict))
+ status = Column(Enum(*STATES, name='deployment_modification_status'))
+ @declared_attr
+ def deployment(cls):
+ return one_to_many_relationship(cls,
+ Deployment,
+ cls.deployment_fk,
+ backreference='modifications')
-class Relationship(Model):
+
+class Node(SQLModelBase):
"""
- A Model which represents a relationship
+ Node model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- source_id = Field(type=basestring)
- target_id = Field(type=basestring)
- source_interfaces = Field(type=dict)
- source_operations = Field(type=dict)
- target_interfaces = Field(type=dict)
- target_operations = Field(type=dict)
- type = Field(type=basestring)
- type_hierarchy = Field(type=list)
- properties = Field(type=dict)
-
-
-class Node(Model):
+ __tablename__ = 'nodes'
+
+ # See base class for an explanation on these properties
+ is_id_unique = False
+
+ _private_fields = ['deployment_fk']
+ deployment_fk = foreign_key(Deployment.storage_id)
+
+ deployment_id = association_proxy('deployment', 'id')
+ blueprint_id = association_proxy('blueprint', 'id')
+
+ @declared_attr
+ def deployment(cls):
+ return one_to_many_relationship(cls, Deployment, cls.deployment_fk)
+
+ deploy_number_of_instances = Column(Integer, nullable=False)
+ # TODO: This probably should be a foreign key, but there's no guarantee
+ # in the code, currently, that the host will be created beforehand
+ host_id = Column(Text)
+ max_number_of_instances = Column(Integer, nullable=False)
+ min_number_of_instances = Column(Integer, nullable=False)
+ number_of_instances = Column(Integer, nullable=False)
+ planned_number_of_instances = Column(Integer, nullable=False)
+ plugins = Column(MutableDict.as_mutable(Dict))
+ plugins_to_install = Column(MutableDict.as_mutable(Dict))
+ properties = Column(MutableDict.as_mutable(Dict))
+ operations = Column(MutableDict.as_mutable(Dict))
+ type = Column(Text, nullable=False, index=True)
+ type_hierarchy = Column(PickleType)
+
+
+class Relationship(SQLModelBase):
"""
- A Model which represents a node
+ Relationship model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- blueprint_id = Field(type=basestring)
- type = Field(type=basestring)
- type_hierarchy = Field()
- number_of_instances = Field(type=int)
- planned_number_of_instances = Field(type=int)
- deploy_number_of_instances = Field(type=int)
- host_id = Field(type=basestring, default=None)
- properties = Field(type=dict)
- operations = Field(type=dict)
- plugins = Field(type=list, default=())
- relationships = IterPointerField(type=Relationship)
- plugins_to_install = Field(type=list, default=())
- min_number_of_instances = Field(type=int)
- max_number_of_instances = Field(type=int)
-
- def relationships_by_target(self, target_id):
- """
- Retreives all of the relationship by target.
- :param target_id: the node id of the target of the relationship
- :yields: a relationship which target and node with the specified target_id
- """
- for relationship in self.relationships:
- if relationship.target_id == target_id:
- yield relationship
- # todo: maybe add here Exception if isn't exists (didn't yield one's)
+ __tablename__ = 'relationships'
+ blueprint_id = association_proxy('blueprint', 'id')
+ deployment_id = association_proxy('deployment', 'id')
-class RelationshipInstance(Model):
- """
- A Model which represents a relationship instance
- """
- id = Field(type=basestring, default=uuid_generator)
- target_id = Field(type=basestring)
- target_name = Field(type=basestring)
- source_id = Field(type=basestring)
- source_name = Field(type=basestring)
- type = Field(type=basestring)
- relationship = PointerField(type=Relationship)
+ _private_fields = ['source_node_fk', 'target_node_fk']
+
+ source_node_fk = foreign_key(Node.storage_id)
+ target_node_fk = foreign_key(Node.storage_id)
+
+ @declared_attr
+ def source_node(cls):
+ return one_to_many_relationship(cls, Node, cls.source_node_fk, 'outbound_relationships')
+ @declared_attr
+ def target_node(cls):
+ return one_to_many_relationship(cls, Node, cls.target_node_fk, 'inbound_relationships')
-class NodeInstance(Model):
+ source_interfaces = Column(MutableDict.as_mutable(Dict))
+ source_operations = Column(MutableDict.as_mutable(Dict))
+ target_interfaces = Column(MutableDict.as_mutable(Dict))
+ target_operations = Column(MutableDict.as_mutable(Dict))
+ type = Column(String)
+ type_hierarchy = Column(PickleType)
+ properties = Column(MutableDict.as_mutable(Dict))
+
+
+class NodeInstance(SQLModelBase):
"""
- A Model which represents a node instance
+ Node instance model representation.
"""
- # todo: add statuses
- UNINITIALIZED = 'uninitialized'
- INITIALIZING = 'initializing'
- CREATING = 'creating'
- CONFIGURING = 'configuring'
- STARTING = 'starting'
- DELETED = 'deleted'
- STOPPING = 'stopping'
- DELETING = 'deleting'
- STATES = (
- UNINITIALIZED,
- INITIALIZING,
- CREATING,
- CONFIGURING,
- STARTING,
- DELETED,
- STOPPING,
- DELETING
- )
+ __tablename__ = 'node_instances'
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- runtime_properties = Field(type=dict)
- state = Field(type=basestring, choices=STATES, default=UNINITIALIZED)
- version = Field(type=(basestring, NoneType))
- relationship_instances = IterPointerField(type=RelationshipInstance)
- node = PointerField(type=Node)
- host_id = Field(type=basestring, default=None)
- scaling_groups = Field(default=())
-
- def relationships_by_target(self, target_id):
- """
- Retreives all of the relationship by target.
- :param target_id: the instance id of the target of the relationship
- :yields: a relationship instance which target and node with the specified target_id
- """
- for relationship_instance in self.relationship_instances:
- if relationship_instance.target_id == target_id:
- yield relationship_instance
- # todo: maybe add here Exception if isn't exists (didn't yield one's)
+ node_fk = foreign_key(Node.storage_id)
+ deployment_fk = foreign_key(Deployment.storage_id)
+ _private_fields = ['node_fk', 'deployment_fk']
+
+ node_id = association_proxy('node', 'id')
+ deployment_id = association_proxy('node', 'deployment_id')
+
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
+ id = Column(Text, index=True)
+
+ # TODO: This probably should be a foreign key, but there's no guarantee
+ # in the code, currently, that the host will be created beforehand
+ host_id = Column(Text)
+ runtime_properties = Column(MutableDict.as_mutable(Dict))
+ scaling_groups = Column(MutableDict.as_mutable(Dict))
+ state = Column(Text, nullable=False)
+ version = Column(Integer, default=1)
+ @declared_attr
+ def node(cls):
+ return one_to_many_relationship(cls, Node, cls.node_fk)
-class DeploymentModification(Model):
+
+class RelationshipInstance(SQLModelBase):
"""
- A Model which represents a deployment modification
+ Relationship instance model representation.
"""
- STARTED = 'started'
- FINISHED = 'finished'
- ROLLEDBACK = 'rolledback'
- END_STATES = [FINISHED, ROLLEDBACK]
+ __tablename__ = 'relationship_instances'
+
+ blueprint_id = association_proxy('blueprint', 'id')
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779014
--- Diff: aria/storage/structures.py ---
@@ -27,281 +27,189 @@
* Model - abstract model implementation.
"""
import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
+
+from sqlalchemy import VARCHAR
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+# pylint: disable=unused-import
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import (
+ schema,
+ Column,
+ Integer,
+ Text,
+ DateTime,
+ Boolean,
+ Enum,
+ String,
+ PickleType,
+ Float,
+ TypeDecorator,
+ ForeignKey,
+ orm,
)
+Model = declarative_base()
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+def foreign_key(foreign_key_column, nullable=False):
+ """Return a ForeignKey object with the relevant
-class Field(ValidatorMixin):
+ :param foreign_key_column: Unique id column in the parent table
+ :param nullable: Should the column be allowed to remain empty
"""
- A single field implementation
+ return Column(
+ ForeignKey(foreign_key_column, ondelete='CASCADE'),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(child_class,
+ parent_class,
+ foreign_key_column,
+ backreference=None):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param parent_class: Class of the parent table
+ :param child_class: Class of the child table
+ :param foreign_key_column: The column of the foreign key
+ :param backreference: The name to give to the reference to the child
"""
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
-
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
-
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
-
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ backreference = backreference or child_class.__tablename__
+ return relationship(
+ parent_class,
+ primaryjoin=lambda: parent_class.storage_id == foreign_key_column,
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=backref(backreference, cascade='all')
+ )
-class IterField(Field):
+class Dict(TypeDecorator):
"""
- Represents an iterable field.
+ Dict representation of type.
"""
- def __init__(self, **kwargs):
- """
- Simple iterable field manager.
- This field type don't have choices option.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- super(IterField, self).__init__(choices=(), **kwargs)
+ def process_literal_param(self, value, dialect):
+ pass
- def validate_value(self, name, values, *args):
- """
- Validates the value of each iterable value.
+ @property
+ def python_type(self):
+ return dict
- :param name: the name of the field.
- :param values: the values of the field.
- """
- for value in values:
- self.validate_instance(name, value, self.type)
+ impl = VARCHAR
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ value = json.dumps(value)
+ return value
-class PointerField(Field):
- """
- A single pointer field implementation.
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = json.loads(value)
+ return value
- Any PointerField points via id to another document.
+
+class MutableDict(Mutable, dict):
+ """
+ Enables tracking for dict values.
"""
+ @classmethod
+ def coerce(cls, key, value):
+ "Convert plain dictionaries to MutableDict."
- def __init__(self, type, **kwargs):
- assert issubclass(type, Model)
- super(PointerField, self).__init__(type=type, **kwargs)
+ if not isinstance(value, MutableDict):
+ if isinstance(value, dict):
+ return MutableDict(value)
+ # this call will raise ValueError
+ return Mutable.coerce(key, value)
+ else:
+ return value
-class IterPointerField(IterField, PointerField):
- """
- An iterable pointers field.
+ def __setitem__(self, key, value):
+ "Detect dictionary set events and emit change events."
- Any IterPointerField points via id to other documents.
- """
- pass
+ dict.__setitem__(self, key, value)
+ self.changed()
+ def __delitem__(self, key):
+ "Detect dictionary del events and emit change events."
-class Model(object):
- """
- Base class for all of the storage models.
- """
- id = None
+ dict.__delitem__(self, key)
+ self.changed()
- def __init__(self, **fields):
- """
- Abstract class for any model in the storage.
- The Initializer creates attributes according to the (keyword arguments) that given
- Each value is validated according to the Field.
- Each model has to have and ID Field.
- :param fields: each item is validated and transformed into instance attributes.
- """
- self._assert_model_have_id_field(**fields)
- missing_fields, unexpected_fields = self._setup_fields(fields)
+class SQLModelBase(Model):
+ """
+ Abstract base class for all SQL models that allows [de]serialization
+ """
+ # SQLAlchemy syntax
+ __abstract__ = True
- if missing_fields:
- raise StorageError(
- 'Model {name} got missing keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=missing_fields))
+ # Does the class represent a resource (Blueprint, Deployment, etc.) or a
+ # management table (User, Tenant, etc.), as they are handled differently
+ is_resource = False
- if unexpected_fields:
- raise StorageError(
- 'Model {name} got unexpected keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=unexpected_fields))
+ # This would be overridden once the models are created.
--- End diff --
tell the world why i really did it
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by ran-z <gi...@git.apache.org>.
Github user ran-z commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90774474
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
+ query = self._get_query(include, filters)
+ if locking:
+ query = query.with_for_update()
+ result = query.first()
+
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with ID `{1}` was not found'
+ .format(self.model_cls.__name__, entry_id)
+ )
+ return result
+
+ def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+ """Return a (possibly empty) list of `model_class` results
+ """
+ query = self._get_query(include, filters, sort)
+
+ results, _, _, _ = self._paginate(query, pagination)
+
+ for result in results:
+ yield result
+
+ def put(self, entry, **kwargs):
+ """Create a `model_class` instance from a serializable `model` object
+
+ :param entry: A dict with relevant kwargs, or an instance of a class
+ that has a `to_dict` method, and whose attributes match the columns
+ of `model_class` (might also my just an instance of `model_class`)
+ :return: An instance of `model_class`
+ """
+ self._session.add(entry)
+ self._safe_commit()
+ return entry
+
+ def delete(self, entry_id, filters=None, **kwargs):
+ """Delete a single result based on the model class and element ID
+ """
+ try:
+ instance = self.get(
+ entry_id,
+ filters=filters
+ )
+ except exceptions.StorageError:
+ raise exceptions.StorageError(
+ 'Could not delete {0} with ID `{1}` - element not found'
+ .format(
+ self.model_cls.__name__,
+ entry_id
+ )
+ )
+ self._load_properties(instance)
+ self._session.delete(instance)
+ self._safe_commit()
+ return instance
+
+ # TODO: this might need rework
+ def update(self, entry, **kwargs):
+ """Add `instance` to the DB session, and attempt to commit
+
+ :return: The updated instance
+ """
+ return self.put(entry)
+
+ def refresh(self, entry):
+ """Reload the instance with fresh information from the DB
+
+ :param entry: Instance to be re-loaded from the DB
+ :return: The refreshed instance
+ """
+ self._session.refresh(entry)
+ self._load_properties(entry)
+ return entry
+
+ def _destroy_connection(self):
--- End diff --
move
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779945
--- Diff: tests/orchestrator/workflows/builtin/test_heal.py ---
@@ -52,8 +47,9 @@ def test_heal_dependent_node(ctx):
assert_node_uninstall_operations(dependent_node_uninstall_tasks, with_relationships=True)
assert_node_install_operations(dependent_node_install_tasks, with_relationships=True)
+def test_heal_dependency_node():
+ ctx = mock.context.simple()
--- End diff --
check fixture
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90778551
--- Diff: aria/storage/models.py ---
@@ -148,265 +302,220 @@ def __lt__(self, other):
return False
-class DeploymentUpdate(Model):
+class DeploymentModification(SQLModelBase):
"""
- A Model which represents a deployment update
+ Deployment modification model representation.
"""
- INITIALIZING = 'initializing'
- SUCCESSFUL = 'successful'
- UPDATING = 'updating'
- FINALIZING = 'finalizing'
- EXECUTING_WORKFLOW = 'executing_workflow'
- FAILED = 'failed'
+ __tablename__ = 'deployment_modifications'
- STATES = [
- INITIALIZING,
- SUCCESSFUL,
- UPDATING,
- FINALIZING,
- EXECUTING_WORKFLOW,
- FAILED,
- ]
-
- # '{0}-{1}'.format(kwargs['deployment_id'], uuid4())
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- state = Field(type=basestring, choices=STATES, default=INITIALIZING)
- deployment_plan = Field()
- deployment_update_nodes = Field(default=None)
- deployment_update_node_instances = Field(default=None)
- deployment_update_deployment = Field(default=None)
- modified_entity_ids = Field(default=None)
- execution_id = Field(type=basestring)
- steps = IterPointerField(type=DeploymentUpdateStep, default=())
-
-
-class Execution(Model):
- """
- A Model which represents an execution
- """
+ STARTED = 'started'
+ FINISHED = 'finished'
+ ROLLEDBACK = 'rolledback'
- class _Validation(object):
-
- @staticmethod
- def execution_status_transition_validation(_, value, instance):
- """Validation function that verifies execution status transitions are OK"""
- try:
- current_status = instance.status
- except AttributeError:
- return
- valid_transitions = Execution.VALID_TRANSITIONS.get(current_status, [])
- if current_status != value and value not in valid_transitions:
- raise ValueError('Cannot change execution status from {current} to {new}'.format(
- current=current_status,
- new=value))
+ STATES = [STARTED, FINISHED, ROLLEDBACK]
+ END_STATES = [FINISHED, ROLLEDBACK]
- TERMINATED = 'terminated'
- FAILED = 'failed'
- CANCELLED = 'cancelled'
- PENDING = 'pending'
- STARTED = 'started'
- CANCELLING = 'cancelling'
- STATES = (
- TERMINATED,
- FAILED,
- CANCELLED,
- PENDING,
- STARTED,
- CANCELLING,
- )
- END_STATES = [TERMINATED, FAILED, CANCELLED]
- ACTIVE_STATES = [state for state in STATES if state not in END_STATES]
- VALID_TRANSITIONS = {
- PENDING: [STARTED, CANCELLED],
- STARTED: END_STATES + [CANCELLING],
- CANCELLING: END_STATES
- }
+ deployment_fk = foreign_key(Deployment.storage_id)
+ _private_fields = ['deployment_fk']
+ deployment_id = association_proxy('deployment', 'id')
- id = Field(type=basestring, default=uuid_generator)
- status = Field(type=basestring, choices=STATES,
- validation_func=_Validation.execution_status_transition_validation)
- deployment_id = Field(type=basestring)
- workflow_id = Field(type=basestring)
- blueprint_id = Field(type=basestring)
- created_at = Field(type=datetime, default=datetime.utcnow)
- started_at = Field(type=datetime, default=None)
- ended_at = Field(type=datetime, default=None)
- error = Field(type=basestring, default=None)
- parameters = Field()
+ context = Column(MutableDict.as_mutable(Dict))
+ created_at = Column(DateTime, nullable=False, index=True)
+ ended_at = Column(DateTime, index=True)
+ modified_nodes = Column(MutableDict.as_mutable(Dict))
+ node_instances = Column(MutableDict.as_mutable(Dict))
+ status = Column(Enum(*STATES, name='deployment_modification_status'))
+ @declared_attr
+ def deployment(cls):
+ return one_to_many_relationship(cls,
+ Deployment,
+ cls.deployment_fk,
+ backreference='modifications')
-class Relationship(Model):
+
+class Node(SQLModelBase):
"""
- A Model which represents a relationship
+ Node model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- source_id = Field(type=basestring)
- target_id = Field(type=basestring)
- source_interfaces = Field(type=dict)
- source_operations = Field(type=dict)
- target_interfaces = Field(type=dict)
- target_operations = Field(type=dict)
- type = Field(type=basestring)
- type_hierarchy = Field(type=list)
- properties = Field(type=dict)
-
-
-class Node(Model):
+ __tablename__ = 'nodes'
+
+ # See base class for an explanation on these properties
+ is_id_unique = False
+
+ _private_fields = ['deployment_fk']
+ deployment_fk = foreign_key(Deployment.storage_id)
+
+ deployment_id = association_proxy('deployment', 'id')
+ blueprint_id = association_proxy('blueprint', 'id')
+
+ @declared_attr
+ def deployment(cls):
+ return one_to_many_relationship(cls, Deployment, cls.deployment_fk)
+
+ deploy_number_of_instances = Column(Integer, nullable=False)
+ # TODO: This probably should be a foreign key, but there's no guarantee
+ # in the code, currently, that the host will be created beforehand
+ host_id = Column(Text)
+ max_number_of_instances = Column(Integer, nullable=False)
+ min_number_of_instances = Column(Integer, nullable=False)
+ number_of_instances = Column(Integer, nullable=False)
+ planned_number_of_instances = Column(Integer, nullable=False)
+ plugins = Column(MutableDict.as_mutable(Dict))
+ plugins_to_install = Column(MutableDict.as_mutable(Dict))
+ properties = Column(MutableDict.as_mutable(Dict))
+ operations = Column(MutableDict.as_mutable(Dict))
+ type = Column(Text, nullable=False, index=True)
+ type_hierarchy = Column(PickleType)
+
+
+class Relationship(SQLModelBase):
"""
- A Model which represents a node
+ Relationship model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- blueprint_id = Field(type=basestring)
- type = Field(type=basestring)
- type_hierarchy = Field()
- number_of_instances = Field(type=int)
- planned_number_of_instances = Field(type=int)
- deploy_number_of_instances = Field(type=int)
- host_id = Field(type=basestring, default=None)
- properties = Field(type=dict)
- operations = Field(type=dict)
- plugins = Field(type=list, default=())
- relationships = IterPointerField(type=Relationship)
- plugins_to_install = Field(type=list, default=())
- min_number_of_instances = Field(type=int)
- max_number_of_instances = Field(type=int)
-
- def relationships_by_target(self, target_id):
- """
- Retreives all of the relationship by target.
- :param target_id: the node id of the target of the relationship
- :yields: a relationship which target and node with the specified target_id
- """
- for relationship in self.relationships:
- if relationship.target_id == target_id:
- yield relationship
- # todo: maybe add here Exception if isn't exists (didn't yield one's)
+ __tablename__ = 'relationships'
+ blueprint_id = association_proxy('blueprint', 'id')
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90504075
--- Diff: aria/storage/models.py ---
@@ -60,353 +73,612 @@
)
# todo: sort this, maybe move from mgr or move from aria???
-ACTION_TYPES = ()
-ENTITY_TYPES = ()
+# TODO: this must change
+ACTION_TYPES = ('a')
--- End diff --
move into deployment update step
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90780099
--- Diff: tests/storage/test_model_storage.py ---
@@ -13,150 +13,91 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import tempfile
+import shutil
+
import pytest
+
from aria.storage import (
- Storage,
ModelStorage,
models,
+ exceptions,
+ sql_mapi,
)
-from aria.storage import structures
-from aria.storage.exceptions import StorageError
-from aria.storage.structures import Model, Field, PointerField
from aria import application_model_storage
+from tests.storage import get_sqlite_api_params
+
+temp_dir = tempfile.mkdtemp()
+
-from . import InMemoryModelDriver
+@pytest.fixture
+def storage():
+ return ModelStorage(sql_mapi.SQLAlchemyModelAPI, api_params=get_sqlite_api_params())
-def test_storage_base():
- driver = InMemoryModelDriver()
- storage = Storage(driver)
+@pytest.fixture(autouse=True)
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90501421
--- Diff: aria/storage/api.py ---
@@ -0,0 +1,219 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+General storage API
+"""
+from contextlib import contextmanager
+
+from . import exceptions
+
+
+class StorageAPI(object):
+ """
+ General storage Base API
+ """
+ def create(self, **kwargs):
+ """
+ Create a storage API.
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract create method')
+
+ @contextmanager
+ def connect(self):
+ """
+ Established a connection and destroys it after use.
+ :return:
+ """
+ try:
+ self._establish_connection()
+ yield self
+ except BaseException as e:
+ raise exceptions.StorageError(str(e))
+ finally:
+ self._destroy_connection()
+
+ def _establish_connection(self):
+ """
+ Establish a conenction. used in the 'connect' contextmanager.
+ :return:
+ """
+ pass
+
+ def _destroy_connection(self):
+ """
+ Destroy a connection. used in the 'connect' contextmanager.
+ :return:
+ """
+ pass
+
+ def __getattr__(self, item):
+ try:
+ return self.registered[item]
+ except KeyError:
+ return super(StorageAPI, self).__getattribute__(item)
+
+
+class ModelAPI(StorageAPI):
+ """
+ A Base object for the model.
+ """
+ def __init__(self, model_cls, name=None, **kwargs):
+ """
+ Base model API
+
+ :param model_cls: the representing class of the model
+ :param str name: the name of the model
+ :param kwargs:
+ """
+ super(ModelAPI, self).__init__(**kwargs)
+ self._model_cls = model_cls
+ self._name = name or generate_lower_name(model_cls)
+
+ @property
+ def name(self):
+ """
+ The name of the class
+ :return: name of the class
+ """
+ return self._name
+
+ @property
+ def model_cls(self):
+ """
+ The class represting the model
+ :return:
+ """
+ return self._model_cls
+
+ def get(self, entry_id, filters=None, **kwargs):
+ """
+ Get entry from storage.
+
+ :param entry_id:
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract get method')
+
+ def store(self, entry, **kwargs):
--- End diff --
put
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by asfgit <gi...@git.apache.org>.
Github user asfgit closed the pull request at:
https://github.com/apache/incubator-ariatosca/pull/31
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90780094
--- Diff: tests/storage/test_model_storage.py ---
@@ -13,150 +13,91 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import tempfile
+import shutil
+
import pytest
+
from aria.storage import (
- Storage,
ModelStorage,
models,
+ exceptions,
+ sql_mapi,
)
-from aria.storage import structures
-from aria.storage.exceptions import StorageError
-from aria.storage.structures import Model, Field, PointerField
from aria import application_model_storage
+from tests.storage import get_sqlite_api_params
+
+temp_dir = tempfile.mkdtemp()
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90879741
--- Diff: aria/orchestrator/context/toolbelt.py ---
@@ -33,7 +33,7 @@ def dependent_node_instances(self):
:return:
"""
assert isinstance(self._op_context, operation.NodeOperationContext)
- filters = {'target_node_instance_fk': self._op_context.node_instance.storage_id}
--- End diff --
check with Ran: remove unness toolbelt functions
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779796
--- Diff: tests/mock/context.py ---
@@ -13,21 +13,59 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import pytest
+
+
from aria import application_model_storage
from aria.orchestrator import context
+from aria.storage.sql_mapi import SQLAlchemyModelAPI
+
+from tests.storage import get_sqlite_api_params
from . import models
-from ..storage import InMemoryModelDriver
+@pytest.fixture
def simple(**kwargs):
- storage = application_model_storage(InMemoryModelDriver())
- storage.setup()
- storage.blueprint.store(models.get_blueprint())
- storage.deployment.store(models.get_deployment())
+ api_params = get_sqlite_api_params()
+ model_storage = application_model_storage(SQLAlchemyModelAPI, api_params=api_params)
--- End diff --
api_kwargs
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by ran-z <gi...@git.apache.org>.
Github user ran-z commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90774557
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
+ query = self._get_query(include, filters)
+ if locking:
+ query = query.with_for_update()
+ result = query.first()
+
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with ID `{1}` was not found'
+ .format(self.model_cls.__name__, entry_id)
+ )
+ return result
+
+ def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+ """Return a (possibly empty) list of `model_class` results
+ """
+ query = self._get_query(include, filters, sort)
+
+ results, _, _, _ = self._paginate(query, pagination)
+
+ for result in results:
+ yield result
+
+ def put(self, entry, **kwargs):
+ """Create a `model_class` instance from a serializable `model` object
+
+ :param entry: A dict with relevant kwargs, or an instance of a class
+ that has a `to_dict` method, and whose attributes match the columns
+ of `model_class` (might also my just an instance of `model_class`)
+ :return: An instance of `model_class`
+ """
+ self._session.add(entry)
+ self._safe_commit()
+ return entry
+
+ def delete(self, entry_id, filters=None, **kwargs):
+ """Delete a single result based on the model class and element ID
+ """
+ try:
+ instance = self.get(
+ entry_id,
+ filters=filters
+ )
+ except exceptions.StorageError:
+ raise exceptions.StorageError(
+ 'Could not delete {0} with ID `{1}` - element not found'
+ .format(
+ self.model_cls.__name__,
+ entry_id
+ )
+ )
+ self._load_properties(instance)
+ self._session.delete(instance)
+ self._safe_commit()
+ return instance
+
+ # TODO: this might need rework
+ def update(self, entry, **kwargs):
+ """Add `instance` to the DB session, and attempt to commit
+
+ :return: The updated instance
+ """
+ return self.put(entry)
+
+ def refresh(self, entry):
+ """Reload the instance with fresh information from the DB
+
+ :param entry: Instance to be re-loaded from the DB
+ :return: The refreshed instance
+ """
+ self._session.refresh(entry)
+ self._load_properties(entry)
+ return entry
+
+ def _destroy_connection(self):
+ pass
+
+ def _establish_connection(self):
+ pass
+
+ def create(self):
--- End diff --
add use_existing flag, defaulting to true
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90883153
--- Diff: tests/orchestrator/workflows/builtin/test_execute_operation.py ---
@@ -13,16 +13,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import pytest
+
from aria.orchestrator.workflows.api import task
from aria.orchestrator.workflows.builtin.execute_operation import execute_operation
from tests import mock
-def test_execute_operation():
- ctx = mock.context.simple()
+@pytest.fixture
+def ctx():
+ context = mock.context.simple()
+ yield context
+ context.model.drop()
+
+
+def test_execute_operation(ctx):
+ node_instance = ctx.model.node_instance.list(filters={
--- End diff --
get_by_name
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90778697
--- Diff: aria/storage/models.py ---
@@ -148,265 +302,220 @@ def __lt__(self, other):
return False
-class DeploymentUpdate(Model):
+class DeploymentModification(SQLModelBase):
"""
- A Model which represents a deployment update
+ Deployment modification model representation.
"""
- INITIALIZING = 'initializing'
- SUCCESSFUL = 'successful'
- UPDATING = 'updating'
- FINALIZING = 'finalizing'
- EXECUTING_WORKFLOW = 'executing_workflow'
- FAILED = 'failed'
+ __tablename__ = 'deployment_modifications'
- STATES = [
- INITIALIZING,
- SUCCESSFUL,
- UPDATING,
- FINALIZING,
- EXECUTING_WORKFLOW,
- FAILED,
- ]
-
- # '{0}-{1}'.format(kwargs['deployment_id'], uuid4())
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- state = Field(type=basestring, choices=STATES, default=INITIALIZING)
- deployment_plan = Field()
- deployment_update_nodes = Field(default=None)
- deployment_update_node_instances = Field(default=None)
- deployment_update_deployment = Field(default=None)
- modified_entity_ids = Field(default=None)
- execution_id = Field(type=basestring)
- steps = IterPointerField(type=DeploymentUpdateStep, default=())
-
-
-class Execution(Model):
- """
- A Model which represents an execution
- """
+ STARTED = 'started'
+ FINISHED = 'finished'
+ ROLLEDBACK = 'rolledback'
- class _Validation(object):
-
- @staticmethod
- def execution_status_transition_validation(_, value, instance):
- """Validation function that verifies execution status transitions are OK"""
- try:
- current_status = instance.status
- except AttributeError:
- return
- valid_transitions = Execution.VALID_TRANSITIONS.get(current_status, [])
- if current_status != value and value not in valid_transitions:
- raise ValueError('Cannot change execution status from {current} to {new}'.format(
- current=current_status,
- new=value))
+ STATES = [STARTED, FINISHED, ROLLEDBACK]
+ END_STATES = [FINISHED, ROLLEDBACK]
- TERMINATED = 'terminated'
- FAILED = 'failed'
- CANCELLED = 'cancelled'
- PENDING = 'pending'
- STARTED = 'started'
- CANCELLING = 'cancelling'
- STATES = (
- TERMINATED,
- FAILED,
- CANCELLED,
- PENDING,
- STARTED,
- CANCELLING,
- )
- END_STATES = [TERMINATED, FAILED, CANCELLED]
- ACTIVE_STATES = [state for state in STATES if state not in END_STATES]
- VALID_TRANSITIONS = {
- PENDING: [STARTED, CANCELLED],
- STARTED: END_STATES + [CANCELLING],
- CANCELLING: END_STATES
- }
+ deployment_fk = foreign_key(Deployment.storage_id)
+ _private_fields = ['deployment_fk']
+ deployment_id = association_proxy('deployment', 'id')
- id = Field(type=basestring, default=uuid_generator)
- status = Field(type=basestring, choices=STATES,
- validation_func=_Validation.execution_status_transition_validation)
- deployment_id = Field(type=basestring)
- workflow_id = Field(type=basestring)
- blueprint_id = Field(type=basestring)
- created_at = Field(type=datetime, default=datetime.utcnow)
- started_at = Field(type=datetime, default=None)
- ended_at = Field(type=datetime, default=None)
- error = Field(type=basestring, default=None)
- parameters = Field()
+ context = Column(MutableDict.as_mutable(Dict))
+ created_at = Column(DateTime, nullable=False, index=True)
+ ended_at = Column(DateTime, index=True)
+ modified_nodes = Column(MutableDict.as_mutable(Dict))
+ node_instances = Column(MutableDict.as_mutable(Dict))
+ status = Column(Enum(*STATES, name='deployment_modification_status'))
+ @declared_attr
+ def deployment(cls):
+ return one_to_many_relationship(cls,
+ Deployment,
+ cls.deployment_fk,
+ backreference='modifications')
-class Relationship(Model):
+
+class Node(SQLModelBase):
"""
- A Model which represents a relationship
+ Node model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- source_id = Field(type=basestring)
- target_id = Field(type=basestring)
- source_interfaces = Field(type=dict)
- source_operations = Field(type=dict)
- target_interfaces = Field(type=dict)
- target_operations = Field(type=dict)
- type = Field(type=basestring)
- type_hierarchy = Field(type=list)
- properties = Field(type=dict)
-
-
-class Node(Model):
+ __tablename__ = 'nodes'
+
+ # See base class for an explanation on these properties
+ is_id_unique = False
+
+ _private_fields = ['deployment_fk']
+ deployment_fk = foreign_key(Deployment.storage_id)
+
+ deployment_id = association_proxy('deployment', 'id')
+ blueprint_id = association_proxy('blueprint', 'id')
+
+ @declared_attr
+ def deployment(cls):
+ return one_to_many_relationship(cls, Deployment, cls.deployment_fk)
+
+ deploy_number_of_instances = Column(Integer, nullable=False)
+ # TODO: This probably should be a foreign key, but there's no guarantee
+ # in the code, currently, that the host will be created beforehand
+ host_id = Column(Text)
+ max_number_of_instances = Column(Integer, nullable=False)
+ min_number_of_instances = Column(Integer, nullable=False)
+ number_of_instances = Column(Integer, nullable=False)
+ planned_number_of_instances = Column(Integer, nullable=False)
+ plugins = Column(MutableDict.as_mutable(Dict))
+ plugins_to_install = Column(MutableDict.as_mutable(Dict))
+ properties = Column(MutableDict.as_mutable(Dict))
+ operations = Column(MutableDict.as_mutable(Dict))
+ type = Column(Text, nullable=False, index=True)
+ type_hierarchy = Column(PickleType)
+
+
+class Relationship(SQLModelBase):
"""
- A Model which represents a node
+ Relationship model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- blueprint_id = Field(type=basestring)
- type = Field(type=basestring)
- type_hierarchy = Field()
- number_of_instances = Field(type=int)
- planned_number_of_instances = Field(type=int)
- deploy_number_of_instances = Field(type=int)
- host_id = Field(type=basestring, default=None)
- properties = Field(type=dict)
- operations = Field(type=dict)
- plugins = Field(type=list, default=())
- relationships = IterPointerField(type=Relationship)
- plugins_to_install = Field(type=list, default=())
- min_number_of_instances = Field(type=int)
- max_number_of_instances = Field(type=int)
-
- def relationships_by_target(self, target_id):
- """
- Retreives all of the relationship by target.
- :param target_id: the node id of the target of the relationship
- :yields: a relationship which target and node with the specified target_id
- """
- for relationship in self.relationships:
- if relationship.target_id == target_id:
- yield relationship
- # todo: maybe add here Exception if isn't exists (didn't yield one's)
+ __tablename__ = 'relationships'
+ blueprint_id = association_proxy('blueprint', 'id')
+ deployment_id = association_proxy('deployment', 'id')
-class RelationshipInstance(Model):
- """
- A Model which represents a relationship instance
- """
- id = Field(type=basestring, default=uuid_generator)
- target_id = Field(type=basestring)
- target_name = Field(type=basestring)
- source_id = Field(type=basestring)
- source_name = Field(type=basestring)
- type = Field(type=basestring)
- relationship = PointerField(type=Relationship)
+ _private_fields = ['source_node_fk', 'target_node_fk']
+
+ source_node_fk = foreign_key(Node.storage_id)
+ target_node_fk = foreign_key(Node.storage_id)
+
+ @declared_attr
+ def source_node(cls):
+ return one_to_many_relationship(cls, Node, cls.source_node_fk, 'outbound_relationships')
+ @declared_attr
+ def target_node(cls):
+ return one_to_many_relationship(cls, Node, cls.target_node_fk, 'inbound_relationships')
-class NodeInstance(Model):
+ source_interfaces = Column(MutableDict.as_mutable(Dict))
+ source_operations = Column(MutableDict.as_mutable(Dict))
+ target_interfaces = Column(MutableDict.as_mutable(Dict))
+ target_operations = Column(MutableDict.as_mutable(Dict))
+ type = Column(String)
+ type_hierarchy = Column(PickleType)
+ properties = Column(MutableDict.as_mutable(Dict))
+
+
+class NodeInstance(SQLModelBase):
"""
- A Model which represents a node instance
+ Node instance model representation.
"""
- # todo: add statuses
- UNINITIALIZED = 'uninitialized'
- INITIALIZING = 'initializing'
- CREATING = 'creating'
- CONFIGURING = 'configuring'
- STARTING = 'starting'
- DELETED = 'deleted'
- STOPPING = 'stopping'
- DELETING = 'deleting'
- STATES = (
- UNINITIALIZED,
- INITIALIZING,
- CREATING,
- CONFIGURING,
- STARTING,
- DELETED,
- STOPPING,
- DELETING
- )
+ __tablename__ = 'node_instances'
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- runtime_properties = Field(type=dict)
- state = Field(type=basestring, choices=STATES, default=UNINITIALIZED)
- version = Field(type=(basestring, NoneType))
- relationship_instances = IterPointerField(type=RelationshipInstance)
- node = PointerField(type=Node)
- host_id = Field(type=basestring, default=None)
- scaling_groups = Field(default=())
-
- def relationships_by_target(self, target_id):
- """
- Retreives all of the relationship by target.
- :param target_id: the instance id of the target of the relationship
- :yields: a relationship instance which target and node with the specified target_id
- """
- for relationship_instance in self.relationship_instances:
- if relationship_instance.target_id == target_id:
- yield relationship_instance
- # todo: maybe add here Exception if isn't exists (didn't yield one's)
+ node_fk = foreign_key(Node.storage_id)
+ deployment_fk = foreign_key(Deployment.storage_id)
+ _private_fields = ['node_fk', 'deployment_fk']
+
+ node_id = association_proxy('node', 'id')
+ deployment_id = association_proxy('node', 'deployment_id')
+
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
+ id = Column(Text, index=True)
+
+ # TODO: This probably should be a foreign key, but there's no guarantee
+ # in the code, currently, that the host will be created beforehand
+ host_id = Column(Text)
+ runtime_properties = Column(MutableDict.as_mutable(Dict))
+ scaling_groups = Column(MutableDict.as_mutable(Dict))
+ state = Column(Text, nullable=False)
+ version = Column(Integer, default=1)
+ @declared_attr
+ def node(cls):
+ return one_to_many_relationship(cls, Node, cls.node_fk)
-class DeploymentModification(Model):
+
+class RelationshipInstance(SQLModelBase):
"""
- A Model which represents a deployment modification
+ Relationship instance model representation.
"""
- STARTED = 'started'
- FINISHED = 'finished'
- ROLLEDBACK = 'rolledback'
- END_STATES = [FINISHED, ROLLEDBACK]
+ __tablename__ = 'relationship_instances'
+
+ blueprint_id = association_proxy('blueprint', 'id')
+ deployment_id = association_proxy('deployment', 'id')
+
+ relationship_fk = foreign_key(Relationship.storage_id)
+ source_node_instance_fk = foreign_key(NodeInstance.storage_id)
+ target_node_instance_fk = foreign_key(NodeInstance.storage_id)
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- modified_nodes = Field(type=(dict, NoneType))
- added_and_related = IterPointerField(type=NodeInstance)
- removed_and_related = IterPointerField(type=NodeInstance)
- extended_and_related = IterPointerField(type=NodeInstance)
- reduced_and_related = IterPointerField(type=NodeInstance)
- # before_modification = IterPointerField(type=NodeInstance)
- status = Field(type=basestring, choices=(STARTED, FINISHED, ROLLEDBACK))
- created_at = Field(type=datetime)
- ended_at = Field(type=(datetime, NoneType))
- context = Field()
-
-
-class ProviderContext(Model):
+ _private_fields = ['relationship_storage_fk',
+ 'source_node_instance_fk',
+ 'target_node_instance_fk']
+
+ @declared_attr
+ def source_node_instance(cls):
+ return one_to_many_relationship(cls,
+ NodeInstance,
+ cls.source_node_instance_fk,
+ 'outbound_relationship_instances')
+
+ @declared_attr
+ def target_node_instance(cls):
+ return one_to_many_relationship(cls,
+ NodeInstance,
+ cls.target_node_instance_fk,
+ 'inbound_relationship_instances')
+
+ @declared_attr
+ def relationship(cls):
+ return one_to_many_relationship(cls, Relationship, cls.relationship_fk)
+
+
+class ProviderContext(SQLModelBase):
"""
- A Model which represents a provider context
+ Provider context model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- context = Field(type=dict)
- name = Field(type=basestring)
+ __tablename__ = 'provider_context'
+
+ name = Column(Text, nullable=False)
+ context = Column(MutableDict.as_mutable(Dict), nullable=False)
-class Plugin(Model):
+class Plugin(SQLModelBase):
"""
- A Model which represents a plugin
+ Plugin model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- package_name = Field(type=basestring)
- archive_name = Field(type=basestring)
- package_source = Field(type=dict)
- package_version = Field(type=basestring)
- supported_platform = Field(type=basestring)
- distribution = Field(type=basestring)
- distribution_version = Field(type=basestring)
- distribution_release = Field(type=basestring)
- wheels = Field()
- excluded_wheels = Field()
- supported_py_versions = Field(type=list)
- uploaded_at = Field(type=datetime)
-
-
-class Task(Model):
+ __tablename__ = 'plugins'
+
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
--- End diff --
remvoe
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90502146
--- Diff: aria/storage/filesystem_api.py ---
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Filesystem based API Base
+"""
+from multiprocessing import RLock
--- End diff --
move to filesystem rapi
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90778725
--- Diff: aria/storage/models.py ---
@@ -422,23 +531,55 @@ def validate_max_attempts(_, value, *args):
SUCCESS,
FAILED,
)
+
WAIT_STATES = [PENDING, RETRYING]
END_STATES = [SUCCESS, FAILED]
+
+ @validates('max_attempts')
+ def validate_max_attempts(self, _, value): # pylint: disable=no-self-use
+ """Validates that max attempts is either -1 or a positive number"""
+ if value < 1 and value != Task.INFINITE_RETRIES:
+ raise ValueError('Max attempts can be either -1 (infinite) or any positive number. '
+ 'Got {value}'.format(value=value))
+ return value
+
INFINITE_RETRIES = -1
- id = Field(type=basestring, default=uuid_generator)
- status = Field(type=basestring, choices=STATES, default=PENDING)
- execution_id = Field(type=basestring)
- due_at = Field(type=datetime, default=datetime.utcnow)
- started_at = Field(type=datetime, default=None)
- ended_at = Field(type=datetime, default=None)
- max_attempts = Field(type=int, default=1, validation_func=_Validation.validate_max_attempts)
- retry_count = Field(type=int, default=0)
- retry_interval = Field(type=(int, float), default=0)
- ignore_failure = Field(type=bool, default=False)
+ status = Column(Enum(*STATES), name='status', default=PENDING)
+
+ execution_id = Column(String)
+ due_at = Column(DateTime, default=datetime.utcnow, nullable=True)
+ started_at = Column(DateTime, default=None, nullable=True)
+ ended_at = Column(DateTime, default=None, nullable=True)
+ max_attempts = Column(Integer, default=1)
+ retry_count = Column(Integer, default=0)
+ retry_interval = Column(Float, default=0)
+ ignore_failure = Column(Boolean, default=False)
# Operation specific fields
- name = Field(type=basestring)
- operation_mapping = Field(type=basestring)
- actor = Field()
- inputs = Field(type=dict, default=lambda: {})
+ name = Column(String)
+ operation_mapping = Column(String)
+ inputs = Column(MutableDict.as_mutable(Dict))
+
+ @property
+ def actor_storage_id(self):
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779244
--- Diff: aria/storage/structures.py ---
@@ -27,281 +27,189 @@
* Model - abstract model implementation.
"""
import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
+
+from sqlalchemy import VARCHAR
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+# pylint: disable=unused-import
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import (
+ schema,
+ Column,
+ Integer,
+ Text,
+ DateTime,
+ Boolean,
+ Enum,
+ String,
+ PickleType,
+ Float,
+ TypeDecorator,
+ ForeignKey,
+ orm,
)
+Model = declarative_base()
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+def foreign_key(foreign_key_column, nullable=False):
+ """Return a ForeignKey object with the relevant
-class Field(ValidatorMixin):
+ :param foreign_key_column: Unique id column in the parent table
+ :param nullable: Should the column be allowed to remain empty
"""
- A single field implementation
+ return Column(
+ ForeignKey(foreign_key_column, ondelete='CASCADE'),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(child_class,
+ parent_class,
+ foreign_key_column,
+ backreference=None):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param parent_class: Class of the parent table
+ :param child_class: Class of the child table
+ :param foreign_key_column: The column of the foreign key
+ :param backreference: The name to give to the reference to the child
"""
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
-
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
-
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
-
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ backreference = backreference or child_class.__tablename__
+ return relationship(
+ parent_class,
+ primaryjoin=lambda: parent_class.storage_id == foreign_key_column,
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=backref(backreference, cascade='all')
+ )
-class IterField(Field):
+class Dict(TypeDecorator):
"""
- Represents an iterable field.
+ Dict representation of type.
"""
- def __init__(self, **kwargs):
- """
- Simple iterable field manager.
- This field type don't have choices option.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- super(IterField, self).__init__(choices=(), **kwargs)
+ def process_literal_param(self, value, dialect):
+ pass
- def validate_value(self, name, values, *args):
- """
- Validates the value of each iterable value.
+ @property
+ def python_type(self):
+ return dict
- :param name: the name of the field.
- :param values: the values of the field.
- """
- for value in values:
- self.validate_instance(name, value, self.type)
+ impl = VARCHAR
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ value = json.dumps(value)
+ return value
-class PointerField(Field):
- """
- A single pointer field implementation.
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = json.loads(value)
+ return value
- Any PointerField points via id to another document.
+
+class MutableDict(Mutable, dict):
+ """
+ Enables tracking for dict values.
"""
+ @classmethod
+ def coerce(cls, key, value):
+ "Convert plain dictionaries to MutableDict."
- def __init__(self, type, **kwargs):
- assert issubclass(type, Model)
- super(PointerField, self).__init__(type=type, **kwargs)
+ if not isinstance(value, MutableDict):
+ if isinstance(value, dict):
+ return MutableDict(value)
+ # this call will raise ValueError
+ return Mutable.coerce(key, value)
+ else:
+ return value
-class IterPointerField(IterField, PointerField):
- """
- An iterable pointers field.
+ def __setitem__(self, key, value):
+ "Detect dictionary set events and emit change events."
- Any IterPointerField points via id to other documents.
- """
- pass
+ dict.__setitem__(self, key, value)
+ self.changed()
+ def __delitem__(self, key):
+ "Detect dictionary del events and emit change events."
-class Model(object):
- """
- Base class for all of the storage models.
- """
- id = None
+ dict.__delitem__(self, key)
+ self.changed()
- def __init__(self, **fields):
- """
- Abstract class for any model in the storage.
- The Initializer creates attributes according to the (keyword arguments) that given
- Each value is validated according to the Field.
- Each model has to have and ID Field.
- :param fields: each item is validated and transformed into instance attributes.
- """
- self._assert_model_have_id_field(**fields)
- missing_fields, unexpected_fields = self._setup_fields(fields)
+class SQLModelBase(Model):
+ """
+ Abstract base class for all SQL models that allows [de]serialization
+ """
+ # SQLAlchemy syntax
+ __abstract__ = True
- if missing_fields:
- raise StorageError(
- 'Model {name} got missing keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=missing_fields))
+ # Does the class represent a resource (Blueprint, Deployment, etc.) or a
+ # management table (User, Tenant, etc.), as they are handled differently
+ is_resource = False
- if unexpected_fields:
- raise StorageError(
- 'Model {name} got unexpected keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=unexpected_fields))
+ # This would be overridden once the models are created.
+ __table__ = None
- def __repr__(self):
- return '{name}(fields={0})'.format(sorted(self.fields), name=self.__class__.__name__)
+ _private_fields = []
- def __eq__(self, other):
- return (
- isinstance(other, self.__class__) and
- self.fields_dict == other.fields_dict)
+ # Indicates whether the `id` column in this class should be unique
+ is_id_unique = True
- @property
- def fields(self):
- """
- Iterates over the fields of the model.
- :yields: the class's field name
- """
- for name, field in vars(self.__class__).items():
- if isinstance(field, Field):
- yield name
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
+ id = Column(Text, index=True)
- @property
- def fields_dict(self):
- """
- Transforms the instance attributes into a dict.
+ @classmethod
+ def unique_id(cls):
+ return 'id'
+
+ def to_dict(self, suppress_error=False):
+ """Return a dict representation of the model
+
+ :param suppress_error: If set to True, sets `None` to attributes that
+ it's unable to retrieve (e.g., if a relationship wasn't established
+ yet, and so it's impossible to access a property through it)
+ """
+ if suppress_error:
+ res = dict()
+ for field in self.fields():
+ try:
+ field_value = getattr(self, field)
+ except AttributeError:
+ field_value = None
+ res[field] = field_value
+ else:
+ # Can't simply call here `self.to_response()` because inheriting
+ # class might override it, but we always need the same code here
+ res = dict((f, getattr(self, f)) for f in self.fields())
+ return res
- :return: all fields in dict format.
- :rtype dict
- """
- return dict((name, getattr(self, name)) for name in self.fields)
+ @classmethod
+ def fields(cls):
+ """Return the list of field names for this table
- @property
- def json(self):
- """
- Transform the dict of attributes into json
- :return:
+ Mostly for backwards compatibility in the code (that uses `fields`)
"""
- return json.dumps(self.fields_dict)
+ return set(cls.__table__.columns.keys()) - set(cls._private_fields)
- @classmethod
- def _assert_model_have_id_field(cls, **fields_initializer_values):
- if not getattr(cls, 'id', None):
- raise StorageError('Model {cls.__name__} must have id field'.format(cls=cls))
-
- if cls.id.default == cls.id.NO_DEFAULT and 'id' not in fields_initializer_values:
- raise StorageError(
- 'Model {cls.__name__} is missing required '
- 'keyword-only argument: "id"'.format(cls=cls))
-
- def _setup_fields(self, input_fields):
- missing = []
- for field_name in self.fields:
- try:
- field_obj = input_fields.pop(field_name)
- setattr(self, field_name, field_obj)
- except KeyError:
- field = getattr(self.__class__, field_name)
- if field.default == field.NO_DEFAULT:
- missing.append(field_name)
-
- unexpected_fields = input_fields.keys()
- return missing, unexpected_fields
-
-
-class Storage(LoggerMixin):
- """
- Represents the storage
- """
- def __init__(self, driver, items=(), **kwargs):
- super(Storage, self).__init__(**kwargs)
- self.driver = driver
- self.registered = {}
- for item in items:
- self.register(item)
- self.logger.debug('{name} object is ready: {0!r}'.format(
- self, name=self.__class__.__name__))
+ def __str__(self):
+ id_name, id_value = self.unique_id()
+ return '<{0} {1}=`{2}`>'.format(
+ self.__class__.__name__,
+ id_name,
+ id_value
+ )
def __repr__(self):
- return '{name}(driver={self.driver})'.format(
- name=self.__class__.__name__, self=self)
+ return str(self)
- def __getattr__(self, item):
- try:
- return self.registered[item]
- except KeyError:
- return super(Storage, self).__getattribute__(item)
+ def __unicode__(self):
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90882024
--- Diff: aria/storage/models.py ---
@@ -339,94 +324,100 @@ class Node(SQLModelBase):
Node model representation.
"""
__tablename__ = 'nodes'
+ id = Column(Integer, primary_key=True)
# See base class for an explanation on these properties
is_id_unique = False
- _private_fields = ['deployment_fk']
- deployment_fk = foreign_key(Deployment.storage_id)
-
- deployment_id = association_proxy('deployment', 'id')
- blueprint_id = association_proxy('blueprint', 'id')
+ _private_fields = ['deployment_id', 'host_id']
+ deployment_id = foreign_key(Deployment.id)
+ host_id = foreign_key('nodes.id', nullable=True)
@declared_attr
def deployment(cls):
- return one_to_many_relationship(cls, Deployment, cls.deployment_fk)
+ return one_to_many_relationship(cls, Deployment, cls.deployment_id)
deploy_number_of_instances = Column(Integer, nullable=False)
# TODO: This probably should be a foreign key, but there's no guarantee
# in the code, currently, that the host will be created beforehand
- host_id = Column(Text)
+ _host_id = foreign_key('nodes.id', nullable=True)
max_number_of_instances = Column(Integer, nullable=False)
min_number_of_instances = Column(Integer, nullable=False)
number_of_instances = Column(Integer, nullable=False)
planned_number_of_instances = Column(Integer, nullable=False)
- plugins = Column(MutableDict.as_mutable(Dict))
- plugins_to_install = Column(MutableDict.as_mutable(Dict))
- properties = Column(MutableDict.as_mutable(Dict))
- operations = Column(MutableDict.as_mutable(Dict))
+ plugins = Column(Dict)
+ plugins_to_install = Column(Dict)
+ properties = Column(Dict)
+ operations = Column(Dict)
type = Column(Text, nullable=False, index=True)
type_hierarchy = Column(PickleType)
+ host = relationship('Node',
--- End diff --
pretify
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90502764
--- Diff: aria/storage/mapi/inmemory.py ---
@@ -0,0 +1,148 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
--- End diff --
delete
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90498171
--- Diff: aria/storage/__init__.py ---
@@ -37,354 +37,93 @@
* drivers - module, a pool of Aria standard drivers.
* StorageDriver - class, abstract model implementation.
"""
-# todo: rewrite the above package documentation
-# (something like explaning the two types of storage - models and resources)
-from collections import namedtuple
-
-from .structures import Storage, Field, Model, IterField, PointerField
-from .drivers import (
- ModelDriver,
- ResourceDriver,
- FileSystemResourceDriver,
- FileSystemModelDriver,
+from aria.logger import LoggerMixin
+from . import (
+ models,
+ exceptions,
+ api as storage_api,
+ structures
)
-from . import models, exceptions
+
__all__ = (
'ModelStorage',
- 'ResourceStorage',
- 'FileSystemModelDriver',
'models',
'structures',
- 'Field',
- 'IterField',
- 'PointerField',
- 'Model',
- 'drivers',
- 'ModelDriver',
- 'ResourceDriver',
- 'FileSystemResourceDriver',
)
-# todo: think about package output api's...
-# todo: in all drivers name => entry_type
-# todo: change in documentation str => basestring
-class ModelStorage(Storage):
+class Storage(LoggerMixin):
"""
- Managing the models storage.
+ Represents the storage
"""
- def __init__(self, driver, model_classes=(), **kwargs):
- """
- Simple storage client api for Aria applications.
- The storage instance defines the tables/documents/code api.
-
- :param ModelDriver driver: model storage driver.
- :param model_classes: the models to register.
- """
- assert isinstance(driver, ModelDriver)
- super(ModelStorage, self).__init__(driver, model_classes, **kwargs)
-
- def __getattr__(self, table):
- """
- getattr is a shortcut to simple api
-
- for Example:
- >> storage = ModelStorage(driver=FileSystemModelDriver('/tmp'))
- >> node_table = storage.node
- >> for node in node_table:
- >> print node
-
- :param str table: table name to get
- :return: a storage object that mapped to the table name
- """
- return super(ModelStorage, self).__getattr__(table)
-
- def register(self, model_cls):
- """
- Registers the model type in the resource storage manager.
- :param model_cls: the model to register.
- """
- model_name = generate_lower_name(model_cls)
- model_api = _ModelApi(model_name, self.driver, model_cls)
- self.registered[model_name] = model_api
-
- for pointer_schema_register in model_api.pointer_mapping.values():
- model_cls = pointer_schema_register.model_cls
- self.register(model_cls)
-
-_Pointer = namedtuple('_Pointer', 'name, is_iter')
-
-
-class _ModelApi(object):
- def __init__(self, name, driver, model_cls):
- """
- Managing the model in the storage, using the driver.
-
- :param basestring name: the name of the model.
- :param ModelDriver driver: the driver which supports this model in the storage.
- :param Model model_cls: table/document class model.
- """
- assert isinstance(driver, ModelDriver)
- assert issubclass(model_cls, Model)
- self.name = name
- self.driver = driver
- self.model_cls = model_cls
- self.pointer_mapping = {}
- self._setup_pointers_mapping()
-
- def _setup_pointers_mapping(self):
- for field_name, field_cls in vars(self.model_cls).items():
- if not(isinstance(field_cls, PointerField) and field_cls.type):
- continue
- pointer_key = _Pointer(field_name, is_iter=isinstance(field_cls, IterField))
- self.pointer_mapping[pointer_key] = self.__class__(
- name=generate_lower_name(field_cls.type),
- driver=self.driver,
- model_cls=field_cls.type)
-
- def __iter__(self):
- return self.iter()
+ def __init__(self, api, items=(), api_params=None, **kwargs):
--- End diff --
api_cls, api_params, items
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90499868
--- Diff: aria/storage/__init__.py ---
@@ -37,354 +37,93 @@
* drivers - module, a pool of Aria standard drivers.
* StorageDriver - class, abstract model implementation.
"""
-# todo: rewrite the above package documentation
-# (something like explaning the two types of storage - models and resources)
-from collections import namedtuple
-
-from .structures import Storage, Field, Model, IterField, PointerField
-from .drivers import (
- ModelDriver,
- ResourceDriver,
- FileSystemResourceDriver,
- FileSystemModelDriver,
+from aria.logger import LoggerMixin
+from . import (
+ models,
+ exceptions,
+ api as storage_api,
+ structures
)
-from . import models, exceptions
+
__all__ = (
'ModelStorage',
- 'ResourceStorage',
- 'FileSystemModelDriver',
'models',
'structures',
- 'Field',
- 'IterField',
- 'PointerField',
- 'Model',
- 'drivers',
- 'ModelDriver',
- 'ResourceDriver',
- 'FileSystemResourceDriver',
)
-# todo: think about package output api's...
-# todo: in all drivers name => entry_type
-# todo: change in documentation str => basestring
-class ModelStorage(Storage):
+class Storage(LoggerMixin):
"""
- Managing the models storage.
+ Represents the storage
"""
- def __init__(self, driver, model_classes=(), **kwargs):
- """
- Simple storage client api for Aria applications.
- The storage instance defines the tables/documents/code api.
-
- :param ModelDriver driver: model storage driver.
- :param model_classes: the models to register.
- """
- assert isinstance(driver, ModelDriver)
- super(ModelStorage, self).__init__(driver, model_classes, **kwargs)
-
- def __getattr__(self, table):
- """
- getattr is a shortcut to simple api
-
- for Example:
- >> storage = ModelStorage(driver=FileSystemModelDriver('/tmp'))
- >> node_table = storage.node
- >> for node in node_table:
- >> print node
-
- :param str table: table name to get
- :return: a storage object that mapped to the table name
- """
- return super(ModelStorage, self).__getattr__(table)
-
- def register(self, model_cls):
- """
- Registers the model type in the resource storage manager.
- :param model_cls: the model to register.
- """
- model_name = generate_lower_name(model_cls)
- model_api = _ModelApi(model_name, self.driver, model_cls)
- self.registered[model_name] = model_api
-
- for pointer_schema_register in model_api.pointer_mapping.values():
- model_cls = pointer_schema_register.model_cls
- self.register(model_cls)
-
-_Pointer = namedtuple('_Pointer', 'name, is_iter')
-
-
-class _ModelApi(object):
- def __init__(self, name, driver, model_cls):
- """
- Managing the model in the storage, using the driver.
-
- :param basestring name: the name of the model.
- :param ModelDriver driver: the driver which supports this model in the storage.
- :param Model model_cls: table/document class model.
- """
- assert isinstance(driver, ModelDriver)
- assert issubclass(model_cls, Model)
- self.name = name
- self.driver = driver
- self.model_cls = model_cls
- self.pointer_mapping = {}
- self._setup_pointers_mapping()
-
- def _setup_pointers_mapping(self):
- for field_name, field_cls in vars(self.model_cls).items():
- if not(isinstance(field_cls, PointerField) and field_cls.type):
- continue
- pointer_key = _Pointer(field_name, is_iter=isinstance(field_cls, IterField))
- self.pointer_mapping[pointer_key] = self.__class__(
- name=generate_lower_name(field_cls.type),
- driver=self.driver,
- model_cls=field_cls.type)
-
- def __iter__(self):
- return self.iter()
+ def __init__(self, api, items=(), api_params=None, **kwargs):
+ self._api_params = api_params or {}
+ super(Storage, self).__init__(**kwargs)
+ self.api = api
+ self.registered = {}
+ for item in items:
+ self.register(item)
+ self.logger.debug('{name} object is ready: {0!r}'.format(
+ self, name=self.__class__.__name__))
def __repr__(self):
- return '{self.name}(driver={self.driver}, model={self.model_cls})'.format(self=self)
-
- def create(self):
- """
- Creates the model in the storage.
- """
- with self.driver as connection:
- connection.create(self.name)
-
- def get(self, entry_id, **kwargs):
- """
- Getter for the model from the storage.
-
- :param basestring entry_id: the id of the table/document.
- :return: model instance
- :rtype: Model
- """
- with self.driver as connection:
- data = connection.get(
- name=self.name,
- entry_id=entry_id,
- **kwargs)
- data.update(self._get_pointers(data, **kwargs))
- return self.model_cls(**data)
+ return '{name}(api={self.api})'.format(name=self.__class__.__name__, self=self)
- def store(self, entry, **kwargs):
- """
- Setter for the model in the storage.
-
- :param Model entry: the table/document to store.
- """
- assert isinstance(entry, self.model_cls)
- with self.driver as connection:
- data = entry.fields_dict
- data.update(self._store_pointers(data, **kwargs))
- connection.store(
- name=self.name,
- entry_id=entry.id,
- entry=data,
- **kwargs)
-
- def delete(self, entry_id, **kwargs):
- """
- Delete the model from storage.
-
- :param basestring entry_id: id of the entity to delete from storage.
- """
- entry = self.get(entry_id)
- with self.driver as connection:
- self._delete_pointers(entry, **kwargs)
- connection.delete(
- name=self.name,
- entry_id=entry_id,
- **kwargs)
-
- def iter(self, **kwargs):
- """
- Generator over the entries of model in storage.
- """
- with self.driver as connection:
- for data in connection.iter(name=self.name, **kwargs):
- data.update(self._get_pointers(data, **kwargs))
- yield self.model_cls(**data)
+ def __getattr__(self, item):
+ try:
+ return self.registered[item]
+ except KeyError:
+ return super(Storage, self).__getattribute__(item)
- def update(self, entry_id, **kwargs):
+ def register(self, entry):
"""
- Updates and entry in storage.
-
- :param str entry_id: the id of the table/document.
- :param kwargs: the fields to update.
+ Register the entry to the storage
+ :param name:
:return:
"""
- with self.driver as connection:
- connection.update(
- name=self.name,
- entry_id=entry_id,
- **kwargs
- )
-
- def _get_pointers(self, data, **kwargs):
- pointers = {}
- for field, schema in self.pointer_mapping.items():
- if field.is_iter:
- pointers[field.name] = [
- schema.get(entry_id=pointer_id, **kwargs)
- for pointer_id in data[field.name]
- if pointer_id]
- elif data[field.name]:
- pointers[field.name] = schema.get(entry_id=data[field.name], **kwargs)
- return pointers
-
- def _store_pointers(self, data, **kwargs):
- pointers = {}
- for field, model_api in self.pointer_mapping.items():
- if field.is_iter:
- pointers[field.name] = []
- for iter_entity in data[field.name]:
- pointers[field.name].append(iter_entity.id)
- model_api.store(iter_entity, **kwargs)
- else:
- pointers[field.name] = data[field.name].id
- model_api.store(data[field.name], **kwargs)
- return pointers
+ raise NotImplementedError('Subclass must implement abstract register method')
- def _delete_pointers(self, entry, **kwargs):
- for field, schema in self.pointer_mapping.items():
- if field.is_iter:
- for iter_entry in getattr(entry, field.name):
- schema.delete(iter_entry.id, **kwargs)
- else:
- schema.delete(getattr(entry, field.name).id, **kwargs)
-
-class ResourceApi(object):
+class ResourceStorage(Storage):
"""
- Managing the resource in the storage, using the driver.
-
- :param basestring name: the name of the resource.
- :param ResourceDriver driver: the driver which supports this resource in the storage.
+ Represents resource storage.
"""
- def __init__(self, driver, resource_name):
- """
- Managing the resources in the storage, using the driver.
-
- :param ResourceDriver driver: the driver which supports this model in the storage.
- :param basestring resource_name: the type of the entry this resourceAPI manages.
- """
- assert isinstance(driver, ResourceDriver)
- self.driver = driver
- self.resource_name = resource_name
-
- def __repr__(self):
- return '{name}(driver={self.driver}, resource={self.resource_name})'.format(
- name=self.__class__.__name__, self=self)
-
- def create(self):
- """
- Create the resource dir in the storage.
- """
- with self.driver as connection:
- connection.create(self.resource_name)
-
- def data(self, entry_id, path=None, **kwargs):
+ def register(self, name):
"""
- Retrieve the content of a storage resource.
-
- :param basestring entry_id: the id of the entry.
- :param basestring path: path of the resource on the storage.
- :param kwargs: resources to be passed to the driver..
- :return the content of a single file:
- """
- with self.driver as connection:
- return connection.data(
- entry_type=self.resource_name,
- entry_id=entry_id,
- path=path,
- **kwargs)
-
- def download(self, entry_id, destination, path=None, **kwargs):
- """
- Download a file/dir from the resource storage.
-
- :param basestring entry_id: the id of the entry.
- :param basestring destination: the destination of the file/dir.
- :param basestring path: path of the resource on the storage.
- """
- with self.driver as connection:
- connection.download(
- entry_type=self.resource_name,
- entry_id=entry_id,
- destination=destination,
- path=path,
- **kwargs)
-
- def upload(self, entry_id, source, path=None, **kwargs):
- """
- Upload a file/dir from the resource storage.
-
- :param basestring entry_id: the id of the entry.
- :param basestring source: the source path of the file to upload.
- :param basestring path: the destination of the file, relative to the root dir
- of the resource
+ Register the resource type to resource storage.
+ :param name:
+ :return:
"""
- with self.driver as connection:
- connection.upload(
- entry_type=self.resource_name,
- entry_id=entry_id,
- source=source,
- path=path,
- **kwargs)
+ self.registered[name] = self.api(name=name, **self._api_params)
+ self.registered[name].create()
+ self.logger.debug('setup {name} in storage {self!r}'.format(name=name, self=self))
-def generate_lower_name(model_cls):
- """
- Generates the name of the class from the class object. e.g. SomeClass -> some_class
- :param model_cls: the class to evaluate.
- :return: lower name
- :rtype: basestring
- """
- return ''.join(
- character if character.islower() else '_{0}'.format(character.lower())
- for character in model_cls.__name__)[1:]
-
-
-class ResourceStorage(Storage):
+class ModelStorage(Storage):
"""
- Managing the resource storage.
+ Represents model storage.
"""
- def __init__(self, driver, resources=(), **kwargs):
- """
- Simple storage client api for Aria applications.
- The storage instance defines the tables/documents/code api.
-
- :param ResourceDriver driver: resource storage driver
- :param resources: the resources to register.
- """
- assert isinstance(driver, ResourceDriver)
- super(ResourceStorage, self).__init__(driver, resources, **kwargs)
-
- def register(self, resource):
+ def register(self, model):
--- End diff --
model_cls
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90778954
--- Diff: aria/storage/structures.py ---
@@ -27,281 +27,189 @@
* Model - abstract model implementation.
"""
import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
+
+from sqlalchemy import VARCHAR
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+# pylint: disable=unused-import
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import (
+ schema,
+ Column,
+ Integer,
+ Text,
+ DateTime,
+ Boolean,
+ Enum,
+ String,
+ PickleType,
+ Float,
+ TypeDecorator,
+ ForeignKey,
+ orm,
)
+Model = declarative_base()
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+def foreign_key(foreign_key_column, nullable=False):
+ """Return a ForeignKey object with the relevant
-class Field(ValidatorMixin):
+ :param foreign_key_column: Unique id column in the parent table
+ :param nullable: Should the column be allowed to remain empty
"""
- A single field implementation
+ return Column(
+ ForeignKey(foreign_key_column, ondelete='CASCADE'),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(child_class,
+ parent_class,
+ foreign_key_column,
+ backreference=None):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param parent_class: Class of the parent table
+ :param child_class: Class of the child table
+ :param foreign_key_column: The column of the foreign key
+ :param backreference: The name to give to the reference to the child
"""
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
-
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
-
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
-
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ backreference = backreference or child_class.__tablename__
+ return relationship(
+ parent_class,
+ primaryjoin=lambda: parent_class.storage_id == foreign_key_column,
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=backref(backreference, cascade='all')
+ )
-class IterField(Field):
+class Dict(TypeDecorator):
"""
- Represents an iterable field.
+ Dict representation of type.
"""
- def __init__(self, **kwargs):
- """
- Simple iterable field manager.
- This field type don't have choices option.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- super(IterField, self).__init__(choices=(), **kwargs)
+ def process_literal_param(self, value, dialect):
+ pass
- def validate_value(self, name, values, *args):
- """
- Validates the value of each iterable value.
+ @property
+ def python_type(self):
+ return dict
- :param name: the name of the field.
- :param values: the values of the field.
- """
- for value in values:
- self.validate_instance(name, value, self.type)
+ impl = VARCHAR
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ value = json.dumps(value)
+ return value
-class PointerField(Field):
- """
- A single pointer field implementation.
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = json.loads(value)
+ return value
- Any PointerField points via id to another document.
+
+class MutableDict(Mutable, dict):
--- End diff --
use single attributes `ATTR? = MutableDict.as_mutable(Dict)`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90502574
--- Diff: aria/storage/mapi/filesystem.py ---
@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
--- End diff --
delete
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779218
--- Diff: aria/storage/structures.py ---
@@ -27,281 +27,189 @@
* Model - abstract model implementation.
"""
import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
+
+from sqlalchemy import VARCHAR
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+# pylint: disable=unused-import
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import (
+ schema,
+ Column,
+ Integer,
+ Text,
+ DateTime,
+ Boolean,
+ Enum,
+ String,
+ PickleType,
+ Float,
+ TypeDecorator,
+ ForeignKey,
+ orm,
)
+Model = declarative_base()
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+def foreign_key(foreign_key_column, nullable=False):
+ """Return a ForeignKey object with the relevant
-class Field(ValidatorMixin):
+ :param foreign_key_column: Unique id column in the parent table
+ :param nullable: Should the column be allowed to remain empty
"""
- A single field implementation
+ return Column(
+ ForeignKey(foreign_key_column, ondelete='CASCADE'),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(child_class,
+ parent_class,
+ foreign_key_column,
+ backreference=None):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param parent_class: Class of the parent table
+ :param child_class: Class of the child table
+ :param foreign_key_column: The column of the foreign key
+ :param backreference: The name to give to the reference to the child
"""
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
-
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
-
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
-
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ backreference = backreference or child_class.__tablename__
+ return relationship(
+ parent_class,
+ primaryjoin=lambda: parent_class.storage_id == foreign_key_column,
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=backref(backreference, cascade='all')
+ )
-class IterField(Field):
+class Dict(TypeDecorator):
"""
- Represents an iterable field.
+ Dict representation of type.
"""
- def __init__(self, **kwargs):
- """
- Simple iterable field manager.
- This field type don't have choices option.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- super(IterField, self).__init__(choices=(), **kwargs)
+ def process_literal_param(self, value, dialect):
+ pass
- def validate_value(self, name, values, *args):
- """
- Validates the value of each iterable value.
+ @property
+ def python_type(self):
+ return dict
- :param name: the name of the field.
- :param values: the values of the field.
- """
- for value in values:
- self.validate_instance(name, value, self.type)
+ impl = VARCHAR
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ value = json.dumps(value)
+ return value
-class PointerField(Field):
- """
- A single pointer field implementation.
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = json.loads(value)
+ return value
- Any PointerField points via id to another document.
+
+class MutableDict(Mutable, dict):
+ """
+ Enables tracking for dict values.
"""
+ @classmethod
+ def coerce(cls, key, value):
+ "Convert plain dictionaries to MutableDict."
- def __init__(self, type, **kwargs):
- assert issubclass(type, Model)
- super(PointerField, self).__init__(type=type, **kwargs)
+ if not isinstance(value, MutableDict):
+ if isinstance(value, dict):
+ return MutableDict(value)
+ # this call will raise ValueError
+ return Mutable.coerce(key, value)
+ else:
+ return value
-class IterPointerField(IterField, PointerField):
- """
- An iterable pointers field.
+ def __setitem__(self, key, value):
+ "Detect dictionary set events and emit change events."
- Any IterPointerField points via id to other documents.
- """
- pass
+ dict.__setitem__(self, key, value)
+ self.changed()
+ def __delitem__(self, key):
+ "Detect dictionary del events and emit change events."
-class Model(object):
- """
- Base class for all of the storage models.
- """
- id = None
+ dict.__delitem__(self, key)
+ self.changed()
- def __init__(self, **fields):
- """
- Abstract class for any model in the storage.
- The Initializer creates attributes according to the (keyword arguments) that given
- Each value is validated according to the Field.
- Each model has to have and ID Field.
- :param fields: each item is validated and transformed into instance attributes.
- """
- self._assert_model_have_id_field(**fields)
- missing_fields, unexpected_fields = self._setup_fields(fields)
+class SQLModelBase(Model):
+ """
+ Abstract base class for all SQL models that allows [de]serialization
+ """
+ # SQLAlchemy syntax
+ __abstract__ = True
- if missing_fields:
- raise StorageError(
- 'Model {name} got missing keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=missing_fields))
+ # Does the class represent a resource (Blueprint, Deployment, etc.) or a
+ # management table (User, Tenant, etc.), as they are handled differently
+ is_resource = False
- if unexpected_fields:
- raise StorageError(
- 'Model {name} got unexpected keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=unexpected_fields))
+ # This would be overridden once the models are created.
+ __table__ = None
- def __repr__(self):
- return '{name}(fields={0})'.format(sorted(self.fields), name=self.__class__.__name__)
+ _private_fields = []
- def __eq__(self, other):
- return (
- isinstance(other, self.__class__) and
- self.fields_dict == other.fields_dict)
+ # Indicates whether the `id` column in this class should be unique
+ is_id_unique = True
- @property
- def fields(self):
- """
- Iterates over the fields of the model.
- :yields: the class's field name
- """
- for name, field in vars(self.__class__).items():
- if isinstance(field, Field):
- yield name
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
+ id = Column(Text, index=True)
- @property
- def fields_dict(self):
- """
- Transforms the instance attributes into a dict.
+ @classmethod
+ def unique_id(cls):
+ return 'id'
+
+ def to_dict(self, suppress_error=False):
+ """Return a dict representation of the model
+
+ :param suppress_error: If set to True, sets `None` to attributes that
+ it's unable to retrieve (e.g., if a relationship wasn't established
+ yet, and so it's impossible to access a property through it)
+ """
+ if suppress_error:
+ res = dict()
+ for field in self.fields():
+ try:
+ field_value = getattr(self, field)
+ except AttributeError:
+ field_value = None
+ res[field] = field_value
+ else:
+ # Can't simply call here `self.to_response()` because inheriting
+ # class might override it, but we always need the same code here
+ res = dict((f, getattr(self, f)) for f in self.fields())
+ return res
- :return: all fields in dict format.
- :rtype dict
- """
- return dict((name, getattr(self, name)) for name in self.fields)
+ @classmethod
+ def fields(cls):
--- End diff --
assoc proxies + tests
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779347
--- Diff: aria/storage/structures.py ---
@@ -27,281 +27,189 @@
* Model - abstract model implementation.
"""
import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
+
+from sqlalchemy import VARCHAR
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+# pylint: disable=unused-import
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import (
+ schema,
+ Column,
+ Integer,
+ Text,
+ DateTime,
+ Boolean,
+ Enum,
+ String,
+ PickleType,
+ Float,
+ TypeDecorator,
+ ForeignKey,
+ orm,
)
+Model = declarative_base()
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+def foreign_key(foreign_key_column, nullable=False):
+ """Return a ForeignKey object with the relevant
-class Field(ValidatorMixin):
+ :param foreign_key_column: Unique id column in the parent table
+ :param nullable: Should the column be allowed to remain empty
"""
- A single field implementation
+ return Column(
+ ForeignKey(foreign_key_column, ondelete='CASCADE'),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(child_class,
+ parent_class,
+ foreign_key_column,
+ backreference=None):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param parent_class: Class of the parent table
+ :param child_class: Class of the child table
+ :param foreign_key_column: The column of the foreign key
+ :param backreference: The name to give to the reference to the child
"""
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
-
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
-
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
-
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ backreference = backreference or child_class.__tablename__
+ return relationship(
+ parent_class,
+ primaryjoin=lambda: parent_class.storage_id == foreign_key_column,
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=backref(backreference, cascade='all')
+ )
-class IterField(Field):
+class Dict(TypeDecorator):
"""
- Represents an iterable field.
+ Dict representation of type.
"""
- def __init__(self, **kwargs):
- """
- Simple iterable field manager.
- This field type don't have choices option.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- super(IterField, self).__init__(choices=(), **kwargs)
+ def process_literal_param(self, value, dialect):
+ pass
- def validate_value(self, name, values, *args):
- """
- Validates the value of each iterable value.
+ @property
+ def python_type(self):
+ return dict
- :param name: the name of the field.
- :param values: the values of the field.
- """
- for value in values:
- self.validate_instance(name, value, self.type)
+ impl = VARCHAR
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ value = json.dumps(value)
+ return value
-class PointerField(Field):
- """
- A single pointer field implementation.
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = json.loads(value)
+ return value
- Any PointerField points via id to another document.
+
+class MutableDict(Mutable, dict):
+ """
+ Enables tracking for dict values.
"""
+ @classmethod
+ def coerce(cls, key, value):
+ "Convert plain dictionaries to MutableDict."
- def __init__(self, type, **kwargs):
- assert issubclass(type, Model)
- super(PointerField, self).__init__(type=type, **kwargs)
+ if not isinstance(value, MutableDict):
+ if isinstance(value, dict):
+ return MutableDict(value)
+ # this call will raise ValueError
+ return Mutable.coerce(key, value)
+ else:
+ return value
-class IterPointerField(IterField, PointerField):
- """
- An iterable pointers field.
+ def __setitem__(self, key, value):
+ "Detect dictionary set events and emit change events."
- Any IterPointerField points via id to other documents.
- """
- pass
+ dict.__setitem__(self, key, value)
+ self.changed()
+ def __delitem__(self, key):
+ "Detect dictionary del events and emit change events."
-class Model(object):
- """
- Base class for all of the storage models.
- """
- id = None
+ dict.__delitem__(self, key)
+ self.changed()
- def __init__(self, **fields):
- """
- Abstract class for any model in the storage.
- The Initializer creates attributes according to the (keyword arguments) that given
- Each value is validated according to the Field.
- Each model has to have and ID Field.
- :param fields: each item is validated and transformed into instance attributes.
- """
- self._assert_model_have_id_field(**fields)
- missing_fields, unexpected_fields = self._setup_fields(fields)
+class SQLModelBase(Model):
+ """
+ Abstract base class for all SQL models that allows [de]serialization
+ """
+ # SQLAlchemy syntax
+ __abstract__ = True
- if missing_fields:
- raise StorageError(
- 'Model {name} got missing keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=missing_fields))
+ # Does the class represent a resource (Blueprint, Deployment, etc.) or a
+ # management table (User, Tenant, etc.), as they are handled differently
+ is_resource = False
- if unexpected_fields:
- raise StorageError(
- 'Model {name} got unexpected keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=unexpected_fields))
+ # This would be overridden once the models are created.
+ __table__ = None
- def __repr__(self):
- return '{name}(fields={0})'.format(sorted(self.fields), name=self.__class__.__name__)
+ _private_fields = []
- def __eq__(self, other):
- return (
- isinstance(other, self.__class__) and
- self.fields_dict == other.fields_dict)
+ # Indicates whether the `id` column in this class should be unique
+ is_id_unique = True
- @property
- def fields(self):
- """
- Iterates over the fields of the model.
- :yields: the class's field name
- """
- for name, field in vars(self.__class__).items():
- if isinstance(field, Field):
- yield name
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
+ id = Column(Text, index=True)
- @property
- def fields_dict(self):
- """
- Transforms the instance attributes into a dict.
+ @classmethod
+ def unique_id(cls):
+ return 'id'
+
+ def to_dict(self, suppress_error=False):
+ """Return a dict representation of the model
+
+ :param suppress_error: If set to True, sets `None` to attributes that
+ it's unable to retrieve (e.g., if a relationship wasn't established
+ yet, and so it's impossible to access a property through it)
+ """
+ if suppress_error:
+ res = dict()
+ for field in self.fields():
+ try:
+ field_value = getattr(self, field)
+ except AttributeError:
+ field_value = None
+ res[field] = field_value
+ else:
+ # Can't simply call here `self.to_response()` because inheriting
+ # class might override it, but we always need the same code here
+ res = dict((f, getattr(self, f)) for f in self.fields())
+ return res
- :return: all fields in dict format.
- :rtype dict
- """
- return dict((name, getattr(self, name)) for name in self.fields)
+ @classmethod
+ def fields(cls):
+ """Return the list of field names for this table
- @property
- def json(self):
- """
- Transform the dict of attributes into json
- :return:
+ Mostly for backwards compatibility in the code (that uses `fields`)
"""
- return json.dumps(self.fields_dict)
+ return set(cls.__table__.columns.keys()) - set(cls._private_fields)
- @classmethod
- def _assert_model_have_id_field(cls, **fields_initializer_values):
- if not getattr(cls, 'id', None):
- raise StorageError('Model {cls.__name__} must have id field'.format(cls=cls))
-
- if cls.id.default == cls.id.NO_DEFAULT and 'id' not in fields_initializer_values:
- raise StorageError(
- 'Model {cls.__name__} is missing required '
- 'keyword-only argument: "id"'.format(cls=cls))
-
- def _setup_fields(self, input_fields):
- missing = []
- for field_name in self.fields:
- try:
- field_obj = input_fields.pop(field_name)
- setattr(self, field_name, field_obj)
- except KeyError:
- field = getattr(self.__class__, field_name)
- if field.default == field.NO_DEFAULT:
- missing.append(field_name)
-
- unexpected_fields = input_fields.keys()
- return missing, unexpected_fields
-
-
-class Storage(LoggerMixin):
- """
- Represents the storage
- """
- def __init__(self, driver, items=(), **kwargs):
- super(Storage, self).__init__(**kwargs)
- self.driver = driver
- self.registered = {}
- for item in items:
- self.register(item)
- self.logger.debug('{name} object is ready: {0!r}'.format(
- self, name=self.__class__.__name__))
+ def __str__(self):
+ id_name, id_value = self.unique_id()
+ return '<{0} {1}=`{2}`>'.format(
+ self.__class__.__name__,
+ id_name,
+ id_value
+ )
def __repr__(self):
- return '{name}(driver={self.driver})'.format(
- name=self.__class__.__name__, self=self)
+ return str(self)
- def __getattr__(self, item):
- try:
- return self.registered[item]
- except KeyError:
- return super(Storage, self).__getattribute__(item)
+ def __unicode__(self):
+ return str(self)
- def setup(self):
- """
- Setup and create all storage items
- """
- for name, api in self.registered.iteritems():
- try:
- api.create()
- self.logger.debug(
- 'setup {name} in storage {self!r}'.format(name=name, self=self))
- except StorageError:
- pass
+ def __eq__(self, other):
--- End diff --
think of alternative. possibly remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90496245
--- Diff: aria/orchestrator/context/workflow.py ---
@@ -64,19 +66,27 @@ def nodes(self):
"""
Iterator over nodes
"""
- return self.model.node.iter(filters={'blueprint_id': self.blueprint.id})
+ return self.model.node.iter(
+ filters={
+ 'deployment_storage_id': self.deployment.storage_id
--- End diff --
create an issue considering unique id for different models
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90898173
--- Diff: aria/orchestrator/context/toolbelt.py ---
@@ -33,7 +33,7 @@ def dependent_node_instances(self):
:return:
"""
assert isinstance(self._op_context, operation.NodeOperationContext)
- filters = {'target_node_instance_fk': self._op_context.node_instance.storage_id}
--- End diff --
leave one as reference
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90882506
--- Diff: aria/storage/sql_mapi.py ---
@@ -54,19 +53,47 @@ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
)
return result
- def iter(self,
+ def get_by_name(self, entry_name, include=None, **kwargs):
+ result = self.list(include=include, filters={'name': entry_name})
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with NAME `{1}` was not found'
+ .format(self.model_cls.__name__, entry_name)
+ )
+ elif len(result) > 1:
+ raise exceptions.StorageError(
+ 'Requested {0} with NAME `{1}` returned more than 1 value'
+ .format(self.model_cls.__name__, entry_name)
+ )
+ else:
+ return result[0]
+
+ def list(self,
include=None,
filters=None,
pagination=None,
sort=None,
**kwargs):
- """Return a (possibly empty) list of `model_class` results
- """
query = self._get_query(include, filters, sort)
- results, _, _, _ = self._paginate(query, pagination)
+ results, total, size, offset = self._paginate(query, pagination)
- for result in results:
+ return ListResult(
+ items=results,
+ metadata=dict(total=total,
+ size=size,
+ offset=offset)
+ )
+
+ def iter(self,
+ include=None,
+ filters=None,
+ sort=None,
+ **kwargs):
+ """Return a (possibly empty) list of `model_class` results
+ """
+ query = self._get_query(include, filters, sort)
--- End diff --
return from here
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90883621
--- Diff: tests/storage/__init__.py ---
@@ -31,9 +32,10 @@ def teardown_method(self):
rmtree(self.path, ignore_errors=True)
-def get_sqlite_api_params():
+def get_sqlite_api_kwargs():
engine = create_engine('sqlite:///:memory:',
connect_args={'check_same_thread': False},
poolclass=StaticPool)
session = orm.sessionmaker(bind=engine)()
+ MetaData().create_all(engine)
--- End diff --
try through base
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779099
--- Diff: aria/storage/structures.py ---
@@ -27,281 +27,189 @@
* Model - abstract model implementation.
"""
import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
+
+from sqlalchemy import VARCHAR
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+# pylint: disable=unused-import
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import (
+ schema,
+ Column,
+ Integer,
+ Text,
+ DateTime,
+ Boolean,
+ Enum,
+ String,
+ PickleType,
+ Float,
+ TypeDecorator,
+ ForeignKey,
+ orm,
)
+Model = declarative_base()
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+def foreign_key(foreign_key_column, nullable=False):
+ """Return a ForeignKey object with the relevant
-class Field(ValidatorMixin):
+ :param foreign_key_column: Unique id column in the parent table
+ :param nullable: Should the column be allowed to remain empty
"""
- A single field implementation
+ return Column(
+ ForeignKey(foreign_key_column, ondelete='CASCADE'),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(child_class,
+ parent_class,
+ foreign_key_column,
+ backreference=None):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param parent_class: Class of the parent table
+ :param child_class: Class of the child table
+ :param foreign_key_column: The column of the foreign key
+ :param backreference: The name to give to the reference to the child
"""
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
-
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
-
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
-
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ backreference = backreference or child_class.__tablename__
+ return relationship(
+ parent_class,
+ primaryjoin=lambda: parent_class.storage_id == foreign_key_column,
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=backref(backreference, cascade='all')
+ )
-class IterField(Field):
+class Dict(TypeDecorator):
"""
- Represents an iterable field.
+ Dict representation of type.
"""
- def __init__(self, **kwargs):
- """
- Simple iterable field manager.
- This field type don't have choices option.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- super(IterField, self).__init__(choices=(), **kwargs)
+ def process_literal_param(self, value, dialect):
+ pass
- def validate_value(self, name, values, *args):
- """
- Validates the value of each iterable value.
+ @property
+ def python_type(self):
+ return dict
- :param name: the name of the field.
- :param values: the values of the field.
- """
- for value in values:
- self.validate_instance(name, value, self.type)
+ impl = VARCHAR
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ value = json.dumps(value)
+ return value
-class PointerField(Field):
- """
- A single pointer field implementation.
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = json.loads(value)
+ return value
- Any PointerField points via id to another document.
+
+class MutableDict(Mutable, dict):
+ """
+ Enables tracking for dict values.
"""
+ @classmethod
+ def coerce(cls, key, value):
+ "Convert plain dictionaries to MutableDict."
- def __init__(self, type, **kwargs):
- assert issubclass(type, Model)
- super(PointerField, self).__init__(type=type, **kwargs)
+ if not isinstance(value, MutableDict):
+ if isinstance(value, dict):
+ return MutableDict(value)
+ # this call will raise ValueError
+ return Mutable.coerce(key, value)
+ else:
+ return value
-class IterPointerField(IterField, PointerField):
- """
- An iterable pointers field.
+ def __setitem__(self, key, value):
+ "Detect dictionary set events and emit change events."
- Any IterPointerField points via id to other documents.
- """
- pass
+ dict.__setitem__(self, key, value)
+ self.changed()
+ def __delitem__(self, key):
+ "Detect dictionary del events and emit change events."
-class Model(object):
- """
- Base class for all of the storage models.
- """
- id = None
+ dict.__delitem__(self, key)
+ self.changed()
- def __init__(self, **fields):
- """
- Abstract class for any model in the storage.
- The Initializer creates attributes according to the (keyword arguments) that given
- Each value is validated according to the Field.
- Each model has to have and ID Field.
- :param fields: each item is validated and transformed into instance attributes.
- """
- self._assert_model_have_id_field(**fields)
- missing_fields, unexpected_fields = self._setup_fields(fields)
+class SQLModelBase(Model):
+ """
+ Abstract base class for all SQL models that allows [de]serialization
+ """
+ # SQLAlchemy syntax
+ __abstract__ = True
- if missing_fields:
- raise StorageError(
- 'Model {name} got missing keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=missing_fields))
+ # Does the class represent a resource (Blueprint, Deployment, etc.) or a
+ # management table (User, Tenant, etc.), as they are handled differently
+ is_resource = False
- if unexpected_fields:
- raise StorageError(
- 'Model {name} got unexpected keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=unexpected_fields))
+ # This would be overridden once the models are created.
+ __table__ = None
- def __repr__(self):
- return '{name}(fields={0})'.format(sorted(self.fields), name=self.__class__.__name__)
+ _private_fields = []
- def __eq__(self, other):
- return (
- isinstance(other, self.__class__) and
- self.fields_dict == other.fields_dict)
+ # Indicates whether the `id` column in this class should be unique
+ is_id_unique = True
- @property
- def fields(self):
- """
- Iterates over the fields of the model.
- :yields: the class's field name
- """
- for name, field in vars(self.__class__).items():
- if isinstance(field, Field):
- yield name
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
+ id = Column(Text, index=True)
- @property
- def fields_dict(self):
- """
- Transforms the instance attributes into a dict.
+ @classmethod
+ def unique_id(cls):
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90452398
--- Diff: aria/__init__.py ---
@@ -58,37 +57,37 @@ def install_aria_extensions():
del sys.modules[module_name]
-def application_model_storage(driver):
+def application_model_storage(api, api_params=None):
--- End diff --
remove _model_storage
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by ran-z <gi...@git.apache.org>.
Github user ran-z commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90774280
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
+ query = self._get_query(include, filters)
+ if locking:
+ query = query.with_for_update()
+ result = query.first()
+
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with ID `{1}` was not found'
+ .format(self.model_cls.__name__, entry_id)
+ )
+ return result
+
+ def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+ """Return a (possibly empty) list of `model_class` results
+ """
+ query = self._get_query(include, filters, sort)
+
+ results, _, _, _ = self._paginate(query, pagination)
+
+ for result in results:
+ yield result
+
+ def put(self, entry, **kwargs):
+ """Create a `model_class` instance from a serializable `model` object
+
+ :param entry: A dict with relevant kwargs, or an instance of a class
+ that has a `to_dict` method, and whose attributes match the columns
+ of `model_class` (might also my just an instance of `model_class`)
+ :return: An instance of `model_class`
+ """
+ self._session.add(entry)
+ self._safe_commit()
+ return entry
+
+ def delete(self, entry_id, filters=None, **kwargs):
--- End diff --
check usage for get/delete for filters
probably remove both
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90504139
--- Diff: aria/storage/models.py ---
@@ -60,353 +73,612 @@
)
# todo: sort this, maybe move from mgr or move from aria???
-ACTION_TYPES = ()
-ENTITY_TYPES = ()
+# TODO: this must change
+ACTION_TYPES = ('a')
+ENTITY_TYPES = ('b')
+
+
+def uuid_generator():
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by ran-z <gi...@git.apache.org>.
Github user ran-z commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90773942
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
--- End diff --
entry_id should always be used
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90780155
--- Diff: tests/storage/test_models.py ---
@@ -349,12 +168,11 @@ def create_execution(status):
def test_task_max_attempts_validation():
def create_task(max_attempts):
- Task(execution_id='eid',
- name='name',
- operation_mapping='',
- inputs={},
- actor=models.get_dependency_node_instance(),
- max_attempts=max_attempts)
+ return Task(execution_id='eid',
--- End diff --
no return
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90778768
--- Diff: aria/storage/models.py ---
@@ -422,23 +531,55 @@ def validate_max_attempts(_, value, *args):
SUCCESS,
FAILED,
)
+
WAIT_STATES = [PENDING, RETRYING]
END_STATES = [SUCCESS, FAILED]
+
+ @validates('max_attempts')
+ def validate_max_attempts(self, _, value): # pylint: disable=no-self-use
+ """Validates that max attempts is either -1 or a positive number"""
+ if value < 1 and value != Task.INFINITE_RETRIES:
+ raise ValueError('Max attempts can be either -1 (infinite) or any positive number. '
+ 'Got {value}'.format(value=value))
+ return value
+
INFINITE_RETRIES = -1
- id = Field(type=basestring, default=uuid_generator)
- status = Field(type=basestring, choices=STATES, default=PENDING)
- execution_id = Field(type=basestring)
- due_at = Field(type=datetime, default=datetime.utcnow)
- started_at = Field(type=datetime, default=None)
- ended_at = Field(type=datetime, default=None)
- max_attempts = Field(type=int, default=1, validation_func=_Validation.validate_max_attempts)
- retry_count = Field(type=int, default=0)
- retry_interval = Field(type=(int, float), default=0)
- ignore_failure = Field(type=bool, default=False)
+ status = Column(Enum(*STATES), name='status', default=PENDING)
+
+ execution_id = Column(String)
+ due_at = Column(DateTime, default=datetime.utcnow, nullable=True)
+ started_at = Column(DateTime, default=None, nullable=True)
+ ended_at = Column(DateTime, default=None, nullable=True)
+ max_attempts = Column(Integer, default=1)
+ retry_count = Column(Integer, default=0)
+ retry_interval = Column(Float, default=0)
+ ignore_failure = Column(Boolean, default=False)
# Operation specific fields
- name = Field(type=basestring)
- operation_mapping = Field(type=basestring)
- actor = Field()
- inputs = Field(type=dict, default=lambda: {})
+ name = Column(String)
+ operation_mapping = Column(String)
+ inputs = Column(MutableDict.as_mutable(Dict))
+
+ @property
+ def actor_storage_id(self):
+ """
+ Return the actor storage id of the task
+ :return:
+ """
+ return self.node_instance_fk or self.relationship_instance_fk
+
+ @property
+ def actor(self):
+ """
+ Return the actor of the task
+ :return:
+ """
+ return self.node_instance or self.relationship_instance
+
+ def __init__(self, actor=None, **kwargs):
--- End diff --
2 class methods
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90781017
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
+ query = self._get_query(include, filters)
+ if locking:
+ query = query.with_for_update()
+ result = query.first()
+
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with ID `{1}` was not found'
--- End diff --
no need anymore
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90777224
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
+ query = self._get_query(include, filters)
+ if locking:
+ query = query.with_for_update()
+ result = query.first()
+
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with ID `{1}` was not found'
+ .format(self.model_cls.__name__, entry_id)
+ )
+ return result
+
+ def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+ """Return a (possibly empty) list of `model_class` results
+ """
+ query = self._get_query(include, filters, sort)
+
+ results, _, _, _ = self._paginate(query, pagination)
+
+ for result in results:
+ yield result
+
+ def put(self, entry, **kwargs):
+ """Create a `model_class` instance from a serializable `model` object
+
+ :param entry: A dict with relevant kwargs, or an instance of a class
+ that has a `to_dict` method, and whose attributes match the columns
+ of `model_class` (might also my just an instance of `model_class`)
+ :return: An instance of `model_class`
+ """
+ self._session.add(entry)
+ self._safe_commit()
+ return entry
+
+ def delete(self, entry_id, filters=None, **kwargs):
+ """Delete a single result based on the model class and element ID
+ """
+ try:
+ instance = self.get(
+ entry_id,
+ filters=filters
+ )
+ except exceptions.StorageError:
+ raise exceptions.StorageError(
+ 'Could not delete {0} with ID `{1}` - element not found'
+ .format(
+ self.model_cls.__name__,
+ entry_id
+ )
+ )
+ self._load_properties(instance)
+ self._session.delete(instance)
+ self._safe_commit()
+ return instance
+
+ # TODO: this might need rework
+ def update(self, entry, **kwargs):
+ """Add `instance` to the DB session, and attempt to commit
+
+ :return: The updated instance
+ """
+ return self.put(entry)
+
+ def refresh(self, entry):
+ """Reload the instance with fresh information from the DB
+
+ :param entry: Instance to be re-loaded from the DB
+ :return: The refreshed instance
+ """
+ self._session.refresh(entry)
+ self._load_properties(entry)
+ return entry
+
+ def _destroy_connection(self):
+ pass
+
+ def _establish_connection(self):
+ pass
+
+ def create(self):
+ self.model_cls.__table__.create(self._engine)
+
+ def drop(self):
+ """
+ Drop the table from the storage.
+ :return:
+ """
+ self.model_cls.__table__.drop(self._engine)
+
+ def _safe_commit(self):
+ """Try to commit changes in the session. Roll back if exception raised
+ Excepts SQLAlchemy errors and rollbacks if they're caught
+ """
+ try:
+ self._session.commit()
+ except SQLAlchemyError as e:
+ self._session.rollback()
+ raise exceptions.StorageError('SQL Storage error: {0}'.format(str(e)))
+
+ def _get_base_query(self, include, joins):
+ """Create the initial query from the model class and included columns
+
+ :param include: A (possibly empty) list of columns to include in
+ the query
+ :param joins: A (possibly empty) list of models on which the query
+ should join
+ :return: An SQLAlchemy AppenderQuery object
+ """
+
+ # If only some columns are included, query through the session object
+ if include:
+ query = self._session.query(*include)
+ else:
+ # If all columns should be returned, query directly from the model
+ query = self._session.query(self.model_cls)
+
+ # Add any joins that might be necessary
+ for join_model in joins:
+ query = query.join(join_model)
+
+ return query
+
+ @staticmethod
+ def _sort_query(query, sort=None):
+ """Add sorting clauses to the query
+
+ :param query: Base SQL query
+ :param sort: An optional dictionary where keys are column names to
+ sort by, and values are the order (asc/desc)
+ :return: An SQLAlchemy AppenderQuery object
+ """
+ if sort:
+ for column, order in sort.items():
+ if order == 'desc':
+ column = column.desc()
+ query = query.order_by(column)
+ return query
+
+ @staticmethod
+ def _filter_query(query, filters):
+ """Add filter clauses to the query
+
+ :param query: Base SQL query
+ :param filters: An optional dictionary where keys are column names to
+ filter by, and values are values applicable for those columns (or lists
+ of such values)
+ :return: An SQLAlchemy AppenderQuery object
+ """
+ for column, value in filters.items():
+ # If there are multiple values, use `in_`, otherwise, use `eq`
+ if isinstance(value, (list, tuple)):
+ query = query.filter(column.in_(value))
+ else:
+ query = query.filter(column == value)
+
+ return query
+
+ def _get_query(self,
+ include=None,
+ filters=None,
+ sort=None):
+ """Get an SQL query object based on the params passed
+
+ :param include: An optional list of columns to include in the query
+ :param filters: An optional dictionary where keys are column names to
+ filter by, and values are values applicable for those columns (or lists
+ of such values)
+ :param sort: An optional dictionary where keys are column names to
+ sort by, and values are the order (asc/desc)
+ :return: A sorted and filtered query with only the relevant
+ columns
+ """
+
+ include = include or []
+ filters = filters or dict()
+ sort = sort or OrderedDict()
+
+ joins = self._get_join_models_list(include, filters, sort)
+ include, filters, sort = self._get_columns_from_field_names(
+ include, filters, sort
+ )
+
+ query = self._get_base_query(include, joins)
+ query = self._filter_query(query, filters)
+ query = self._sort_query(query, sort)
+ return query
+
+ def _get_columns_from_field_names(self,
+ include,
+ filters,
+ sort):
+ """Go over the optional parameters (include, filters, sort), and
+ replace column names with actual SQLA column objects
+ """
+ all_includes = [self._get_column(c) for c in include]
+ include = []
+ # Columns that are inferred from properties (Labels) should be included
+ # last for the following joins to work properly
+ for col in all_includes:
+ if isinstance(col, Label):
+ include.append(col)
+ else:
+ include.insert(0, col)
+
+ filters = dict((self._get_column(c), filters[c]) for c in filters)
+ sort = OrderedDict((self._get_column(c), sort[c]) for c in sort)
+
+ return include, filters, sort
+
+ def _get_join_models_list(self, include, filters, sort):
+ """Return a list of models on which the query should be joined, as
+ inferred from the include, filter and sort column names
+ """
+ if not self.model_cls.is_resource:
+ return []
+
+ all_column_names = include + filters.keys() + sort.keys()
+ join_columns = set(column_name for column_name in all_column_names
+ if self._is_join_column(column_name))
+
+ # If the only columns included are the columns on which we would
+ # normally join, there isn't actually a need to join, as the FROM
+ # clause in the query will be generated from the relevant models anyway
+ if include == list(join_columns):
+ return []
+
+ # Initializing a set, because the same model can appear in several
+ # join lists
+ join_models = set()
+ for column_name in join_columns:
+ join_models.update(
+ self.model_cls.join_properties[column_name]['models']
+ )
+ # Sort the models by their correct join order
+ join_models = sorted(join_models,
+ key=lambda model: model.join_order, reverse=True)
+
+ return join_models
+
+ def _is_join_column(self, column_name):
--- End diff --
check Cloudify
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by ran-z <gi...@git.apache.org>.
Github user ran-z commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90774547
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
+ query = self._get_query(include, filters)
+ if locking:
+ query = query.with_for_update()
+ result = query.first()
+
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with ID `{1}` was not found'
+ .format(self.model_cls.__name__, entry_id)
+ )
+ return result
+
+ def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+ """Return a (possibly empty) list of `model_class` results
+ """
+ query = self._get_query(include, filters, sort)
+
+ results, _, _, _ = self._paginate(query, pagination)
+
+ for result in results:
+ yield result
+
+ def put(self, entry, **kwargs):
+ """Create a `model_class` instance from a serializable `model` object
+
+ :param entry: A dict with relevant kwargs, or an instance of a class
+ that has a `to_dict` method, and whose attributes match the columns
+ of `model_class` (might also my just an instance of `model_class`)
+ :return: An instance of `model_class`
+ """
+ self._session.add(entry)
+ self._safe_commit()
+ return entry
+
+ def delete(self, entry_id, filters=None, **kwargs):
+ """Delete a single result based on the model class and element ID
+ """
+ try:
+ instance = self.get(
+ entry_id,
+ filters=filters
+ )
+ except exceptions.StorageError:
+ raise exceptions.StorageError(
+ 'Could not delete {0} with ID `{1}` - element not found'
+ .format(
+ self.model_cls.__name__,
+ entry_id
+ )
+ )
+ self._load_properties(instance)
+ self._session.delete(instance)
+ self._safe_commit()
+ return instance
+
+ # TODO: this might need rework
+ def update(self, entry, **kwargs):
+ """Add `instance` to the DB session, and attempt to commit
+
+ :return: The updated instance
+ """
+ return self.put(entry)
+
+ def refresh(self, entry):
+ """Reload the instance with fresh information from the DB
+
+ :param entry: Instance to be re-loaded from the DB
+ :return: The refreshed instance
+ """
+ self._session.refresh(entry)
+ self._load_properties(entry)
+ return entry
+
+ def _destroy_connection(self):
+ pass
+
+ def _establish_connection(self):
+ pass
+
+ def create(self):
--- End diff --
use create_all for tests, have it with the engine creation
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90503715
--- Diff: aria/storage/mapi/sql.py ---
@@ -0,0 +1,368 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+try:
+ from psycopg2 import DatabaseError as Psycopg2DBError
+ sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
+except ImportError:
+ sql_errors = (SQLAlchemyError, SQLiteDBError)
+ Psycopg2DBError = None
+
+from aria.utils.collections import OrderedDict
+
+from ... import storage
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(storage.api.ModelAPI):
--- End diff --
get back on Sunday
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90504471
--- Diff: aria/storage/models.py ---
@@ -60,353 +73,612 @@
)
# todo: sort this, maybe move from mgr or move from aria???
-ACTION_TYPES = ()
-ENTITY_TYPES = ()
+# TODO: this must change
+ACTION_TYPES = ('a')
+ENTITY_TYPES = ('b')
+
+
+def uuid_generator():
+ """
+ wrapper function which generates ids
+ """
+ return str(uuid4())
-class Blueprint(Model):
+class Blueprint(SQLModelBase):
"""
- A Model which represents a blueprint
+ Blueprint model representation.
"""
- plan = Field(type=dict)
- id = Field(type=basestring, default=uuid_generator)
- description = Field(type=(basestring, NoneType))
- created_at = Field(type=datetime)
- updated_at = Field(type=datetime)
- main_file_name = Field(type=basestring)
+ __tablename__ = 'blueprints'
+
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
+ id = Column(Text, index=True)
+ created_at = Column(DateTime, nullable=False, index=True)
--- End diff --
remove index
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90777213
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
+ query = self._get_query(include, filters)
+ if locking:
+ query = query.with_for_update()
+ result = query.first()
+
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with ID `{1}` was not found'
+ .format(self.model_cls.__name__, entry_id)
+ )
+ return result
+
+ def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+ """Return a (possibly empty) list of `model_class` results
+ """
+ query = self._get_query(include, filters, sort)
+
+ results, _, _, _ = self._paginate(query, pagination)
+
+ for result in results:
+ yield result
+
+ def put(self, entry, **kwargs):
+ """Create a `model_class` instance from a serializable `model` object
+
+ :param entry: A dict with relevant kwargs, or an instance of a class
+ that has a `to_dict` method, and whose attributes match the columns
+ of `model_class` (might also my just an instance of `model_class`)
+ :return: An instance of `model_class`
+ """
+ self._session.add(entry)
+ self._safe_commit()
+ return entry
+
+ def delete(self, entry_id, filters=None, **kwargs):
+ """Delete a single result based on the model class and element ID
+ """
+ try:
+ instance = self.get(
+ entry_id,
+ filters=filters
+ )
+ except exceptions.StorageError:
+ raise exceptions.StorageError(
+ 'Could not delete {0} with ID `{1}` - element not found'
+ .format(
+ self.model_cls.__name__,
+ entry_id
+ )
+ )
+ self._load_properties(instance)
+ self._session.delete(instance)
+ self._safe_commit()
+ return instance
+
+ # TODO: this might need rework
+ def update(self, entry, **kwargs):
+ """Add `instance` to the DB session, and attempt to commit
+
+ :return: The updated instance
+ """
+ return self.put(entry)
+
+ def refresh(self, entry):
+ """Reload the instance with fresh information from the DB
+
+ :param entry: Instance to be re-loaded from the DB
+ :return: The refreshed instance
+ """
+ self._session.refresh(entry)
+ self._load_properties(entry)
+ return entry
+
+ def _destroy_connection(self):
+ pass
+
+ def _establish_connection(self):
+ pass
+
+ def create(self):
+ self.model_cls.__table__.create(self._engine)
+
+ def drop(self):
+ """
+ Drop the table from the storage.
+ :return:
+ """
+ self.model_cls.__table__.drop(self._engine)
+
+ def _safe_commit(self):
+ """Try to commit changes in the session. Roll back if exception raised
+ Excepts SQLAlchemy errors and rollbacks if they're caught
+ """
+ try:
+ self._session.commit()
+ except SQLAlchemyError as e:
+ self._session.rollback()
+ raise exceptions.StorageError('SQL Storage error: {0}'.format(str(e)))
+
+ def _get_base_query(self, include, joins):
+ """Create the initial query from the model class and included columns
+
+ :param include: A (possibly empty) list of columns to include in
+ the query
+ :param joins: A (possibly empty) list of models on which the query
+ should join
+ :return: An SQLAlchemy AppenderQuery object
+ """
+
+ # If only some columns are included, query through the session object
+ if include:
+ query = self._session.query(*include)
+ else:
+ # If all columns should be returned, query directly from the model
+ query = self._session.query(self.model_cls)
+
+ # Add any joins that might be necessary
+ for join_model in joins:
+ query = query.join(join_model)
+
+ return query
+
+ @staticmethod
+ def _sort_query(query, sort=None):
+ """Add sorting clauses to the query
+
+ :param query: Base SQL query
+ :param sort: An optional dictionary where keys are column names to
+ sort by, and values are the order (asc/desc)
+ :return: An SQLAlchemy AppenderQuery object
+ """
+ if sort:
+ for column, order in sort.items():
+ if order == 'desc':
+ column = column.desc()
+ query = query.order_by(column)
+ return query
+
+ @staticmethod
+ def _filter_query(query, filters):
+ """Add filter clauses to the query
+
+ :param query: Base SQL query
+ :param filters: An optional dictionary where keys are column names to
+ filter by, and values are values applicable for those columns (or lists
+ of such values)
+ :return: An SQLAlchemy AppenderQuery object
+ """
+ for column, value in filters.items():
+ # If there are multiple values, use `in_`, otherwise, use `eq`
+ if isinstance(value, (list, tuple)):
+ query = query.filter(column.in_(value))
+ else:
+ query = query.filter(column == value)
+
+ return query
+
+ def _get_query(self,
+ include=None,
+ filters=None,
+ sort=None):
+ """Get an SQL query object based on the params passed
+
+ :param include: An optional list of columns to include in the query
+ :param filters: An optional dictionary where keys are column names to
+ filter by, and values are values applicable for those columns (or lists
+ of such values)
+ :param sort: An optional dictionary where keys are column names to
+ sort by, and values are the order (asc/desc)
+ :return: A sorted and filtered query with only the relevant
+ columns
+ """
+
+ include = include or []
+ filters = filters or dict()
+ sort = sort or OrderedDict()
+
+ joins = self._get_join_models_list(include, filters, sort)
+ include, filters, sort = self._get_columns_from_field_names(
+ include, filters, sort
+ )
+
+ query = self._get_base_query(include, joins)
+ query = self._filter_query(query, filters)
+ query = self._sort_query(query, sort)
+ return query
+
+ def _get_columns_from_field_names(self,
+ include,
+ filters,
+ sort):
+ """Go over the optional parameters (include, filters, sort), and
+ replace column names with actual SQLA column objects
+ """
+ all_includes = [self._get_column(c) for c in include]
+ include = []
+ # Columns that are inferred from properties (Labels) should be included
+ # last for the following joins to work properly
+ for col in all_includes:
+ if isinstance(col, Label):
+ include.append(col)
+ else:
+ include.insert(0, col)
+
+ filters = dict((self._get_column(c), filters[c]) for c in filters)
+ sort = OrderedDict((self._get_column(c), sort[c]) for c in sort)
+
+ return include, filters, sort
+
+ def _get_join_models_list(self, include, filters, sort):
+ """Return a list of models on which the query should be joined, as
+ inferred from the include, filter and sort column names
+ """
+ if not self.model_cls.is_resource:
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90880532
--- Diff: aria/orchestrator/workflows/core/task.py ---
@@ -106,24 +106,25 @@ class OperationTask(BaseTask):
def __init__(self, api_task, *args, **kwargs):
super(OperationTask, self).__init__(id=api_task.id, **kwargs)
self._workflow_context = api_task._workflow_context
- task_model = api_task._workflow_context.model.task.model_cls
+ base_task_model = api_task._workflow_context.model.task.model_cls
if isinstance(api_task.actor, models.NodeInstance):
context_class = operation_context.NodeOperationContext
+ task_model_cls = base_task_model.as_node_instance
elif isinstance(api_task.actor, models.RelationshipInstance):
context_class = operation_context.RelationshipOperationContext
+ task_model_cls = base_task_model.as_relationship_instance
else:
- raise RuntimeError('No operation context could be created for {0}'
- .format(api_task.actor.model_cls))
+ raise RuntimeError('No operation context could be created for {actor.model_cls}'
+ .format(actor=api_task.actor))
- operation_task = task_model(
- id=api_task.id,
+ operation_task = task_model_cls(
name=api_task.name,
operation_mapping=api_task.operation_mapping,
- actor=api_task.actor,
+ instance_id=api_task.actor.id,
inputs=api_task.inputs,
- status=task_model.PENDING,
- execution_id=self._workflow_context._execution_id,
+ status=base_task_model.PENDING,
+ execution_id=self._workflow_context._execution_name,
--- End diff --
execution_name=...
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779972
--- Diff: tests/orchestrator/workflows/core/test_engine.py ---
@@ -233,12 +212,13 @@ def mock_workflow(ctx, graph):
class TestCancel(BaseTest):
+ # TODO: what is up with this test?
def test_cancel_started_execution(self, workflow_context, executor):
number_of_tasks = 100
@workflow
def mock_workflow(ctx, graph):
- return graph.sequence(*(self._op(mock_sleep_task, ctx, inputs={'seconds': 0.1})
+ return graph.sequence(*(self._op(mock_sleep_task, ctx, inputs={'seconds': 1})
--- End diff --
get back to
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90505762
--- Diff: aria/storage/models.py ---
@@ -60,353 +73,612 @@
)
# todo: sort this, maybe move from mgr or move from aria???
-ACTION_TYPES = ()
-ENTITY_TYPES = ()
+# TODO: this must change
+ACTION_TYPES = ('a')
+ENTITY_TYPES = ('b')
+
+
+def uuid_generator():
+ """
+ wrapper function which generates ids
+ """
+ return str(uuid4())
-class Blueprint(Model):
+class Blueprint(SQLModelBase):
"""
- A Model which represents a blueprint
+ Blueprint model representation.
"""
- plan = Field(type=dict)
- id = Field(type=basestring, default=uuid_generator)
- description = Field(type=(basestring, NoneType))
- created_at = Field(type=datetime)
- updated_at = Field(type=datetime)
- main_file_name = Field(type=basestring)
+ __tablename__ = 'blueprints'
+
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
+ id = Column(Text, index=True)
+ created_at = Column(DateTime, nullable=False, index=True)
+ main_file_name = Column(Text, nullable=False)
+ plan = Column(MutableDict.as_mutable(Dict), nullable=False)
+ updated_at = Column(DateTime)
+ description = Column(Text)
-class Snapshot(Model):
+
+class Snapshot(SQLModelBase):
"""
- A Model which represents a snapshot
+ Snapshot model representation.
"""
+ __tablename__ = 'snapshots'
+
CREATED = 'created'
FAILED = 'failed'
CREATING = 'creating'
UPLOADED = 'uploaded'
+
+ STATES = [CREATED, FAILED, CREATING, UPLOADED]
END_STATES = [CREATED, FAILED, UPLOADED]
- id = Field(type=basestring, default=uuid_generator)
- created_at = Field(type=datetime)
- status = Field(type=basestring)
- error = Field(type=basestring, default=None)
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
+ id = Column(Text, index=True)
+ created_at = Column(DateTime, nullable=False, index=True)
+ status = Column(Enum(*STATES, name='snapshot_status'))
+ error = Column(Text)
-class Deployment(Model):
- """
- A Model which represents a deployment
- """
- id = Field(type=basestring, default=uuid_generator)
- description = Field(type=(basestring, NoneType))
- created_at = Field(type=datetime)
- updated_at = Field(type=datetime)
- blueprint_id = Field(type=basestring)
- workflows = Field(type=dict)
- inputs = Field(type=dict, default=lambda: {})
- policy_types = Field(type=dict, default=lambda: {})
- policy_triggers = Field(type=dict, default=lambda: {})
- groups = Field(type=dict, default=lambda: {})
- outputs = Field(type=dict, default=lambda: {})
- scaling_groups = Field(type=dict, default=lambda: {})
-
-
-class DeploymentUpdateStep(Model):
+
+class Deployment(SQLModelBase):
"""
- A Model which represents a deployment update step
+ Deployment model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- action = Field(type=basestring, choices=ACTION_TYPES)
- entity_type = Field(type=basestring, choices=ENTITY_TYPES)
- entity_id = Field(type=basestring)
- supported = Field(type=bool, default=True)
-
- def __hash__(self):
- return hash((self.id, self.entity_id))
+ __tablename__ = 'deployments'
+
+ # See base class for an explanation on these properties
+ join_properties = {
+ 'blueprint_id': {
+ # No need to provide the Blueprint table, as it's already joined
+ 'models': [Blueprint],
+ 'column': Blueprint.id.label('blueprint_id')
+ },
+ }
+ join_order = 2
+
+ _private_fields = ['blueprint_storage_id']
--- End diff --
add comment in base
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90880920
--- Diff: aria/storage/models.py ---
@@ -339,94 +324,100 @@ class Node(SQLModelBase):
Node model representation.
"""
__tablename__ = 'nodes'
+ id = Column(Integer, primary_key=True)
# See base class for an explanation on these properties
is_id_unique = False
- _private_fields = ['deployment_fk']
- deployment_fk = foreign_key(Deployment.storage_id)
-
- deployment_id = association_proxy('deployment', 'id')
- blueprint_id = association_proxy('blueprint', 'id')
+ _private_fields = ['deployment_id', 'host_id']
+ deployment_id = foreign_key(Deployment.id)
+ host_id = foreign_key('nodes.id', nullable=True)
@declared_attr
def deployment(cls):
- return one_to_many_relationship(cls, Deployment, cls.deployment_fk)
+ return one_to_many_relationship(cls, Deployment, cls.deployment_id)
deploy_number_of_instances = Column(Integer, nullable=False)
# TODO: This probably should be a foreign key, but there's no guarantee
# in the code, currently, that the host will be created beforehand
- host_id = Column(Text)
+ _host_id = foreign_key('nodes.id', nullable=True)
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90879281
--- Diff: aria/orchestrator/context/common.py ---
@@ -44,8 +44,8 @@ def __init__(
self._model = model_storage
self._resource = resource_storage
self._deployment_id = deployment_id
- self._workflow_id = workflow_id
- self._execution_id = execution_id or str(uuid4())
+ self._workflow_name = workflow_name
+ self._execution_name = execution_name or str(uuid4())
--- End diff --
remove name in executions
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90882255
--- Diff: aria/storage/sql_mapi.py ---
@@ -38,11 +38,10 @@ def __init__(self,
self._engine = engine
self._session = session
- def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ def get(self, entry_id, include=None, locking=False, **kwargs):
"""Return a single result based on the model class and element ID
"""
- filters = filters or {'id': entry_id}
- query = self._get_query(include, filters)
+ query = self._get_query(include, {'id': entry_id})
if locking:
--- End diff --
context manager
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90778979
--- Diff: aria/storage/structures.py ---
@@ -27,281 +27,189 @@
* Model - abstract model implementation.
"""
import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
+
+from sqlalchemy import VARCHAR
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+# pylint: disable=unused-import
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import (
+ schema,
+ Column,
+ Integer,
+ Text,
+ DateTime,
+ Boolean,
+ Enum,
+ String,
+ PickleType,
+ Float,
+ TypeDecorator,
+ ForeignKey,
+ orm,
)
+Model = declarative_base()
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+def foreign_key(foreign_key_column, nullable=False):
+ """Return a ForeignKey object with the relevant
-class Field(ValidatorMixin):
+ :param foreign_key_column: Unique id column in the parent table
+ :param nullable: Should the column be allowed to remain empty
"""
- A single field implementation
+ return Column(
+ ForeignKey(foreign_key_column, ondelete='CASCADE'),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(child_class,
+ parent_class,
+ foreign_key_column,
+ backreference=None):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param parent_class: Class of the parent table
+ :param child_class: Class of the child table
+ :param foreign_key_column: The column of the foreign key
+ :param backreference: The name to give to the reference to the child
"""
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
-
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
-
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
-
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ backreference = backreference or child_class.__tablename__
+ return relationship(
+ parent_class,
+ primaryjoin=lambda: parent_class.storage_id == foreign_key_column,
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=backref(backreference, cascade='all')
+ )
-class IterField(Field):
+class Dict(TypeDecorator):
"""
- Represents an iterable field.
+ Dict representation of type.
"""
- def __init__(self, **kwargs):
- """
- Simple iterable field manager.
- This field type don't have choices option.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- super(IterField, self).__init__(choices=(), **kwargs)
+ def process_literal_param(self, value, dialect):
+ pass
- def validate_value(self, name, values, *args):
- """
- Validates the value of each iterable value.
+ @property
+ def python_type(self):
+ return dict
- :param name: the name of the field.
- :param values: the values of the field.
- """
- for value in values:
- self.validate_instance(name, value, self.type)
+ impl = VARCHAR
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ value = json.dumps(value)
+ return value
-class PointerField(Field):
- """
- A single pointer field implementation.
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = json.loads(value)
+ return value
- Any PointerField points via id to another document.
+
+class MutableDict(Mutable, dict):
+ """
+ Enables tracking for dict values.
"""
+ @classmethod
+ def coerce(cls, key, value):
+ "Convert plain dictionaries to MutableDict."
- def __init__(self, type, **kwargs):
- assert issubclass(type, Model)
- super(PointerField, self).__init__(type=type, **kwargs)
+ if not isinstance(value, MutableDict):
+ if isinstance(value, dict):
+ return MutableDict(value)
+ # this call will raise ValueError
+ return Mutable.coerce(key, value)
+ else:
+ return value
-class IterPointerField(IterField, PointerField):
- """
- An iterable pointers field.
+ def __setitem__(self, key, value):
+ "Detect dictionary set events and emit change events."
- Any IterPointerField points via id to other documents.
- """
- pass
+ dict.__setitem__(self, key, value)
+ self.changed()
+ def __delitem__(self, key):
+ "Detect dictionary del events and emit change events."
-class Model(object):
- """
- Base class for all of the storage models.
- """
- id = None
+ dict.__delitem__(self, key)
+ self.changed()
- def __init__(self, **fields):
- """
- Abstract class for any model in the storage.
- The Initializer creates attributes according to the (keyword arguments) that given
- Each value is validated according to the Field.
- Each model has to have and ID Field.
- :param fields: each item is validated and transformed into instance attributes.
- """
- self._assert_model_have_id_field(**fields)
- missing_fields, unexpected_fields = self._setup_fields(fields)
+class SQLModelBase(Model):
+ """
+ Abstract base class for all SQL models that allows [de]serialization
+ """
+ # SQLAlchemy syntax
+ __abstract__ = True
- if missing_fields:
- raise StorageError(
- 'Model {name} got missing keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=missing_fields))
+ # Does the class represent a resource (Blueprint, Deployment, etc.) or a
+ # management table (User, Tenant, etc.), as they are handled differently
+ is_resource = False
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90778961
--- Diff: aria/storage/structures.py ---
@@ -27,281 +27,189 @@
* Model - abstract model implementation.
"""
import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
+
+from sqlalchemy import VARCHAR
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+# pylint: disable=unused-import
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import (
+ schema,
+ Column,
+ Integer,
+ Text,
+ DateTime,
+ Boolean,
+ Enum,
+ String,
+ PickleType,
+ Float,
+ TypeDecorator,
+ ForeignKey,
+ orm,
)
+Model = declarative_base()
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+def foreign_key(foreign_key_column, nullable=False):
+ """Return a ForeignKey object with the relevant
-class Field(ValidatorMixin):
+ :param foreign_key_column: Unique id column in the parent table
+ :param nullable: Should the column be allowed to remain empty
"""
- A single field implementation
+ return Column(
+ ForeignKey(foreign_key_column, ondelete='CASCADE'),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(child_class,
+ parent_class,
+ foreign_key_column,
+ backreference=None):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param parent_class: Class of the parent table
+ :param child_class: Class of the child table
+ :param foreign_key_column: The column of the foreign key
+ :param backreference: The name to give to the reference to the child
"""
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
-
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
-
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
-
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ backreference = backreference or child_class.__tablename__
+ return relationship(
+ parent_class,
+ primaryjoin=lambda: parent_class.storage_id == foreign_key_column,
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=backref(backreference, cascade='all')
+ )
-class IterField(Field):
+class Dict(TypeDecorator):
--- End diff --
verify recursive. if not, cry and fix
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90501812
--- Diff: aria/storage/api.py ---
@@ -0,0 +1,219 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+General storage API
+"""
+from contextlib import contextmanager
+
+from . import exceptions
+
+
+class StorageAPI(object):
+ """
+ General storage Base API
+ """
+ def create(self, **kwargs):
+ """
+ Create a storage API.
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract create method')
+
+ @contextmanager
+ def connect(self):
+ """
+ Established a connection and destroys it after use.
+ :return:
+ """
+ try:
+ self._establish_connection()
+ yield self
+ except BaseException as e:
+ raise exceptions.StorageError(str(e))
+ finally:
+ self._destroy_connection()
+
+ def _establish_connection(self):
+ """
+ Establish a conenction. used in the 'connect' contextmanager.
+ :return:
+ """
+ pass
+
+ def _destroy_connection(self):
+ """
+ Destroy a connection. used in the 'connect' contextmanager.
+ :return:
+ """
+ pass
+
+ def __getattr__(self, item):
+ try:
+ return self.registered[item]
+ except KeyError:
+ return super(StorageAPI, self).__getattribute__(item)
+
+
+class ModelAPI(StorageAPI):
+ """
+ A Base object for the model.
+ """
+ def __init__(self, model_cls, name=None, **kwargs):
+ """
+ Base model API
+
+ :param model_cls: the representing class of the model
+ :param str name: the name of the model
+ :param kwargs:
+ """
+ super(ModelAPI, self).__init__(**kwargs)
+ self._model_cls = model_cls
+ self._name = name or generate_lower_name(model_cls)
+
+ @property
+ def name(self):
+ """
+ The name of the class
+ :return: name of the class
+ """
+ return self._name
+
+ @property
+ def model_cls(self):
+ """
+ The class represting the model
+ :return:
+ """
+ return self._model_cls
+
+ def get(self, entry_id, filters=None, **kwargs):
+ """
+ Get entry from storage.
+
+ :param entry_id:
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract get method')
+
+ def store(self, entry, **kwargs):
+ """
+ Store entry in storage
+
+ :param entry:
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract store method')
+
+ def delete(self, entry_id, **kwargs):
+ """
+ Delete entry from storage.
+
+ :param entry_id:
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract delete method')
+
+ def __iter__(self):
+ return self.iter()
+
+ def iter(self, **kwargs):
+ """
+ Iter over the entries in storage.
+
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract iter method')
+
+ def update(self, entry, **kwargs):
+ """
+ Update entry in storage.
+
+ :param entry:
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract update method')
+
+
+class ResourceAPI(StorageAPI):
+ """
+ A Base object for the resource.
+ """
+ def __init__(self, name):
+ """
+ Base resource API
+ :param str name: the resource type
+ """
+ self._name = name
+
+ @property
+ def name(self):
+ """
+ The name of the resource
+ :return:
+ """
+ return self._name
+
+ def data(self, entry_id, path=None, **kwargs):
+ """
+ Get a bytesteam from the storagee.
--- End diff --
storage
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by ran-z <gi...@git.apache.org>.
Github user ran-z commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90774304
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
+ query = self._get_query(include, filters)
+ if locking:
+ query = query.with_for_update()
+ result = query.first()
+
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with ID `{1}` was not found'
+ .format(self.model_cls.__name__, entry_id)
+ )
+ return result
+
+ def iter(self,
+ include=None,
+ filters=None,
+ pagination=None,
+ sort=None,
+ **kwargs):
+ """Return a (possibly empty) list of `model_class` results
+ """
+ query = self._get_query(include, filters, sort)
+
+ results, _, _, _ = self._paginate(query, pagination)
+
+ for result in results:
+ yield result
+
+ def put(self, entry, **kwargs):
+ """Create a `model_class` instance from a serializable `model` object
+
+ :param entry: A dict with relevant kwargs, or an instance of a class
+ that has a `to_dict` method, and whose attributes match the columns
+ of `model_class` (might also my just an instance of `model_class`)
+ :return: An instance of `model_class`
+ """
+ self._session.add(entry)
+ self._safe_commit()
+ return entry
+
+ def delete(self, entry_id, filters=None, **kwargs):
--- End diff --
get entry not entry_id
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779283
--- Diff: aria/storage/structures.py ---
@@ -27,281 +27,189 @@
* Model - abstract model implementation.
"""
import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
+
+from sqlalchemy import VARCHAR
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+# pylint: disable=unused-import
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import (
+ schema,
+ Column,
+ Integer,
+ Text,
+ DateTime,
+ Boolean,
+ Enum,
+ String,
+ PickleType,
+ Float,
+ TypeDecorator,
+ ForeignKey,
+ orm,
)
+Model = declarative_base()
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+def foreign_key(foreign_key_column, nullable=False):
+ """Return a ForeignKey object with the relevant
-class Field(ValidatorMixin):
+ :param foreign_key_column: Unique id column in the parent table
+ :param nullable: Should the column be allowed to remain empty
"""
- A single field implementation
+ return Column(
+ ForeignKey(foreign_key_column, ondelete='CASCADE'),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(child_class,
+ parent_class,
+ foreign_key_column,
+ backreference=None):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param parent_class: Class of the parent table
+ :param child_class: Class of the child table
+ :param foreign_key_column: The column of the foreign key
+ :param backreference: The name to give to the reference to the child
"""
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
-
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
-
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
-
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ backreference = backreference or child_class.__tablename__
+ return relationship(
+ parent_class,
+ primaryjoin=lambda: parent_class.storage_id == foreign_key_column,
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=backref(backreference, cascade='all')
+ )
-class IterField(Field):
+class Dict(TypeDecorator):
"""
- Represents an iterable field.
+ Dict representation of type.
"""
- def __init__(self, **kwargs):
- """
- Simple iterable field manager.
- This field type don't have choices option.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- super(IterField, self).__init__(choices=(), **kwargs)
+ def process_literal_param(self, value, dialect):
+ pass
- def validate_value(self, name, values, *args):
- """
- Validates the value of each iterable value.
+ @property
+ def python_type(self):
+ return dict
- :param name: the name of the field.
- :param values: the values of the field.
- """
- for value in values:
- self.validate_instance(name, value, self.type)
+ impl = VARCHAR
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ value = json.dumps(value)
+ return value
-class PointerField(Field):
- """
- A single pointer field implementation.
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = json.loads(value)
+ return value
- Any PointerField points via id to another document.
+
+class MutableDict(Mutable, dict):
+ """
+ Enables tracking for dict values.
"""
+ @classmethod
+ def coerce(cls, key, value):
+ "Convert plain dictionaries to MutableDict."
- def __init__(self, type, **kwargs):
- assert issubclass(type, Model)
- super(PointerField, self).__init__(type=type, **kwargs)
+ if not isinstance(value, MutableDict):
+ if isinstance(value, dict):
+ return MutableDict(value)
+ # this call will raise ValueError
+ return Mutable.coerce(key, value)
+ else:
+ return value
-class IterPointerField(IterField, PointerField):
- """
- An iterable pointers field.
+ def __setitem__(self, key, value):
+ "Detect dictionary set events and emit change events."
- Any IterPointerField points via id to other documents.
- """
- pass
+ dict.__setitem__(self, key, value)
+ self.changed()
+ def __delitem__(self, key):
+ "Detect dictionary del events and emit change events."
-class Model(object):
- """
- Base class for all of the storage models.
- """
- id = None
+ dict.__delitem__(self, key)
+ self.changed()
- def __init__(self, **fields):
- """
- Abstract class for any model in the storage.
- The Initializer creates attributes according to the (keyword arguments) that given
- Each value is validated according to the Field.
- Each model has to have and ID Field.
- :param fields: each item is validated and transformed into instance attributes.
- """
- self._assert_model_have_id_field(**fields)
- missing_fields, unexpected_fields = self._setup_fields(fields)
+class SQLModelBase(Model):
+ """
+ Abstract base class for all SQL models that allows [de]serialization
+ """
+ # SQLAlchemy syntax
+ __abstract__ = True
- if missing_fields:
- raise StorageError(
- 'Model {name} got missing keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=missing_fields))
+ # Does the class represent a resource (Blueprint, Deployment, etc.) or a
+ # management table (User, Tenant, etc.), as they are handled differently
+ is_resource = False
- if unexpected_fields:
- raise StorageError(
- 'Model {name} got unexpected keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=unexpected_fields))
+ # This would be overridden once the models are created.
+ __table__ = None
- def __repr__(self):
- return '{name}(fields={0})'.format(sorted(self.fields), name=self.__class__.__name__)
+ _private_fields = []
- def __eq__(self, other):
- return (
- isinstance(other, self.__class__) and
- self.fields_dict == other.fields_dict)
+ # Indicates whether the `id` column in this class should be unique
+ is_id_unique = True
- @property
- def fields(self):
- """
- Iterates over the fields of the model.
- :yields: the class's field name
- """
- for name, field in vars(self.__class__).items():
- if isinstance(field, Field):
- yield name
+ storage_id = Column(Integer, primary_key=True, autoincrement=True)
+ id = Column(Text, index=True)
- @property
- def fields_dict(self):
- """
- Transforms the instance attributes into a dict.
+ @classmethod
+ def unique_id(cls):
+ return 'id'
+
+ def to_dict(self, suppress_error=False):
+ """Return a dict representation of the model
+
+ :param suppress_error: If set to True, sets `None` to attributes that
+ it's unable to retrieve (e.g., if a relationship wasn't established
+ yet, and so it's impossible to access a property through it)
+ """
+ if suppress_error:
+ res = dict()
+ for field in self.fields():
+ try:
+ field_value = getattr(self, field)
+ except AttributeError:
+ field_value = None
+ res[field] = field_value
+ else:
+ # Can't simply call here `self.to_response()` because inheriting
+ # class might override it, but we always need the same code here
+ res = dict((f, getattr(self, f)) for f in self.fields())
+ return res
- :return: all fields in dict format.
- :rtype dict
- """
- return dict((name, getattr(self, name)) for name in self.fields)
+ @classmethod
+ def fields(cls):
+ """Return the list of field names for this table
- @property
- def json(self):
- """
- Transform the dict of attributes into json
- :return:
+ Mostly for backwards compatibility in the code (that uses `fields`)
"""
- return json.dumps(self.fields_dict)
+ return set(cls.__table__.columns.keys()) - set(cls._private_fields)
- @classmethod
- def _assert_model_have_id_field(cls, **fields_initializer_values):
- if not getattr(cls, 'id', None):
- raise StorageError('Model {cls.__name__} must have id field'.format(cls=cls))
-
- if cls.id.default == cls.id.NO_DEFAULT and 'id' not in fields_initializer_values:
- raise StorageError(
- 'Model {cls.__name__} is missing required '
- 'keyword-only argument: "id"'.format(cls=cls))
-
- def _setup_fields(self, input_fields):
- missing = []
- for field_name in self.fields:
- try:
- field_obj = input_fields.pop(field_name)
- setattr(self, field_name, field_obj)
- except KeyError:
- field = getattr(self.__class__, field_name)
- if field.default == field.NO_DEFAULT:
- missing.append(field_name)
-
- unexpected_fields = input_fields.keys()
- return missing, unexpected_fields
-
-
-class Storage(LoggerMixin):
- """
- Represents the storage
- """
- def __init__(self, driver, items=(), **kwargs):
- super(Storage, self).__init__(**kwargs)
- self.driver = driver
- self.registered = {}
- for item in items:
- self.register(item)
- self.logger.debug('{name} object is ready: {0!r}'.format(
- self, name=self.__class__.__name__))
+ def __str__(self):
+ id_name, id_value = self.unique_id()
+ return '<{0} {1}=`{2}`>'.format(
+ self.__class__.__name__,
+ id_name,
+ id_value
+ )
def __repr__(self):
--- End diff --
only implement repr
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779747
--- Diff: aria/storage/models.py ---
@@ -59,66 +76,203 @@
'Plugin',
)
-# todo: sort this, maybe move from mgr or move from aria???
-ACTION_TYPES = ()
-ENTITY_TYPES = ()
+
+def uuid_generator():
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90495335
--- Diff: aria/orchestrator/context/workflow.py ---
@@ -64,19 +66,27 @@ def nodes(self):
"""
Iterator over nodes
"""
- return self.model.node.iter(filters={'blueprint_id': self.blueprint.id})
+ return self.model.node.iter(
+ filters={
+ 'deployment_storage_id': self.deployment.storage_id
--- End diff --
storage_id -> id
id -> name (not unique)
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r91278499
--- Diff: aria/orchestrator/context/operation.py ---
@@ -62,15 +63,15 @@ def node(self):
the node of the current operation
:return:
"""
- return self._actor.node
+ return self.node_instance.node
@property
def node_instance(self):
"""
The node instance of the current operation
:return:
"""
- return self._actor
+ return self.model.node_instance.get(self._task_id)
--- End diff --
self._actor_id
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r91277019
--- Diff: tests/mock/context.py ---
@@ -15,19 +15,15 @@
import pytest
-
from aria import application_model_storage
from aria.orchestrator import context
from aria.storage.sql_mapi import SQLAlchemyModelAPI
-from tests.storage import get_sqlite_api_kwargs
-
from . import models
@pytest.fixture
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90501110
--- Diff: aria/storage/api.py ---
@@ -0,0 +1,219 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+General storage API
+"""
+from contextlib import contextmanager
+
+from . import exceptions
+
+
+class StorageAPI(object):
+ """
+ General storage Base API
+ """
+ def create(self, **kwargs):
+ """
+ Create a storage API.
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract create method')
+
+ @contextmanager
+ def connect(self):
+ """
+ Established a connection and destroys it after use.
+ :return:
+ """
+ try:
+ self._establish_connection()
+ yield self
+ except BaseException as e:
+ raise exceptions.StorageError(str(e))
+ finally:
+ self._destroy_connection()
+
+ def _establish_connection(self):
+ """
+ Establish a conenction. used in the 'connect' contextmanager.
+ :return:
+ """
+ pass
+
+ def _destroy_connection(self):
+ """
+ Destroy a connection. used in the 'connect' contextmanager.
+ :return:
+ """
+ pass
+
+ def __getattr__(self, item):
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90503314
--- Diff: aria/storage/mapi/sql.py ---
@@ -0,0 +1,368 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlite3 import DatabaseError as SQLiteDBError
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+try:
+ from psycopg2 import DatabaseError as Psycopg2DBError
+ sql_errors = (SQLAlchemyError, SQLiteDBError, Psycopg2DBError)
--- End diff --
check why all these errors are imported
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90779028
--- Diff: aria/storage/structures.py ---
@@ -27,281 +27,189 @@
* Model - abstract model implementation.
"""
import json
-from itertools import count
-from uuid import uuid4
-
-from .exceptions import StorageError
-from ..logger import LoggerMixin
-from ..utils.validation import ValidatorMixin
-
-__all__ = (
- 'uuid_generator',
- 'Field',
- 'IterField',
- 'PointerField',
- 'IterPointerField',
- 'Model',
- 'Storage',
+
+from sqlalchemy import VARCHAR
+from sqlalchemy.ext.mutable import Mutable
+from sqlalchemy.orm import relationship, backref
+from sqlalchemy.ext.declarative import declarative_base
+# pylint: disable=unused-import
+from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy import (
+ schema,
+ Column,
+ Integer,
+ Text,
+ DateTime,
+ Boolean,
+ Enum,
+ String,
+ PickleType,
+ Float,
+ TypeDecorator,
+ ForeignKey,
+ orm,
)
+Model = declarative_base()
-def uuid_generator():
- """
- wrapper function which generates ids
- """
- return str(uuid4())
+def foreign_key(foreign_key_column, nullable=False):
+ """Return a ForeignKey object with the relevant
-class Field(ValidatorMixin):
+ :param foreign_key_column: Unique id column in the parent table
+ :param nullable: Should the column be allowed to remain empty
"""
- A single field implementation
+ return Column(
+ ForeignKey(foreign_key_column, ondelete='CASCADE'),
+ nullable=nullable
+ )
+
+
+def one_to_many_relationship(child_class,
+ parent_class,
+ foreign_key_column,
+ backreference=None):
+ """Return a one-to-many SQL relationship object
+ Meant to be used from inside the *child* object
+
+ :param parent_class: Class of the parent table
+ :param child_class: Class of the child table
+ :param foreign_key_column: The column of the foreign key
+ :param backreference: The name to give to the reference to the child
"""
- NO_DEFAULT = 'NO_DEFAULT'
-
- try:
- # python 3 syntax
- _next_id = count().__next__
- except AttributeError:
- # python 2 syntax
- _next_id = count().next
- _ATTRIBUTE_NAME = '_cache_{0}'.format
-
- def __init__(
- self,
- type=None,
- choices=(),
- validation_func=None,
- default=NO_DEFAULT,
- **kwargs):
- """
- Simple field manager.
-
- :param type: possible type of the field.
- :param choices: a set of possible field values.
- :param default: default field value.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- self.type = type
- self.choices = choices
- self.default = default
- self.validation_func = validation_func
- super(Field, self).__init__(**kwargs)
-
- def __get__(self, instance, owner):
- if instance is None:
- return self
- field_name = self._field_name(instance)
- try:
- return getattr(instance, self._ATTRIBUTE_NAME(field_name))
- except AttributeError as exc:
- if self.default == self.NO_DEFAULT:
- raise AttributeError(
- str(exc).replace(self._ATTRIBUTE_NAME(field_name), field_name))
-
- default_value = self.default() if callable(self.default) else self.default
- setattr(instance, self._ATTRIBUTE_NAME(field_name), default_value)
- return default_value
-
- def __set__(self, instance, value):
- field_name = self._field_name(instance)
- self.validate_value(field_name, value, instance)
- setattr(instance, self._ATTRIBUTE_NAME(field_name), value)
-
- def validate_value(self, name, value, instance):
- """
- Validates the value of the field.
-
- :param name: the name of the field.
- :param value: the value of the field.
- :param instance: the instance containing the field.
- """
- if self.default != self.NO_DEFAULT and value == self.default:
- return
- if self.type:
- self.validate_instance(name, value, self.type)
- if self.choices:
- self.validate_in_choice(name, value, self.choices)
- if self.validation_func:
- self.validation_func(name, value, instance)
-
- def _field_name(self, instance):
- """
- retrieves the field name from the instance.
-
- :param Field instance: the instance which holds the field.
- :return: name of the field
- :rtype: basestring
- """
- for name, member in vars(instance.__class__).iteritems():
- if member is self:
- return name
+ backreference = backreference or child_class.__tablename__
+ return relationship(
+ parent_class,
+ primaryjoin=lambda: parent_class.storage_id == foreign_key_column,
+ # The following line make sure that when the *parent* is
+ # deleted, all its connected children are deleted as well
+ backref=backref(backreference, cascade='all')
+ )
-class IterField(Field):
+class Dict(TypeDecorator):
"""
- Represents an iterable field.
+ Dict representation of type.
"""
- def __init__(self, **kwargs):
- """
- Simple iterable field manager.
- This field type don't have choices option.
- :param kwargs: kwargs to be passed to next in line classes.
- """
- super(IterField, self).__init__(choices=(), **kwargs)
+ def process_literal_param(self, value, dialect):
+ pass
- def validate_value(self, name, values, *args):
- """
- Validates the value of each iterable value.
+ @property
+ def python_type(self):
+ return dict
- :param name: the name of the field.
- :param values: the values of the field.
- """
- for value in values:
- self.validate_instance(name, value, self.type)
+ impl = VARCHAR
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ value = json.dumps(value)
+ return value
-class PointerField(Field):
- """
- A single pointer field implementation.
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = json.loads(value)
+ return value
- Any PointerField points via id to another document.
+
+class MutableDict(Mutable, dict):
+ """
+ Enables tracking for dict values.
"""
+ @classmethod
+ def coerce(cls, key, value):
+ "Convert plain dictionaries to MutableDict."
- def __init__(self, type, **kwargs):
- assert issubclass(type, Model)
- super(PointerField, self).__init__(type=type, **kwargs)
+ if not isinstance(value, MutableDict):
+ if isinstance(value, dict):
+ return MutableDict(value)
+ # this call will raise ValueError
+ return Mutable.coerce(key, value)
+ else:
+ return value
-class IterPointerField(IterField, PointerField):
- """
- An iterable pointers field.
+ def __setitem__(self, key, value):
+ "Detect dictionary set events and emit change events."
- Any IterPointerField points via id to other documents.
- """
- pass
+ dict.__setitem__(self, key, value)
+ self.changed()
+ def __delitem__(self, key):
+ "Detect dictionary del events and emit change events."
-class Model(object):
- """
- Base class for all of the storage models.
- """
- id = None
+ dict.__delitem__(self, key)
+ self.changed()
- def __init__(self, **fields):
- """
- Abstract class for any model in the storage.
- The Initializer creates attributes according to the (keyword arguments) that given
- Each value is validated according to the Field.
- Each model has to have and ID Field.
- :param fields: each item is validated and transformed into instance attributes.
- """
- self._assert_model_have_id_field(**fields)
- missing_fields, unexpected_fields = self._setup_fields(fields)
+class SQLModelBase(Model):
+ """
+ Abstract base class for all SQL models that allows [de]serialization
+ """
+ # SQLAlchemy syntax
+ __abstract__ = True
- if missing_fields:
- raise StorageError(
- 'Model {name} got missing keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=missing_fields))
+ # Does the class represent a resource (Blueprint, Deployment, etc.) or a
+ # management table (User, Tenant, etc.), as they are handled differently
+ is_resource = False
- if unexpected_fields:
- raise StorageError(
- 'Model {name} got unexpected keyword arguments: {fields}'.format(
- name=self.__class__.__name__, fields=unexpected_fields))
+ # This would be overridden once the models are created.
+ __table__ = None
- def __repr__(self):
- return '{name}(fields={0})'.format(sorted(self.fields), name=self.__class__.__name__)
+ _private_fields = []
- def __eq__(self, other):
- return (
- isinstance(other, self.__class__) and
- self.fields_dict == other.fields_dict)
+ # Indicates whether the `id` column in this class should be unique
+ is_id_unique = True
--- End diff --
remove
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90778632
--- Diff: aria/storage/models.py ---
@@ -148,265 +302,220 @@ def __lt__(self, other):
return False
-class DeploymentUpdate(Model):
+class DeploymentModification(SQLModelBase):
"""
- A Model which represents a deployment update
+ Deployment modification model representation.
"""
- INITIALIZING = 'initializing'
- SUCCESSFUL = 'successful'
- UPDATING = 'updating'
- FINALIZING = 'finalizing'
- EXECUTING_WORKFLOW = 'executing_workflow'
- FAILED = 'failed'
+ __tablename__ = 'deployment_modifications'
- STATES = [
- INITIALIZING,
- SUCCESSFUL,
- UPDATING,
- FINALIZING,
- EXECUTING_WORKFLOW,
- FAILED,
- ]
-
- # '{0}-{1}'.format(kwargs['deployment_id'], uuid4())
- id = Field(type=basestring, default=uuid_generator)
- deployment_id = Field(type=basestring)
- state = Field(type=basestring, choices=STATES, default=INITIALIZING)
- deployment_plan = Field()
- deployment_update_nodes = Field(default=None)
- deployment_update_node_instances = Field(default=None)
- deployment_update_deployment = Field(default=None)
- modified_entity_ids = Field(default=None)
- execution_id = Field(type=basestring)
- steps = IterPointerField(type=DeploymentUpdateStep, default=())
-
-
-class Execution(Model):
- """
- A Model which represents an execution
- """
+ STARTED = 'started'
+ FINISHED = 'finished'
+ ROLLEDBACK = 'rolledback'
- class _Validation(object):
-
- @staticmethod
- def execution_status_transition_validation(_, value, instance):
- """Validation function that verifies execution status transitions are OK"""
- try:
- current_status = instance.status
- except AttributeError:
- return
- valid_transitions = Execution.VALID_TRANSITIONS.get(current_status, [])
- if current_status != value and value not in valid_transitions:
- raise ValueError('Cannot change execution status from {current} to {new}'.format(
- current=current_status,
- new=value))
+ STATES = [STARTED, FINISHED, ROLLEDBACK]
+ END_STATES = [FINISHED, ROLLEDBACK]
- TERMINATED = 'terminated'
- FAILED = 'failed'
- CANCELLED = 'cancelled'
- PENDING = 'pending'
- STARTED = 'started'
- CANCELLING = 'cancelling'
- STATES = (
- TERMINATED,
- FAILED,
- CANCELLED,
- PENDING,
- STARTED,
- CANCELLING,
- )
- END_STATES = [TERMINATED, FAILED, CANCELLED]
- ACTIVE_STATES = [state for state in STATES if state not in END_STATES]
- VALID_TRANSITIONS = {
- PENDING: [STARTED, CANCELLED],
- STARTED: END_STATES + [CANCELLING],
- CANCELLING: END_STATES
- }
+ deployment_fk = foreign_key(Deployment.storage_id)
+ _private_fields = ['deployment_fk']
+ deployment_id = association_proxy('deployment', 'id')
- id = Field(type=basestring, default=uuid_generator)
- status = Field(type=basestring, choices=STATES,
- validation_func=_Validation.execution_status_transition_validation)
- deployment_id = Field(type=basestring)
- workflow_id = Field(type=basestring)
- blueprint_id = Field(type=basestring)
- created_at = Field(type=datetime, default=datetime.utcnow)
- started_at = Field(type=datetime, default=None)
- ended_at = Field(type=datetime, default=None)
- error = Field(type=basestring, default=None)
- parameters = Field()
+ context = Column(MutableDict.as_mutable(Dict))
+ created_at = Column(DateTime, nullable=False, index=True)
+ ended_at = Column(DateTime, index=True)
+ modified_nodes = Column(MutableDict.as_mutable(Dict))
+ node_instances = Column(MutableDict.as_mutable(Dict))
+ status = Column(Enum(*STATES, name='deployment_modification_status'))
+ @declared_attr
+ def deployment(cls):
+ return one_to_many_relationship(cls,
+ Deployment,
+ cls.deployment_fk,
+ backreference='modifications')
-class Relationship(Model):
+
+class Node(SQLModelBase):
"""
- A Model which represents a relationship
+ Node model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- source_id = Field(type=basestring)
- target_id = Field(type=basestring)
- source_interfaces = Field(type=dict)
- source_operations = Field(type=dict)
- target_interfaces = Field(type=dict)
- target_operations = Field(type=dict)
- type = Field(type=basestring)
- type_hierarchy = Field(type=list)
- properties = Field(type=dict)
-
-
-class Node(Model):
+ __tablename__ = 'nodes'
+
+ # See base class for an explanation on these properties
+ is_id_unique = False
+
+ _private_fields = ['deployment_fk']
+ deployment_fk = foreign_key(Deployment.storage_id)
+
+ deployment_id = association_proxy('deployment', 'id')
+ blueprint_id = association_proxy('blueprint', 'id')
+
+ @declared_attr
+ def deployment(cls):
+ return one_to_many_relationship(cls, Deployment, cls.deployment_fk)
+
+ deploy_number_of_instances = Column(Integer, nullable=False)
+ # TODO: This probably should be a foreign key, but there's no guarantee
+ # in the code, currently, that the host will be created beforehand
+ host_id = Column(Text)
+ max_number_of_instances = Column(Integer, nullable=False)
+ min_number_of_instances = Column(Integer, nullable=False)
+ number_of_instances = Column(Integer, nullable=False)
+ planned_number_of_instances = Column(Integer, nullable=False)
+ plugins = Column(MutableDict.as_mutable(Dict))
+ plugins_to_install = Column(MutableDict.as_mutable(Dict))
+ properties = Column(MutableDict.as_mutable(Dict))
+ operations = Column(MutableDict.as_mutable(Dict))
+ type = Column(Text, nullable=False, index=True)
+ type_hierarchy = Column(PickleType)
+
+
+class Relationship(SQLModelBase):
"""
- A Model which represents a node
+ Relationship model representation.
"""
- id = Field(type=basestring, default=uuid_generator)
- blueprint_id = Field(type=basestring)
- type = Field(type=basestring)
- type_hierarchy = Field()
- number_of_instances = Field(type=int)
- planned_number_of_instances = Field(type=int)
- deploy_number_of_instances = Field(type=int)
- host_id = Field(type=basestring, default=None)
- properties = Field(type=dict)
- operations = Field(type=dict)
- plugins = Field(type=list, default=())
- relationships = IterPointerField(type=Relationship)
- plugins_to_install = Field(type=list, default=())
- min_number_of_instances = Field(type=int)
- max_number_of_instances = Field(type=int)
-
- def relationships_by_target(self, target_id):
- """
- Retreives all of the relationship by target.
- :param target_id: the node id of the target of the relationship
- :yields: a relationship which target and node with the specified target_id
- """
- for relationship in self.relationships:
- if relationship.target_id == target_id:
- yield relationship
- # todo: maybe add here Exception if isn't exists (didn't yield one's)
+ __tablename__ = 'relationships'
+ blueprint_id = association_proxy('blueprint', 'id')
+ deployment_id = association_proxy('deployment', 'id')
-class RelationshipInstance(Model):
- """
- A Model which represents a relationship instance
- """
- id = Field(type=basestring, default=uuid_generator)
- target_id = Field(type=basestring)
- target_name = Field(type=basestring)
- source_id = Field(type=basestring)
- source_name = Field(type=basestring)
- type = Field(type=basestring)
- relationship = PointerField(type=Relationship)
+ _private_fields = ['source_node_fk', 'target_node_fk']
+
+ source_node_fk = foreign_key(Node.storage_id)
--- End diff --
consider replacing fk for id
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90780128
--- Diff: tests/storage/test_models.py ---
@@ -187,111 +99,18 @@ def test_deployment_update_step_model():
assert remove_rel < remove_node < add_node < add_rel
assert not add_node < None
- # TODO fix logic here so that pylint is happy
- # assert not modify_node < modify_rel and not modify_rel < modify_node
-
-
-def _relationship(id=''):
- return Relationship(
- id='rel{0}'.format(id),
- target_id='target{0}'.format(id),
- source_id='source{0}'.format(id),
- source_interfaces={},
- source_operations={},
- target_interfaces={},
- target_operations={},
- type='type{0}'.format(id),
- type_hierarchy=[],
- properties={})
-
-
-def test_relationships():
- relationships = [_relationship(index) for index in xrange(3)]
-
- node = Node(
- blueprint_id='blueprint_id',
- type='type',
- type_hierarchy=None,
- number_of_instances=1,
- planned_number_of_instances=1,
- deploy_number_of_instances=1,
- properties={},
- operations={},
- relationships=relationships,
- min_number_of_instances=1,
- max_number_of_instances=1)
-
- for index in xrange(3):
- assert relationships[index] is \
- next(node.relationships_by_target('target{0}'.format(index)))
-
- relationship = _relationship()
-
- node = Node(
--- End diff --
test every model and every field in every model
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90881263
--- Diff: aria/storage/models.py ---
@@ -339,94 +324,100 @@ class Node(SQLModelBase):
Node model representation.
"""
__tablename__ = 'nodes'
+ id = Column(Integer, primary_key=True)
# See base class for an explanation on these properties
is_id_unique = False
- _private_fields = ['deployment_fk']
- deployment_fk = foreign_key(Deployment.storage_id)
-
- deployment_id = association_proxy('deployment', 'id')
- blueprint_id = association_proxy('blueprint', 'id')
+ _private_fields = ['deployment_id', 'host_id']
+ deployment_id = foreign_key(Deployment.id)
+ host_id = foreign_key('nodes.id', nullable=True)
@declared_attr
def deployment(cls):
- return one_to_many_relationship(cls, Deployment, cls.deployment_fk)
+ return one_to_many_relationship(cls, Deployment, cls.deployment_id)
deploy_number_of_instances = Column(Integer, nullable=False)
# TODO: This probably should be a foreign key, but there's no guarantee
# in the code, currently, that the host will be created beforehand
- host_id = Column(Text)
+ _host_id = foreign_key('nodes.id', nullable=True)
max_number_of_instances = Column(Integer, nullable=False)
min_number_of_instances = Column(Integer, nullable=False)
number_of_instances = Column(Integer, nullable=False)
planned_number_of_instances = Column(Integer, nullable=False)
- plugins = Column(MutableDict.as_mutable(Dict))
- plugins_to_install = Column(MutableDict.as_mutable(Dict))
- properties = Column(MutableDict.as_mutable(Dict))
- operations = Column(MutableDict.as_mutable(Dict))
+ plugins = Column(Dict)
+ plugins_to_install = Column(Dict)
+ properties = Column(Dict)
+ operations = Column(Dict)
type = Column(Text, nullable=False, index=True)
type_hierarchy = Column(PickleType)
+ host = relationship('Node',
+ foreign_keys=[host_id],
+ remote_side=[id],
+ backref=orm.backref('guests'))
--- End diff --
remove backref
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90500929
--- Diff: aria/storage/api.py ---
@@ -0,0 +1,219 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+General storage API
+"""
+from contextlib import contextmanager
+
+from . import exceptions
+
+
+class StorageAPI(object):
+ """
+ General storage Base API
+ """
+ def create(self, **kwargs):
+ """
+ Create a storage API.
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract create method')
+
+ @contextmanager
+ def connect(self):
--- End diff --
consider removing
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r91277752
--- Diff: tests/storage/__init__.py ---
@@ -33,10 +35,30 @@ def teardown_method(self):
rmtree(self.path, ignore_errors=True)
-def get_sqlite_api_kwargs():
- engine = create_engine('sqlite:///:memory:',
- connect_args={'check_same_thread': False},
- poolclass=StaticPool)
- session = orm.sessionmaker(bind=engine)()
+def get_sqlite_api_kwargs(base_dir=None, filename='memory'):
+ """
+ Create sql params. works in in-memory and in filesystem mode.
+ If base_dir is passed, the mode will be filesystem mode. while the default mode is in-memory.
+ :param str base_dir: The base dir for the filesystem memory file.
+ :param str filename: the file name - defaults to 'memory'.
--- End diff --
...
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by ran-z <gi...@git.apache.org>.
Github user ran-z commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90773988
--- Diff: aria/storage/sql_mapi.py ---
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+SQLAlchemy based MAPI
+"""
+
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.sql.elements import Label
+
+from aria.utils.collections import OrderedDict
+
+from aria.storage import (
+ api,
+ exceptions
+)
+
+
+DEFAULT_SQL_DIALECT = 'sqlite'
+
+
+class SQLAlchemyModelAPI(api.ModelAPI):
+ """
+ SQL based MAPI.
+ """
+
+ def __init__(self,
+ engine,
+ session,
+ **kwargs):
+ super(SQLAlchemyModelAPI, self).__init__(**kwargs)
+ self._engine = engine
+ self._session = session
+
+ def get(self, entry_id, include=None, filters=None, locking=False, **kwargs):
+ """Return a single result based on the model class and element ID
+ """
+ filters = filters or {'id': entry_id}
+ query = self._get_query(include, filters)
+ if locking:
+ query = query.with_for_update()
+ result = query.first()
+
+ if not result:
+ raise exceptions.StorageError(
+ 'Requested {0} with ID `{1}` was not found'
--- End diff --
add filters
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90501806
--- Diff: aria/storage/api.py ---
@@ -0,0 +1,219 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+General storage API
+"""
+from contextlib import contextmanager
+
+from . import exceptions
+
+
+class StorageAPI(object):
+ """
+ General storage Base API
+ """
+ def create(self, **kwargs):
+ """
+ Create a storage API.
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract create method')
+
+ @contextmanager
+ def connect(self):
+ """
+ Established a connection and destroys it after use.
+ :return:
+ """
+ try:
+ self._establish_connection()
+ yield self
+ except BaseException as e:
+ raise exceptions.StorageError(str(e))
+ finally:
+ self._destroy_connection()
+
+ def _establish_connection(self):
+ """
+ Establish a conenction. used in the 'connect' contextmanager.
+ :return:
+ """
+ pass
+
+ def _destroy_connection(self):
+ """
+ Destroy a connection. used in the 'connect' contextmanager.
+ :return:
+ """
+ pass
+
+ def __getattr__(self, item):
+ try:
+ return self.registered[item]
+ except KeyError:
+ return super(StorageAPI, self).__getattribute__(item)
+
+
+class ModelAPI(StorageAPI):
+ """
+ A Base object for the model.
+ """
+ def __init__(self, model_cls, name=None, **kwargs):
+ """
+ Base model API
+
+ :param model_cls: the representing class of the model
+ :param str name: the name of the model
+ :param kwargs:
+ """
+ super(ModelAPI, self).__init__(**kwargs)
+ self._model_cls = model_cls
+ self._name = name or generate_lower_name(model_cls)
+
+ @property
+ def name(self):
+ """
+ The name of the class
+ :return: name of the class
+ """
+ return self._name
+
+ @property
+ def model_cls(self):
+ """
+ The class represting the model
+ :return:
+ """
+ return self._model_cls
+
+ def get(self, entry_id, filters=None, **kwargs):
+ """
+ Get entry from storage.
+
+ :param entry_id:
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract get method')
+
+ def store(self, entry, **kwargs):
+ """
+ Store entry in storage
+
+ :param entry:
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract store method')
+
+ def delete(self, entry_id, **kwargs):
+ """
+ Delete entry from storage.
+
+ :param entry_id:
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract delete method')
+
+ def __iter__(self):
+ return self.iter()
+
+ def iter(self, **kwargs):
+ """
+ Iter over the entries in storage.
+
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract iter method')
+
+ def update(self, entry, **kwargs):
+ """
+ Update entry in storage.
+
+ :param entry:
+ :param kwargs:
+ :return:
+ """
+ raise NotImplementedError('Subclass must implement abstract update method')
+
+
+class ResourceAPI(StorageAPI):
+ """
+ A Base object for the resource.
+ """
+ def __init__(self, name):
+ """
+ Base resource API
+ :param str name: the resource type
+ """
+ self._name = name
+
+ @property
+ def name(self):
+ """
+ The name of the resource
+ :return:
+ """
+ return self._name
+
+ def data(self, entry_id, path=None, **kwargs):
--- End diff --
read
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
[GitHub] incubator-ariatosca pull request #31: ARIA-30-SQL-based-storage-implementati...
Posted by mxmrlv <gi...@git.apache.org>.
Github user mxmrlv commented on a diff in the pull request:
https://github.com/apache/incubator-ariatosca/pull/31#discussion_r90499710
--- Diff: aria/storage/__init__.py ---
@@ -37,354 +37,93 @@
* drivers - module, a pool of Aria standard drivers.
* StorageDriver - class, abstract model implementation.
"""
-# todo: rewrite the above package documentation
-# (something like explaning the two types of storage - models and resources)
-from collections import namedtuple
-
-from .structures import Storage, Field, Model, IterField, PointerField
-from .drivers import (
- ModelDriver,
- ResourceDriver,
- FileSystemResourceDriver,
- FileSystemModelDriver,
+from aria.logger import LoggerMixin
+from . import (
+ models,
+ exceptions,
+ api as storage_api,
+ structures
)
-from . import models, exceptions
+
__all__ = (
'ModelStorage',
- 'ResourceStorage',
- 'FileSystemModelDriver',
'models',
'structures',
- 'Field',
- 'IterField',
- 'PointerField',
- 'Model',
- 'drivers',
- 'ModelDriver',
- 'ResourceDriver',
- 'FileSystemResourceDriver',
)
-# todo: think about package output api's...
-# todo: in all drivers name => entry_type
-# todo: change in documentation str => basestring
-class ModelStorage(Storage):
+class Storage(LoggerMixin):
"""
- Managing the models storage.
+ Represents the storage
"""
- def __init__(self, driver, model_classes=(), **kwargs):
- """
- Simple storage client api for Aria applications.
- The storage instance defines the tables/documents/code api.
-
- :param ModelDriver driver: model storage driver.
- :param model_classes: the models to register.
- """
- assert isinstance(driver, ModelDriver)
- super(ModelStorage, self).__init__(driver, model_classes, **kwargs)
-
- def __getattr__(self, table):
- """
- getattr is a shortcut to simple api
-
- for Example:
- >> storage = ModelStorage(driver=FileSystemModelDriver('/tmp'))
- >> node_table = storage.node
- >> for node in node_table:
- >> print node
-
- :param str table: table name to get
- :return: a storage object that mapped to the table name
- """
- return super(ModelStorage, self).__getattr__(table)
-
- def register(self, model_cls):
- """
- Registers the model type in the resource storage manager.
- :param model_cls: the model to register.
- """
- model_name = generate_lower_name(model_cls)
- model_api = _ModelApi(model_name, self.driver, model_cls)
- self.registered[model_name] = model_api
-
- for pointer_schema_register in model_api.pointer_mapping.values():
- model_cls = pointer_schema_register.model_cls
- self.register(model_cls)
-
-_Pointer = namedtuple('_Pointer', 'name, is_iter')
-
-
-class _ModelApi(object):
- def __init__(self, name, driver, model_cls):
- """
- Managing the model in the storage, using the driver.
-
- :param basestring name: the name of the model.
- :param ModelDriver driver: the driver which supports this model in the storage.
- :param Model model_cls: table/document class model.
- """
- assert isinstance(driver, ModelDriver)
- assert issubclass(model_cls, Model)
- self.name = name
- self.driver = driver
- self.model_cls = model_cls
- self.pointer_mapping = {}
- self._setup_pointers_mapping()
-
- def _setup_pointers_mapping(self):
- for field_name, field_cls in vars(self.model_cls).items():
- if not(isinstance(field_cls, PointerField) and field_cls.type):
- continue
- pointer_key = _Pointer(field_name, is_iter=isinstance(field_cls, IterField))
- self.pointer_mapping[pointer_key] = self.__class__(
- name=generate_lower_name(field_cls.type),
- driver=self.driver,
- model_cls=field_cls.type)
-
- def __iter__(self):
- return self.iter()
+ def __init__(self, api, items=(), api_params=None, **kwargs):
+ self._api_params = api_params or {}
+ super(Storage, self).__init__(**kwargs)
+ self.api = api
+ self.registered = {}
+ for item in items:
+ self.register(item)
+ self.logger.debug('{name} object is ready: {0!r}'.format(
+ self, name=self.__class__.__name__))
def __repr__(self):
- return '{self.name}(driver={self.driver}, model={self.model_cls})'.format(self=self)
-
- def create(self):
- """
- Creates the model in the storage.
- """
- with self.driver as connection:
- connection.create(self.name)
-
- def get(self, entry_id, **kwargs):
- """
- Getter for the model from the storage.
-
- :param basestring entry_id: the id of the table/document.
- :return: model instance
- :rtype: Model
- """
- with self.driver as connection:
- data = connection.get(
- name=self.name,
- entry_id=entry_id,
- **kwargs)
- data.update(self._get_pointers(data, **kwargs))
- return self.model_cls(**data)
+ return '{name}(api={self.api})'.format(name=self.__class__.__name__, self=self)
- def store(self, entry, **kwargs):
- """
- Setter for the model in the storage.
-
- :param Model entry: the table/document to store.
- """
- assert isinstance(entry, self.model_cls)
- with self.driver as connection:
- data = entry.fields_dict
- data.update(self._store_pointers(data, **kwargs))
- connection.store(
- name=self.name,
- entry_id=entry.id,
- entry=data,
- **kwargs)
-
- def delete(self, entry_id, **kwargs):
- """
- Delete the model from storage.
-
- :param basestring entry_id: id of the entity to delete from storage.
- """
- entry = self.get(entry_id)
- with self.driver as connection:
- self._delete_pointers(entry, **kwargs)
- connection.delete(
- name=self.name,
- entry_id=entry_id,
- **kwargs)
-
- def iter(self, **kwargs):
- """
- Generator over the entries of model in storage.
- """
- with self.driver as connection:
- for data in connection.iter(name=self.name, **kwargs):
- data.update(self._get_pointers(data, **kwargs))
- yield self.model_cls(**data)
+ def __getattr__(self, item):
+ try:
+ return self.registered[item]
+ except KeyError:
+ return super(Storage, self).__getattribute__(item)
- def update(self, entry_id, **kwargs):
+ def register(self, entry):
"""
- Updates and entry in storage.
-
- :param str entry_id: the id of the table/document.
- :param kwargs: the fields to update.
+ Register the entry to the storage
+ :param name:
:return:
"""
- with self.driver as connection:
- connection.update(
- name=self.name,
- entry_id=entry_id,
- **kwargs
- )
-
- def _get_pointers(self, data, **kwargs):
- pointers = {}
- for field, schema in self.pointer_mapping.items():
- if field.is_iter:
- pointers[field.name] = [
- schema.get(entry_id=pointer_id, **kwargs)
- for pointer_id in data[field.name]
- if pointer_id]
- elif data[field.name]:
- pointers[field.name] = schema.get(entry_id=data[field.name], **kwargs)
- return pointers
-
- def _store_pointers(self, data, **kwargs):
- pointers = {}
- for field, model_api in self.pointer_mapping.items():
- if field.is_iter:
- pointers[field.name] = []
- for iter_entity in data[field.name]:
- pointers[field.name].append(iter_entity.id)
- model_api.store(iter_entity, **kwargs)
- else:
- pointers[field.name] = data[field.name].id
- model_api.store(data[field.name], **kwargs)
- return pointers
+ raise NotImplementedError('Subclass must implement abstract register method')
- def _delete_pointers(self, entry, **kwargs):
- for field, schema in self.pointer_mapping.items():
- if field.is_iter:
- for iter_entry in getattr(entry, field.name):
- schema.delete(iter_entry.id, **kwargs)
- else:
- schema.delete(getattr(entry, field.name).id, **kwargs)
-
-class ResourceApi(object):
+class ResourceStorage(Storage):
"""
- Managing the resource in the storage, using the driver.
-
- :param basestring name: the name of the resource.
- :param ResourceDriver driver: the driver which supports this resource in the storage.
+ Represents resource storage.
"""
- def __init__(self, driver, resource_name):
- """
- Managing the resources in the storage, using the driver.
-
- :param ResourceDriver driver: the driver which supports this model in the storage.
- :param basestring resource_name: the type of the entry this resourceAPI manages.
- """
- assert isinstance(driver, ResourceDriver)
- self.driver = driver
- self.resource_name = resource_name
-
- def __repr__(self):
- return '{name}(driver={self.driver}, resource={self.resource_name})'.format(
- name=self.__class__.__name__, self=self)
-
- def create(self):
- """
- Create the resource dir in the storage.
- """
- with self.driver as connection:
- connection.create(self.resource_name)
-
- def data(self, entry_id, path=None, **kwargs):
+ def register(self, name):
"""
- Retrieve the content of a storage resource.
-
- :param basestring entry_id: the id of the entry.
- :param basestring path: path of the resource on the storage.
- :param kwargs: resources to be passed to the driver..
- :return the content of a single file:
- """
- with self.driver as connection:
- return connection.data(
- entry_type=self.resource_name,
- entry_id=entry_id,
- path=path,
- **kwargs)
-
- def download(self, entry_id, destination, path=None, **kwargs):
- """
- Download a file/dir from the resource storage.
-
- :param basestring entry_id: the id of the entry.
- :param basestring destination: the destination of the file/dir.
- :param basestring path: path of the resource on the storage.
- """
- with self.driver as connection:
- connection.download(
- entry_type=self.resource_name,
- entry_id=entry_id,
- destination=destination,
- path=path,
- **kwargs)
-
- def upload(self, entry_id, source, path=None, **kwargs):
- """
- Upload a file/dir from the resource storage.
-
- :param basestring entry_id: the id of the entry.
- :param basestring source: the source path of the file to upload.
- :param basestring path: the destination of the file, relative to the root dir
- of the resource
+ Register the resource type to resource storage.
+ :param name:
+ :return:
"""
- with self.driver as connection:
- connection.upload(
- entry_type=self.resource_name,
- entry_id=entry_id,
- source=source,
- path=path,
- **kwargs)
+ self.registered[name] = self.api(name=name, **self._api_params)
--- End diff --
move __init__ to core
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---