You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ariatosca.apache.org by "Maxim Orlov (JIRA)" <ji...@apache.org> on 2017/06/07 14:23:18 UTC
[jira] [Comment Edited] (ARIA-213) Sporadic tests failures over
locked database issue
[ https://issues.apache.org/jira/browse/ARIA-213?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16040948#comment-16040948 ]
Maxim Orlov edited comment on ARIA-213 at 6/7/17 2:22 PM:
----------------------------------------------------------
database is a know issue when working with sqlalcehmy as pysqlite as a driver. pysqlite tries to "guess" where to start and end transactions. pysqlite sometimes fails in this prediction which might result in database_locked issue. Overriding this behavior is possible via sqlalchemy. However, this raises other issues. (ref: http://docs.sqlalchemy.org/en/latest/dialects/sqlite.html#serializable-isolation-savepoints-transactional-ddl)
Another way to try and solve this locking issue is to increase the timeout for acquiring locks. Thus, we increased the timeout from the default (5 secs) to 15.
was (Author: mxmrlv):
database is a know issue when working with sqlalcehmy as pysqlite as a driver. pysqlite tries to "guess" where to start and end transactions. pysqlite sometimes fails in this prediction which might result in database_locked issue. Overriding this behavior is possible via sqlalchemy. However, this raises other issues. (ref: http://docs.sqlalchemy.org/en/latest/dialects/sqlite.html#serializable-isolation-savepoints-transactional-ddl)
Another way to try and solve this locking issue is to increase the timeout for aquiring locks. Thus, we increased the timeout from the default (5 secs) to 15.
> Sporadic tests failures over locked database issue
> --------------------------------------------------
>
> Key: ARIA-213
> URL: https://issues.apache.org/jira/browse/ARIA-213
> Project: AriaTosca
> Issue Type: Bug
> Reporter: Ran Ziv
> Assignee: Maxim Orlov
> Priority: Minor
> Fix For: 0.1.0
>
>
> Unit tests sporadically fail over an issue of locked database.
> Here are two stack traces for the same issue that I found on a single run of all of the tests:
> {code}
> _________________ TestLocalRunScript.test_retry_with_interval __________________
> self = <tests.orchestrator.execution_plugin.test_local.TestLocalRunScript object at 0x7f27682af910>
> executor = <aria.orchestrator.workflows.executor.process.ProcessExecutor object at 0x7f2768250d90>
> workflow_context = WorkflowContext(deployment_id=1, workflow_name=test_workflow, execution_id=1)
> tmpdir = local('/tmp/pytest-of-travis/pytest-0/test_retry_with_interval0')
> def test_retry_with_interval(self, executor, workflow_context, tmpdir):
> script_path = self._create_script(
> tmpdir,
> linux_script='''#! /bin/bash -e
> ctx task retry retry-message @100
> ''',
> windows_script='''
> ctx task retry retry-message @100
> ''')
> exception = self._run_and_get_task_exception(
> executor, workflow_context,
> > script_path=script_path)
> tests/orchestrator/execution_plugin/test_local.py:323:
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> tests/orchestrator/execution_plugin/test_local.py:452: in _run_and_get_task_exception
> self._run(*args, **kwargs)
> tests/orchestrator/execution_plugin/test_local.py:503: in _run
> eng.execute()
> aria/orchestrator/workflows/core/engine.py:60: in execute
> for task in self._ended_tasks():
> aria/orchestrator/workflows/core/engine.py:97: in <genexpr>
> return (task for task in self._tasks_iter() if task.has_ended())
> aria/orchestrator/workflows/core/engine.py:110: in _tasks_iter
> self._workflow_context.model.task.refresh(task.model_task)
> aria/orchestrator/workflows/core/task.py:191: in model_task
> return self._workflow_context.model.task.get(self._task_id)
> aria/storage/sql_mapi.py:59: in get
> result = query.first()
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/orm/query.py:2731: in first
> ret = list(self[0:1])
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/orm/query.py:2523: in __getitem__
> return list(res)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/orm/query.py:2831: in __iter__
> return self._execute_and_instances(context)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/orm/query.py:2854: in _execute_and_instances
> result = conn.execute(querycontext.statement, self._params)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/base.py:945: in execute
> return meth(self, multiparams, params)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/sql/elements.py:263: in _execute_on_connection
> return connection._execute_clauseelement(self, multiparams, params)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/base.py:1053: in _execute_clauseelement
> compiled_sql, distilled_params
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/base.py:1189: in _execute_context
> context)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/base.py:1393: in _handle_dbapi_exception
> exc_info
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/util/compat.py:203: in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb, cause=cause)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/base.py:1182: in _execute_context
> context)
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> self = <sqlalchemy.dialects.sqlite.pysqlite.SQLiteDialect_pysqlite object at 0x7f27683238d0>
> cursor = <sqlite3.Cursor object at 0x7f27681f25e0>
> statement = 'SELECT task.id AS task_id, task.name AS task_name, task.implementation AS task_implementation, task.max_attempts AS t...hip_fk AS task_relationship_fk, task.execution_fk AS task_execution_fk
> FROM task
> WHERE task.id = ?
> LIMIT ? OFFSET ?'
> parameters = (1, 1, 0)
> context = <sqlalchemy.dialects.sqlite.base.SQLiteExecutionContext object at 0x7f2768227590>
> def do_execute(self, cursor, statement, parameters, context=None):
> > cursor.execute(statement, parameters)
> E OperationalError: (sqlite3.OperationalError) database is locked [SQL: u'SELECT task.id AS task_id, task.name AS task_name, task.implementation AS task_implementation, task.max_attempts AS task_max_attempts, task.retry_interval AS task_retry_interval, task.ignore_failure AS task_ignore_failure, task.status AS task_status, task.due_at AS task_due_at, task.started_at AS task_started_at, task.ended_at AS task_ended_at, task.attempts_count AS task_attempts_count, task.plugin_fk AS task_plugin_fk, task.node_fk AS task_node_fk, task.relationship_fk AS task_relationship_fk, task.execution_fk AS task_execution_fk \nFROM task \nWHERE task.id = ?\n LIMIT ? OFFSET ?'] [parameters: (1, 1, 0)]
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/default.py:470: OperationalError
> {code}
> {code}
> _______________ TestLocalRunScript.test_crash_abort_after_abort ________________
> self = <tests.orchestrator.execution_plugin.test_local.TestLocalRunScript object at 0x7f277b3b0050>
> executor = <aria.orchestrator.workflows.executor.process.ProcessExecutor object at 0x7f277b3b0d50>
> workflow_context = WorkflowContext(deployment_id=1, workflow_name=test_workflow, execution_id=1)
> tmpdir = local('/tmp/pytest-of-travis/pytest-0/test_crash_abort_after_abort0')
> def test_crash_abort_after_abort(self, executor, workflow_context, tmpdir):
> script_path = self._create_script(
> tmpdir,
> linux_script='''#! /bin/bash
> ctx task abort abort-message
> ctx task abort should-raise-a-runtime-error
> ''',
> windows_script='''
> ctx task abort abort-message
> ctx task abort should-raise-a-runtime-error
> ''')
> exception = self._run_and_get_task_exception(
> executor, workflow_context,
> > script_path=script_path)
> tests/orchestrator/execution_plugin/test_local.py:375:
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> tests/orchestrator/execution_plugin/test_local.py:452: in _run_and_get_task_exception
> self._run(*args, **kwargs)
> tests/orchestrator/execution_plugin/test_local.py:503: in _run
> eng.execute()
> aria/orchestrator/workflows/core/engine.py:57: in execute
> cancel = self._is_cancel()
> aria/orchestrator/workflows/core/engine.py:86: in _is_cancel
> return self._workflow_context.execution.status in (models.Execution.CANCELLING,
> aria/orchestrator/context/workflow.py:61: in execution
> return self.model.execution.get(self._execution_id)
> aria/storage/sql_mapi.py:59: in get
> result = query.first()
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/orm/query.py:2731: in first
> ret = list(self[0:1])
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/orm/query.py:2523: in __getitem__
> return list(res)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/orm/query.py:2831: in __iter__
> return self._execute_and_instances(context)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/orm/query.py:2854: in _execute_and_instances
> result = conn.execute(querycontext.statement, self._params)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/base.py:945: in execute
> return meth(self, multiparams, params)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/sql/elements.py:263: in _execute_on_connection
> return connection._execute_clauseelement(self, multiparams, params)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/base.py:1053: in _execute_clauseelement
> compiled_sql, distilled_params
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/base.py:1189: in _execute_context
> context)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/base.py:1393: in _handle_dbapi_exception
> exc_info
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/util/compat.py:203: in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb, cause=cause)
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/base.py:1182: in _execute_context
> context)
> _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> self = <sqlalchemy.dialects.sqlite.pysqlite.SQLiteDialect_pysqlite object at 0x7f277a84e610>
> cursor = <sqlite3.Cursor object at 0x7f276839c2d0>
> statement = 'SELECT execution.id AS execution_id, execution.name AS execution_name, execution.created_at AS execution_created_at, ..._workflow_name, execution.service_fk AS execution_service_fk
> FROM execution
> WHERE execution.id = ?
> LIMIT ? OFFSET ?'
> parameters = (1, 1, 0)
> context = <sqlalchemy.dialects.sqlite.base.SQLiteExecutionContext object at 0x7f277a7c8510>
> def do_execute(self, cursor, statement, parameters, context=None):
> > cursor.execute(statement, parameters)
> E OperationalError: (sqlite3.OperationalError) database is locked [SQL: u'SELECT execution.id AS execution_id, execution.name AS execution_name, execution.created_at AS execution_created_at, execution.started_at AS execution_started_at, execution.ended_at AS execution_ended_at, execution.error AS execution_error, execution.status AS execution_status, execution.workflow_name AS execution_workflow_name, execution.service_fk AS execution_service_fk \nFROM execution \nWHERE execution.id = ?\n LIMIT ? OFFSET ?'] [parameters: (1, 1, 0)]
> .tox/py27/lib/python2.7/site-packages/sqlalchemy/engine/default.py:470: OperationalError
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)