You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by "t oo (Jira)" <ji...@apache.org> on 2019/09/16 18:04:00 UTC
[jira] [Created] (AIRFLOW-5505) task_instance table errors in
metastore db with localexecutor
t oo created AIRFLOW-5505:
-----------------------------
Summary: task_instance table errors in metastore db with localexecutor
Key: AIRFLOW-5505
URL: https://issues.apache.org/jira/browse/AIRFLOW-5505
Project: Apache Airflow
Issue Type: Bug
Components: database, scheduler
Affects Versions: 1.10.3
Reporter: t oo
using v1.10.3, localexecutor, mysql backend. MySQL CPU usage is around 50%, peaking at 70%. I externally trigger 30 DAGs in parallel (different execution dates but same dagid). I repeat that same pattern for 20 different DAGids.
ie dagidA - run execdate 1-30sep in parallel
let those 30 runs finish then:
dagidB - run execdate 1-30sep in parallel
let those 30 runs finish then:
dagidC - run execdate 1-30sep in parallel
..etc
I face these errors approx 50 times a day.
Facing several error messages, all related to task_instance table.
1.
[2019-09-15 22:09:14,475] \{__init__.py:305} INFO - Filling up the DagBag from /home/ec2-user/airflow/dags
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 32, in <module>
args.func(args)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py", line 233, in trigger_dag
execution_date=args.exec_date)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/client/local_client.py", line 33, in trigger_dag
execution_date=execution_date)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/common/experimental/trigger_dag.py", line 101, in trigger_dag
replace_microseconds=replace_microseconds,
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/common/experimental/trigger_dag.py", line 77, in _trigger_dag
external_trigger=True,
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 73, in wrapper
return func(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 4095, in create_dagrun
run.verify_integrity(session=session)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 69, in wrapper
return func(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 4934, in verify_integrity
session.commit()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 1023, in commit
self.transaction.commit()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 487, in commit
self._prepare_impl()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 466, in _prepare_impl
self.session.flush()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2446, in flush
self._flush(objects)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2584, in _flush
transaction.rollback(_capture_exception=True)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 67, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2544, in _flush
flush_context.execute()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 416, in execute
rec.execute(self)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 583, in execute
uow,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
insert,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1063, in _emit_insert_statements
c = cached_connections[connection].execute(statement, multiparams)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 980, in execute
return meth(self, multiparams, params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
distilled_params,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
e, statement, parameters, cursor, context
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1216, in _execute_context
cursor, statement, parameters, context
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 107, in do_executemany
rowcount = cursor.executemany(statement, parameters)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 237, in executemany
self._get_db().encoding)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 264, in _do_execute_many
rows += self.execute(sql + postfix)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 209, in execute
res = self._query(query)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 315, in _query
db.query(q)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/connections.py", line 226, in query
_mysql.connection.query(self, query)
sqlalchemy.exc.OperationalError: (MySQLdb._exceptions.OperationalError) (1205, 'Lock wait timeout exceeded; try restarting transaction') [SQL: u'INSERT INTO task_instance (task_id, dag_id, execution_date, start_date, end_date, duration, state, try_number, max_tries, hostname, unixname, job_id, pool, queue, priority_weight, operator, queued_dttm, pid, executor_config) VALUES (%!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING))'] [parameters: ((u'co
2.
[2019-09-15 08:31:15,401] \{__init__.py:305} INFO - Filling up the DagBag from /home/ec2-user/airflow/dags
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 32, in <module>
args.func(args)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py", line 233, in trigger_dag
execution_date=args.exec_date)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/client/local_client.py", line 33, in trigger_dag
execution_date=execution_date)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/common/experimental/trigger_dag.py", line 101, in trigger_dag
replace_microseconds=replace_microseconds,
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/common/experimental/trigger_dag.py", line 77, in _trigger_dag
external_trigger=True,
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 73, in wrapper
return func(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 4095, in create_dagrun
run.verify_integrity(session=session)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 69, in wrapper
return func(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 4934, in verify_integrity
session.commit()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 1023, in commit
self.transaction.commit()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 487, in commit
self._prepare_impl()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 466, in _prepare_impl
self.session.flush()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2446, in flush
self._flush(objects)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2584, in _flush
transaction.rollback(_capture_exception=True)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 67, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2544, in _flush
flush_context.execute()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 416, in execute
rec.execute(self)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 583, in execute
uow,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
insert,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1063, in _emit_insert_statements
c = cached_connections[connection].execute(statement, multiparams)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 980, in execute
return meth(self, multiparams, params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
distilled_params,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
e, statement, parameters, cursor, context
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1216, in _execute_context
cursor, statement, parameters, context
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 107, in do_executemany
rowcount = cursor.executemany(statement, parameters)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 237, in executemany
self._get_db().encoding)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 264, in _do_execute_many
rows += self.execute(sql + postfix)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 209, in execute
res = self._query(query)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 315, in _query
db.query(q)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/connections.py", line 226, in query
_mysql.connection.query(self, query)
sqlalchemy.exc.OperationalError: (MySQLdb._exceptions.OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting transaction') [SQL: u'INSERT INTO task_instance (task_id, dag_id, execution_date, start_date, end_date, duration, state, try_number, max_tries, hostname, unixname, job_id, pool, queue, priority_weight, operator, queued_dttm, pid, executor_config) VALUES (%!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING))'] [parameters: ((u'J
3.
[2019-09-15 22:30:48,375] \{__init__.py:305} INFO - Filling up the DagBag from /home/ec2-user/airflow/dags
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 32, in <module>
args.func(args)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py", line 709, in clear
include_parentdag=not args.exclude_parentdag,
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 3811, in clear_dags
dry_run=False,
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 73, in wrapper
return func(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 3756, in clear
session.commit()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 1023, in commit
self.transaction.commit()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 487, in commit
self._prepare_impl()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 466, in _prepare_impl
self.session.flush()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2446, in flush
self._flush(objects)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2584, in _flush
transaction.rollback(_capture_exception=True)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 67, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2544, in _flush
flush_context.execute()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 416, in execute
rec.execute(self)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 583, in execute
uow,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 236, in save_obj
update,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 976, in _emit_update_statements
statement, multiparams
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 980, in execute
return meth(self, multiparams, params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
distilled_params,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
e, statement, parameters, cursor, context
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1216, in _execute_context
cursor, statement, parameters, context
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 107, in do_executemany
rowcount = cursor.executemany(statement, parameters)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 239, in executemany
self.rowcount = sum(self.execute(query, arg) for arg in args)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 239, in <genexpr>
self.rowcount = sum(self.execute(query, arg) for arg in args)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 209, in execute
res = self._query(query)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 315, in _query
db.query(q)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/connections.py", line 226, in query
_mysql.connection.query(self, query)
sqlalchemy.exc.OperationalError: (MySQLdb._exceptions.OperationalError) (1205, 'Lock wait timeout exceeded; try restarting transaction') [SQL: u'UPDATE task_instance SET state=%!s(MISSING), max_tries=%!s(MISSING) WHERE task_instance.task_id = %!s(MISSING) AND task_instance.dag_id = %!s(MISSING) AND task_instance.execution_date = %!s(MISSING)'] [parame
4.
[2019-09-15 22:30:47,354] \{__init__.py:305} INFO - Filling up the DagBag from /home/ec2-user/airflow/dags
Traceback (most recent call last):
File "/home/ec2-user/venv/bin/airflow", line 32, in <module>
args.func(args)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper
return f(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py", line 709, in clear
include_parentdag=not args.exclude_parentdag,
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 3811, in clear_dags
dry_run=False,
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 73, in wrapper
return func(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 3756, in clear
session.commit()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 1023, in commit
self.transaction.commit()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 487, in commit
self._prepare_impl()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 466, in _prepare_impl
self.session.flush()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2446, in flush
self._flush(objects)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2584, in _flush
transaction.rollback(_capture_exception=True)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 67, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2544, in _flush
flush_context.execute()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 416, in execute
rec.execute(self)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 583, in execute
uow,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 236, in save_obj
update,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 976, in _emit_update_statements
statement, multiparams
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 980, in execute
return meth(self, multiparams, params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
distilled_params,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
e, statement, parameters, cursor, context
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1216, in _execute_context
cursor, statement, parameters, context
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 107, in do_executemany
rowcount = cursor.executemany(statement, parameters)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 239, in executemany
self.rowcount = sum(self.execute(query, arg) for arg in args)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 239, in <genexpr>
self.rowcount = sum(self.execute(query, arg) for arg in args)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 209, in execute
res = self._query(query)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 315, in _query
db.query(q)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/connections.py", line 226, in query
_mysql.connection.query(self, query)
sqlalchemy.exc.OperationalError: (MySQLdb._exceptions.OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting transaction') [SQL: u'UPDATE task_instance SET state=%!s(MISSING), max_tries=%!s(MISSING) WHERE task_instance.task_id = %!s(MISSING) AND task_instance.dag_id = %!s(MISSING) AND task_instance.execution_date = %!s(MISSING)'] [parame
5.
[2019-09-15 02:09:02,594] \{jobs.py:1537} ERROR - Exception when executing execute_helper
Traceback (most recent call last):
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 1535, in _execute
self._execute_helper()
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 1562, in _execute_helper
self.reset_state_for_orphaned_tasks()
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 73, in wrapper
return func(*args, **kwargs)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 294, in reset_state_for_orphaned_tasks
self.max_tis_per_query)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/helpers.py", line 146, in reduce_in_chunks
return reduce(fn, chunks(iterable, chunk_size), initializer)
File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 283, in query
.filter(or_(*filter_for_tis), TI.state.in_(resettable_states))
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2925, in all
return list(self)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 3081, in __iter__
return self._execute_and_instances(context)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 3106, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 980, in execute
return meth(self, multiparams, params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
distilled_params,
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
e, statement, parameters, cursor, context
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1236, in _execute_context
cursor, statement, parameters, context
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/default.py", line 536, in do_execute
cursor.execute(statement, parameters)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 209, in execute
res = self._query(query)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 316, in _query
self._do_get_result(db)
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 147, in _do_get_result
self._result = result = self._get_result()
File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 349, in _get_result
return self._get_db().store_result()
OperationalError: (MySQLdb._exceptions.OperationalError) (1205, 'Lock wait timeout exceeded; try restarting transaction') [SQL: u'SELECT task_instance.try_number AS task_instance_try_number, task_instance.task_id AS task_instance_task_id, task_instance.dag_id AS task_instance_dag_id, task_instance.execution_date AS task_instance_execution_date, task_instance.start_date AS task_instance_start_date, task_instance.end_date AS task_instance_end_date, task_instance.duration AS task_instance_duration, task_instance.state AS task_instance_state, task_instance.max_tries AS task_instance_max_tries, task_instance.hostname AS task_instance_hostname, task_instance.unixname AS task_instance_unixname, task_instance.job_id AS task_instance_job_id, task_instance.pool AS task_instance_pool, task_instance.queue AS task_instance_queue, task_instance.priority_weight AS task_instance_priority_weight, task_instance.operator AS task_instance_operator, task_instance.queued_dttm AS task_instance_queued_dttm, task_instance.pid AS task_instance_pid, task_instance.executor_config AS task_instance_executor_config \nFROM task_instance \nWHERE (task_instance.dag_id = %s AND task_instance.task_id = %s AND task_instance.execution_date = %s OR task_instance.dag_id = %s AND task_instance.task_id = %s AND task_instance.execution_date = %s OR task_ins
--
This message was sent by Atlassian Jira
(v8.3.2#803003)