You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by "Jarek Potiuk (Jira)" <ji...@apache.org> on 2020/01/13 19:25:00 UTC
[jira] [Resolved] (AIRFLOW-5505) task_instance table errors in
metastore db with localexecutor/mysql
[ https://issues.apache.org/jira/browse/AIRFLOW-5505?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Jarek Potiuk resolved AIRFLOW-5505.
-----------------------------------
Fix Version/s: 1.10.8
Resolution: Duplicate
> task_instance table errors in metastore db with localexecutor/mysql
> -------------------------------------------------------------------
>
> Key: AIRFLOW-5505
> URL: https://issues.apache.org/jira/browse/AIRFLOW-5505
> Project: Apache Airflow
> Issue Type: Bug
> Components: database, scheduler
> Affects Versions: 1.10.3
> Reporter: t oo
> Priority: Major
> Fix For: 1.10.8
>
>
> using v1.10.3, localexecutor, mysql backend. MySQL CPU usage is around 50%, peaking at 70%. I externally trigger 30 DAGs in parallel (different execution dates but same dagid). I repeat that same pattern for 20 different DAGids.
> ie dagidA - run execdate 1-30sep in parallel
> let those 30 runs finish then:
> dagidB - run execdate 1-30sep in parallel
> let those 30 runs finish then:
> dagidC - run execdate 1-30sep in parallel
> ..etc
>
> I face these errors approx 50 times a day.
>
> Facing several error messages, all related to task_instance table.
>
> 1.
> [2019-09-15 22:09:14,475] \{__init__.py:305} INFO - Filling up the DagBag from /home/ec2-user/airflow/dags
> Traceback (most recent call last):
> File "/home/ec2-user/venv/bin/airflow", line 32, in <module>
> args.func(args)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper
> return f(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py", line 233, in trigger_dag
> execution_date=args.exec_date)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/client/local_client.py", line 33, in trigger_dag
> execution_date=execution_date)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/common/experimental/trigger_dag.py", line 101, in trigger_dag
> replace_microseconds=replace_microseconds,
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/common/experimental/trigger_dag.py", line 77, in _trigger_dag
> external_trigger=True,
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 73, in wrapper
> return func(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 4095, in create_dagrun
> run.verify_integrity(session=session)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 69, in wrapper
> return func(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 4934, in verify_integrity
> session.commit()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 1023, in commit
> self.transaction.commit()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 487, in commit
> self._prepare_impl()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 466, in _prepare_impl
> self.session.flush()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2446, in flush
> self._flush(objects)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2584, in _flush
> transaction.rollback(_capture_exception=True)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 67, in __exit__
> compat.reraise(exc_type, exc_value, exc_tb)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2544, in _flush
> flush_context.execute()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 416, in execute
> rec.execute(self)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 583, in execute
> uow,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
> insert,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1063, in _emit_insert_statements
> c = cached_connections[connection].execute(statement, multiparams)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 980, in execute
> return meth(self, multiparams, params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
> return connection._execute_clauseelement(self, multiparams, params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
> distilled_params,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
> e, statement, parameters, cursor, context
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
> util.raise_from_cause(sqlalchemy_exception, exc_info)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb, cause=cause)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1216, in _execute_context
> cursor, statement, parameters, context
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 107, in do_executemany
> rowcount = cursor.executemany(statement, parameters)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 237, in executemany
> self._get_db().encoding)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 264, in _do_execute_many
> rows += self.execute(sql + postfix)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 209, in execute
> res = self._query(query)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 315, in _query
> db.query(q)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/connections.py", line 226, in query
> _mysql.connection.query(self, query)
> sqlalchemy.exc.OperationalError: (MySQLdb._exceptions.OperationalError) (1205, 'Lock wait timeout exceeded; try restarting transaction') [SQL: u'INSERT INTO task_instance (task_id, dag_id, execution_date, start_date, end_date, duration, state, try_number, max_tries, hostname, unixname, job_id, pool, queue, priority_weight, operator, queued_dttm, pid, executor_config) VALUES (%!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING))'] [parameters: ((u'co
>
> 2.
> [2019-09-15 08:31:15,401] \{__init__.py:305} INFO - Filling up the DagBag from /home/ec2-user/airflow/dags
> Traceback (most recent call last):
> File "/home/ec2-user/venv/bin/airflow", line 32, in <module>
> args.func(args)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper
> return f(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py", line 233, in trigger_dag
> execution_date=args.exec_date)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/client/local_client.py", line 33, in trigger_dag
> execution_date=execution_date)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/common/experimental/trigger_dag.py", line 101, in trigger_dag
> replace_microseconds=replace_microseconds,
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/api/common/experimental/trigger_dag.py", line 77, in _trigger_dag
> external_trigger=True,
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 73, in wrapper
> return func(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 4095, in create_dagrun
> run.verify_integrity(session=session)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 69, in wrapper
> return func(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 4934, in verify_integrity
> session.commit()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 1023, in commit
> self.transaction.commit()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 487, in commit
> self._prepare_impl()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 466, in _prepare_impl
> self.session.flush()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2446, in flush
> self._flush(objects)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2584, in _flush
> transaction.rollback(_capture_exception=True)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 67, in __exit__
> compat.reraise(exc_type, exc_value, exc_tb)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2544, in _flush
> flush_context.execute()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 416, in execute
> rec.execute(self)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 583, in execute
> uow,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
> insert,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 1063, in _emit_insert_statements
> c = cached_connections[connection].execute(statement, multiparams)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 980, in execute
> return meth(self, multiparams, params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
> return connection._execute_clauseelement(self, multiparams, params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
> distilled_params,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
> e, statement, parameters, cursor, context
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
> util.raise_from_cause(sqlalchemy_exception, exc_info)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb, cause=cause)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1216, in _execute_context
> cursor, statement, parameters, context
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 107, in do_executemany
> rowcount = cursor.executemany(statement, parameters)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 237, in executemany
> self._get_db().encoding)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 264, in _do_execute_many
> rows += self.execute(sql + postfix)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 209, in execute
> res = self._query(query)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 315, in _query
> db.query(q)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/connections.py", line 226, in query
> _mysql.connection.query(self, query)
> sqlalchemy.exc.OperationalError: (MySQLdb._exceptions.OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting transaction') [SQL: u'INSERT INTO task_instance (task_id, dag_id, execution_date, start_date, end_date, duration, state, try_number, max_tries, hostname, unixname, job_id, pool, queue, priority_weight, operator, queued_dttm, pid, executor_config) VALUES (%!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING), %!s(MISSING))'] [parameters: ((u'J
>
> 3.
> [2019-09-15 22:30:48,375] \{__init__.py:305} INFO - Filling up the DagBag from /home/ec2-user/airflow/dags
> Traceback (most recent call last):
> File "/home/ec2-user/venv/bin/airflow", line 32, in <module>
> args.func(args)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper
> return f(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py", line 709, in clear
> include_parentdag=not args.exclude_parentdag,
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 3811, in clear_dags
> dry_run=False,
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 73, in wrapper
> return func(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 3756, in clear
> session.commit()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 1023, in commit
> self.transaction.commit()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 487, in commit
> self._prepare_impl()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 466, in _prepare_impl
> self.session.flush()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2446, in flush
> self._flush(objects)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2584, in _flush
> transaction.rollback(_capture_exception=True)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 67, in __exit__
> compat.reraise(exc_type, exc_value, exc_tb)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2544, in _flush
> flush_context.execute()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 416, in execute
> rec.execute(self)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 583, in execute
> uow,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 236, in save_obj
> update,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 976, in _emit_update_statements
> statement, multiparams
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 980, in execute
> return meth(self, multiparams, params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
> return connection._execute_clauseelement(self, multiparams, params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
> distilled_params,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
> e, statement, parameters, cursor, context
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
> util.raise_from_cause(sqlalchemy_exception, exc_info)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb, cause=cause)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1216, in _execute_context
> cursor, statement, parameters, context
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 107, in do_executemany
> rowcount = cursor.executemany(statement, parameters)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 239, in executemany
> self.rowcount = sum(self.execute(query, arg) for arg in args)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 239, in <genexpr>
> self.rowcount = sum(self.execute(query, arg) for arg in args)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 209, in execute
> res = self._query(query)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 315, in _query
> db.query(q)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/connections.py", line 226, in query
> _mysql.connection.query(self, query)
> sqlalchemy.exc.OperationalError: (MySQLdb._exceptions.OperationalError) (1205, 'Lock wait timeout exceeded; try restarting transaction') [SQL: u'UPDATE task_instance SET state=%!s(MISSING), max_tries=%!s(MISSING) WHERE task_instance.task_id = %!s(MISSING) AND task_instance.dag_id = %!s(MISSING) AND task_instance.execution_date = %!s(MISSING)'] [parame
>
> 4.
> [2019-09-15 22:30:47,354] \{__init__.py:305} INFO - Filling up the DagBag from /home/ec2-user/airflow/dags
> Traceback (most recent call last):
> File "/home/ec2-user/venv/bin/airflow", line 32, in <module>
> args.func(args)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/cli.py", line 74, in wrapper
> return f(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py", line 709, in clear
> include_parentdag=not args.exclude_parentdag,
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 3811, in clear_dags
> dry_run=False,
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 73, in wrapper
> return func(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/models/__init__.py", line 3756, in clear
> session.commit()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 1023, in commit
> self.transaction.commit()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 487, in commit
> self._prepare_impl()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 466, in _prepare_impl
> self.session.flush()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2446, in flush
> self._flush(objects)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2584, in _flush
> transaction.rollback(_capture_exception=True)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 67, in __exit__
> compat.reraise(exc_type, exc_value, exc_tb)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 2544, in _flush
> flush_context.execute()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 416, in execute
> rec.execute(self)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 583, in execute
> uow,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 236, in save_obj
> update,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 976, in _emit_update_statements
> statement, multiparams
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 980, in execute
> return meth(self, multiparams, params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
> return connection._execute_clauseelement(self, multiparams, params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
> distilled_params,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
> e, statement, parameters, cursor, context
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
> util.raise_from_cause(sqlalchemy_exception, exc_info)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb, cause=cause)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1216, in _execute_context
> cursor, statement, parameters, context
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/dialects/mysql/mysqldb.py", line 107, in do_executemany
> rowcount = cursor.executemany(statement, parameters)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 239, in executemany
> self.rowcount = sum(self.execute(query, arg) for arg in args)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 239, in <genexpr>
> self.rowcount = sum(self.execute(query, arg) for arg in args)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 209, in execute
> res = self._query(query)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 315, in _query
> db.query(q)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/connections.py", line 226, in query
> _mysql.connection.query(self, query)
> sqlalchemy.exc.OperationalError: (MySQLdb._exceptions.OperationalError) (1213, 'Deadlock found when trying to get lock; try restarting transaction') [SQL: u'UPDATE task_instance SET state=%!s(MISSING), max_tries=%!s(MISSING) WHERE task_instance.task_id = %!s(MISSING) AND task_instance.dag_id = %!s(MISSING) AND task_instance.execution_date = %!s(MISSING)'] [parame
>
> 5.
> [2019-09-15 02:09:02,594] \{jobs.py:1537} ERROR - Exception when executing execute_helper
> Traceback (most recent call last):
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 1535, in _execute
> self._execute_helper()
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 1562, in _execute_helper
> self.reset_state_for_orphaned_tasks()
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 73, in wrapper
> return func(*args, **kwargs)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 294, in reset_state_for_orphaned_tasks
> self.max_tis_per_query)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/utils/helpers.py", line 146, in reduce_in_chunks
> return reduce(fn, chunks(iterable, chunk_size), initializer)
> File "/home/ec2-user/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 283, in query
> .filter(or_(*filter_for_tis), TI.state.in_(resettable_states))
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2925, in all
> return list(self)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 3081, in __iter__
> return self._execute_and_instances(context)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 3106, in _execute_and_instances
> result = conn.execute(querycontext.statement, self._params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 980, in execute
> return meth(self, multiparams, params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/sql/elements.py", line 273, in _execute_on_connection
> return connection._execute_clauseelement(self, multiparams, params)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1099, in _execute_clauseelement
> distilled_params,
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1240, in _execute_context
> e, statement, parameters, cursor, context
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1458, in _handle_dbapi_exception
> util.raise_from_cause(sqlalchemy_exception, exc_info)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/util/compat.py", line 296, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb, cause=cause)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 1236, in _execute_context
> cursor, statement, parameters, context
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/sqlalchemy/engine/default.py", line 536, in do_execute
> cursor.execute(statement, parameters)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 209, in execute
> res = self._query(query)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 316, in _query
> self._do_get_result(db)
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 147, in _do_get_result
> self._result = result = self._get_result()
> File "/home/ec2-user/venv/local/lib64/python2.7/site-packages/MySQLdb/cursors.py", line 349, in _get_result
> return self._get_db().store_result()
> OperationalError: (MySQLdb._exceptions.OperationalError) (1205, 'Lock wait timeout exceeded; try restarting transaction') [SQL: u'SELECT task_instance.try_number AS task_instance_try_number, task_instance.task_id AS task_instance_task_id, task_instance.dag_id AS task_instance_dag_id, task_instance.execution_date AS task_instance_execution_date, task_instance.start_date AS task_instance_start_date, task_instance.end_date AS task_instance_end_date, task_instance.duration AS task_instance_duration, task_instance.state AS task_instance_state, task_instance.max_tries AS task_instance_max_tries, task_instance.hostname AS task_instance_hostname, task_instance.unixname AS task_instance_unixname, task_instance.job_id AS task_instance_job_id, task_instance.pool AS task_instance_pool, task_instance.queue AS task_instance_queue, task_instance.priority_weight AS task_instance_priority_weight, task_instance.operator AS task_instance_operator, task_instance.queued_dttm AS task_instance_queued_dttm, task_instance.pid AS task_instance_pid, task_instance.executor_config AS task_instance_executor_config \nFROM task_instance \nWHERE (task_instance.dag_id = %s AND task_instance.task_id = %s AND task_instance.execution_date = %s OR task_instance.dag_id = %s AND task_instance.task_id = %s AND task_instance.execution_date = %s OR task_ins
--
This message was sent by Atlassian Jira
(v8.3.4#803005)