You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@airflow.apache.org by ra...@gmail.com,
ra...@gmail.com on 2018/05/10 04:30:48 UTC
Airflow Scheduler Crashing
Hi All,
We are running airflow version 1.9 in LocalExecutor mode. We are observing that scheduler is crashed after few hours with below stack logs(Seems to be an issue with Mysql Connection. Is there any fix or workaround for this)
Traceback (most recent call last):
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 371, in helper
pickle_dags)
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 50, in wrapper
result = func(*args, **kwargs)
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 1762, in process_file
dag.sync_to_db()
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 50, in wrapper
result = func(*args, **kwargs)
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/models.py", line 3806, in sync_to_db
DagModel).filter(DagModel.dag_id == self.dag_id).first()
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2825, in first
ret = list(self[0:1])
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2617, in __getitem__
return list(res)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2925, in __iter__
return self._execute_and_instances(context)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2946, in _execute_and_instances
close_with_result=True)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2955, in _get_bind_args
**kw
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2937, in _connection_from_session
conn = self.session.connection(**kw)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 1035, in connection
execution_options=execution_options)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 1040, in _connection_for_bind
engine, execution_options)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 409, in _connection_for_bind
conn = bind.contextual_connect()
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 2123, in contextual_connect
self._wrap_pool_connect(self.pool.connect, None),
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 2158, in _wrap_pool_connect
return fn()
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 403, in connect
return _ConnectionFairy._checkout(self)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 788, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 532, in checkout
rec = pool._do_get()
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 1193, in _do_get
self._dec_overflow()
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 1190, in _do_get
return self._create_connection()
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 350, in _create_connection
return _ConnectionRecord(self)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 477, in __init__
self.__connect(first_connect_check=True)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 671, in __connect
connection = pool._invoke_creator(self)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/strategies.py", line 106, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 410, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/src/venv/local/lib/python2.7/site-packages/MySQLdb/__init__.py", line 86, in Connect
return Connection(*args, **kwargs)
File "/usr/src/venv/local/lib/python2.7/site-packages/MySQLdb/connections.py", line 209, in __init__
self._server_version = tuple([ numeric_part(n) for n in self.get_server_info().split('.')[:2] ])
File "/usr/src/venv/local/lib/python2.7/site-packages/MySQLdb/connections.py", line 69, in numeric_part
m = re_numeric_part.match(s)
File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py", line 73, in sigint_handler
sys.exit(0)
SystemExit: 0
Re: Airflow Scheduler Crashing
Posted by Bolke de Bruin <bd...@gmail.com>.
Sigint means someone is pressing ctrl-c or sends a sigint to the scheduler. This is outside airflow.
B.
Sent from my iPhone
> On 10 May 2018, at 06:30, ramandumcs@gmail.com <ra...@gmail.com> wrote:
>
> Hi All,
>
> We are running airflow version 1.9 in LocalExecutor mode. We are observing that scheduler is crashed after few hours with below stack logs(Seems to be an issue with Mysql Connection. Is there any fix or workaround for this)
> Traceback (most recent call last):
> File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 371, in helper
> pickle_dags)
> File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 50, in wrapper
> result = func(*args, **kwargs)
> File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/jobs.py", line 1762, in process_file
> dag.sync_to_db()
> File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/utils/db.py", line 50, in wrapper
> result = func(*args, **kwargs)
> File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/models.py", line 3806, in sync_to_db
> DagModel).filter(DagModel.dag_id == self.dag_id).first()
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2825, in first
> ret = list(self[0:1])
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2617, in __getitem__
> return list(res)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2925, in __iter__
> return self._execute_and_instances(context)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2946, in _execute_and_instances
> close_with_result=True)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2955, in _get_bind_args
> **kw
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2937, in _connection_from_session
> conn = self.session.connection(**kw)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 1035, in connection
> execution_options=execution_options)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 1040, in _connection_for_bind
> engine, execution_options)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 409, in _connection_for_bind
> conn = bind.contextual_connect()
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 2123, in contextual_connect
> self._wrap_pool_connect(self.pool.connect, None),
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 2158, in _wrap_pool_connect
> return fn()
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 403, in connect
> return _ConnectionFairy._checkout(self)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 788, in _checkout
> fairy = _ConnectionRecord.checkout(pool)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 532, in checkout
> rec = pool._do_get()
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 1193, in _do_get
> self._dec_overflow()
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
> compat.reraise(exc_type, exc_value, exc_tb)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 1190, in _do_get
> return self._create_connection()
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 350, in _create_connection
> return _ConnectionRecord(self)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 477, in __init__
> self.__connect(first_connect_check=True)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/pool.py", line 671, in __connect
> connection = pool._invoke_creator(self)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/strategies.py", line 106, in connect
> return dialect.connect(*cargs, **cparams)
> File "/usr/src/venv/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 410, in connect
> return self.dbapi.connect(*cargs, **cparams)
> File "/usr/src/venv/local/lib/python2.7/site-packages/MySQLdb/__init__.py", line 86, in Connect
> return Connection(*args, **kwargs)
> File "/usr/src/venv/local/lib/python2.7/site-packages/MySQLdb/connections.py", line 209, in __init__
> self._server_version = tuple([ numeric_part(n) for n in self.get_server_info().split('.')[:2] ])
> File "/usr/src/venv/local/lib/python2.7/site-packages/MySQLdb/connections.py", line 69, in numeric_part
> m = re_numeric_part.match(s)
> File "/usr/src/venv/local/lib/python2.7/site-packages/airflow/bin/cli.py", line 73, in sigint_handler
> sys.exit(0)
> SystemExit: 0