You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by "Aakash Bhardwaj (JIRA)" <ji...@apache.org> on 2017/07/12 08:13:00 UTC
[jira] [Updated] (AIRFLOW-1405) Airflow v 1.8.1 unable to properly
initialize with MySQL
[ https://issues.apache.org/jira/browse/AIRFLOW-1405?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Aakash Bhardwaj updated AIRFLOW-1405:
-------------------------------------
Description:
While working on a CentOS7 system, I was trying to configure Airflow version 1.8.1 to run with MySql in the backend.
I have installed Airflow in a Virtual Environment, and the MySQL has a database named airflow (default).
But on running the command -
{code:none}
airflow initdb
{code}
the following error is reported
{noformat}
[2017-07-12 13:22:36,558] {__init__.py:57} INFO - Using executor LocalExecutor
DB: mysql://airflow:***@localhost/airflow
[2017-07-12 13:22:37,218] {db.py:287} INFO - Creating tables
INFO [alembic.runtime.migration] Context impl MySQLImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
INFO [alembic.runtime.migration] Running upgrade f2ca10b85618 -> 4addfa1236f1, Add fractional seconds to mysql tables
Traceback (most recent call last):
File "/opt/airflow_virtual_environment/airflow_venv/bin/airflow", line 28, in <module>
args.func(args)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/bin/cli.py", line 951, in initdb
db_utils.initdb()
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/utils/db.py", line 106, in initdb
upgradedb()
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/utils/db.py", line 294, in upgradedb
command.upgrade(config, 'heads')
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/command.py", line 174, in upgrade
script.run_env()
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/script/base.py", line 416, in run_env
util.load_python_file(self.dir, 'env.py')
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/util/pyfiles.py", line 93, in load_python_file
module = load_module_py(module_id, path)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/util/compat.py", line 79, in load_module_py
mod = imp.load_source(module_id, path, fp)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/migrations/env.py", line 86, in <module>
run_migrations_online()
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/migrations/env.py", line 81, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/runtime/environment.py", line 807, in run_migrations
self.get_context().run_migrations(**kw)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/runtime/migration.py", line 321, in run_migrations
step.migration_fn(**kw)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py", line 36, in upgrade
op.alter_column(table_name='dag', column_name='last_scheduler_run', type_=mysql.DATETIME(fsp=6))
File "<string>", line 8, in alter_column
File "<string>", line 3, in alter_column
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/operations/ops.py", line 1420, in alter_column
return operations.invoke(alt)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/operations/base.py", line 318, in invoke
return fn(self, operation)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/operations/toimpl.py", line 53, in alter_column
**operation.kw
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/ddl/mysql.py", line 67, in alter_column
else existing_autoincrement
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/ddl/impl.py", line 118, in _exec
return conn.execute(construct, *multiparams, **params)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 945, in execute
return meth(self, multiparams, params)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/sql/ddl.py", line 68, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1002, in _execute_ddl
compiled
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
context)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
exc_info
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
context)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 470, in do_execute
cursor.execute(statement, parameters)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/MySQLdb/cursors.py", line 205, in execute
self.errorhandler(self, exc, value)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
sqlalchemy.exc.ProgrammingError: (_mysql_exceptions.ProgrammingError) (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '(6) NULL' at line 1") [SQL: u'ALTER TABLE dag MODIFY last_scheduler_run DATETIME(6) NULL']
{noformat}
Please note that airflow is able to connect to the DB, as new tables are created in the airflow database, but this query is not working.
Airflow version=1.8.1
MySQL version = Ver 14.14 Distrib 5.1.73, for redhat-linux-gnu (x86_64) using readline 5.1
was:
While working on a CentOS7 system, I was trying to configure Airflow version 1.8.1 to run with MySql in the backend.
I have installed Airflow in a Virtual Environment, and the MySQL has a database named airflow (default).
But on running the command -
{code:shell}
airflow initdb
{code}
the following error is reported
{noformat}
[2017-07-12 13:22:36,558] {__init__.py:57} INFO - Using executor LocalExecutor
DB: mysql://airflow:***@localhost/airflow
[2017-07-12 13:22:37,218] {db.py:287} INFO - Creating tables
INFO [alembic.runtime.migration] Context impl MySQLImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
INFO [alembic.runtime.migration] Running upgrade f2ca10b85618 -> 4addfa1236f1, Add fractional seconds to mysql tables
Traceback (most recent call last):
File "/opt/airflow_virtual_environment/airflow_venv/bin/airflow", line 28, in <module>
args.func(args)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/bin/cli.py", line 951, in initdb
db_utils.initdb()
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/utils/db.py", line 106, in initdb
upgradedb()
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/utils/db.py", line 294, in upgradedb
command.upgrade(config, 'heads')
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/command.py", line 174, in upgrade
script.run_env()
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/script/base.py", line 416, in run_env
util.load_python_file(self.dir, 'env.py')
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/util/pyfiles.py", line 93, in load_python_file
module = load_module_py(module_id, path)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/util/compat.py", line 79, in load_module_py
mod = imp.load_source(module_id, path, fp)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/migrations/env.py", line 86, in <module>
run_migrations_online()
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/migrations/env.py", line 81, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/runtime/environment.py", line 807, in run_migrations
self.get_context().run_migrations(**kw)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/runtime/migration.py", line 321, in run_migrations
step.migration_fn(**kw)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py", line 36, in upgrade
op.alter_column(table_name='dag', column_name='last_scheduler_run', type_=mysql.DATETIME(fsp=6))
File "<string>", line 8, in alter_column
File "<string>", line 3, in alter_column
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/operations/ops.py", line 1420, in alter_column
return operations.invoke(alt)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/operations/base.py", line 318, in invoke
return fn(self, operation)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/operations/toimpl.py", line 53, in alter_column
**operation.kw
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/ddl/mysql.py", line 67, in alter_column
else existing_autoincrement
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/ddl/impl.py", line 118, in _exec
return conn.execute(construct, *multiparams, **params)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 945, in execute
return meth(self, multiparams, params)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/sql/ddl.py", line 68, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1002, in _execute_ddl
compiled
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
context)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
exc_info
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
context)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 470, in do_execute
cursor.execute(statement, parameters)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/MySQLdb/cursors.py", line 205, in execute
self.errorhandler(self, exc, value)
File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
sqlalchemy.exc.ProgrammingError: (_mysql_exceptions.ProgrammingError) (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '(6) NULL' at line 1") [SQL: u'ALTER TABLE dag MODIFY last_scheduler_run DATETIME(6) NULL']
{noformat}
Please note that airflow is able to connect to the DB, as new tables are created in the airflow database, but this query is not working.
Airflow version=1.8.1
MySQL version = Ver 14.14 Distrib 5.1.73, for redhat-linux-gnu (x86_64) using readline 5.1
> Airflow v 1.8.1 unable to properly initialize with MySQL
> --------------------------------------------------------
>
> Key: AIRFLOW-1405
> URL: https://issues.apache.org/jira/browse/AIRFLOW-1405
> Project: Apache Airflow
> Issue Type: Bug
> Components: db
> Affects Versions: 1.8.1
> Environment: CentOS7
> Reporter: Aakash Bhardwaj
> Fix For: 1.8.1
>
> Attachments: error_log.txt
>
>
> While working on a CentOS7 system, I was trying to configure Airflow version 1.8.1 to run with MySql in the backend.
> I have installed Airflow in a Virtual Environment, and the MySQL has a database named airflow (default).
> But on running the command -
> {code:none}
> airflow initdb
> {code}
> the following error is reported
> {noformat}
> [2017-07-12 13:22:36,558] {__init__.py:57} INFO - Using executor LocalExecutor
> DB: mysql://airflow:***@localhost/airflow
> [2017-07-12 13:22:37,218] {db.py:287} INFO - Creating tables
> INFO [alembic.runtime.migration] Context impl MySQLImpl.
> INFO [alembic.runtime.migration] Will assume non-transactional DDL.
> INFO [alembic.runtime.migration] Running upgrade f2ca10b85618 -> 4addfa1236f1, Add fractional seconds to mysql tables
> Traceback (most recent call last):
> File "/opt/airflow_virtual_environment/airflow_venv/bin/airflow", line 28, in <module>
> args.func(args)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/bin/cli.py", line 951, in initdb
> db_utils.initdb()
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/utils/db.py", line 106, in initdb
> upgradedb()
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/utils/db.py", line 294, in upgradedb
> command.upgrade(config, 'heads')
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/command.py", line 174, in upgrade
> script.run_env()
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/script/base.py", line 416, in run_env
> util.load_python_file(self.dir, 'env.py')
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/util/pyfiles.py", line 93, in load_python_file
> module = load_module_py(module_id, path)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/util/compat.py", line 79, in load_module_py
> mod = imp.load_source(module_id, path, fp)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/migrations/env.py", line 86, in <module>
> run_migrations_online()
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/migrations/env.py", line 81, in run_migrations_online
> context.run_migrations()
> File "<string>", line 8, in run_migrations
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/runtime/environment.py", line 807, in run_migrations
> self.get_context().run_migrations(**kw)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/runtime/migration.py", line 321, in run_migrations
> step.migration_fn(**kw)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py", line 36, in upgrade
> op.alter_column(table_name='dag', column_name='last_scheduler_run', type_=mysql.DATETIME(fsp=6))
> File "<string>", line 8, in alter_column
> File "<string>", line 3, in alter_column
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/operations/ops.py", line 1420, in alter_column
> return operations.invoke(alt)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/operations/base.py", line 318, in invoke
> return fn(self, operation)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/operations/toimpl.py", line 53, in alter_column
> **operation.kw
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/ddl/mysql.py", line 67, in alter_column
> else existing_autoincrement
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/alembic/ddl/impl.py", line 118, in _exec
> return conn.execute(construct, *multiparams, **params)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 945, in execute
> return meth(self, multiparams, params)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/sql/ddl.py", line 68, in _execute_on_connection
> return connection._execute_ddl(self, multiparams, params)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1002, in _execute_ddl
> compiled
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
> context)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
> exc_info
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb, cause=cause)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
> context)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 470, in do_execute
> cursor.execute(statement, parameters)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/MySQLdb/cursors.py", line 205, in execute
> self.errorhandler(self, exc, value)
> File "/opt/airflow_virtual_environment/airflow_venv/lib/python2.7/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
> raise errorclass, errorvalue
> sqlalchemy.exc.ProgrammingError: (_mysql_exceptions.ProgrammingError) (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '(6) NULL' at line 1") [SQL: u'ALTER TABLE dag MODIFY last_scheduler_run DATETIME(6) NULL']
> {noformat}
> Please note that airflow is able to connect to the DB, as new tables are created in the airflow database, but this query is not working.
> Airflow version=1.8.1
> MySQL version = Ver 14.14 Distrib 5.1.73, for redhat-linux-gnu (x86_64) using readline 5.1
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)