You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@airflow.apache.org by David Montgomery <da...@gmail.com> on 2016/08/21 22:09:26 UTC
airflow does not like mysql - ProgrammingError: (mysql.connector.errors.ProgrammingError)
Failed processing pyformat-parameters;
Hi,
Below is my config.
sql_alchemy_conn = mysql+mysqlconnector://root:xxx@localhost:3306/airflow
when I run initdb tables are present
mysql> use airflow;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
mysql> show tables;
+-------------------+
| Tables_in_airflow |
+-------------------+
| alembic_version |
| chart |
| connection |
| dag |
| dag_pickle |
| dag_run |
| import_error |
| job |
| known_event |
| known_event_type |
| log |
| sla_miss |
| slot_pool |
| task_instance |
| users |
| variable |
| xcom |
+-------------------+
17 rows in set (0.00 sec)
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line
1817, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line
1477, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line
1381, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line
1475, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line
1461, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py",
line 68, in inner
return self._run_view(f, *args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py",
line 367, in _run_view
return fn(self, *args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line
755, in decorated_view
return func(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py",
line 213, in view_func
return f(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py",
line 116, in wrapper
session.commit()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
line 801, in commit
self.transaction.commit()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
line 392, in commit
self._prepare_impl()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
line 372, in _prepare_impl
self.session.flush()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
line 2019, in flush
self._flush(objects)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
line 2137, in _flush
transaction.rollback(_capture_exception=True)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py",
line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
line 2101, in _flush
flush_context.execute()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py",
line 373, in execute
rec.execute(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py",
line 532, in execute
uow
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py",
line 174, in save_obj
mapper, table, insert)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py",
line 800, in _emit_insert_statements
execute(statement, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py",
line 914, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py",
line 323, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py",
line 1010, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py",
line 1146, in _execute_context
context)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py",
line 1341, in _handle_dbapi_exception
exc_info
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py",
line 202, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py",
line 1139, in _execute_context
context)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py",
line 450, in do_execute
cursor.execute(statement, parameters)
File "/usr/lib/python2.7/dist-packages/mysql/connector/cursor.py",
line 500, in execute
for key, value in self._process_params_dict(params).items():
File "/usr/lib/python2.7/dist-packages/mysql/connector/cursor.py",
line 368, in _process_params_dict
"Failed processing pyformat-parameters; %s" % err)
ProgrammingError: (mysql.connector.errors.ProgrammingError) Failed
processing pyformat-parameters; Python 'newstr' cannot be converted to
a MySQL type [SQL: u'INSERT INTO log (dttm, dag_id, task_id, event,
execution_date, owner, extra) VALUES (%(dttm)s, %(dag_id)s,
%(task_id)s, %(event)s, %(execution_date)s, %(owner)s, %(extra)s)']
[parameters: {'task_id': None, 'extra': "[('dag_id', u'drug')]",
'execution_date': None, 'dttm': datetime.datetime(2016, 8, 21, 15, 5,
55, 38365), 'owner': 'anonymous', 'event': 'graph', 'dag_id':
u'drug'}]
Re: airflow does not like mysql - ProgrammingError:
(mysql.connector.errors.ProgrammingError) Failed processing pyformat-parameters;
Posted by Jeremiah Lowin <jl...@apache.org>.
We've seen this before... unfortunately as far as I can tell the discussion
(and resolution) was swallowed by the move from GitHub issues to JIRA!
IIRC, some mysql libraries have an explicit type check for strings and our
py2/py3 compatible string type fails. It looks like you're using
mysql.connector -- can you try with mysqlclient as described here:
https://cwiki.apache.org/confluence/display/AIRFLOW/Common+Pitfalls? I run
Airflow on Google Cloud SQL using the mysqldb dialect with no issue.
Thanks,
J
On Sun, Aug 21, 2016 at 6:09 PM David Montgomery <da...@gmail.com>
wrote:
> Hi,
>
>
>
> Below is my config.
>
> sql_alchemy_conn = mysql+mysqlconnector://root:xxx@localhost:3306/airflow
>
> when I run initdb tables are present
>
>
> mysql> use airflow;
> Reading table information for completion of table and column names
> You can turn off this feature to get a quicker startup with -A
>
> Database changed
> mysql> show tables;
> +-------------------+
> | Tables_in_airflow |
> +-------------------+
> | alembic_version |
> | chart |
> | connection |
> | dag |
> | dag_pickle |
> | dag_run |
> | import_error |
> | job |
> | known_event |
> | known_event_type |
> | log |
> | sla_miss |
> | slot_pool |
> | task_instance |
> | users |
> | variable |
> | xcom |
> +-------------------+
> 17 rows in set (0.00 sec)
>
>
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line
> 1817, in wsgi_app
> response = self.full_dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line
> 1477, in full_dispatch_request
> rv = self.handle_user_exception(e)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line
> 1381, in handle_user_exception
> reraise(exc_type, exc_value, tb)
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line
> 1475, in full_dispatch_request
> rv = self.dispatch_request()
> File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line
> 1461, in dispatch_request
> return self.view_functions[rule.endpoint](**req.view_args)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py",
> line 68, in inner
> return self._run_view(f, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py",
> line 367, in _run_view
> return fn(self, *args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line
> 755, in decorated_view
> return func(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py",
> line 213, in view_func
> return f(*args, **kwargs)
> File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py",
> line 116, in wrapper
> session.commit()
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
> line 801, in commit
> self.transaction.commit()
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
> line 392, in commit
> self._prepare_impl()
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
> line 372, in _prepare_impl
> self.session.flush()
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
> line 2019, in flush
> self._flush(objects)
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
> line 2137, in _flush
> transaction.rollback(_capture_exception=True)
> File
> "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py",
> line 60, in __exit__
> compat.reraise(exc_type, exc_value, exc_tb)
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py",
> line 2101, in _flush
> flush_context.execute()
> File
> "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py",
> line 373, in execute
> rec.execute(self)
> File
> "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py",
> line 532, in execute
> uow
> File
> "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py",
> line 174, in save_obj
> mapper, table, insert)
> File
> "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py",
> line 800, in _emit_insert_statements
> execute(statement, params)
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py",
> line 914, in execute
> return meth(self, multiparams, params)
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py",
> line 323, in _execute_on_connection
> return connection._execute_clauseelement(self, multiparams, params)
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py",
> line 1010, in _execute_clauseelement
> compiled_sql, distilled_params
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py",
> line 1146, in _execute_context
> context)
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py",
> line 1341, in _handle_dbapi_exception
> exc_info
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py",
> line 202, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb, cause=cause)
> File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py",
> line 1139, in _execute_context
> context)
> File
> "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py",
> line 450, in do_execute
> cursor.execute(statement, parameters)
> File "/usr/lib/python2.7/dist-packages/mysql/connector/cursor.py",
> line 500, in execute
> for key, value in self._process_params_dict(params).items():
> File "/usr/lib/python2.7/dist-packages/mysql/connector/cursor.py",
> line 368, in _process_params_dict
> "Failed processing pyformat-parameters; %s" % err)
> ProgrammingError: (mysql.connector.errors.ProgrammingError) Failed
> processing pyformat-parameters; Python 'newstr' cannot be converted to
> a MySQL type [SQL: u'INSERT INTO log (dttm, dag_id, task_id, event,
> execution_date, owner, extra) VALUES (%(dttm)s, %(dag_id)s,
> %(task_id)s, %(event)s, %(execution_date)s, %(owner)s, %(extra)s)']
> [parameters: {'task_id': None, 'extra': "[('dag_id', u'drug')]",
> 'execution_date': None, 'dttm': datetime.datetime(2016, 8, 21, 15, 5,
> 55, 38365), 'owner': 'anonymous', 'event': 'graph', 'dag_id':
> u'drug'}]
>