You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by jl...@apache.org on 2016/11/04 13:42:49 UTC
incubator-airflow git commit: [AIRFLOW-179] Fix DbApiHook with
non-ASCII chars
Repository: incubator-airflow
Updated Branches:
refs/heads/master 9ae583eb1 -> 4fb86f920
[AIRFLOW-179] Fix DbApiHook with non-ASCII chars
String serialization fails when string contains non-ASCII characters
Closes #1553 from
johnbodley/dbapi_hook_serialization-remedy
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/4fb86f92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/4fb86f92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/4fb86f92
Branch: refs/heads/master
Commit: 4fb86f920a6ea0b31efc4d5302065ab28c90a954
Parents: 9ae583e
Author: John Bodley <jo...@airbnb.com>
Authored: Fri Nov 4 09:41:44 2016 -0400
Committer: jlowin <jl...@apache.org>
Committed: Fri Nov 4 09:41:46 2016 -0400
----------------------------------------------------------------------
airflow/hooks/dbapi_hook.py | 15 +++++++++++++--
airflow/hooks/mysql_hook.py | 15 +++++++++++++++
airflow/hooks/postgres_hook.py | 13 ++++++++++++-
3 files changed, 40 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/4fb86f92/airflow/hooks/dbapi_hook.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/dbapi_hook.py b/airflow/hooks/dbapi_hook.py
index 9d6ddf0..939bae2 100644
--- a/airflow/hooks/dbapi_hook.py
+++ b/airflow/hooks/dbapi_hook.py
@@ -212,7 +212,7 @@ class DbApiHook(BaseHook):
i += 1
l = []
for cell in row:
- l.append(self._serialize_cell(cell))
+ l.append(self._serialize_cell(cell, conn))
values = tuple(l)
sql = "INSERT INTO {0} {1} VALUES ({2});".format(
table,
@@ -230,7 +230,18 @@ class DbApiHook(BaseHook):
"Done loading. Loaded a total of {i} rows".format(**locals()))
@staticmethod
- def _serialize_cell(cell):
+ def _serialize_cell(cell, conn=None):
+ """
+ Returns the SQL literal of the cell as a string.
+
+ :param cell: The cell to insert into the table
+ :type cell: object
+ :param conn: The database connection
+ :type conn: connection object
+ :return: The serialized cell
+ :rtype: str
+ """
+
if isinstance(cell, basestring):
return "'" + str(cell).replace("'", "''") + "'"
elif cell is None:
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/4fb86f92/airflow/hooks/mysql_hook.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/mysql_hook.py b/airflow/hooks/mysql_hook.py
index 9c8224c..e4f9533 100644
--- a/airflow/hooks/mysql_hook.py
+++ b/airflow/hooks/mysql_hook.py
@@ -80,3 +80,18 @@ class MySqlHook(DbApiHook):
INTO TABLE {table}
""".format(**locals()))
conn.commit()
+
+ @staticmethod
+ def _serialize_cell(cell, conn):
+ """
+ Returns the MySQL literal of the cell as a string.
+
+ :param cell: The cell to insert into the table
+ :type cell: object
+ :param conn: The database connection
+ :type conn: connection object
+ :return: The serialized cell
+ :rtype: str
+ """
+
+ return conn.literal(cell)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/4fb86f92/airflow/hooks/postgres_hook.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/postgres_hook.py b/airflow/hooks/postgres_hook.py
index ede1d22..d096d75 100644
--- a/airflow/hooks/postgres_hook.py
+++ b/airflow/hooks/postgres_hook.py
@@ -46,5 +46,16 @@ class PostgresHook(DbApiHook):
return psycopg2_conn
@staticmethod
- def _serialize_cell(cell):
+ def _serialize_cell(cell, conn):
+ """
+ Returns the Postgres literal of the cell as a string.
+
+ :param cell: The cell to insert into the table
+ :type cell: object
+ :param conn: The database connection
+ :type conn: connection object
+ :return: The serialized cell
+ :rtype: str
+ """
+
return psycopg2.extensions.adapt(cell).getquoted().decode('utf-8')