You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ar...@apache.org on 2016/05/27 19:09:30 UTC
incubator-airflow git commit: Revert "[AIRFLOW-179] DbApiHook string
serialization fails when string contains non-ASCII characters"
Repository: incubator-airflow
Updated Branches:
refs/heads/master 87b4b8fa1 -> 8f6364058
Revert "[AIRFLOW-179] DbApiHook string serialization fails when string contains non-ASCII characters"
This reverts commit 87b4b8fa19cb660317198d74f6d51fdde0a7e067.
Reverting as the method used in the dbapi hook is actually package
specific to MySQLdb and would break the sqlite and mssql hooks.
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/8f636405
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/8f636405
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/8f636405
Branch: refs/heads/master
Commit: 8f63640584ca2dcd15bcd361d1f9a0d995bad315
Parents: 87b4b8f
Author: Arthur Wiedmer <ar...@gmail.com>
Authored: Fri May 27 11:38:57 2016 -0700
Committer: Arthur Wiedmer <ar...@gmail.com>
Committed: Fri May 27 11:38:57 2016 -0700
----------------------------------------------------------------------
airflow/hooks/dbapi_hook.py | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8f636405/airflow/hooks/dbapi_hook.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/dbapi_hook.py b/airflow/hooks/dbapi_hook.py
index 9e128a2..e5de92e 100644
--- a/airflow/hooks/dbapi_hook.py
+++ b/airflow/hooks/dbapi_hook.py
@@ -1,5 +1,8 @@
+from builtins import str
from past.builtins import basestring
+from datetime import datetime
+import numpy
import logging
from airflow.hooks.base_hook import BaseHook
@@ -168,7 +171,10 @@ class DbApiHook(BaseHook):
i = 0
for row in rows:
i += 1
- values = [conn.literal(cell) for cell in row]
+ l = []
+ for cell in row:
+ l.append(self._serialize_cell(cell))
+ values = tuple(l)
sql = "INSERT INTO {0} {1} VALUES ({2});".format(
table,
target_fields,
@@ -184,6 +190,19 @@ class DbApiHook(BaseHook):
logging.info(
"Done loading. Loaded a total of {i} rows".format(**locals()))
+ @staticmethod
+ def _serialize_cell(cell):
+ if isinstance(cell, basestring):
+ return "'" + str(cell).replace("'", "''") + "'"
+ elif cell is None:
+ return 'NULL'
+ elif isinstance(cell, numpy.datetime64):
+ return "'" + str(cell) + "'"
+ elif isinstance(cell, datetime):
+ return "'" + cell.isoformat() + "'"
+ else:
+ return str(cell)
+
def bulk_dump(self, table, tmp_file):
"""
Dumps a database table into a tab-delimited file