You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/06/11 18:50:11 UTC
incubator-airflow git commit: [AIRFLOW-2587] Add TIMESTAMP type
mapping to MySqlToHiveTransfer
Repository: incubator-airflow
Updated Branches:
refs/heads/master cae918e91 -> f1df9575b
[AIRFLOW-2587] Add TIMESTAMP type mapping to MySqlToHiveTransfer
This PR fixes MySqlToHiveTransfer to map
MySQL TIMESTAMP to Hive TIMESTAMP
(not STRING) so that users are not
required to cast it explicitly.
Closes #3486 from sekikn/AIRFLOW-2587
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/f1df9575
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/f1df9575
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/f1df9575
Branch: refs/heads/master
Commit: f1df9575bd3ac2bd81187d88d3c9665c2755d2c0
Parents: cae918e
Author: Kengo Seki <se...@apache.org>
Authored: Mon Jun 11 20:50:04 2018 +0200
Committer: Fokko Driesprong <fo...@godatadriven.com>
Committed: Mon Jun 11 20:50:04 2018 +0200
----------------------------------------------------------------------
airflow/operators/mysql_to_hive.py | 1 +
tests/operators/operators.py | 4 +++-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f1df9575/airflow/operators/mysql_to_hive.py
----------------------------------------------------------------------
diff --git a/airflow/operators/mysql_to_hive.py b/airflow/operators/mysql_to_hive.py
index ab7b7fa..34b5337 100644
--- a/airflow/operators/mysql_to_hive.py
+++ b/airflow/operators/mysql_to_hive.py
@@ -109,6 +109,7 @@ class MySqlToHiveTransfer(BaseOperator):
t.SHORT: 'INT',
t.TINY: 'SMALLINT',
t.YEAR: 'INT',
+ t.TIMESTAMP: 'TIMESTAMP',
}
return d[mysql_type] if mysql_type in d else 'STRING'
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f1df9575/tests/operators/operators.py
----------------------------------------------------------------------
diff --git a/tests/operators/operators.py b/tests/operators/operators.py
index 07b1396..c343d5c 100644
--- a/tests/operators/operators.py
+++ b/tests/operators/operators.py
@@ -334,7 +334,8 @@ class TransferTests(unittest.TestCase):
c1 SMALLINT,
c2 MEDIUMINT,
c3 INT,
- c4 BIGINT
+ c4 BIGINT,
+ c5 TIMESTAMP
)
""".format(mysql_table))
@@ -355,6 +356,7 @@ class TransferTests(unittest.TestCase):
d["c2"] = "INT"
d["c3"] = "BIGINT"
d["c4"] = "DECIMAL(38,0)"
+ d["c5"] = "TIMESTAMP"
self.assertEqual(mock_load_file.call_args[1]["field_dict"], d)
finally:
with m.get_conn() as c: