You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by da...@apache.org on 2016/05/24 00:01:11 UTC
[3/3] incubator-airflow git commit: Updated HiveServer2Hook.to_csv()
to add fetch_size
Updated HiveServer2Hook.to_csv() to add fetch_size
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/6f7ea90a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/6f7ea90a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/6f7ea90a
Branch: refs/heads/airbnb_rb1.7.1_4
Commit: 6f7ea90ae1f465d7ab1aa710ef05226a5226c862
Parents: 8bedf33
Author: Michael Musson <mi...@airbnb.com>
Authored: Thu May 19 16:31:27 2016 -0700
Committer: Dan Davydov <da...@airbnb.com>
Committed: Mon May 23 17:00:54 2016 -0700
----------------------------------------------------------------------
airflow/hooks/hive_hooks.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/6f7ea90a/airflow/hooks/hive_hooks.py
----------------------------------------------------------------------
diff --git a/airflow/hooks/hive_hooks.py b/airflow/hooks/hive_hooks.py
index 6ea5ba7..d712e16 100644
--- a/airflow/hooks/hive_hooks.py
+++ b/airflow/hooks/hive_hooks.py
@@ -489,7 +489,8 @@ class HiveServer2Hook(BaseHook):
schema='default',
delimiter=',',
lineterminator='\r\n',
- output_header=True):
+ output_header=True,
+ fetch_size=1000):
schema = schema or 'default'
with self.get_conn() as conn:
with conn.cursor() as cur:
@@ -504,7 +505,7 @@ class HiveServer2Hook(BaseHook):
for c in cur.description])
i = 0
while True:
- rows = [row for row in cur.fetchmany() if row]
+ rows = [row for row in cur.fetchmany(fetch_size) if row]
if not rows:
break