You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by sa...@apache.org on 2016/09/29 23:58:10 UTC
incubator-airflow git commit: [AIRFLOW-525] Update template_fields in
Qubole Op
Repository: incubator-airflow
Updated Branches:
refs/heads/master f0db42c62 -> 8ca8f66d8
[AIRFLOW-525] Update template_fields in Qubole Op
There were couple of more fields in Qubole
Operator which requires
support of Jinja templating, so added these missed
out fields as well
to template_fields. Also added a missing doc
(about notify) and an
example of using macros.
Closes #1808 from msumit/AIRFLOW-525
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/8ca8f66d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/8ca8f66d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/8ca8f66d
Branch: refs/heads/master
Commit: 8ca8f66d8f60513ee2933d3bd5db1d65d81f8d3d
Parents: f0db42c
Author: Sumit Maheshwari <su...@qubole.com>
Authored: Thu Sep 29 16:56:36 2016 -0700
Committer: Siddharth Anand <si...@yahoo.com>
Committed: Thu Sep 29 16:57:42 2016 -0700
----------------------------------------------------------------------
airflow/contrib/example_dags/example_qubole_operator.py | 2 ++
airflow/contrib/operators/qubole_operator.py | 10 +++++++---
2 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8ca8f66d/airflow/contrib/example_dags/example_qubole_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/example_dags/example_qubole_operator.py b/airflow/contrib/example_dags/example_qubole_operator.py
index ff363e6..b482cf4 100644
--- a/airflow/contrib/example_dags/example_qubole_operator.py
+++ b/airflow/contrib/example_dags/example_qubole_operator.py
@@ -58,6 +58,8 @@ t2 = QuboleOperator(
script_location="s3n://public-qubole/qbol-library/scripts/show_table.hql",
notfiy=True,
tags=['tag1', 'tag2'],
+ # If the script at s3 location has any qubole specific macros to be replaced
+ # macros='[{"date": "{{ ds }}"}, {"name" : "abc"}]',
trigger_rule="all_done",
dag=dag)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/8ca8f66d/airflow/contrib/operators/qubole_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/qubole_operator.py b/airflow/contrib/operators/qubole_operator.py
index a27458e..623899d 100755
--- a/airflow/contrib/operators/qubole_operator.py
+++ b/airflow/contrib/operators/qubole_operator.py
@@ -29,6 +29,7 @@ class QuboleOperator(BaseOperator):
:tags: array of tags to be assigned with the command
:cluster_label: cluster label on which the command will be executed
:name: name to be given to command
+ :notify: whether to send email on command completion or not (default is False)
**Arguments specific to command types**
@@ -100,8 +101,9 @@ class QuboleOperator(BaseOperator):
.. note:: Following fields are template-supported : ``query``, ``script_location``,
``sub_command``, ``script``, ``files``, ``archives``, ``program``, ``cmdline``,
``sql``, ``where_clause``, ``extract_query``, ``boundary_query``, ``macros``,
- ``tags``, ``name``, ``parameters``. You can also use ``.txt`` files for template
- driven use cases.
+ ``tags``, ``name``, ``parameters``, ``dbtap_id``, ``hive_table``, ``db_table``,
+ ``split_column``, ``db_update_keys``, ``export_dir``, ``partition_spec``. You
+ can also use ``.txt`` files for template driven use cases.
.. note:: In QuboleOperator there is a default handler for task failures and retries,
which generally kills the command running at QDS for the corresponding task
@@ -111,7 +113,9 @@ class QuboleOperator(BaseOperator):
template_fields = ('query', 'script_location', 'sub_command', 'script', 'files',
'archives', 'program', 'cmdline', 'sql', 'where_clause', 'tags',
- 'extract_query', 'boundary_query', 'macros', 'name', 'parameters')
+ 'extract_query', 'boundary_query', 'macros', 'name', 'parameters',
+ 'dbtap_id', 'hive_table', 'db_table', 'split_column',
+ 'db_update_keys', 'export_dir', 'partition_spec')
template_ext = ('.txt',)
ui_color = '#3064A1'
ui_fgcolor = '#fff'