You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/05/04 07:02:40 UTC
[44/50] incubator-airflow git commit: [AIRFLOW-2411] add
dataproc_jars to templated_fields
[AIRFLOW-2411] add dataproc_jars to templated_fields
This commit makes it possible to use jinja
templates when passing
JAR file URIs to the DataProc operators that
require JAR files,
specifically the DataProc Hive, Pig, SparkSql,
Spark, Hadoop and
PySpark operators.
Closes #3305 from mchalek/template-dataproc-jars
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/c5fa8cd4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/c5fa8cd4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/c5fa8cd4
Branch: refs/heads/v1-10-test
Commit: c5fa8cd411ca67889eafd109988d2472ccfbba10
Parents: 6b3f6ce
Author: Kevin McHale <mc...@gmail.com>
Authored: Thu May 3 22:55:10 2018 +0200
Committer: Fokko Driesprong <fo...@godatadriven.com>
Committed: Thu May 3 22:55:10 2018 +0200
----------------------------------------------------------------------
airflow/contrib/operators/dataproc_operator.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/c5fa8cd4/airflow/contrib/operators/dataproc_operator.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py
index 56ebb91..ad0aa09 100644
--- a/airflow/contrib/operators/dataproc_operator.py
+++ b/airflow/contrib/operators/dataproc_operator.py
@@ -481,7 +481,7 @@ class DataProcPigOperator(BaseOperator):
:param region: The specified region where the dataproc cluster is created.
:type region: string
"""
- template_fields = ['query', 'variables', 'job_name', 'cluster_name']
+ template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars']
template_ext = ('.pg', '.pig',)
ui_color = '#0273d4'
@@ -561,7 +561,7 @@ class DataProcHiveOperator(BaseOperator):
:param region: The specified region where the dataproc cluster is created.
:type region: string
"""
- template_fields = ['query', 'variables', 'job_name', 'cluster_name']
+ template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars']
template_ext = ('.q',)
ui_color = '#0273d4'
@@ -642,7 +642,7 @@ class DataProcSparkSqlOperator(BaseOperator):
:param region: The specified region where the dataproc cluster is created.
:type region: string
"""
- template_fields = ['query', 'variables', 'job_name', 'cluster_name']
+ template_fields = ['query', 'variables', 'job_name', 'cluster_name', 'dataproc_jars']
template_ext = ('.q',)
ui_color = '#0273d4'
@@ -731,7 +731,7 @@ class DataProcSparkOperator(BaseOperator):
:type region: string
"""
- template_fields = ['arguments', 'job_name', 'cluster_name']
+ template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars']
ui_color = '#0273d4'
@apply_defaults
@@ -821,7 +821,7 @@ class DataProcHadoopOperator(BaseOperator):
:type region: string
"""
- template_fields = ['arguments', 'job_name', 'cluster_name']
+ template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars']
ui_color = '#0273d4'
@apply_defaults
@@ -911,7 +911,7 @@ class DataProcPySparkOperator(BaseOperator):
:type region: string
"""
- template_fields = ['arguments', 'job_name', 'cluster_name']
+ template_fields = ['arguments', 'job_name', 'cluster_name', 'dataproc_jars']
ui_color = '#0273d4'
@staticmethod