You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/04/02 21:59:31 UTC
[airflow] branch master updated: A bunch of
template_fields_renderers additions (#15130)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 9b76b94 A bunch of template_fields_renderers additions (#15130)
9b76b94 is described below
commit 9b76b94c940d472290861930a1d5860b43b3b2b2
Author: Tzu-ping Chung <tp...@astronomer.io>
AuthorDate: Sat Apr 3 05:59:21 2021 +0800
A bunch of template_fields_renderers additions (#15130)
See #11177. Mostly SQL fields and Python options. Two notable execeptions:
* `WinRMOperator.command` is marked as a new renderer `powershell`.
* A new `jinja` renderer is implemented. The Pygment lexer is called `DjangoLexer`, but [Pygments suggets rendering Jinja2 with it](https://pygments.org/docs/lexers/#pygments.lexers.templates.DjangoLexer).
Fix #14543.
---
airflow/operators/generic_transfer.py | 1 +
airflow/operators/sql.py | 1 +
airflow/operators/trigger_dagrun.py | 1 +
airflow/providers/amazon/aws/operators/athena.py | 1 +
airflow/providers/amazon/aws/operators/batch.py | 1 +
airflow/providers/amazon/aws/operators/datasync.py | 7 +++++++
airflow/providers/amazon/aws/operators/ecs.py | 1 +
airflow/providers/amazon/aws/operators/emr_create_job_flow.py | 1 +
airflow/providers/amazon/aws/operators/glue.py | 1 +
airflow/providers/amazon/aws/operators/sagemaker_base.py | 1 +
airflow/providers/amazon/aws/operators/sns.py | 1 +
airflow/providers/amazon/aws/transfers/mongo_to_s3.py | 1 +
airflow/providers/amazon/aws/transfers/mysql_to_s3.py | 1 +
airflow/providers/grpc/operators/grpc.py | 1 +
.../microsoft/azure/operators/azure_container_instances.py | 1 +
.../microsoft/azure/transfers/oracle_to_azure_data_lake.py | 1 +
airflow/providers/microsoft/winrm/operators/winrm.py | 1 +
airflow/providers/mysql/transfers/presto_to_mysql.py | 1 +
airflow/providers/mysql/transfers/vertica_to_mysql.py | 4 ++++
airflow/providers/oracle/transfers/oracle_to_oracle.py | 1 +
airflow/providers/singularity/operators/singularity.py | 1 +
airflow/providers/snowflake/transfers/snowflake_to_slack.py | 1 +
airflow/providers/ssh/operators/ssh.py | 1 +
airflow/www/utils.py | 2 ++
24 files changed, 34 insertions(+)
diff --git a/airflow/operators/generic_transfer.py b/airflow/operators/generic_transfer.py
index b8f6214..a5242b3 100644
--- a/airflow/operators/generic_transfer.py
+++ b/airflow/operators/generic_transfer.py
@@ -49,6 +49,7 @@ class GenericTransfer(BaseOperator):
'.sql',
'.hql',
)
+ template_fields_renderers = {"preoperator": "sql"}
ui_color = '#b0f07c'
@apply_defaults
diff --git a/airflow/operators/sql.py b/airflow/operators/sql.py
index d6cd208..7c7851c 100644
--- a/airflow/operators/sql.py
+++ b/airflow/operators/sql.py
@@ -281,6 +281,7 @@ class SQLIntervalCheckOperator(BaseSQLOperator):
".hql",
".sql",
)
+ template_fields_renderers = {"sql1": "sql", "sql2": "sql"}
ui_color = "#fff7e6"
ratio_formulas = {
diff --git a/airflow/operators/trigger_dagrun.py b/airflow/operators/trigger_dagrun.py
index 3cf7a4f..26202db 100644
--- a/airflow/operators/trigger_dagrun.py
+++ b/airflow/operators/trigger_dagrun.py
@@ -71,6 +71,7 @@ class TriggerDagRunOperator(BaseOperator):
"""
template_fields = ("trigger_dag_id", "execution_date", "conf")
+ template_fields_renderers = {"conf": "py"}
ui_color = "#ffefeb"
@property
diff --git a/airflow/providers/amazon/aws/operators/athena.py b/airflow/providers/amazon/aws/operators/athena.py
index c789e28..961fc8a 100644
--- a/airflow/providers/amazon/aws/operators/athena.py
+++ b/airflow/providers/amazon/aws/operators/athena.py
@@ -58,6 +58,7 @@ class AWSAthenaOperator(BaseOperator):
ui_color = '#44b5e2'
template_fields = ('query', 'database', 'output_location')
template_ext = ('.sql',)
+ template_fields_renderers = {"query": "sql"}
@apply_defaults
def __init__( # pylint: disable=too-many-arguments
diff --git a/airflow/providers/amazon/aws/operators/batch.py b/airflow/providers/amazon/aws/operators/batch.py
index 8580bfe..1d3e453 100644
--- a/airflow/providers/amazon/aws/operators/batch.py
+++ b/airflow/providers/amazon/aws/operators/batch.py
@@ -100,6 +100,7 @@ class AwsBatchOperator(BaseOperator):
"overrides",
"parameters",
)
+ template_fields_renderers = {"overrides": "py", "parameters": "py"}
@apply_defaults
def __init__(
diff --git a/airflow/providers/amazon/aws/operators/datasync.py b/airflow/providers/amazon/aws/operators/datasync.py
index 96a51c4..e312a46 100644
--- a/airflow/providers/amazon/aws/operators/datasync.py
+++ b/airflow/providers/amazon/aws/operators/datasync.py
@@ -116,6 +116,13 @@ class AWSDataSyncOperator(BaseOperator):
"update_task_kwargs",
"task_execution_kwargs",
)
+ template_fields_renderers = {
+ "create_task_kwargs": "py",
+ "create_source_location_kwargs": "py",
+ "create_destination_location_kwargs": "py",
+ "update_task_kwargs": "py",
+ "task_execution_kwargs": "py",
+ }
ui_color = "#44b5e2"
@apply_defaults
diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py
index 50ab958..cf834ee 100644
--- a/airflow/providers/amazon/aws/operators/ecs.py
+++ b/airflow/providers/amazon/aws/operators/ecs.py
@@ -143,6 +143,7 @@ class ECSOperator(BaseOperator): # pylint: disable=too-many-instance-attributes
ui_color = '#f0ede4'
template_fields = ('overrides',)
+ template_fields_renderers = {"overrides": "py"}
@apply_defaults
def __init__(
diff --git a/airflow/providers/amazon/aws/operators/emr_create_job_flow.py b/airflow/providers/amazon/aws/operators/emr_create_job_flow.py
index 3b8e9f7..ccd3d54 100644
--- a/airflow/providers/amazon/aws/operators/emr_create_job_flow.py
+++ b/airflow/providers/amazon/aws/operators/emr_create_job_flow.py
@@ -43,6 +43,7 @@ class EmrCreateJobFlowOperator(BaseOperator):
template_fields = ['job_flow_overrides']
template_ext = ('.json',)
+ template_fields_renderers = {"job_flow_overrides": "json"}
ui_color = '#f9c915'
@apply_defaults
diff --git a/airflow/providers/amazon/aws/operators/glue.py b/airflow/providers/amazon/aws/operators/glue.py
index b0feb6d..855a6a5 100644
--- a/airflow/providers/amazon/aws/operators/glue.py
+++ b/airflow/providers/amazon/aws/operators/glue.py
@@ -57,6 +57,7 @@ class AwsGlueJobOperator(BaseOperator):
template_fields = ('script_args',)
template_ext = ()
+ template_fields_renderers = {"script_args": "py"}
ui_color = '#ededed'
@apply_defaults
diff --git a/airflow/providers/amazon/aws/operators/sagemaker_base.py b/airflow/providers/amazon/aws/operators/sagemaker_base.py
index 6efd634..86b347b 100644
--- a/airflow/providers/amazon/aws/operators/sagemaker_base.py
+++ b/airflow/providers/amazon/aws/operators/sagemaker_base.py
@@ -41,6 +41,7 @@ class SageMakerBaseOperator(BaseOperator):
template_fields = ['config']
template_ext = ()
+ template_fields_renderers = {"config": "py"}
ui_color = '#ededed'
integer_fields = [] # type: Iterable[Iterable[str]]
diff --git a/airflow/providers/amazon/aws/operators/sns.py b/airflow/providers/amazon/aws/operators/sns.py
index 1e88913..1912bba 100644
--- a/airflow/providers/amazon/aws/operators/sns.py
+++ b/airflow/providers/amazon/aws/operators/sns.py
@@ -43,6 +43,7 @@ class SnsPublishOperator(BaseOperator):
template_fields = ['message', 'subject', 'message_attributes']
template_ext = ()
+ template_fields_renderers = {"message_attributes": "py"}
@apply_defaults
def __init__(
diff --git a/airflow/providers/amazon/aws/transfers/mongo_to_s3.py b/airflow/providers/amazon/aws/transfers/mongo_to_s3.py
index 41c8098..0dc24dc 100644
--- a/airflow/providers/amazon/aws/transfers/mongo_to_s3.py
+++ b/airflow/providers/amazon/aws/transfers/mongo_to_s3.py
@@ -59,6 +59,7 @@ class MongoToS3Operator(BaseOperator):
template_fields = ('s3_bucket', 's3_key', 'mongo_query', 'mongo_collection')
ui_color = '#589636'
+ template_fields_renderers = {"mongo_query": "py"}
# pylint: disable=too-many-instance-attributes
@apply_defaults
diff --git a/airflow/providers/amazon/aws/transfers/mysql_to_s3.py b/airflow/providers/amazon/aws/transfers/mysql_to_s3.py
index 45b78f0..d2c79ec 100644
--- a/airflow/providers/amazon/aws/transfers/mysql_to_s3.py
+++ b/airflow/providers/amazon/aws/transfers/mysql_to_s3.py
@@ -69,6 +69,7 @@ class MySQLToS3Operator(BaseOperator):
'query',
)
template_ext = ('.sql',)
+ template_fields_renderers = {"query": "sql"}
@apply_defaults
def __init__(
diff --git a/airflow/providers/grpc/operators/grpc.py b/airflow/providers/grpc/operators/grpc.py
index 2d54c2b..a31b388 100644
--- a/airflow/providers/grpc/operators/grpc.py
+++ b/airflow/providers/grpc/operators/grpc.py
@@ -51,6 +51,7 @@ class GrpcOperator(BaseOperator):
"""
template_fields = ('stub_class', 'call_func', 'data')
+ template_fields_renderers = {"data": "py"}
@apply_defaults
def __init__(
diff --git a/airflow/providers/microsoft/azure/operators/azure_container_instances.py b/airflow/providers/microsoft/azure/operators/azure_container_instances.py
index 70355ed..64f7be9 100644
--- a/airflow/providers/microsoft/azure/operators/azure_container_instances.py
+++ b/airflow/providers/microsoft/azure/operators/azure_container_instances.py
@@ -133,6 +133,7 @@ class AzureContainerInstancesOperator(BaseOperator):
"""
template_fields = ('name', 'image', 'command', 'environment_variables')
+ template_fields_renderers = {"command": "bash", "environment_variables": "json"}
# pylint: disable=too-many-arguments
@apply_defaults
diff --git a/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py b/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py
index 5071dbf..3ce1e24 100644
--- a/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py
+++ b/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py
@@ -57,6 +57,7 @@ class OracleToAzureDataLakeOperator(BaseOperator):
"""
template_fields = ('filename', 'sql', 'sql_params')
+ template_fields_renderers = {"sql_params": "py"}
ui_color = '#e08c8c'
# pylint: disable=too-many-arguments
diff --git a/airflow/providers/microsoft/winrm/operators/winrm.py b/airflow/providers/microsoft/winrm/operators/winrm.py
index 79dcaf3..8a29529 100644
--- a/airflow/providers/microsoft/winrm/operators/winrm.py
+++ b/airflow/providers/microsoft/winrm/operators/winrm.py
@@ -56,6 +56,7 @@ class WinRMOperator(BaseOperator):
"""
template_fields = ('command',)
+ template_fields_renderers = {"command": "powershell"}
@apply_defaults
def __init__(
diff --git a/airflow/providers/mysql/transfers/presto_to_mysql.py b/airflow/providers/mysql/transfers/presto_to_mysql.py
index 081ca0a..69d1a05 100644
--- a/airflow/providers/mysql/transfers/presto_to_mysql.py
+++ b/airflow/providers/mysql/transfers/presto_to_mysql.py
@@ -47,6 +47,7 @@ class PrestoToMySqlOperator(BaseOperator):
template_fields = ('sql', 'mysql_table', 'mysql_preoperator')
template_ext = ('.sql',)
+ template_fields_renderers = {"mysql_preoperator": "sql"}
ui_color = '#a0e08c'
@apply_defaults
diff --git a/airflow/providers/mysql/transfers/vertica_to_mysql.py b/airflow/providers/mysql/transfers/vertica_to_mysql.py
index 8abc769..85da9ad 100644
--- a/airflow/providers/mysql/transfers/vertica_to_mysql.py
+++ b/airflow/providers/mysql/transfers/vertica_to_mysql.py
@@ -60,6 +60,10 @@ class VerticaToMySqlOperator(BaseOperator):
template_fields = ('sql', 'mysql_table', 'mysql_preoperator', 'mysql_postoperator')
template_ext = ('.sql',)
+ template_fields_renderers = {
+ "mysql_preoperator": "sql",
+ "mysql_postoperator": "sql",
+ }
ui_color = '#a0e08c'
@apply_defaults
diff --git a/airflow/providers/oracle/transfers/oracle_to_oracle.py b/airflow/providers/oracle/transfers/oracle_to_oracle.py
index 889f7e1..be09874 100644
--- a/airflow/providers/oracle/transfers/oracle_to_oracle.py
+++ b/airflow/providers/oracle/transfers/oracle_to_oracle.py
@@ -43,6 +43,7 @@ class OracleToOracleOperator(BaseOperator):
"""
template_fields = ('source_sql', 'source_sql_params')
+ template_fields_renderers = {"source_sql": "sql", "source_sql_params": "py"}
ui_color = '#e08c8c'
@apply_defaults
diff --git a/airflow/providers/singularity/operators/singularity.py b/airflow/providers/singularity/operators/singularity.py
index 3540eaf..0423a25 100644
--- a/airflow/providers/singularity/operators/singularity.py
+++ b/airflow/providers/singularity/operators/singularity.py
@@ -70,6 +70,7 @@ class SingularityOperator(BaseOperator):
'.sh',
'.bash',
)
+ template_fields_renderers = {"command": "bash", "environment": "json"}
@apply_defaults
def __init__( # pylint: disable=too-many-arguments
diff --git a/airflow/providers/snowflake/transfers/snowflake_to_slack.py b/airflow/providers/snowflake/transfers/snowflake_to_slack.py
index 79eac87..f3716fe 100644
--- a/airflow/providers/snowflake/transfers/snowflake_to_slack.py
+++ b/airflow/providers/snowflake/transfers/snowflake_to_slack.py
@@ -69,6 +69,7 @@ class SnowflakeToSlackOperator(BaseOperator):
template_fields = ['sql', 'slack_message']
template_ext = ['.sql', '.jinja', '.j2']
+ template_fields_renderers = {"slack_message": "jinja"}
times_rendered = 0
@apply_defaults
diff --git a/airflow/providers/ssh/operators/ssh.py b/airflow/providers/ssh/operators/ssh.py
index 79d1552..787fa09 100644
--- a/airflow/providers/ssh/operators/ssh.py
+++ b/airflow/providers/ssh/operators/ssh.py
@@ -57,6 +57,7 @@ class SSHOperator(BaseOperator):
template_fields = ('command', 'remote_host')
template_ext = ('.sh',)
+ template_fields_renderers = {"command": "bash"}
@apply_defaults
def __init__(
diff --git a/airflow/www/utils.py b/airflow/www/utils.py
index bb24e8e..e873c12 100644
--- a/airflow/www/utils.py
+++ b/airflow/www/utils.py
@@ -362,8 +362,10 @@ def get_attr_renderer():
'doc_rst': lambda x: render(x, lexers.RstLexer),
'doc_yaml': lambda x: render(x, lexers.YamlLexer),
'doc_md': wrapped_markdown,
+ 'jinja': lambda x: render(x, lexers.DjangoLexer),
'json': lambda x: json_render(x, lexers.JsonLexer),
'md': wrapped_markdown,
+ 'powershell': lambda x: render(x, lexers.PowerShellLexer),
'py': lambda x: render(get_python_source(x), lexers.PythonLexer),
'python_callable': lambda x: render(get_python_source(x), lexers.PythonLexer),
'rst': lambda x: render(x, lexers.RstLexer),