You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ur...@apache.org on 2021/08/11 05:35:56 UTC
[airflow] branch main updated: Add date format filters to Jinja
environment (#17451)
This is an automated email from the ASF dual-hosted git repository.
uranusjr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e6c61ca Add date format filters to Jinja environment (#17451)
e6c61ca is described below
commit e6c61ca679cc2332eb5bce3ff01f961f55223374
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Wed Aug 11 06:35:27 2021 +0100
Add date format filters to Jinja environment (#17451)
On its own this doesn't add all that much, but this is preparatory
work to be combined with the new data_interval_start template context
variables we are adding, without having to add the ds/ts/no-dash etc
permutations of all of them.
---
airflow/macros/__init__.py | 1 -
airflow/templates.py | 34 ++++++++++++++++
docs/apache-airflow/concepts/operators.rst | 4 +-
docs/apache-airflow/faq.rst | 2 +-
docs/apache-airflow/howto/operator/python.rst | 2 +-
docs/apache-airflow/index.rst | 2 +-
docs/apache-airflow/redirects.txt | 1 +
.../{macros-ref.rst => templates-ref.rst} | 46 ++++++++++++++++------
docs/apache-airflow/tutorial.rst | 2 +-
tests/core/test_templates.py | 17 ++++++++
10 files changed, 91 insertions(+), 20 deletions(-)
diff --git a/airflow/macros/__init__.py b/airflow/macros/__init__.py
index b6efaf5..f6e8512 100644
--- a/airflow/macros/__init__.py
+++ b/airflow/macros/__init__.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""Macros."""
import time # noqa
import uuid # noqa
from datetime import datetime, timedelta
diff --git a/airflow/templates.py b/airflow/templates.py
index e475c86..7cbd556 100644
--- a/airflow/templates.py
+++ b/airflow/templates.py
@@ -22,6 +22,11 @@ import jinja2.sandbox
class SandboxedEnvironment(jinja2.sandbox.SandboxedEnvironment):
"""SandboxedEnvironment for Airflow task templates."""
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+ self.filters.update(FILTERS)
+
def is_safe_attribute(self, obj, attr, value):
"""
Allow access to ``_`` prefix vars (but not ``__``).
@@ -30,3 +35,32 @@ class SandboxedEnvironment(jinja2.sandbox.SandboxedEnvironment):
``_``) whilst still blocking internal or truely private attributes (``__`` prefixed ones).
"""
return not jinja2.sandbox.is_internal_attribute(obj, attr)
+
+
+def ds_filter(value):
+ return value.strftime('%Y-%m-%d')
+
+
+def ds_nodash_filter(value):
+ return value.strftime('%Y%m%d')
+
+
+def ts_filter(value):
+ return value.isoformat()
+
+
+def ts_nodash_filter(value):
+ return value.strftime('%Y%m%dT%H%M%S')
+
+
+def ts_nodash_with_tz_filter(value):
+ return value.isoformat().replace('-', '').replace(':', '')
+
+
+FILTERS = {
+ 'ds': ds_filter,
+ 'ds_nodash': ds_nodash_filter,
+ 'ts': ts_filter,
+ 'ts_nodash': ts_nodash_filter,
+ 'ts_nodash_with_tz': ts_nodash_with_tz_filter,
+}
diff --git a/docs/apache-airflow/concepts/operators.rst b/docs/apache-airflow/concepts/operators.rst
index 8d9473f..02aad4b 100644
--- a/docs/apache-airflow/concepts/operators.rst
+++ b/docs/apache-airflow/concepts/operators.rst
@@ -58,7 +58,7 @@ But there are many, many more - you can see the list of those in our :doc:`apach
Jinja Templating
----------------
-Airflow leverages the power of `Jinja Templating <http://jinja.pocoo.org/docs/dev/>`_ and this can be a powerful tool to use in combination with :doc:`macros </macros-ref>`.
+Airflow leverages the power of `Jinja Templating <http://jinja.pocoo.org/docs/dev/>`_ and this can be a powerful tool to use in combination with :ref:`macros <templates-ref>`.
For example, say you want to pass the execution date as an environment variable to a Bash script using the ``BashOperator``:
@@ -73,7 +73,7 @@ For example, say you want to pass the execution date as an environment variable
env={"EXECUTION_DATE": date},
)
-Here, ``{{ ds }}`` is a macro, and because the ``env`` parameter of the ``BashOperator`` is templated with Jinja, the execution date will be available as an environment variable named ``EXECUTION_DATE`` in your Bash script.
+Here, ``{{ ds }}`` is a templated variable, and because the ``env`` parameter of the ``BashOperator`` is templated with Jinja, the execution date will be available as an environment variable named ``EXECUTION_DATE`` in your Bash script.
You can use Jinja templating with every parameter that is marked as "templated" in the documentation. Template substitution occurs just before the ``pre_execute`` function of your operator is called.
diff --git a/docs/apache-airflow/faq.rst b/docs/apache-airflow/faq.rst
index ad5388e..25be346 100644
--- a/docs/apache-airflow/faq.rst
+++ b/docs/apache-airflow/faq.rst
@@ -220,7 +220,7 @@ Airflow was developed as a solution for ETL needs. In the ETL world, you typical
summarize data for 2016-02-19, You would do it at 2016-02-20 midnight UTC, which would be right after all data for
2016-02-19 becomes available.
-This datetime value is available to you as :ref:`Macros<macros:default_variables>` as various forms in Jinja templated
+This datetime value is available to you as :ref:`Template variables<templates:variables>` with various formats in Jinja templated
fields. They are also included in the context dictionary given to an Operator's execute function.
.. code-block:: python
diff --git a/docs/apache-airflow/howto/operator/python.rst b/docs/apache-airflow/howto/operator/python.rst
index 2f5c49c..212d9a2 100644
--- a/docs/apache-airflow/howto/operator/python.rst
+++ b/docs/apache-airflow/howto/operator/python.rst
@@ -47,7 +47,7 @@ Templating
^^^^^^^^^^
Airflow passes in an additional set of keyword arguments: one for each of the
-:doc:`Jinja template variables <../../macros-ref>` and a ``templates_dict``
+:ref:`Jinja template variables <templates:variables>` and a ``templates_dict``
argument.
The ``templates_dict`` argument is templated, so each value in the dictionary
diff --git a/docs/apache-airflow/index.rst b/docs/apache-airflow/index.rst
index 18b2f7a..6853a03 100644
--- a/docs/apache-airflow/index.rst
+++ b/docs/apache-airflow/index.rst
@@ -110,7 +110,7 @@ unit of work and continuity.
Operators and hooks <operators-and-hooks-ref>
CLI <cli-and-env-variables-ref>
- Macros <macros-ref>
+ Templates <templates-ref>
Python API <python-api-ref>
Stable REST API <stable-rest-api-ref>
deprecated-rest-api-ref
diff --git a/docs/apache-airflow/redirects.txt b/docs/apache-airflow/redirects.txt
index acfd694..fd83364 100644
--- a/docs/apache-airflow/redirects.txt
+++ b/docs/apache-airflow/redirects.txt
@@ -44,6 +44,7 @@ start.rst start/index.rst
cli-ref.rst cli-and-env-variables-ref.rst
_api/index.rst python-api-ref.rst
rest-api-ref.rst deprecated-rest-api-ref.rst
+macros-ref.rst templates-ref.rst
# Concepts
concepts.rst concepts/index.rst
diff --git a/docs/apache-airflow/macros-ref.rst b/docs/apache-airflow/templates-ref.rst
similarity index 83%
rename from docs/apache-airflow/macros-ref.rst
rename to docs/apache-airflow/templates-ref.rst
index 1450ca2..8616b68 100644
--- a/docs/apache-airflow/macros-ref.rst
+++ b/docs/apache-airflow/templates-ref.rst
@@ -15,20 +15,20 @@
specific language governing permissions and limitations
under the License.
-.. _macros:
+.. _templates-ref:
-Macros reference
-================
+Templates reference
+===================
-Variables and macros can be used in templates (see the :ref:`concepts:jinja-templating` section)
+Variables, macros and filters can be used in templates (see the :ref:`concepts:jinja-templating` section)
The following come for free out of the box with Airflow.
Additional custom macros can be added globally through :doc:`plugins`, or at a DAG level through the ``DAG.user_defined_macros`` argument.
-.. _macros:default_variables:
+.. _templates:variables:
-Default Variables
------------------
+Variables
+---------
The Airflow engine passes a few variables by default that are accessible
in all templates
@@ -105,6 +105,28 @@ For example, you could use expressions in your templates like ``{{ conn.my_conn_
Just like with ``var`` it's possible to fetch a connection by string (e.g. ``{{ conn.get('my_conn_id_'+index).host }}``
) or provide defaults (e.g ``{{ conn.get('my_conn_id', {"host": "host1", "login": "user1"}).host }}``)
+Filters
+-------
+
+Airflow defines the some Jinja filters that can be used to format values.
+
+For example, using ``{{ execution_date | ds }}`` will output the execution_date in the ``YYYY-MM-DD`` format.
+
+===================== ============ ==================================================================
+Filter Operates on Description
+===================== ============ ==================================================================
+``ds`` datetime Format the datetime as ``YYYY-MM-DD``
+``ds_nodash`` datetime Format the datetime as ``YYYYMMDD``
+``ts`` datetime Same as ``.isoformat()``, Example: ``2018-01-01T00:00:00+00:00``
+``ts_nodash`` datetime Same as ``ts`` filter without ``-``, ``:`` or TimeZone info.
+ Example: ``20180101T000000``
+``ts_nodash_with_tz`` datetime As ``ts`` filter without ``-`` or ``:``. Example
+ ``20180101T000000+0000``
+===================== ============ ==================================================================
+
+
+.. _templates:macros:
+
Macros
------
Macros are a way to expose objects to your templates and live under the
@@ -112,7 +134,6 @@ Macros are a way to expose objects to your templates and live under the
A few commonly used libraries and methods are made available.
-
================================= ==============================================
Variable Description
================================= ==============================================
@@ -124,13 +145,12 @@ Variable Description
``macros.random`` The standard lib's :mod:`random`
================================= ==============================================
-
Some airflow specific macros are also defined:
.. automodule:: airflow.macros
- :show-inheritance:
:members:
-.. autofunction:: airflow.macros.hive.closest_ds_partition
-.. autofunction:: airflow.macros.hive.max_partition
-.. _pendulum.Pendulum: https://pendulum.eustace.io/docs/1.x/#introduction
+.. automodule:: airflow.macros.hive
+ :members:
+
+.. _pendulum.Pendulum: https://pendulum.eustace.io/docs/2.x/#introduction
diff --git a/docs/apache-airflow/tutorial.rst b/docs/apache-airflow/tutorial.rst
index 951a8be..70f89fd 100644
--- a/docs/apache-airflow/tutorial.rst
+++ b/docs/apache-airflow/tutorial.rst
@@ -178,7 +178,7 @@ regarding custom filters have a look at the
`Jinja Documentation <http://jinja.pocoo.org/docs/dev/api/#writing-filters>`_.
For more information on the variables and macros that can be referenced
-in templates, make sure to read through the :doc:`macros-ref`.
+in templates, make sure to read through the :ref:`templates-ref`.
Adding DAG and Tasks documentation
----------------------------------
diff --git a/tests/core/test_templates.py b/tests/core/test_templates.py
index 160fea4..98c1734 100644
--- a/tests/core/test_templates.py
+++ b/tests/core/test_templates.py
@@ -17,6 +17,7 @@
import jinja2
import jinja2.exceptions
+import pendulum
import pytest
import airflow.templates
@@ -37,3 +38,19 @@ def test_protected_access(env):
def test_private_access(env):
with pytest.raises(jinja2.exceptions.SecurityError):
env.from_string(r'{{ func.__code__ }}').render(func=test_private_access)
+
+
+@pytest.mark.parametrize(
+ ['name', 'expected'],
+ (
+ ('ds', '2012-07-24'),
+ ('ds_nodash', '20120724'),
+ ('ts', '2012-07-24T03:04:52+00:00'),
+ ('ts_nodash', '20120724T030452'),
+ ('ts_nodash_with_tz', '20120724T030452+0000'),
+ ),
+)
+def test_filters(env, name, expected):
+ when = pendulum.datetime(2012, 7, 24, 3, 4, 52, tz='UTC')
+ result = env.from_string('{{ date |' + name + ' }}').render(date=when)
+ assert result == expected