You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/11/21 00:04:22 UTC
[airflow] branch v1-10-test updated (f421543 -> fa4bf45)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a change to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
discard f421543 Add setup.cfg for apache-airflow-upgrade-check (#12517)
discard 9194893 Add upgrade check rule to ensure on "latest" versions (#12514)
discard 81111a3 Add upgrade rule to check for mesos executor and flag to change it. (#11528)
discard ce7f0d5 Silence DagBag INFO logs during upgrade check (#12507)
discard 27e7981 Add readme for upgrade-check "subpackage". (#12506)
discard 0b75a71 Fix connection upgrade rules so they run with SQLite backend: (#12502)
discard 415cda4 Create UndefinedJinjaVariablesRule (Resolves #11144) (#11241)
discard 4b299e8 Fix the default value for VaultBackend's config_path (#12518)
discard e348467 Add 1.10.13 Changelog
new e57798b Fix the default value for VaultBackend's config_path (#12518)
new b2631fc Add 1.10.13 Changelog
new 15b628e Create UndefinedJinjaVariablesRule (Resolves #11144) (#11241)
new 61079d8 Fix connection upgrade rules so they run with SQLite backend: (#12502)
new 7ea8a1e Add readme for upgrade-check "subpackage". (#12506)
new f47efe9 Silence DagBag INFO logs during upgrade check (#12507)
new 60b43ef Add upgrade rule to check for mesos executor and flag to change it. (#11528)
new 41bb8f2 Add upgrade check rule to ensure on "latest" versions (#12514)
new fa4bf45 Add setup.cfg for apache-airflow-upgrade-check (#12517)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f421543)
\
N -- N -- N refs/heads/v1-10-test (fa4bf45)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGELOG.txt | 1 +
1 file changed, 1 insertion(+)
[airflow] 07/09: Add upgrade rule to check for mesos executor and
flag to change it. (#11528)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 60b43ef0474c9e479715f7cf9e1592a5230440de
Author: RaviTeja Pothana <ra...@live.com>
AuthorDate: Fri Nov 20 21:07:53 2020 +0530
Add upgrade rule to check for mesos executor and flag to change it. (#11528)
* add upgrade rule to check for mesos config and flag to remove it.
* change from checking the mesos config section to core/executor config
* remove leading new line and indent in desc
(cherry picked from commit 6739b537a016a81f5da495a894a0fe990c8ad25e)
---
airflow/upgrade/rules/mesos_executor_removed.py | 36 ++++++++++++++++
tests/upgrade/rules/test_mesos_executor_removed.py | 48 ++++++++++++++++++++++
2 files changed, 84 insertions(+)
diff --git a/airflow/upgrade/rules/mesos_executor_removed.py b/airflow/upgrade/rules/mesos_executor_removed.py
new file mode 100644
index 0000000..c0e6b52
--- /dev/null
+++ b/airflow/upgrade/rules/mesos_executor_removed.py
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.upgrade.rules.base_rule import BaseRule
+from airflow.configuration import conf
+
+
+class MesosExecutorRemovedRule(BaseRule):
+ """
+ MesosExecutorRemovedRule class to ease upgrade to Airflow 2.0
+ """
+ title = "Removal of Mesos Executor"
+ description = "The Mesos Executor has been deprecated as it was not widely used and not maintained."
+
+ def check(self):
+ executor_key = conf.get(section="core", key="executor")
+ if executor_key == "MesosExecutor":
+ return (
+ "The Mesos Executor has been deprecated as it was not widely used and not maintained."
+ "Please migrate to any of the supported executors."
+ "See https://airflow.apache.org/docs/stable/executor/index.html for more details."
+ )
diff --git a/tests/upgrade/rules/test_mesos_executor_removed.py b/tests/upgrade/rules/test_mesos_executor_removed.py
new file mode 100644
index 0000000..2b1e530
--- /dev/null
+++ b/tests/upgrade/rules/test_mesos_executor_removed.py
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from unittest import TestCase
+
+from airflow.upgrade.rules.mesos_executor_removed import MesosExecutorRemovedRule
+from tests.test_utils.config import conf_vars
+
+
+class TestMesosExecutorRemovedRule(TestCase):
+ @conf_vars({("core", "executor"): "MesosExecutor"})
+ def test_invalid_check(self):
+ rule = MesosExecutorRemovedRule()
+
+ assert isinstance(rule.description, str)
+ assert isinstance(rule.title, str)
+
+ msg = (
+ "The Mesos Executor has been deprecated as it was not widely used and not maintained."
+ "Please migrate to any of the supported executors."
+ "See https://airflow.apache.org/docs/stable/executor/index.html for more details."
+ )
+
+ response = rule.check()
+ assert response == msg
+
+ @conf_vars({("core", "executor"): "SequentialExecutor"})
+ def test_check(self):
+ rule = MesosExecutorRemovedRule()
+
+ assert isinstance(rule.description, str)
+ assert isinstance(rule.title, str)
+
+ response = rule.check()
+ assert response is None
[airflow] 06/09: Silence DagBag INFO logs during upgrade check
(#12507)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f47efe91a8b4ba5ba43fa8d55990d27516ab0b6c
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Fri Nov 20 15:21:15 2020 +0000
Silence DagBag INFO logs during upgrade check (#12507)
By default, the logs would appear in the middle of the status stream,
which makes it slightly harder to parse the output.
Before:
```
============================================= STATUS =============================================
Legacy UI is deprecated by default......................................................SUCCESS
Users must set a kubernetes.pod_template_file value.....................................FAIL
Changes in import paths of hooks, operators, sensors and others.........................FAIL
Remove airflow.AirflowMacroPlugin class.................................................SUCCESS
[2020-11-20 14:26:04,083] {__init__.py:50} INFO - Using executor SequentialExecutor
[2020-11-20 14:26:04,083] {dagbag.py:417} INFO - Filling up the DagBag from /home/ash/airflow/dags
Jinja Template Variables cannot be undefined............................................SUCCESS
```
After:
```
============================================= STATUS =============================================
Legacy UI is deprecated by default......................................................SUCCESS
Users must set a kubernetes.pod_template_file value.....................................FAIL
Changes in import paths of hooks, operators, sensors and others.........................FAIL
Remove airflow.AirflowMacroPlugin class.................................................SUCCESS
Jinja Template Variables cannot be undefined............................................SUCCESS
```
(cherry picked from commit 8e5f7227a4b00149c326637bb51409b8da6caa81)
---
airflow/upgrade/rules/undefined_jinja_varaibles.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/airflow/upgrade/rules/undefined_jinja_varaibles.py b/airflow/upgrade/rules/undefined_jinja_varaibles.py
index b97cfbc..7e39be4 100644
--- a/airflow/upgrade/rules/undefined_jinja_varaibles.py
+++ b/airflow/upgrade/rules/undefined_jinja_varaibles.py
@@ -17,6 +17,7 @@
from __future__ import absolute_import
+import logging
import re
import jinja2
@@ -131,8 +132,14 @@ The user should do either of the following to fix this -
def check(self, dagbag=None):
if not dagbag:
- dag_folder = conf.get("core", "dags_folder")
- dagbag = DagBag(dag_folder)
+ logger = logging.root
+ old_level = logger.level
+ try:
+ logger.setLevel(logging.ERROR)
+ dag_folder = conf.get("core", "dags_folder")
+ dagbag = DagBag(dag_folder)
+ finally:
+ logger.setLevel(old_level)
dags = dagbag.dags
messages = []
for dag_id, dag in dags.items():
[airflow] 05/09: Add readme for upgrade-check "subpackage". (#12506)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7ea8a1e8fa526b160cd82babe2bec318061aaae8
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Fri Nov 20 15:21:05 2020 +0000
Add readme for upgrade-check "subpackage". (#12506)
The intent here is this content will be visible on the pypi page for
apache-airflow-upgrade-check (once it is published.)
This is far from perfect, but the _first_ step is the hardest :)
(cherry picked from commit 47fc5c474684c9f59924837a7661e44af106943a)
---
airflow/upgrade/README.md | 82 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 82 insertions(+)
diff --git a/airflow/upgrade/README.md b/airflow/upgrade/README.md
new file mode 100644
index 0000000..e5e201d
--- /dev/null
+++ b/airflow/upgrade/README.md
@@ -0,0 +1,82 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Apache Airflow Upgrade Check
+
+[](https://badge.fury.io/py/apache-airflow-upgrade-check)
+[](http://www.apache.org/licenses/LICENSE-2.0.txt)
+[](https://pypi.org/project/apache-airflow-upgrade-check/)
+[](https://pypi.org/project/apache-airflow-upgrade-check/)
+[](https://twitter.com/ApacheAirflow)
+[](https://s.apache.org/airflow-slack)
+
+This package aims to easy the upgrade journey from [Apache Airflow](https://airflow.apache.org/) 1.10 to 2.0.
+
+While we have put a lot of effort in to making this upgrade as painless as possible, with many changes
+providing upgrade path (where the old code continues to work and prints out a deprecation warning) there were
+unfortunately some breaking changes where we couldn't provide a compatibility shim.
+
+The recommended upgrade path to get to Airflow 2.0.0 is to first upgrade to the latest release in the 1.10
+series (at the time of writing: 1.10.13) and to then run this script.
+
+```bash
+pip install apache-airflow-upgrade-check
+airflow upgrade_check
+```
+
+This will then print out a number of action items that you should follow before upgrading to 2.0.0 or above.
+
+The exit code of the command will be 0 (success) if no problems are reported, or 1 otherwise.
+
+For example:
+
+```
+============================================= STATUS =============================================
+
+Legacy UI is deprecated by default......................................................SUCCESS
+Users must set a kubernetes.pod_template_file value.....................................FAIL
+Changes in import paths of hooks, operators, sensors and others.........................FAIL
+Remove airflow.AirflowMacroPlugin class.................................................SUCCESS
+Jinja Template Variables cannot be undefined............................................SUCCESS
+Fernet is enabled by default............................................................FAIL
+Logging configuration has been moved to new section.....................................SUCCESS
+Connection.conn_id is not unique........................................................SUCCESS
+GCP service account key deprecation.....................................................SUCCESS
+Users must delete deprecated configs for KubernetesExecutor.............................FAIL
+Changes in import path of remote task handlers..........................................SUCCESS
+Chain between DAG and operator not allowed..............................................SUCCESS
+SendGrid email uses old airflow.contrib module..........................................SUCCESS
+Connection.conn_type is not nullable....................................................SUCCESS
+Found 16 problems.
+
+======================================== RECOMMENDATIONS =========================================
+
+Users must set a kubernetes.pod_template_file value
+---------------------------------------------------
+In Airflow 2.0, KubernetesExecutor Users need to set a pod_template_file as a base
+value for all pods launched by the KubernetesExecutor
+
+
+Problems:
+
+ 1. Please create a pod_template_file by running `airflow generate_pod_template`.
+This will generate a pod using your aiflow.cfg settings
+
+...
+```
[airflow] 02/09: Add 1.10.13 Changelog
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit b2631fca810840b07c00450cd82082e1cef5d3d8
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Thu Nov 19 21:28:11 2020 +0000
Add 1.10.13 Changelog
---
CHANGELOG.txt | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 113 insertions(+)
diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index 4fb12de..b818fef 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -1,3 +1,115 @@
+Airflow 1.10.13, 2020-11-24
+----------------------------
+
+New Features
+""""""""""""
+
+- Add "already checked" to failed pods in K8sPodOperator (#11368)
+- Pass SQLAlchemy engine options to FAB based UI (#11395)
+- [AIRFLOW-4438] Add Gzip compression to S3_hook (#8571)
+- Add permission "extra_links" for Viewer role and above (#10719)
+- Add generate_yaml command to easily test KubernetesExecutor before deploying pods (#10677)
+- Add Secrets backend for Microsoft Azure Key Vault (#10898)
+
+Bug Fixes
+"""""""""
+
+- SkipMixin: Handle empty branches (#11120)
+- [AIRFLOW-5274] dag loading duration metric name too long (#5890)
+- Handle no Dagrun in DagrunIdDep (#8389) (#11343)
+- Fix Kubernetes Executor logs for long dag names (#10942)
+- Add on_kill support for the KubernetesPodOperator (#10666)
+- KubernetesPodOperator template fix (#10963)
+- Fix displaying of add serialized_dag table migration
+- Fix Start Date tooltip on DAGs page (#10637)
+- URL encode execution date in the Last Run link (#10595)
+- Fixes issue with affinity backcompat in Airflow 1.10
+- Fix KubernetesExecutor import in views.py
+- Fix issues with Gantt View (#12419)
+- Fix Entrypoint and _CMD config variables (#12411)
+- Fix operator field update for SerializedBaseOperator (#10924)
+- Limited cryptography to < 3.2 for python 2.7
+- Install cattr on Python 3.7 - Fix docs build on RTD (#12045)
+- Limit version of marshmallow-sqlalchemy
+- Pin `kubernetes` to a max version of 11.0.0 (#11974)
+- Use snakebite-py3 for HDFS dependency for Python3 (#12340)
+- Removes snakebite kerberos dependency (#10865)
+- Fix failing dependencies for FAB and Celery (#10828)
+- Fix pod_mutation_hook for 1.10.13 (#10850)
+- Fix formatting of Host information
+- Fix Logout Google Auth issue in Non-RBAC UI (#11890)
+- Add missing imports to app.py (#10650)
+- Show Generic Error for Charts & Query View in old UI (#12495)
+- TimeSensor should respect the default_timezone config (#9699)
+- TimeSensor should respect DAG timezone (#9882)
+- Unify user session lifetime configuration (#11970)
+- Handle outdated webserver session timeout gracefully. (#12332)
+
+
+Improvements
+""""""""""""
+
+- Add XCom.deserialize_value to Airflow 1.10.13 (#12328)
+- Mount airflow.cfg to pod_template_file (#12311)
+- All k8s object must comply with JSON Schema (#12003)
+- Validate airflow chart values.yaml & values.schema.json (#11990)
+- Pod template file uses custom custom env variable (#11480)
+- Bump attrs and cattrs dependencies (#11969)
+- Bump attrs to > 20.0 (#11799)
+- [AIRFLOW-3607] Only query DB once per DAG run for TriggerRuleDep (#4751)
+- Rename task with duplicate task_id
+- Manage Flask AppBuilder Tables using Alembic Migrations (#12352)
+- ``airflow test`` only works for tasks in 1.10, not whole dags (#11191)
+- Improve warning messaging for duplicate task_ids in a DAG (#11126)
+- Pins moto to 1.3.14 (#10986)
+- DbApiHook: Support kwargs in get_pandas_df (#9730)
+- Make grace_period_seconds option on K8sPodOperator (#10727)
+- Fix syntax error in Dockerfile 'maintainer' Label (#10899)
+- The entrypoints in Docker Image should be owned by Airflow (#10853)
+- Make dockerfiles Google Shell Guide Compliant (#10734)
+- clean-logs script for Dockerfile: trim logs before sleep (#10685)
+- When sending tasks to celery from a sub-process, reset signal handlers (#11278)
+- SkipMixin: Add missing session.commit() and test (#10421)
+- Webserver: Further Sanitize values passed to origin param (#12459)
+- Security upgrade lodash from 4.17.19 to 4.17.20 (#11095)
+- Log instead of raise an Error for unregistered OperatorLinks (#11959)
+- Mask Password in Log table when using the CLI (#11468)
+- [AIRFLOW-3607] Optimize dep checking when depends on past set and concurrency limit
+- Execute job cancel HTTPRequest in Dataproc Hook (#10361)
+- Use rst lexer to format airflow upgrade check output (#11259)
+- Remove deprecation warning from contrib/kubernetes/pod.py
+- adding body as templated field for CloudSqlImportOperator (#10510)
+- Change log level for User's session to DEBUG (#12414)
+
+Deprecations
+""""""""""""
+
+- Deprecate importing Hooks from plugin-created module (#12133)
+- Deprecate adding Operators and Sensors via plugins (#12069)
+
+Doc only changes
+""""""""""""""""
+
+- [Doc] Correct description for macro task_instance_key_str (#11062)
+- Checks if all the libraries in setup.py are listed in installation.rst file (#12023)
+- Revise "Project Focus" copy (#12011)
+- Move Project focus and Principles higher in the README (#11973)
+- Remove archived link from README.md (#11945)
+- Update download url for Airflow Version (#11800)
+- Add Project URLs for PyPI page (#11801)
+- Move Backport Providers docs to our docsite (#11136)
+- Refactor rebase copy (#11030)
+- Add missing images for kubernetes executor docs (#11083)
+- Fix identation in executor_config example (#10467)
+- Enhanced the Kubernetes Executor doc (#10433)
+- Refactor content to a markdown table (#10863)
+- Rename "Beyond the Horizon" section and refactor content (#10802)
+- Refactor official source section to use bullets (#10801)
+- Add section for official source code (#10678)
+- Add redbubble link to Airflow merchandise (#10359)
+- README Doc: Link to Airflow directory in ASF Directory (#11137)
+- Fix the default value for VaultBackend's config_path (#12518)
+
Airflow 1.10.12, 2020-08-25
----------------------------
@@ -36,6 +148,7 @@ Bug Fixes
- BugFix: K8s Executor Multinamespace mode is evaluated to true by default (#10410)
- Make KubernetesExecutor recognize kubernetes_labels (#10412)
- Fix broken Kubernetes PodRuntimeInfoEnv (#10478)
+- Sync FAB Permissions for all base views (#12162)
Improvements
""""""""""""
[airflow] 01/09: Fix the default value for VaultBackend's
config_path (#12518)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e57798b4ddc3a1321cb1b6b41e4c9c1d61303760
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Fri Nov 20 21:24:50 2020 +0000
Fix the default value for VaultBackend's config_path (#12518)
It is `config` not `configs`
---
airflow/contrib/secrets/hashicorp_vault.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/airflow/contrib/secrets/hashicorp_vault.py b/airflow/contrib/secrets/hashicorp_vault.py
index 536e7f9..edf48c3 100644
--- a/airflow/contrib/secrets/hashicorp_vault.py
+++ b/airflow/contrib/secrets/hashicorp_vault.py
@@ -56,7 +56,7 @@ class VaultBackend(BaseSecretsBackend, LoggingMixin):
(default: 'variables')
:type variables_path: str
:param config_path: Specifies the path of the secret to read Airflow Configurations
- (default: 'configs').
+ (default: 'config').
:type config_path: str
:param url: Base URL for the Vault instance being addressed.
:type url: str
[airflow] 08/09: Add upgrade check rule to ensure on "latest"
versions (#12514)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 41bb8f28f1febbd80109435fdc1dcfc0dce44a76
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Fri Nov 20 21:21:13 2020 +0000
Add upgrade check rule to ensure on "latest" versions (#12514)
This checks against PyPI to make sure this is run with the latest
non-preview release of apache-airflow-upgrade-check, and the latest
1.10.x of apache-airflow
(cherry picked from commit d7ace0267c34bd5520d321a495399710c1c49cd1)
---
airflow/upgrade/rules/__init__.py | 2 +-
airflow/upgrade/rules/aaa_airflow_version_check.py | 87 ++++++++++++++++++++++
setup.py | 2 +
.../rules/test_aaa_airflow_version_check.py | 75 +++++++++++++++++++
4 files changed, 165 insertions(+), 1 deletion(-)
diff --git a/airflow/upgrade/rules/__init__.py b/airflow/upgrade/rules/__init__.py
index 4735c7f..97d0160 100644
--- a/airflow/upgrade/rules/__init__.py
+++ b/airflow/upgrade/rules/__init__.py
@@ -21,7 +21,7 @@ def get_rules():
"""Automatically discover all rules"""
rule_classes = []
path = os.path.dirname(os.path.abspath(__file__))
- for file in os.listdir(path):
+ for file in sorted(os.listdir(path)):
if not file.endswith(".py") or file in ("__init__.py", "base_rule.py"):
continue
py_file = file[:-3]
diff --git a/airflow/upgrade/rules/aaa_airflow_version_check.py b/airflow/upgrade/rules/aaa_airflow_version_check.py
new file mode 100644
index 0000000..ad84eb4
--- /dev/null
+++ b/airflow/upgrade/rules/aaa_airflow_version_check.py
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This module starts with `aaa_` so that it is sorted first alphabetically, but is still a valid python module
+# name (starting with digitis is not valid)
+
+from __future__ import absolute_import
+
+from packaging.version import Version
+import requests
+
+from airflow.upgrade.rules.base_rule import BaseRule
+
+try:
+ import importlib.metadata as importlib_metadata
+except ImportError:
+ import importlib_metadata
+
+
+class VersionCheckRule(BaseRule):
+
+ title = "Check for latest versions of apache-airflow and checker"
+
+ description = """\
+Check that the latest version of apache-airflow-upgrade-check is installed, and
+that you are on the latest 1.10.x release of apache-airflow."""
+
+ def pypi_releases(self, distname):
+ """
+ Get all the non-dev releases of a dist from PyPI
+ """
+
+ resp = requests.get("https://pypi.org/pypi/{}/json".format(distname))
+ resp.raise_for_status()
+
+ for rel_string in resp.json()["releases"].keys():
+ ver = Version(rel_string)
+ if ver.is_devrelease or ver.is_prerelease:
+ continue
+ yield ver
+
+ def check(self):
+
+ current_airflow_version = Version(__import__("airflow").__version__)
+ try:
+ upgrade_check_ver = Version(
+ importlib_metadata.distribution("apache-airflow-upgrade-check").version,
+ )
+ except importlib_metadata.PackageNotFoundError:
+ upgrade_check_ver = Version("0.0.0")
+
+ try:
+ latest_airflow_v1_release = sorted(
+ filter(lambda v: v.major == 1, self.pypi_releases("apache-airflow"))
+ )[-1]
+
+ if current_airflow_version < latest_airflow_v1_release:
+ yield (
+ "There is a more recent version of apache-airflow. Please upgrade to {} and re-run this"
+ " script"
+ ).format(latest_airflow_v1_release)
+
+ latest_upgrade_check_release = sorted(
+ self.pypi_releases("apache-airflow-upgrade-check")
+ )[-1]
+
+ if upgrade_check_ver < latest_upgrade_check_release:
+ yield (
+ "There is a more recent version of apache-airflow-upgrade-check. Please upgrade to {}"
+ " and re-run this script"
+ ).format(latest_upgrade_check_release)
+ except Exception as e:
+ yield "Unable to go ask PyPI.org for latest release information: " + str(e)
diff --git a/setup.py b/setup.py
index 30917d2..9617ac7 100644
--- a/setup.py
+++ b/setup.py
@@ -603,6 +603,7 @@ INSTALL_REQUIREMENTS = [
'future>=0.16.0, <0.19',
'graphviz>=0.12',
'gunicorn>=19.5.0, <21.0',
+ 'importlib-metadata~=2.0; python_version<"3.8"',
'iso8601>=0.1.12',
'jinja2>=2.10.1, <2.12.0',
'json-merge-patch==0.2',
@@ -611,6 +612,7 @@ INSTALL_REQUIREMENTS = [
'markdown>=2.5.2, <3.0',
'marshmallow-sqlalchemy>=0.16.1, <0.24.0;python_version>="3.6"',
'marshmallow-sqlalchemy>=0.16.1, <0.19.0;python_version<"3.6"',
+ 'packaging',
'pandas>=0.17.1, <2.0',
'pendulum==1.4.4',
'pep562~=1.0;python_version<"3.7"',
diff --git a/tests/upgrade/rules/test_aaa_airflow_version_check.py b/tests/upgrade/rules/test_aaa_airflow_version_check.py
new file mode 100644
index 0000000..1c4fce0
--- /dev/null
+++ b/tests/upgrade/rules/test_aaa_airflow_version_check.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.upgrade.rules.aaa_airflow_version_check import VersionCheckRule
+
+
+class TestVersionCheckRule:
+ def test_new_airflow_version_available(self, requests_mock):
+ requests_mock.get(
+ "https://pypi.org/pypi/apache-airflow/json",
+ json={
+ "releases": {
+ # The values here don't matter
+ "1.10.22b1": None,
+ "1.10.20": None,
+ "1.10.21": None,
+ }
+ },
+ )
+ requests_mock.get(
+ "https://pypi.org/pypi/apache-airflow-upgrade-check/json",
+ json={
+ "releases": {
+ "0.0.0": None,
+ }
+ },
+ )
+
+ results = list(VersionCheckRule().check())
+ assert len(results) == 1, results
+
+ msg = results[0]
+ assert " 1.10.21 " in msg
+ assert "apache-airflow" in msg
+ assert "apache-airflow-upgrade-check" not in msg
+
+ def test_new_airflow_upgrade_check_version_available(self, requests_mock):
+ requests_mock.get(
+ "https://pypi.org/pypi/apache-airflow/json",
+ json={
+ "releases": {
+ # The values here don't matter
+ "1.10.0": None,
+ }
+ },
+ )
+ requests_mock.get(
+ "https://pypi.org/pypi/apache-airflow-upgrade-check/json",
+ json={
+ "releases": {
+ "99.0.0": None,
+ }
+ },
+ )
+
+ results = list(VersionCheckRule().check())
+ assert len(results) == 1, results
+
+ msg = results[0]
+ assert " 99.0.0 " in msg
+ assert "apache-airflow-upgrade-check" in msg
[airflow] 04/09: Fix connection upgrade rules so they run with
SQLite backend: (#12502)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 61079d8990e714e4a5bf4522236d455ef8ce2a1c
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Fri Nov 20 12:24:41 2020 +0000
Fix connection upgrade rules so they run with SQLite backend: (#12502)
When testing out the ugprade check command on 1.10 with the default
SQLite backend I got the following error:
```
[2020-11-20 12:10:28,248] {base.py:1372} ERROR - Error closing cursor
Traceback (most recent call last):
File "/home/ash/.virtualenvs/airflow-1-10/lib/python3.7/site-packages/sqlalchemy/engine/result.py", line 1284, in fetchall
l = self.process_rows(self._fetchall_impl())
File "/home/ash/.virtualenvs/airflow-1-10/lib/python3.7/site-packages/sqlalchemy/engine/result.py", line 1230, in _fetchall_impl
return self.cursor.fetchall()
sqlite3.ProgrammingError: Cannot operate on a closed database.
```
This was caused because the `@provide_session` decorator closed the
connection when the function returned, and since we were using a
generator expression, we hadn't yet fetched all the rows.
This changes it so the rows are fetched before returning.
(cherry picked from commit cb0a2902fc7de73e5defc9ce54f5fd1429ec4fc6)
---
airflow/upgrade/rules/conn_id_is_unique.py | 2 +-
airflow/upgrade/rules/conn_type_is_not_nullable.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/airflow/upgrade/rules/conn_id_is_unique.py b/airflow/upgrade/rules/conn_id_is_unique.py
index 8e1e474..edb53c8 100644
--- a/airflow/upgrade/rules/conn_id_is_unique.py
+++ b/airflow/upgrade/rules/conn_id_is_unique.py
@@ -41,5 +41,5 @@ duplicate values in conn_id column.
.having(func.count() > 1)
return (
'Connection.conn_id={} is not unique.'.format(conn_id)
- for conn_id in invalid_connections
+ for conn_id in invalid_connections.all()
)
diff --git a/airflow/upgrade/rules/conn_type_is_not_nullable.py b/airflow/upgrade/rules/conn_type_is_not_nullable.py
index 8f574d9..a411eb0 100644
--- a/airflow/upgrade/rules/conn_type_is_not_nullable.py
+++ b/airflow/upgrade/rules/conn_type_is_not_nullable.py
@@ -42,5 +42,5 @@ If you made any modifications to the table directly, make sure you don't have nu
'Connection<id={}", conn_id={}> have empty conn_type field.'.format(
conn.id, conn.conn_id
)
- for conn in invalid_connections
+ for conn in invalid_connections.all()
)
[airflow] 09/09: Add setup.cfg for apache-airflow-upgrade-check
(#12517)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit fa4bf45522eb4bb68b6d70de663cd7359e0b228a
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Fri Nov 20 21:55:22 2020 +0000
Add setup.cfg for apache-airflow-upgrade-check (#12517)
Nothing currently uses this setup.cfg from this folder -- automation for
that will follow shortly.
Now that there is a place list deps for upgrade-check I have moved
`packaging` and `importlib_meta` to test_requires of the main dist.
Build a py2+py3 wheel.
(cherry picked from commit deb7fc0ffe3ddb9bf9aad6f5f9479d20598e2fb5)
---
airflow/upgrade/setup.cfg | 64 +++++++++++++++++++++++++++++++++++++++++++++++
setup.py | 4 ++-
2 files changed, 67 insertions(+), 1 deletion(-)
diff --git a/airflow/upgrade/setup.cfg b/airflow/upgrade/setup.cfg
new file mode 100644
index 0000000..ddd708c
--- /dev/null
+++ b/airflow/upgrade/setup.cfg
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[metadata]
+version=1.0.0
+name = apache-airflow-upgrade-check
+description = Check for compatibility between Airflow versions
+long_description = file: airflow/upgrade/README.md
+long_description_content_type = text/markdown
+url = https://airflow.apache.org
+author = Apache Airflow PMC
+author-email = dev@airflow.apache.org
+license = Apache License 2.0
+license_files =
+ LICENSE
+ NOTICE
+classifiers =
+ Development Status :: 5 - Production/Stable
+ Intended Audience :: Developers
+ License :: OSI Approved :: Apache Software License
+ Programming Language :: Python :: 2.7
+ Programming Language :: Python :: 3
+ Programming Language :: Python :: 3.6
+ Programming Language :: Python :: 3.7
+ Programming Language :: Python :: 3.8
+keywords = airflow, upgrade
+project_urls =
+ Source Code=https://github.com/apache/airflow
+ Bug Tracker=https://github.com/apache/airflow/issues
+ Documentation=https://airflow.apache.org/docs/
+
+[options]
+packages = find:
+install_requires =
+ apache-airflow>=1.10.13,<3
+ importlib-metadata~=2.0; python_version<"3.8"
+ packaging
+python_requires = >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*
+setup_requires =
+ setuptools>=40.0
+ wheel
+zip_safe = no
+
+[options.packages.find]
+include =
+ airflow.upgrade
+ airflow.upgrade.*
+
+[bdist_wheel]
+universal=1
diff --git a/setup.py b/setup.py
index 9617ac7..f5f2a53 100644
--- a/setup.py
+++ b/setup.py
@@ -426,12 +426,14 @@ devel = [
'flaky',
'freezegun',
'gitpython',
+ 'importlib-metadata~=2.0; python_version<"3.8"',
'ipdb',
'jira',
'mock;python_version<"3.3"',
'mongomock',
'moto==1.3.14', # TODO - fix Datasync issues to get higher version of moto:
# See: https://github.com/apache/airflow/issues/10985
+ 'packaging',
'parameterized',
'paramiko',
'pre-commit',
@@ -445,7 +447,7 @@ devel = [
'pywinrm',
'qds-sdk>=1.9.6',
'requests_mock',
- 'yamllint'
+ 'yamllint',
]
############################################################################################################
# IMPORTANT NOTE!!!!!!!!!!!!!!!
[airflow] 03/09: Create UndefinedJinjaVariablesRule (Resolves
#11144) (#11241)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 15b628e2d7bb51ddca9e62dc7d2299f7b6799e5a
Author: Ashmeet Lamba <as...@gmail.com>
AuthorDate: Thu Nov 19 16:33:06 2020 +0530
Create UndefinedJinjaVariablesRule (Resolves #11144) (#11241)
Adding a rule to check for undefined jinja variables when upgrading to Airflow2.0
(cherry picked from commit 18100a0ec96692bb4d7c9e80f206b66a30c65e0d)
---
airflow/models/dag.py | 4 +-
airflow/upgrade/rules/undefined_jinja_varaibles.py | 153 ++++++++++++++++
.../rules/test_undefined_jinja_varaibles.py | 192 +++++++++++++++++++++
3 files changed, 347 insertions(+), 2 deletions(-)
diff --git a/airflow/models/dag.py b/airflow/models/dag.py
index 348e19d..a1908e3 100644
--- a/airflow/models/dag.py
+++ b/airflow/models/dag.py
@@ -224,7 +224,7 @@ class DAG(BaseDag, LoggingMixin):
end_date=None, # type: Optional[datetime]
full_filepath=None, # type: Optional[str]
template_searchpath=None, # type: Optional[Union[str, Iterable[str]]]
- template_undefined=jinja2.Undefined, # type: Type[jinja2.Undefined]
+ template_undefined=None, # type: Optional[Type[jinja2.Undefined]]
user_defined_macros=None, # type: Optional[Dict]
user_defined_filters=None, # type: Optional[Dict]
default_args=None, # type: Optional[Dict]
@@ -807,7 +807,7 @@ class DAG(BaseDag, LoggingMixin):
# Default values (for backward compatibility)
jinja_env_options = {
'loader': jinja2.FileSystemLoader(searchpath),
- 'undefined': self.template_undefined,
+ 'undefined': self.template_undefined or jinja2.Undefined,
'extensions': ["jinja2.ext.do"],
'cache_size': 0
}
diff --git a/airflow/upgrade/rules/undefined_jinja_varaibles.py b/airflow/upgrade/rules/undefined_jinja_varaibles.py
new file mode 100644
index 0000000..b97cfbc
--- /dev/null
+++ b/airflow/upgrade/rules/undefined_jinja_varaibles.py
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+
+import re
+
+import jinja2
+import six
+
+from airflow import conf
+from airflow.models import DagBag, TaskInstance
+from airflow.upgrade.rules.base_rule import BaseRule
+from airflow.utils import timezone
+
+
+class UndefinedJinjaVariablesRule(BaseRule):
+
+ title = "Jinja Template Variables cannot be undefined"
+
+ description = """\
+The default behavior for DAG's Jinja templates has changed. Now, more restrictive validation
+of non-existent variables is applied - `jinja2.StrictUndefined`.
+
+The user should do either of the following to fix this -
+1. Fix the Jinja Templates by defining every variable or providing default values
+2. Explicitly declare `template_undefined=jinja2.Undefined` while defining the DAG
+"""
+
+ def _check_rendered_content(self, rendered_content, seen_oids=None):
+ """Replicates the logic in BaseOperator.render_template() to
+ cover all the cases needed to be checked.
+ """
+ if isinstance(rendered_content, six.string_types):
+ return set(re.findall(r"{{(.*?)}}", rendered_content))
+
+ elif isinstance(rendered_content, (int, float, bool)):
+ return set()
+
+ elif isinstance(rendered_content, (tuple, list, set)):
+ debug_error_messages = set()
+ for element in rendered_content:
+ debug_error_messages.update(self._check_rendered_content(element))
+ return debug_error_messages
+
+ elif isinstance(rendered_content, dict):
+ debug_error_messages = set()
+ for key, value in rendered_content.items():
+ debug_error_messages.update(self._check_rendered_content(value))
+ return debug_error_messages
+
+ else:
+ if seen_oids is None:
+ seen_oids = set()
+ return self._nested_check_rendered(rendered_content, seen_oids)
+
+ def _nested_check_rendered(self, rendered_content, seen_oids):
+ debug_error_messages = set()
+ if id(rendered_content) not in seen_oids:
+ seen_oids.add(id(rendered_content))
+ nested_template_fields = rendered_content.template_fields
+ for attr_name in nested_template_fields:
+ nested_rendered_content = getattr(rendered_content, attr_name)
+
+ if nested_rendered_content:
+ errors = list(
+ self._check_rendered_content(nested_rendered_content, seen_oids)
+ )
+ for i in range(len(errors)):
+ errors[i].strip()
+ errors[i] += " NestedTemplateField={}".format(attr_name)
+ debug_error_messages.update(errors)
+ return debug_error_messages
+
+ def _render_task_content(self, task, content, context):
+ completed_rendering = False
+ errors_while_rendering = []
+ while not completed_rendering:
+ # Catch errors such as {{ object.element }} where
+ # object is not defined
+ try:
+ renderend_content = task.render_template(content, context)
+ completed_rendering = True
+ except Exception as e:
+ undefined_variable = re.sub(" is undefined", "", str(e))
+ undefined_variable = re.sub("'", "", undefined_variable)
+ context[undefined_variable] = dict()
+ message = "Could not find the object '{}'".format(undefined_variable)
+ errors_while_rendering.append(message)
+ return renderend_content, errors_while_rendering
+
+ def iterate_over_template_fields(self, task):
+ messages = {}
+ task_instance = TaskInstance(task=task, execution_date=timezone.utcnow())
+ context = task_instance.get_template_context()
+ for attr_name in task.template_fields:
+ content = getattr(task, attr_name)
+ if content:
+ rendered_content, errors_while_rendering = self._render_task_content(
+ task, content, context
+ )
+ debug_error_messages = list(
+ self._check_rendered_content(rendered_content, set())
+ )
+ messages[attr_name] = errors_while_rendering + debug_error_messages
+
+ return messages
+
+ def iterate_over_dag_tasks(self, dag):
+ dag.template_undefined = jinja2.DebugUndefined
+ tasks = dag.tasks
+ messages = {}
+ for task in tasks:
+ error_messages = self.iterate_over_template_fields(task)
+ messages[task.task_id] = error_messages
+ return messages
+
+ def check(self, dagbag=None):
+ if not dagbag:
+ dag_folder = conf.get("core", "dags_folder")
+ dagbag = DagBag(dag_folder)
+ dags = dagbag.dags
+ messages = []
+ for dag_id, dag in dags.items():
+ if dag.template_undefined:
+ continue
+ dag_messages = self.iterate_over_dag_tasks(dag)
+
+ for task_id, task_messages in dag_messages.items():
+ for attr_name, error_messages in task_messages.items():
+ for error_message in error_messages:
+ message = (
+ "Possible UndefinedJinjaVariable -> DAG: {}, Task: {}, "
+ "Attribute: {}, Error: {}".format(
+ dag_id, task_id, attr_name, error_message.strip()
+ )
+ )
+ messages.append(message)
+ return messages
diff --git a/tests/upgrade/rules/test_undefined_jinja_varaibles.py b/tests/upgrade/rules/test_undefined_jinja_varaibles.py
new file mode 100644
index 0000000..83f99a3
--- /dev/null
+++ b/tests/upgrade/rules/test_undefined_jinja_varaibles.py
@@ -0,0 +1,192 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from tempfile import mkdtemp
+from unittest import TestCase
+
+import jinja2
+
+from airflow import DAG
+from airflow.models import DagBag
+from airflow.operators.bash_operator import BashOperator
+from airflow.upgrade.rules.undefined_jinja_varaibles import UndefinedJinjaVariablesRule
+from tests.models import DEFAULT_DATE
+
+
+class ClassWithCustomAttributes:
+ """Class for testing purpose: allows to create objects with custom attributes in one single statement."""
+
+ def __init__(self, **kwargs):
+ for key, value in kwargs.items():
+ setattr(self, key, value)
+
+ def __str__(self):
+ return "{}({})".format(ClassWithCustomAttributes.__name__, str(self.__dict__))
+
+ def __repr__(self):
+ return self.__str__()
+
+ def __eq__(self, other):
+ return self.__dict__ == other.__dict__
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+
+class TestUndefinedJinjaVariablesRule(TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.empty_dir = mkdtemp()
+
+ def setUpValidDag(self):
+ self.valid_dag = DAG(
+ dag_id="test-defined-jinja-variables", start_date=DEFAULT_DATE
+ )
+
+ BashOperator(
+ task_id="templated_string",
+ depends_on_past=False,
+ bash_command="echo",
+ env={
+ "integer": "{{ params.integer }}",
+ "float": "{{ params.float }}",
+ "string": "{{ params.string }}",
+ "boolean": "{{ params.boolean }}",
+ },
+ params={
+ "integer": 1,
+ "float": 1.0,
+ "string": "test_string",
+ "boolean": True,
+ },
+ dag=self.valid_dag,
+ )
+
+ def setUpDagToSkip(self):
+ self.skip_dag = DAG(
+ dag_id="test-defined-jinja-variables",
+ start_date=DEFAULT_DATE,
+ template_undefined=jinja2.Undefined,
+ )
+
+ BashOperator(
+ task_id="templated_string",
+ depends_on_past=False,
+ bash_command="{{ undefined }}",
+ dag=self.skip_dag,
+ )
+
+ def setUpInvalidDag(self):
+ self.invalid_dag = DAG(
+ dag_id="test-undefined-jinja-variables", start_date=DEFAULT_DATE
+ )
+
+ invalid_template_command = """
+ {% for i in range(5) %}
+ echo "{{ params.defined_variable }}"
+ echo "{{ execution_date.today }}"
+ echo "{{ execution_date.invalid_element }}"
+ echo "{{ params.undefined_variable }}"
+ echo "{{ foo }}"
+ {% endfor %}
+ """
+
+ nested_validation = ClassWithCustomAttributes(
+ nested1=ClassWithCustomAttributes(
+ att1="{{ nested.undefined }}", template_fields=["att1"]
+ ),
+ nested2=ClassWithCustomAttributes(
+ att2="{{ bar }}", template_fields=["att2"]
+ ),
+ template_fields=["nested1", "nested2"],
+ )
+
+ BashOperator(
+ task_id="templated_string",
+ depends_on_past=False,
+ bash_command=invalid_template_command,
+ env={
+ "undefined_object": "{{ undefined_object.element }}",
+ "nested_object": nested_validation,
+ },
+ params={"defined_variable": "defined_value"},
+ dag=self.invalid_dag,
+ )
+
+ def setUp(self):
+ self.setUpValidDag()
+ self.setUpDagToSkip()
+ self.setUpInvalidDag()
+
+ def test_description_and_title_is_defined(self):
+ rule = UndefinedJinjaVariablesRule()
+ assert isinstance(rule.description, str)
+ assert isinstance(rule.title, str)
+
+ def test_valid_check(self):
+ dagbag = DagBag(dag_folder=self.empty_dir, include_examples=False)
+ dagbag.dags[self.valid_dag.dag_id] = self.valid_dag
+ rule = UndefinedJinjaVariablesRule()
+
+ messages = rule.check(dagbag)
+
+ assert len(messages) == 0
+
+ def test_skipping_dag_check(self):
+ dagbag = DagBag(dag_folder=self.empty_dir, include_examples=False)
+ dagbag.dags[self.skip_dag.dag_id] = self.skip_dag
+ rule = UndefinedJinjaVariablesRule()
+
+ messages = rule.check(dagbag)
+
+ assert len(messages) == 0
+
+ def test_invalid_check(self):
+ dagbag = DagBag(dag_folder=self.empty_dir, include_examples=False)
+ dagbag.dags[self.invalid_dag.dag_id] = self.invalid_dag
+ rule = UndefinedJinjaVariablesRule()
+
+ messages = rule.check(dagbag)
+
+ expected_messages = [
+ "Possible UndefinedJinjaVariable -> DAG: test-undefined-jinja-variables, "
+ "Task: templated_string, Attribute: bash_command, Error: no such element: "
+ "dict object['undefined_variable']",
+ "Possible UndefinedJinjaVariable -> DAG: test-undefined-jinja-variables, "
+ "Task: templated_string, Attribute: bash_command, Error: no such element: "
+ "pendulum.pendulum.Pendulum object['invalid_element']",
+ "Possible UndefinedJinjaVariable -> DAG: test-undefined-jinja-variables, "
+ "Task: templated_string, Attribute: bash_command, Error: foo",
+ "Possible UndefinedJinjaVariable -> DAG: test-undefined-jinja-variables, "
+ "Task: templated_string, Attribute: env, Error: Could not find the "
+ "object 'undefined_object",
+ "Possible UndefinedJinjaVariable -> DAG: test-undefined-jinja-variables, "
+ "Task: templated_string, Attribute: env, Error: Could not find the object 'nested'",
+ "Possible UndefinedJinjaVariable -> DAG: test-undefined-jinja-variables, "
+ "Task: templated_string, Attribute: env, Error: bar NestedTemplateField=att2 "
+ "NestedTemplateField=nested2",
+ "Possible UndefinedJinjaVariable -> DAG: test-undefined-jinja-variables, "
+ "Task: templated_string, Attribute: env, Error: no such element: "
+ "dict object['undefined'] NestedTemplateField=att1 NestedTemplateField=nested1",
+ "Possible UndefinedJinjaVariable -> DAG: test-undefined-jinja-variables, "
+ "Task: templated_string, Attribute: env, Error: no such element: dict object['element']",
+ ]
+
+ assert len(messages) == len(expected_messages)
+ assert [m for m in messages if m in expected_messages], len(messages) == len(
+ expected_messages
+ )