[airflow] branch main updated: Adds capability of Warnings for incompatible community providers (#18020)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new b486a0e  Adds capability of Warnings for incompatible community providers (#18020)
b486a0e is described below

commit b486a0ed86ca4bda2f05105e6d30f470d540daed
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Sep 4 11:12:41 2021 +0200

    Adds capability of Warnings for incompatible community providers (#18020)
    
    When we release providers, we do not know if some future version
    of Airflow will be incompatible with them, so we cannot add hard
    limits there. We have constraints that contain the "latest"
    providers at the moment of release but if someone has an old
    versions of providers installed and just upgrades Airflow, the
    incompatible versions of providers might be still installed.
    
    From now on Airflow will print warnings in case such incompatible
    provider is detected.
---
 airflow/providers_manager.py         | 17 +++++++++++++++++
 tests/core/test_providers_manager.py | 24 +++++++++++++-----------
 2 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py
index 3da37a3..2bc90d0 100644
--- a/airflow/providers_manager.py
+++ b/airflow/providers_manager.py
@@ -29,6 +29,7 @@ from time import perf_counter
 from typing import Any, Callable, Dict, List, MutableMapping, NamedTuple, Optional, Set, TypeVar, Union, cast
 
 import jsonschema
+from packaging import version as packaging_version
 from wtforms import BooleanField, Field, IntegerField, PasswordField, StringField
 
 from airflow.utils import yaml
@@ -53,6 +54,10 @@ else:
 
     cache = lru_cache(maxsize=None)
 
+MIN_PROVIDER_VERSIONS = {
+    "apache-airflow-providers-celery": "2.1.0",
+}
+
 
 class LazyDictWithCache(MutableMapping):
     """
@@ -251,8 +256,20 @@ class ProvidersManager(LoggingMixin):
         # in case of local development
         self._discover_all_airflow_builtin_providers_from_local_sources()
         self._discover_all_providers_from_packages()
+        self._verify_all_providers_all_compatible()
         self._provider_dict = OrderedDict(sorted(self._provider_dict.items()))
 
+    def _verify_all_providers_all_compatible(self):
+        for provider_id, info in self._provider_dict.items():
+            min_version = MIN_PROVIDER_VERSIONS.get(provider_id)
+            if min_version:
+                if packaging_version.parse(min_version) > packaging_version.parse(info.version):
+                    log.warning(
+                        f"The package {provider_id} is not compatible with this version of Airflow. "
+                        f"The package has version {info.version} but the minimum supported version "
+                        f"of the package is {min_version}"
+                    )
+
     @provider_info_cache("hooks")
     def initialize_providers_hooks(self):
         """Lazy initialization of providers hooks."""
diff --git a/tests/core/test_providers_manager.py b/tests/core/test_providers_manager.py
index 6e76a54..f303c1c 100644
--- a/tests/core/test_providers_manager.py
+++ b/tests/core/test_providers_manager.py
@@ -31,17 +31,19 @@ class TestProviderManager(unittest.TestCase):
         self._caplog = caplog
 
     def test_providers_are_loaded(self):
-        provider_manager = ProvidersManager()
-        provider_list = list(provider_manager.providers.keys())
-        # No need to sort the list - it should be sorted alphabetically !
-        for provider in provider_list:
-            package_name = provider_manager.providers[provider][1]['package-name']
-            version = provider_manager.providers[provider][0]
-            assert re.search(r'[0-9]*\.[0-9]*\.[0-9]*.*', version)
-            assert package_name == provider
-        # just a sanity check - no exact number as otherwise we would have to update
-        # several tests if we add new connections/provider which is not ideal
-        assert len(provider_list) > 65
+        with self._caplog.at_level(logging.WARNING):
+            provider_manager = ProvidersManager()
+            provider_list = list(provider_manager.providers.keys())
+            # No need to sort the list - it should be sorted alphabetically !
+            for provider in provider_list:
+                package_name = provider_manager.providers[provider][1]['package-name']
+                version = provider_manager.providers[provider][0]
+                assert re.search(r'[0-9]*\.[0-9]*\.[0-9]*.*', version)
+                assert package_name == provider
+            # just a sanity check - no exact number as otherwise we would have to update
+            # several tests if we add new connections/provider which is not ideal
+            assert len(provider_list) > 65
+            assert [] == self._caplog.records
 
     def test_hooks_deprecation_warnings_generated(self):
         with pytest.warns(expected_warning=DeprecationWarning, match='hook-class-names') as warning_records: