[airflow] branch v2-2-test updated: Move away from legacy importlib.resources API (#19091)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/v2-2-test by this push:
     new 69348ca  Move away from legacy importlib.resources API (#19091)
69348ca is described below

commit 69348cac064076c97ccf8141b11fa4a7efb9f469
Author: Tzu-ping Chung <tp...@astronomer.io>
AuthorDate: Thu Oct 21 19:12:29 2021 +0800

    Move away from legacy importlib.resources API (#19091)
    
    (cherry picked from commit ed40c4b4adfb5736b07c08909ffb2d512d347105)
---
 airflow/providers_manager.py         | 20 ++++++--------------
 setup.cfg                            |  2 +-
 tests/core/test_providers_manager.py | 19 +++++++++----------
 3 files changed, 16 insertions(+), 25 deletions(-)

diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py
index 796fc70..5371ca0 100644
--- a/airflow/providers_manager.py
+++ b/airflow/providers_manager.py
@@ -37,20 +37,12 @@ from airflow.utils.entry_points import entry_points_with_dist
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.module_loading import import_string
 
-try:
-    import importlib.resources as importlib_resources
-except ImportError:
-    # Try back-ported to PY<37 `importlib_resources`.
-    import importlib_resources
-
 log = logging.getLogger(__name__)
 
 if sys.version_info >= (3, 9):
-    from functools import cache
+    from importlib.resources import files as resource_files
 else:
-    from functools import lru_cache
-
-    cache = lru_cache(maxsize=None)
+    from importlib_resources import files as resource_files
 
 MIN_PROVIDER_VERSIONS = {
     "apache-airflow-providers-celery": "2.1.0",
@@ -102,7 +94,8 @@ class LazyDictWithCache(MutableMapping):
 
 def _create_provider_info_schema_validator():
     """Creates JSON schema validator from the provider_info.schema.json"""
-    schema = json.loads(importlib_resources.read_text('airflow', 'provider_info.schema.json'))
+    with resource_files("airflow").joinpath("provider_info.schema.json").open("rb") as f:
+        schema = json.load(f)
     cls = jsonschema.validators.validator_for(schema)
     validator = cls(schema)
     return validator
@@ -110,9 +103,8 @@ def _create_provider_info_schema_validator():
 
 def _create_customized_form_field_behaviours_schema_validator():
     """Creates JSON schema validator from the customized_form_field_behaviours.schema.json"""
-    schema = json.loads(
-        importlib_resources.read_text('airflow', 'customized_form_field_behaviours.schema.json')
-    )
+    with resource_files("airflow").joinpath("customized_form_field_behaviours.schema.json").open("rb") as f:
+        schema = json.load(f)
     cls = jsonschema.validators.validator_for(schema)
     validator = cls(schema)
     return validator
diff --git a/setup.cfg b/setup.cfg
index b4b947e..65c22d2 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -112,7 +112,7 @@ install_requires =
     # We need to limit httpx until https://github.com/apache/airflow/issues/20088 is fixed
     httpx<0.20.0
     importlib_metadata>=1.7;python_version<"3.9"
-    importlib_resources~=5.2;python_version<"3.7"
+    importlib_resources~=5.2;python_version<"3.9"
     # Required by vendored-in connexion
     inflection>=0.3.1
     iso8601>=0.1.12
diff --git a/tests/core/test_providers_manager.py b/tests/core/test_providers_manager.py
index 17356f1..6f7b709 100644
--- a/tests/core/test_providers_manager.py
+++ b/tests/core/test_providers_manager.py
@@ -118,10 +118,9 @@ class TestProviderManager(unittest.TestCase):
 
     @patch('airflow.providers_manager.importlib.import_module')
     def test_hooks(self, mock_import_module):
-        # importlib-resources>=5.2 relies on importing the containing module to
-        # read its content. The provider manager needs to read the validation
-        # schema 'provider_info.schema.json'. Not sure why this needs to be
-        # called twice. Also see comment on the assert at the end of the function.
+        # importlib.resources relies on importing the containing module to read
+        # its content. The provider manager needs to read two validation schemas
+        # 'provider_info.schema.json' and 'customized_form_field_behaviours.schema.json'.
         mock_import_module.side_effect = [airflow, airflow]
 
         with pytest.warns(expected_warning=None) as warning_records:
@@ -132,12 +131,12 @@ class TestProviderManager(unittest.TestCase):
         assert [] == [w.message for w in warning_records.list if "hook-class-names" in str(w.message)]
         assert len(self._caplog.records) == 0
 
-        # The stdlib importlib.resources implementation in 3.7 through 3.9 does
-        # not rely on importlib.import_module, so the function should not be
-        # called. The implementation for 3.10+ and the backport to 3.6 uses it
-        # to import the top-level 'airflow' for 'provider_info.schema.json'.
-        # Also see comment on mocking at the beginning of the function.
-        if (3, 7) <= sys.version_info < (3, 10):
+        # The stdlib importlib.resources implementation in 3.9 does not rely on
+        # importlib.import_module, so the function should not be called. The
+        # implementation for 3.10+ and the backport we use for up to 3.8 uses it
+        # to import the top-level 'airflow' for the two schema files. Also see
+        # comment on mocking at the beginning of the function.
+        if sys.version_info[:2] == (3, 9):
             assert mock_import_module.mock_calls == []
         else:
             assert mock_import_module.mock_calls == [call("airflow"), call("airflow")]