[GitHub] [airflow] ashb commented on a change in pull request #12512: Implement reading provider information from packages/sources

[GitBox <gi...@apache.org>]

ashb commented on a change in pull request #12512:
URL: https://github.com/apache/airflow/pull/12512#discussion_r527867383



##########
File path: airflow/providers_manager.py
##########
@@ -46,42 +48,64 @@ class ProvidersManager:
     """Manages all provider packages."""
 
     def __init__(self):
-        self._provider_directory = {}
-        try:
-            from airflow import providers
-        except ImportError as e:
-            log.warning("No providers are present or error when importing them! :%s", e)
-            return
+        # Keeps list of providers keyed by module name and value is Tuple: version, provider_info
+        self._provider_directory: Dict[str, Tuple[str, Dict]] = {}
         self._validator = _create_validator()
-        self.__find_all_providers(providers.__path__)
-
-    def __find_all_providers(self, paths: str):
-        def onerror(_):
-            exception_string = traceback.format_exc()
-            log.warning(exception_string)
-
-        for module_info in pkgutil.walk_packages(paths, prefix="airflow.providers.", onerror=onerror):
-            try:
-                imported_module = importlib.import_module(module_info.name)
-            except Exception as e:  # noqa pylint: disable=broad-except
-                log.warning("Error when importing %s:%s", module_info.name, e)
-                continue
-            try:
-                provider = importlib_resources.read_text(imported_module, 'provider.yaml')
-                provider_info = yaml.safe_load(provider)
-                self._validator.validate(provider_info)
-                self._provider_directory[provider_info['package-name']] = provider_info
-            except FileNotFoundError:
-                # This is OK - this is not a provider package
-                pass
-            except TypeError as e:
-                if "is not a package" not in str(e):
-                    log.warning("Error when loading 'provider.yaml' file from %s:%s}", module_info.name, e)
-                # Otherwise this is OK - this is likely a module
-            except Exception as e:  # noqa pylint: disable=broad-except
-                log.warning("Error when loading 'provider.yaml' file from %s:%s", module_info.name, e)
+        self.__find_all_providers_from_packages()
+        self.__find_all_airflow_builtin_providers_from_local_sources()
+
+    def __find_all_providers_from_packages(self):
+        for entry_point in pkg_resources.iter_entry_points('apache_airflow_provider'):

Review comment:
       An idea from pytest: to "version" the entrypoint https://docs.pytest.org/en/stable/writing_plugins.html#making-your-plugin-installable-by-others




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org