You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/04/25 12:00:01 UTC

(airflow) branch main updated: Implement run-method on KiotaRequestAdapterHook and move logic away from triggerer to hook (#39237)

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 15c2734c4c Implement run-method on KiotaRequestAdapterHook and move logic away from triggerer to hook (#39237)
15c2734c4c is described below

commit 15c2734c4cac2e2acbdeaf28dfe67d04e28cee90
Author: David Blain <in...@dabla.be>
AuthorDate: Thu Apr 25 13:59:55 2024 +0200

    Implement run-method on KiotaRequestAdapterHook and move logic away from triggerer to hook (#39237)
    
    
    
    ---------
    
    Co-authored-by: David Blain <da...@infrabel.be>
---
 airflow/providers/microsoft/azure/hooks/msgraph.py | 164 ++++++++++++++++++++-
 .../providers/microsoft/azure/triggers/msgraph.py  |  98 ++----------
 .../microsoft/azure/hooks/test_msgraph.py          |  64 +++++++-
 .../microsoft/azure/triggers/test_msgraph.py       |  28 ----
 4 files changed, 231 insertions(+), 123 deletions(-)

diff --git a/airflow/providers/microsoft/azure/hooks/msgraph.py b/airflow/providers/microsoft/azure/hooks/msgraph.py
index 7fcc328f86..84b2252bd2 100644
--- a/airflow/providers/microsoft/azure/hooks/msgraph.py
+++ b/airflow/providers/microsoft/azure/hooks/msgraph.py
@@ -17,28 +17,74 @@
 # under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-from urllib.parse import urljoin, urlparse
+import json
+from http import HTTPStatus
+from io import BytesIO
+from typing import TYPE_CHECKING, Any, Callable
+from urllib.parse import quote, urljoin, urlparse
 
 import httpx
 from azure.identity import ClientSecretCredential
 from httpx import Timeout
+from kiota_abstractions.api_error import APIError
+from kiota_abstractions.method import Method
+from kiota_abstractions.request_information import RequestInformation
+from kiota_abstractions.response_handler import ResponseHandler
 from kiota_authentication_azure.azure_identity_authentication_provider import (
     AzureIdentityAuthenticationProvider,
 )
 from kiota_http.httpx_request_adapter import HttpxRequestAdapter
-from msgraph_core import GraphClientFactory
-from msgraph_core._enums import APIVersion, NationalClouds
+from kiota_http.middleware.options import ResponseHandlerOption
+from msgraph_core import APIVersion, GraphClientFactory
+from msgraph_core._enums import NationalClouds
 
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowBadRequest, AirflowException, AirflowNotFoundException
 from airflow.hooks.base import BaseHook
 
 if TYPE_CHECKING:
     from kiota_abstractions.request_adapter import RequestAdapter
+    from kiota_abstractions.request_information import QueryParams
+    from kiota_abstractions.response_handler import NativeResponseType
+    from kiota_abstractions.serialization import ParsableFactory
+    from kiota_http.httpx_request_adapter import ResponseType
 
     from airflow.models import Connection
 
 
+class CallableResponseHandler(ResponseHandler):
+    """
+    CallableResponseHandler executes the passed callable_function with response as parameter.
+
+    param callable_function: Function that is applied to the response.
+    """
+
+    def __init__(
+        self,
+        callable_function: Callable[[NativeResponseType, dict[str, ParsableFactory | None] | None], Any],
+    ):
+        self.callable_function = callable_function
+
+    async def handle_response_async(
+        self, response: NativeResponseType, error_map: dict[str, ParsableFactory | None] | None = None
+    ) -> Any:
+        """
+        Invoke this callback method when a response is received.
+
+        param response: The type of the native response object.
+        param error_map: The error dict to use in case of a failed request.
+        """
+        value = self.callable_function(response, error_map)
+        if response.status_code not in {200, 201, 202, 204, 302}:
+            message = value or response.reason_phrase
+            status_code = HTTPStatus(response.status_code)
+            if status_code == HTTPStatus.BAD_REQUEST:
+                raise AirflowBadRequest(message)
+            elif status_code == HTTPStatus.NOT_FOUND:
+                raise AirflowNotFoundException(message)
+            raise AirflowException(message)
+        return value
+
+
 class KiotaRequestAdapterHook(BaseHook):
     """
     A Microsoft Graph API interaction hook, a Wrapper around KiotaRequestAdapter.
@@ -54,6 +100,7 @@ class KiotaRequestAdapterHook(BaseHook):
         or you can pass a string as "v1.0" or "beta".
     """
 
+    DEFAULT_HEADERS = {"Accept": "application/json;q=1"}
     cached_request_adapters: dict[str, tuple[APIVersion, RequestAdapter]] = {}
     default_conn_name: str = "msgraph_default"
 
@@ -117,13 +164,16 @@ class KiotaRequestAdapterHook(BaseHook):
                 proxies[cls.format_no_proxy_url(url.strip())] = None
         return proxies
 
-    @classmethod
-    def to_msal_proxies(cls, authority: str | None, proxies: dict):
+    def to_msal_proxies(self, authority: str | None, proxies: dict):
+        self.log.info("authority: %s", authority)
         if authority:
             no_proxies = proxies.get("no")
+            self.log.info("no_proxies: %s", no_proxies)
             if no_proxies:
                 for url in no_proxies.split(","):
+                    self.log.info("url: %s", url)
                     domain_name = urlparse(url).path.replace("*", "")
+                    self.log.info("domain_name: %s", domain_name)
                     if authority.endswith(domain_name):
                         return None
         return proxies
@@ -206,3 +256,103 @@ class KiotaRequestAdapterHook(BaseHook):
             self.cached_request_adapters[self.conn_id] = (api_version, request_adapter)
         self._api_version = api_version
         return request_adapter
+
+    def test_connection(self):
+        """Test HTTP Connection."""
+        try:
+            self.run()
+            return True, "Connection successfully tested"
+        except Exception as e:
+            return False, str(e)
+
+    async def run(
+        self,
+        url: str = "",
+        response_type: ResponseType | None = None,
+        response_handler: Callable[
+            [NativeResponseType, dict[str, ParsableFactory | None] | None], Any
+        ] = lambda response, error_map: response.json(),
+        path_parameters: dict[str, Any] | None = None,
+        method: str = "GET",
+        query_parameters: dict[str, QueryParams] | None = None,
+        headers: dict[str, str] | None = None,
+        data: dict[str, Any] | str | BytesIO | None = None,
+    ):
+        response = await self.get_conn().send_primitive_async(
+            request_info=self.request_information(
+                url=url,
+                response_type=response_type,
+                response_handler=response_handler,
+                path_parameters=path_parameters,
+                method=method,
+                query_parameters=query_parameters,
+                headers=headers,
+                data=data,
+            ),
+            response_type=response_type,
+            error_map=self.error_mapping(),
+        )
+
+        self.log.debug("response: %s", response)
+
+        return response
+
+    def request_information(
+        self,
+        url: str,
+        response_type: ResponseType | None = None,
+        response_handler: Callable[
+            [NativeResponseType, dict[str, ParsableFactory | None] | None], Any
+        ] = lambda response, error_map: response.json(),
+        path_parameters: dict[str, Any] | None = None,
+        method: str = "GET",
+        query_parameters: dict[str, QueryParams] | None = None,
+        headers: dict[str, str] | None = None,
+        data: dict[str, Any] | str | BytesIO | None = None,
+    ) -> RequestInformation:
+        request_information = RequestInformation()
+        request_information.path_parameters = path_parameters or {}
+        request_information.http_method = Method(method.strip().upper())
+        request_information.query_parameters = self.encoded_query_parameters(query_parameters)
+        if url.startswith("http"):
+            request_information.url = url
+        elif request_information.query_parameters.keys():
+            query = ",".join(request_information.query_parameters.keys())
+            request_information.url_template = f"{{+baseurl}}/{self.normalize_url(url)}{{?{query}}}"
+        else:
+            request_information.url_template = f"{{+baseurl}}/{self.normalize_url(url)}"
+        if not response_type:
+            request_information.request_options[ResponseHandlerOption.get_key()] = ResponseHandlerOption(
+                response_handler=CallableResponseHandler(response_handler)
+            )
+        headers = {**self.DEFAULT_HEADERS, **headers} if headers else self.DEFAULT_HEADERS
+        for header_name, header_value in headers.items():
+            request_information.headers.try_add(header_name=header_name, header_value=header_value)
+        self.log.info("data: %s", data)
+        if isinstance(data, BytesIO) or isinstance(data, bytes) or isinstance(data, str):
+            request_information.content = data
+        elif data:
+            request_information.headers.try_add(
+                header_name=RequestInformation.CONTENT_TYPE_HEADER, header_value="application/json"
+            )
+            request_information.content = json.dumps(data).encode("utf-8")
+        return request_information
+
+    @staticmethod
+    def normalize_url(url: str) -> str | None:
+        if url.startswith("/"):
+            return url.replace("/", "", 1)
+        return url
+
+    @staticmethod
+    def encoded_query_parameters(query_parameters) -> dict:
+        if query_parameters:
+            return {quote(key): value for key, value in query_parameters.items()}
+        return {}
+
+    @staticmethod
+    def error_mapping() -> dict[str, ParsableFactory | None]:
+        return {
+            "4XX": APIError,
+            "5XX": APIError,
+        }
diff --git a/airflow/providers/microsoft/azure/triggers/msgraph.py b/airflow/providers/microsoft/azure/triggers/msgraph.py
index 814a91f7c1..1848f969f8 100644
--- a/airflow/providers/microsoft/azure/triggers/msgraph.py
+++ b/airflow/providers/microsoft/azure/triggers/msgraph.py
@@ -22,7 +22,6 @@ import locale
 from base64 import b64encode
 from contextlib import suppress
 from datetime import datetime
-from io import BytesIO
 from json import JSONDecodeError
 from typing import (
     TYPE_CHECKING,
@@ -31,21 +30,17 @@ from typing import (
     Callable,
     Sequence,
 )
-from urllib.parse import quote
 from uuid import UUID
 
 import pendulum
-from kiota_abstractions.api_error import APIError
-from kiota_abstractions.method import Method
-from kiota_abstractions.request_information import RequestInformation
-from kiota_abstractions.response_handler import ResponseHandler
-from kiota_http.middleware.options import ResponseHandlerOption
 
 from airflow.providers.microsoft.azure.hooks.msgraph import KiotaRequestAdapterHook
 from airflow.triggers.base import BaseTrigger, TriggerEvent
 from airflow.utils.module_loading import import_string
 
 if TYPE_CHECKING:
+    from io import BytesIO
+
     from kiota_abstractions.request_adapter import RequestAdapter
     from kiota_abstractions.request_information import QueryParams
     from kiota_abstractions.response_handler import NativeResponseType
@@ -87,31 +82,6 @@ class ResponseSerializer:
         return response
 
 
-class CallableResponseHandler(ResponseHandler):
-    """
-    CallableResponseHandler executes the passed callable_function with response as parameter.
-
-    param callable_function: Function that is applied to the response.
-    """
-
-    def __init__(
-        self,
-        callable_function: Callable[[NativeResponseType, dict[str, ParsableFactory | None] | None], Any],
-    ):
-        self.callable_function = callable_function
-
-    async def handle_response_async(
-        self, response: NativeResponseType, error_map: dict[str, ParsableFactory | None] | None = None
-    ) -> Any:
-        """
-        Invoke this callback method when a response is received.
-
-        param response: The type of the native response object.
-        param error_map: The error dict to use in case of a failed request.
-        """
-        return self.callable_function(response, error_map)
-
-
 class MSGraphTrigger(BaseTrigger):
     """
     A Microsoft Graph API trigger which allows you to execute an async REST call to the Microsoft Graph API.
@@ -134,7 +104,6 @@ class MSGraphTrigger(BaseTrigger):
         Bytes will be base64 encoded into a string, so it can be stored as an XCom.
     """
 
-    DEFAULT_HEADERS = {"Accept": "application/json;q=1"}
     template_fields: Sequence[str] = (
         "url",
         "response_type",
@@ -235,7 +204,16 @@ class MSGraphTrigger(BaseTrigger):
     async def run(self) -> AsyncIterator[TriggerEvent]:
         """Make a series of asynchronous HTTP calls via a KiotaRequestAdapterHook."""
         try:
-            response = await self.execute()
+            response = await self.hook.run(
+                url=self.url,
+                response_type=self.response_type,
+                response_handler=self.response_handler,
+                path_parameters=self.path_parameters,
+                method=self.method,
+                query_parameters=self.query_parameters,
+                headers=self.headers,
+                data=self.data,
+            )
 
             self.log.debug("response: %s", response)
 
@@ -262,55 +240,3 @@ class MSGraphTrigger(BaseTrigger):
         except Exception as e:
             self.log.exception("An error occurred: %s", e)
             yield TriggerEvent({"status": "failure", "message": str(e)})
-
-    def normalize_url(self) -> str | None:
-        if self.url.startswith("/"):
-            return self.url.replace("/", "", 1)
-        return self.url
-
-    def encoded_query_parameters(self) -> dict:
-        if self.query_parameters:
-            return {quote(key): value for key, value in self.query_parameters.items()}
-        return {}
-
-    def request_information(self) -> RequestInformation:
-        request_information = RequestInformation()
-        request_information.path_parameters = self.path_parameters or {}
-        request_information.http_method = Method(self.method.strip().upper())
-        request_information.query_parameters = self.encoded_query_parameters()
-        if self.url.startswith("http"):
-            request_information.url = self.url
-        elif request_information.query_parameters.keys():
-            query = ",".join(request_information.query_parameters.keys())
-            request_information.url_template = f"{{+baseurl}}/{self.normalize_url()}{{?{query}}}"
-        else:
-            request_information.url_template = f"{{+baseurl}}/{self.normalize_url()}"
-        if not self.response_type:
-            request_information.request_options[ResponseHandlerOption.get_key()] = ResponseHandlerOption(
-                response_handler=CallableResponseHandler(self.response_handler)
-            )
-        headers = {**self.DEFAULT_HEADERS, **self.headers} if self.headers else self.DEFAULT_HEADERS
-        for header_name, header_value in headers.items():
-            request_information.headers.try_add(header_name=header_name, header_value=header_value)
-        if isinstance(self.data, BytesIO) or isinstance(self.data, bytes) or isinstance(self.data, str):
-            request_information.content = self.data
-        elif self.data:
-            request_information.headers.try_add(
-                header_name=RequestInformation.CONTENT_TYPE_HEADER, header_value="application/json"
-            )
-            request_information.content = json.dumps(self.data).encode("utf-8")
-        return request_information
-
-    @staticmethod
-    def error_mapping() -> dict[str, ParsableFactory | None]:
-        return {
-            "4XX": APIError,
-            "5XX": APIError,
-        }
-
-    async def execute(self) -> AsyncIterator[TriggerEvent]:
-        return await self.get_conn().send_primitive_async(
-            request_info=self.request_information(),
-            response_type=self.response_type,
-            error_map=self.error_mapping(),
-        )
diff --git a/tests/providers/microsoft/azure/hooks/test_msgraph.py b/tests/providers/microsoft/azure/hooks/test_msgraph.py
index 1c1046e1fa..9d2db07acf 100644
--- a/tests/providers/microsoft/azure/hooks/test_msgraph.py
+++ b/tests/providers/microsoft/azure/hooks/test_msgraph.py
@@ -16,13 +16,21 @@
 # under the License.
 from __future__ import annotations
 
+import asyncio
 from unittest.mock import patch
 
+import pytest
 from kiota_http.httpx_request_adapter import HttpxRequestAdapter
 from msgraph_core import APIVersion, NationalClouds
 
-from airflow.providers.microsoft.azure.hooks.msgraph import KiotaRequestAdapterHook
-from tests.providers.microsoft.conftest import get_airflow_connection, mock_connection
+from airflow.exceptions import AirflowBadRequest, AirflowException, AirflowNotFoundException
+from airflow.providers.microsoft.azure.hooks.msgraph import CallableResponseHandler, KiotaRequestAdapterHook
+from tests.providers.microsoft.conftest import (
+    get_airflow_connection,
+    load_json,
+    mock_connection,
+    mock_json_response,
+)
 
 
 class TestKiotaRequestAdapterHook:
@@ -77,3 +85,55 @@ class TestKiotaRequestAdapterHook:
         actual = KiotaRequestAdapterHook.get_host(connection)
 
         assert actual == NationalClouds.Global.value
+
+    def test_encoded_query_parameters(self):
+        actual = KiotaRequestAdapterHook.encoded_query_parameters(
+            query_parameters={"$expand": "reports,users,datasets,dataflows,dashboards", "$top": 5000},
+        )
+
+        assert actual == {"%24expand": "reports,users,datasets,dataflows,dashboards", "%24top": 5000}
+
+
+class TestResponseHandler:
+    def test_handle_response_async_when_ok(self):
+        users = load_json("resources", "users.json")
+        response = mock_json_response(200, users)
+
+        actual = asyncio.run(
+            CallableResponseHandler(lambda response, error_map: response.json()).handle_response_async(
+                response, None
+            )
+        )
+
+        assert isinstance(actual, dict)
+        assert actual == users
+
+    def test_handle_response_async_when_bad_request(self):
+        response = mock_json_response(400, {})
+
+        with pytest.raises(AirflowBadRequest):
+            asyncio.run(
+                CallableResponseHandler(lambda response, error_map: response.json()).handle_response_async(
+                    response, None
+                )
+            )
+
+    def test_handle_response_async_when_not_found(self):
+        response = mock_json_response(404, {})
+
+        with pytest.raises(AirflowNotFoundException):
+            asyncio.run(
+                CallableResponseHandler(lambda response, error_map: response.json()).handle_response_async(
+                    response, None
+                )
+            )
+
+    def test_handle_response_async_when_internal_server_error(self):
+        response = mock_json_response(500, {})
+
+        with pytest.raises(AirflowException):
+            asyncio.run(
+                CallableResponseHandler(lambda response, error_map: response.json()).handle_response_async(
+                    response, None
+                )
+            )
diff --git a/tests/providers/microsoft/azure/triggers/test_msgraph.py b/tests/providers/microsoft/azure/triggers/test_msgraph.py
index a4e2c7f0fd..23085563cf 100644
--- a/tests/providers/microsoft/azure/triggers/test_msgraph.py
+++ b/tests/providers/microsoft/azure/triggers/test_msgraph.py
@@ -16,7 +16,6 @@
 # under the License.
 from __future__ import annotations
 
-import asyncio
 import json
 import locale
 from base64 import b64decode, b64encode
@@ -28,7 +27,6 @@ import pendulum
 
 from airflow.exceptions import AirflowException
 from airflow.providers.microsoft.azure.triggers.msgraph import (
-    CallableResponseHandler,
     MSGraphTrigger,
     ResponseSerializer,
 )
@@ -132,32 +130,6 @@ class TestMSGraphTrigger(Base):
         for template_field in MSGraphTrigger.template_fields:
             getattr(trigger, template_field)
 
-    def test_encoded_query_parameters(self):
-        trigger = MSGraphTrigger(
-            url="myorg/admin/groups",
-            conn_id="msgraph_api",
-            query_parameters={"$expand": "reports,users,datasets,dataflows,dashboards", "$top": 5000},
-        )
-
-        actual = trigger.encoded_query_parameters()
-
-        assert actual == {"%24expand": "reports,users,datasets,dataflows,dashboards", "%24top": 5000}
-
-
-class TestResponseHandler:
-    def test_handle_response_async(self):
-        users = load_json("resources", "users.json")
-        response = mock_json_response(200, users)
-
-        actual = asyncio.run(
-            CallableResponseHandler(lambda response, error_map: response.json()).handle_response_async(
-                response, None
-            )
-        )
-
-        assert isinstance(actual, dict)
-        assert actual == users
-
 
 class TestResponseSerializer:
     def test_serialize_when_bytes_then_base64_encoded(self):