You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by hu...@apache.org on 2023/09/20 18:35:57 UTC

[airflow] branch main updated: fix(providers/microsoft-azure): respect soft_fail argument when exception is raised (#34494)

This is an automated email from the ASF dual-hosted git repository.

husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 6aa30f2ad6 fix(providers/microsoft-azure): respect soft_fail argument when exception is raised (#34494)
6aa30f2ad6 is described below

commit 6aa30f2ad647ef5167205fabe3bebbee4594b6e3
Author: Wei Lee <we...@gmail.com>
AuthorDate: Thu Sep 21 02:35:51 2023 +0800

    fix(providers/microsoft-azure): respect soft_fail argument when exception is raised (#34494)
---
 .../microsoft/azure/sensors/data_factory.py         | 17 ++++++++++++++---
 airflow/providers/microsoft/azure/sensors/wasb.py   |  8 +++++++-
 .../azure/sensors/test_azure_data_factory.py        | 18 +++++++++++++-----
 .../providers/microsoft/azure/sensors/test_wasb.py  | 21 +++++++++++++--------
 4 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/airflow/providers/microsoft/azure/sensors/data_factory.py b/airflow/providers/microsoft/azure/sensors/data_factory.py
index b3e6947f0e..5cede76ad9 100644
--- a/airflow/providers/microsoft/azure/sensors/data_factory.py
+++ b/airflow/providers/microsoft/azure/sensors/data_factory.py
@@ -21,7 +21,7 @@ from functools import cached_property
 from typing import TYPE_CHECKING, Sequence
 
 from airflow.configuration import conf
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException, AirflowSkipException
 from airflow.providers.microsoft.azure.hooks.data_factory import (
     AzureDataFactoryHook,
     AzureDataFactoryPipelineRunException,
@@ -85,10 +85,18 @@ class AzureDataFactoryPipelineRunStatusSensor(BaseSensorOperator):
         )
 
         if pipeline_run_status == AzureDataFactoryPipelineRunStatus.FAILED:
-            raise AzureDataFactoryPipelineRunException(f"Pipeline run {self.run_id} has failed.")
+            # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+            message = f"Pipeline run {self.run_id} has failed."
+            if self.soft_fail:
+                raise AirflowSkipException(message)
+            raise AzureDataFactoryPipelineRunException(message)
 
         if pipeline_run_status == AzureDataFactoryPipelineRunStatus.CANCELLED:
-            raise AzureDataFactoryPipelineRunException(f"Pipeline run {self.run_id} has been cancelled.")
+            # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+            message = f"Pipeline run {self.run_id} has been cancelled."
+            if self.soft_fail:
+                raise AirflowSkipException(message)
+            raise AzureDataFactoryPipelineRunException(message)
 
         return pipeline_run_status == AzureDataFactoryPipelineRunStatus.SUCCEEDED
 
@@ -122,6 +130,9 @@ class AzureDataFactoryPipelineRunStatusSensor(BaseSensorOperator):
         """
         if event:
             if event["status"] == "error":
+                # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+                if self.soft_fail:
+                    raise AirflowSkipException(event["message"])
                 raise AirflowException(event["message"])
             self.log.info(event["message"])
         return None
diff --git a/airflow/providers/microsoft/azure/sensors/wasb.py b/airflow/providers/microsoft/azure/sensors/wasb.py
index 5abd88a129..3cd3457527 100644
--- a/airflow/providers/microsoft/azure/sensors/wasb.py
+++ b/airflow/providers/microsoft/azure/sensors/wasb.py
@@ -22,7 +22,7 @@ from datetime import timedelta
 from typing import TYPE_CHECKING, Any, Sequence
 
 from airflow.configuration import conf
-from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
+from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowSkipException
 from airflow.providers.microsoft.azure.hooks.wasb import WasbHook
 from airflow.providers.microsoft.azure.triggers.wasb import WasbBlobSensorTrigger, WasbPrefixSensorTrigger
 from airflow.sensors.base import BaseSensorOperator
@@ -102,6 +102,9 @@ class WasbBlobSensor(BaseSensorOperator):
         """
         if event:
             if event["status"] == "error":
+                # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+                if self.soft_fail:
+                    raise AirflowSkipException(event["message"])
                 raise AirflowException(event["message"])
             self.log.info(event["message"])
         else:
@@ -203,6 +206,9 @@ class WasbPrefixSensor(BaseSensorOperator):
         """
         if event:
             if event["status"] == "error":
+                # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+                if self.soft_fail:
+                    raise AirflowSkipException(event["message"])
                 raise AirflowException(event["message"])
             self.log.info(event["message"])
         else:
diff --git a/tests/providers/microsoft/azure/sensors/test_azure_data_factory.py b/tests/providers/microsoft/azure/sensors/test_azure_data_factory.py
index ca67444b5a..fb489c7ad7 100644
--- a/tests/providers/microsoft/azure/sensors/test_azure_data_factory.py
+++ b/tests/providers/microsoft/azure/sensors/test_azure_data_factory.py
@@ -21,7 +21,7 @@ from unittest.mock import patch
 
 import pytest
 
-from airflow.exceptions import AirflowException, TaskDeferred
+from airflow.exceptions import AirflowException, AirflowSkipException, TaskDeferred
 from airflow.providers.microsoft.azure.hooks.data_factory import (
     AzureDataFactoryHook,
     AzureDataFactoryPipelineRunException,
@@ -111,10 +111,14 @@ class TestAzureDataFactoryPipelineRunStatusSensor:
             self.defered_sensor.execute_complete(context={}, event={"status": "success", "message": msg})
         mock_log_info.assert_called_with(msg)
 
-    def test_adf_pipeline_status_sensor_execute_complete_failure(self):
+    @pytest.mark.parametrize(
+        "soft_fail, expected_exception", ((False, AirflowException), (True, AirflowSkipException))
+    )
+    def test_adf_pipeline_status_sensor_execute_complete_failure(self, soft_fail, expected_exception):
         """Assert execute_complete method fail"""
 
-        with pytest.raises(AirflowException):
+        self.defered_sensor.soft_fail = soft_fail
+        with pytest.raises(expected_exception):
             self.defered_sensor.execute_complete(context={}, event={"status": "error", "message": ""})
 
 
@@ -142,8 +146,12 @@ class TestAzureDataFactoryPipelineRunStatusSensorWithAsync:
             self.SENSOR.execute_complete(context={}, event={"status": "success", "message": msg})
         mock_log_info.assert_called_with(msg)
 
-    def test_adf_pipeline_status_sensor_execute_complete_failure(self):
+    @pytest.mark.parametrize(
+        "soft_fail, expected_exception", ((False, AirflowException), (True, AirflowSkipException))
+    )
+    def test_adf_pipeline_status_sensor_execute_complete_failure(self, soft_fail, expected_exception):
         """Assert execute_complete method fail"""
 
-        with pytest.raises(AirflowException):
+        self.SENSOR.soft_fail = soft_fail
+        with pytest.raises(expected_exception):
             self.SENSOR.execute_complete(context={}, event={"status": "error", "message": ""})
diff --git a/tests/providers/microsoft/azure/sensors/test_wasb.py b/tests/providers/microsoft/azure/sensors/test_wasb.py
index a94fd0c07b..96b24f8cc8 100644
--- a/tests/providers/microsoft/azure/sensors/test_wasb.py
+++ b/tests/providers/microsoft/azure/sensors/test_wasb.py
@@ -24,15 +24,12 @@ from unittest import mock
 import pendulum
 import pytest
 
-from airflow.exceptions import AirflowException, TaskDeferred
+from airflow.exceptions import AirflowException, AirflowSkipException, TaskDeferred
 from airflow.models import Connection
 from airflow.models.dag import DAG
 from airflow.models.dagrun import DagRun
 from airflow.models.taskinstance import TaskInstance
-from airflow.providers.microsoft.azure.sensors.wasb import (
-    WasbBlobSensor,
-    WasbPrefixSensor,
-)
+from airflow.providers.microsoft.azure.sensors.wasb import WasbBlobSensor, WasbPrefixSensor
 from airflow.providers.microsoft.azure.triggers.wasb import WasbBlobSensorTrigger, WasbPrefixSensorTrigger
 from airflow.utils import timezone
 from airflow.utils.types import DagRunType
@@ -163,10 +160,14 @@ class TestWasbBlobAsyncSensor:
                 self.SENSOR.execute_complete(context={}, event=event)
             mock_log_info.assert_called_with(event["message"])
 
-    def test_wasb_blob_sensor_execute_complete_failure(self):
+    @pytest.mark.parametrize(
+        "soft_fail, expected_exception", ((False, AirflowException), (True, AirflowSkipException))
+    )
+    def test_wasb_blob_sensor_execute_complete_failure(self, soft_fail, expected_exception):
         """Assert execute_complete method raises an exception when the triggerer fires an error event."""
 
-        with pytest.raises(AirflowException):
+        self.SENSOR.soft_fail = soft_fail
+        with pytest.raises(expected_exception):
             self.SENSOR.execute_complete(context={}, event={"status": "error", "message": ""})
 
 
@@ -288,8 +289,12 @@ class TestWasbPrefixAsyncSensor:
                 self.SENSOR.execute_complete(context={}, event=event)
             mock_log_info.assert_called_with(event["message"])
 
-    def test_wasb_prefix_sensor_execute_complete_failure(self):
+    @pytest.mark.parametrize(
+        "soft_fail, expected_exception", ((False, AirflowException), (True, AirflowSkipException))
+    )
+    def test_wasb_prefix_sensor_execute_complete_failure(self, soft_fail, expected_exception):
         """Assert execute_complete method raises an exception when the triggerer fires an error event."""
 
+        self.SENSOR.soft_fail = soft_fail
         with pytest.raises(AirflowException):
             self.SENSOR.execute_complete(context={}, event={"status": "error", "message": ""})