You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by hu...@apache.org on 2023/09/20 18:35:57 UTC
[airflow] branch main updated: fix(providers/microsoft-azure): respect soft_fail argument when exception is raised (#34494)
This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6aa30f2ad6 fix(providers/microsoft-azure): respect soft_fail argument when exception is raised (#34494)
6aa30f2ad6 is described below
commit 6aa30f2ad647ef5167205fabe3bebbee4594b6e3
Author: Wei Lee <we...@gmail.com>
AuthorDate: Thu Sep 21 02:35:51 2023 +0800
fix(providers/microsoft-azure): respect soft_fail argument when exception is raised (#34494)
---
.../microsoft/azure/sensors/data_factory.py | 17 ++++++++++++++---
airflow/providers/microsoft/azure/sensors/wasb.py | 8 +++++++-
.../azure/sensors/test_azure_data_factory.py | 18 +++++++++++++-----
.../providers/microsoft/azure/sensors/test_wasb.py | 21 +++++++++++++--------
4 files changed, 47 insertions(+), 17 deletions(-)
diff --git a/airflow/providers/microsoft/azure/sensors/data_factory.py b/airflow/providers/microsoft/azure/sensors/data_factory.py
index b3e6947f0e..5cede76ad9 100644
--- a/airflow/providers/microsoft/azure/sensors/data_factory.py
+++ b/airflow/providers/microsoft/azure/sensors/data_factory.py
@@ -21,7 +21,7 @@ from functools import cached_property
from typing import TYPE_CHECKING, Sequence
from airflow.configuration import conf
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException, AirflowSkipException
from airflow.providers.microsoft.azure.hooks.data_factory import (
AzureDataFactoryHook,
AzureDataFactoryPipelineRunException,
@@ -85,10 +85,18 @@ class AzureDataFactoryPipelineRunStatusSensor(BaseSensorOperator):
)
if pipeline_run_status == AzureDataFactoryPipelineRunStatus.FAILED:
- raise AzureDataFactoryPipelineRunException(f"Pipeline run {self.run_id} has failed.")
+ # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+ message = f"Pipeline run {self.run_id} has failed."
+ if self.soft_fail:
+ raise AirflowSkipException(message)
+ raise AzureDataFactoryPipelineRunException(message)
if pipeline_run_status == AzureDataFactoryPipelineRunStatus.CANCELLED:
- raise AzureDataFactoryPipelineRunException(f"Pipeline run {self.run_id} has been cancelled.")
+ # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+ message = f"Pipeline run {self.run_id} has been cancelled."
+ if self.soft_fail:
+ raise AirflowSkipException(message)
+ raise AzureDataFactoryPipelineRunException(message)
return pipeline_run_status == AzureDataFactoryPipelineRunStatus.SUCCEEDED
@@ -122,6 +130,9 @@ class AzureDataFactoryPipelineRunStatusSensor(BaseSensorOperator):
"""
if event:
if event["status"] == "error":
+ # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+ if self.soft_fail:
+ raise AirflowSkipException(event["message"])
raise AirflowException(event["message"])
self.log.info(event["message"])
return None
diff --git a/airflow/providers/microsoft/azure/sensors/wasb.py b/airflow/providers/microsoft/azure/sensors/wasb.py
index 5abd88a129..3cd3457527 100644
--- a/airflow/providers/microsoft/azure/sensors/wasb.py
+++ b/airflow/providers/microsoft/azure/sensors/wasb.py
@@ -22,7 +22,7 @@ from datetime import timedelta
from typing import TYPE_CHECKING, Any, Sequence
from airflow.configuration import conf
-from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
+from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowSkipException
from airflow.providers.microsoft.azure.hooks.wasb import WasbHook
from airflow.providers.microsoft.azure.triggers.wasb import WasbBlobSensorTrigger, WasbPrefixSensorTrigger
from airflow.sensors.base import BaseSensorOperator
@@ -102,6 +102,9 @@ class WasbBlobSensor(BaseSensorOperator):
"""
if event:
if event["status"] == "error":
+ # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+ if self.soft_fail:
+ raise AirflowSkipException(event["message"])
raise AirflowException(event["message"])
self.log.info(event["message"])
else:
@@ -203,6 +206,9 @@ class WasbPrefixSensor(BaseSensorOperator):
"""
if event:
if event["status"] == "error":
+ # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
+ if self.soft_fail:
+ raise AirflowSkipException(event["message"])
raise AirflowException(event["message"])
self.log.info(event["message"])
else:
diff --git a/tests/providers/microsoft/azure/sensors/test_azure_data_factory.py b/tests/providers/microsoft/azure/sensors/test_azure_data_factory.py
index ca67444b5a..fb489c7ad7 100644
--- a/tests/providers/microsoft/azure/sensors/test_azure_data_factory.py
+++ b/tests/providers/microsoft/azure/sensors/test_azure_data_factory.py
@@ -21,7 +21,7 @@ from unittest.mock import patch
import pytest
-from airflow.exceptions import AirflowException, TaskDeferred
+from airflow.exceptions import AirflowException, AirflowSkipException, TaskDeferred
from airflow.providers.microsoft.azure.hooks.data_factory import (
AzureDataFactoryHook,
AzureDataFactoryPipelineRunException,
@@ -111,10 +111,14 @@ class TestAzureDataFactoryPipelineRunStatusSensor:
self.defered_sensor.execute_complete(context={}, event={"status": "success", "message": msg})
mock_log_info.assert_called_with(msg)
- def test_adf_pipeline_status_sensor_execute_complete_failure(self):
+ @pytest.mark.parametrize(
+ "soft_fail, expected_exception", ((False, AirflowException), (True, AirflowSkipException))
+ )
+ def test_adf_pipeline_status_sensor_execute_complete_failure(self, soft_fail, expected_exception):
"""Assert execute_complete method fail"""
- with pytest.raises(AirflowException):
+ self.defered_sensor.soft_fail = soft_fail
+ with pytest.raises(expected_exception):
self.defered_sensor.execute_complete(context={}, event={"status": "error", "message": ""})
@@ -142,8 +146,12 @@ class TestAzureDataFactoryPipelineRunStatusSensorWithAsync:
self.SENSOR.execute_complete(context={}, event={"status": "success", "message": msg})
mock_log_info.assert_called_with(msg)
- def test_adf_pipeline_status_sensor_execute_complete_failure(self):
+ @pytest.mark.parametrize(
+ "soft_fail, expected_exception", ((False, AirflowException), (True, AirflowSkipException))
+ )
+ def test_adf_pipeline_status_sensor_execute_complete_failure(self, soft_fail, expected_exception):
"""Assert execute_complete method fail"""
- with pytest.raises(AirflowException):
+ self.SENSOR.soft_fail = soft_fail
+ with pytest.raises(expected_exception):
self.SENSOR.execute_complete(context={}, event={"status": "error", "message": ""})
diff --git a/tests/providers/microsoft/azure/sensors/test_wasb.py b/tests/providers/microsoft/azure/sensors/test_wasb.py
index a94fd0c07b..96b24f8cc8 100644
--- a/tests/providers/microsoft/azure/sensors/test_wasb.py
+++ b/tests/providers/microsoft/azure/sensors/test_wasb.py
@@ -24,15 +24,12 @@ from unittest import mock
import pendulum
import pytest
-from airflow.exceptions import AirflowException, TaskDeferred
+from airflow.exceptions import AirflowException, AirflowSkipException, TaskDeferred
from airflow.models import Connection
from airflow.models.dag import DAG
from airflow.models.dagrun import DagRun
from airflow.models.taskinstance import TaskInstance
-from airflow.providers.microsoft.azure.sensors.wasb import (
- WasbBlobSensor,
- WasbPrefixSensor,
-)
+from airflow.providers.microsoft.azure.sensors.wasb import WasbBlobSensor, WasbPrefixSensor
from airflow.providers.microsoft.azure.triggers.wasb import WasbBlobSensorTrigger, WasbPrefixSensorTrigger
from airflow.utils import timezone
from airflow.utils.types import DagRunType
@@ -163,10 +160,14 @@ class TestWasbBlobAsyncSensor:
self.SENSOR.execute_complete(context={}, event=event)
mock_log_info.assert_called_with(event["message"])
- def test_wasb_blob_sensor_execute_complete_failure(self):
+ @pytest.mark.parametrize(
+ "soft_fail, expected_exception", ((False, AirflowException), (True, AirflowSkipException))
+ )
+ def test_wasb_blob_sensor_execute_complete_failure(self, soft_fail, expected_exception):
"""Assert execute_complete method raises an exception when the triggerer fires an error event."""
- with pytest.raises(AirflowException):
+ self.SENSOR.soft_fail = soft_fail
+ with pytest.raises(expected_exception):
self.SENSOR.execute_complete(context={}, event={"status": "error", "message": ""})
@@ -288,8 +289,12 @@ class TestWasbPrefixAsyncSensor:
self.SENSOR.execute_complete(context={}, event=event)
mock_log_info.assert_called_with(event["message"])
- def test_wasb_prefix_sensor_execute_complete_failure(self):
+ @pytest.mark.parametrize(
+ "soft_fail, expected_exception", ((False, AirflowException), (True, AirflowSkipException))
+ )
+ def test_wasb_prefix_sensor_execute_complete_failure(self, soft_fail, expected_exception):
"""Assert execute_complete method raises an exception when the triggerer fires an error event."""
+ self.SENSOR.soft_fail = soft_fail
with pytest.raises(AirflowException):
self.SENSOR.execute_complete(context={}, event={"status": "error", "message": ""})