You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ds...@apache.org on 2022/11/05 01:18:58 UTC

[airflow] branch main updated: Suppress any Exception in wasb task handler (#27495)

This is an automated email from the ASF dual-hosted git repository.

dstandish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 59da943428 Suppress any Exception in wasb task handler (#27495)
59da943428 is described below

commit 59da94342813d382a768d064ac9cfd0245825679
Author: Daniel Standish <15...@users.noreply.github.com>
AuthorDate: Fri Nov 4 18:18:37 2022 -0700

    Suppress any Exception in wasb task handler (#27495)
    
    It's important that problems with the log handler don't cause the task itself to fail. We discovered that sometimes the blob client will throw an error from azure.core.AzureError (currently we catch from azure.common). Rather than just add azure.core.AzureError, it seems sensible to just catch anything.
    
    Co-authored-by: Jed Cunningham <66...@users.noreply.github.com>
---
 .../providers/microsoft/azure/log/wasb_task_handler.py    | 15 +++++++--------
 .../microsoft/azure/log/test_wasb_task_handler.py         | 12 +++---------
 2 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/airflow/providers/microsoft/azure/log/wasb_task_handler.py b/airflow/providers/microsoft/azure/log/wasb_task_handler.py
index d0512982ce..8c0fe22083 100644
--- a/airflow/providers/microsoft/azure/log/wasb_task_handler.py
+++ b/airflow/providers/microsoft/azure/log/wasb_task_handler.py
@@ -21,8 +21,6 @@ import os
 import shutil
 from typing import Any
 
-from azure.common import AzureHttpError
-
 from airflow.compat.functools import cached_property
 from airflow.configuration import conf
 from airflow.utils.log.file_task_handler import FileTaskHandler
@@ -62,11 +60,12 @@ class WasbTaskHandler(FileTaskHandler, LoggingMixin):
             from airflow.providers.microsoft.azure.hooks.wasb import WasbHook
 
             return WasbHook(remote_conn_id)
-        except AzureHttpError:
+        except Exception:
             self.log.exception(
-                'Could not create an WasbHook with connection id "%s".'
-                " Please make sure that apache-airflow[azure] is installed"
-                " and the Wasb connection exists.",
+                "Could not create a WasbHook with connection id '%s'. "
+                "Do you have apache-airflow[azure] installed? "
+                "Does connection the connection exist, and is it "
+                "configured properly?",
                 remote_conn_id,
             )
             return None
@@ -158,7 +157,7 @@ class WasbTaskHandler(FileTaskHandler, LoggingMixin):
         """
         try:
             return self.hook.read_file(self.wasb_container, remote_log_location)
-        except AzureHttpError:
+        except Exception:
             msg = f"Could not read logs from {remote_log_location}"
             self.log.exception(msg)
             # return error if needed
@@ -182,5 +181,5 @@ class WasbTaskHandler(FileTaskHandler, LoggingMixin):
 
         try:
             self.hook.load_string(log, self.wasb_container, remote_log_location, overwrite=True)
-        except AzureHttpError:
+        except Exception:
             self.log.exception("Could not write logs to %s", remote_log_location)
diff --git a/tests/providers/microsoft/azure/log/test_wasb_task_handler.py b/tests/providers/microsoft/azure/log/test_wasb_task_handler.py
index 98f03a4ec5..3f29ea0c86 100644
--- a/tests/providers/microsoft/azure/log/test_wasb_task_handler.py
+++ b/tests/providers/microsoft/azure/log/test_wasb_task_handler.py
@@ -68,21 +68,15 @@ class TestWasbTaskHandler:
         assert isinstance(self.wasb_task_handler.hook, WasbHook)
 
     @conf_vars({("logging", "remote_log_conn_id"): "wasb_default"})
-    def test_hook_raises(self):
+    def test_hook_warns(self):
         handler = self.wasb_task_handler
-        with mock.patch.object(handler.log, "error") as mock_error:
+        with mock.patch.object(handler.log, "exception") as mock_exc:
             with mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbHook") as mock_hook:
                 mock_hook.side_effect = AzureHttpError("failed to connect", 404)
                 # Initialize the hook
                 handler.hook
 
-            mock_error.assert_called_once_with(
-                'Could not create an WasbHook with connection id "%s".'
-                " Please make sure that apache-airflow[azure] is installed"
-                " and the Wasb connection exists.",
-                "wasb_default",
-                exc_info=True,
-            )
+        assert "Could not create a WasbHook with connection id '%s'" in mock_exc.call_args[0][0]
 
     def test_set_context_raw(self, ti):
         ti.raw = True