You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by sh...@apache.org on 2024/09/27 08:11:15 UTC

(airflow) branch main updated: Refactor ``bucket.get_blob`` calls in ``GCSHook`` to handle validation for non-existent objects. (#42474)

This is an automated email from the ASF dual-hosted git repository.

shahar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new b0234cbaf7 Refactor ``bucket.get_blob`` calls in ``GCSHook`` to handle validation for non-existent objects. (#42474)
b0234cbaf7 is described below

commit b0234cbaf73bdf28640b77e2f1c378d5c489b26a
Author: Jason <46...@users.noreply.github.com>
AuthorDate: Fri Sep 27 15:11:08 2024 +0700

    Refactor ``bucket.get_blob`` calls in ``GCSHook`` to handle validation for non-existent objects. (#42474)
    
    * Fix implicit exception in GCSToLocalFilesystemOperator when reading non-existing files.
    
    * change method to internal method
    
    * Fix unit test expectations
    
    * Refactor bucket.get_blob calls in GCSHook to handle validation for non-existent objects
---
 airflow/providers/google/cloud/hooks/gcs.py    | 43 ++++++++++--------
 newsfragments/42439.bugfix.rst                 |  1 +
 tests/providers/google/cloud/hooks/test_gcs.py | 60 ++++++++++++++++++--------
 3 files changed, 68 insertions(+), 36 deletions(-)

diff --git a/airflow/providers/google/cloud/hooks/gcs.py b/airflow/providers/google/cloud/hooks/gcs.py
index 83118e7079..fb48fcd190 100644
--- a/airflow/providers/google/cloud/hooks/gcs.py
+++ b/airflow/providers/google/cloud/hooks/gcs.py
@@ -59,6 +59,7 @@ if TYPE_CHECKING:
 
     from aiohttp import ClientSession
     from google.api_core.retry import Retry
+    from google.cloud.storage.blob import Blob
 
 
 RT = TypeVar("RT")
@@ -597,11 +598,7 @@ class GCSHook(GoogleBaseHook):
         :param object_name: The name of the blob to get updated time from the Google cloud
             storage bucket.
         """
-        client = self.get_conn()
-        bucket = client.bucket(bucket_name)
-        blob = bucket.get_blob(blob_name=object_name)
-        if blob is None:
-            raise ValueError(f"Object ({object_name}) not found in Bucket ({bucket_name})")
+        blob = self._get_blob(bucket_name, object_name)
         return blob.updated
 
     def is_updated_after(self, bucket_name: str, object_name: str, ts: datetime) -> bool:
@@ -957,19 +954,35 @@ class GCSHook(GoogleBaseHook):
                 break
         return ids
 
-    def get_size(self, bucket_name: str, object_name: str) -> int:
+    def _get_blob(self, bucket_name: str, object_name: str) -> Blob:
         """
-        Get the size of a file in Google Cloud Storage.
+        Get a blob object in Google Cloud Storage.
 
         :param bucket_name: The Google Cloud Storage bucket where the blob_name is.
         :param object_name: The name of the object to check in the Google
             cloud storage bucket_name.
 
         """
-        self.log.info("Checking the file size of object: %s in bucket_name: %s", object_name, bucket_name)
         client = self.get_conn()
         bucket = client.bucket(bucket_name)
         blob = bucket.get_blob(blob_name=object_name)
+
+        if blob is None:
+            raise ValueError(f"Object ({object_name}) not found in Bucket ({bucket_name})")
+
+        return blob
+
+    def get_size(self, bucket_name: str, object_name: str) -> int:
+        """
+        Get the size of a file in Google Cloud Storage.
+
+        :param bucket_name: The Google Cloud Storage bucket where the blob_name is.
+        :param object_name: The name of the object to check in the Google
+            cloud storage bucket_name.
+
+        """
+        self.log.info("Checking the file size of object: %s in bucket_name: %s", object_name, bucket_name)
+        blob = self._get_blob(bucket_name, object_name)
         blob_size = blob.size
         self.log.info("The file size of %s is %s bytes.", object_name, blob_size)
         return blob_size
@@ -987,9 +1000,7 @@ class GCSHook(GoogleBaseHook):
             object_name,
             bucket_name,
         )
-        client = self.get_conn()
-        bucket = client.bucket(bucket_name)
-        blob = bucket.get_blob(blob_name=object_name)
+        blob = self._get_blob(bucket_name, object_name)
         blob_crc32c = blob.crc32c
         self.log.info("The crc32c checksum of %s is %s", object_name, blob_crc32c)
         return blob_crc32c
@@ -1003,9 +1014,7 @@ class GCSHook(GoogleBaseHook):
             storage bucket_name.
         """
         self.log.info("Retrieving the MD5 hash of object: %s in bucket: %s", object_name, bucket_name)
-        client = self.get_conn()
-        bucket = client.bucket(bucket_name)
-        blob = bucket.get_blob(blob_name=object_name)
+        blob = self._get_blob(bucket_name, object_name)
         blob_md5hash = blob.md5_hash
         self.log.info("The md5Hash of %s is %s", object_name, blob_md5hash)
         return blob_md5hash
@@ -1019,11 +1028,7 @@ class GCSHook(GoogleBaseHook):
         :return: The metadata associated with the object
         """
         self.log.info("Retrieving the metadata dict of object (%s) in bucket (%s)", object_name, bucket_name)
-        client = self.get_conn()
-        bucket = client.bucket(bucket_name)
-        blob = bucket.get_blob(blob_name=object_name)
-        if blob is None:
-            raise ValueError("Object (%s) not found in bucket (%s)", object_name, bucket_name)
+        blob = self._get_blob(bucket_name, object_name)
         blob_metadata = blob.metadata
         if blob_metadata:
             self.log.info("Retrieved metadata of object (%s) with %s fields", object_name, len(blob_metadata))
diff --git a/newsfragments/42439.bugfix.rst b/newsfragments/42439.bugfix.rst
new file mode 100644
index 0000000000..4ecd73eb63
--- /dev/null
+++ b/newsfragments/42439.bugfix.rst
@@ -0,0 +1 @@
+Refactor ``bucket.get_blob`` calls in ``GCSHook`` to handle validation for non-existent objects.
diff --git a/tests/providers/google/cloud/hooks/test_gcs.py b/tests/providers/google/cloud/hooks/test_gcs.py
index 5dee8cbf8c..5d2735834a 100644
--- a/tests/providers/google/cloud/hooks/test_gcs.py
+++ b/tests/providers/google/cloud/hooks/test_gcs.py
@@ -523,57 +523,57 @@ class TestGCSHook:
         mock_service.return_value.bucket.return_value.delete.assert_called_once()
         assert "Bucket test bucket not exist" in caplog.text
 
-    @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+    @mock.patch(GCS_STRING.format("GCSHook._get_blob"))
     def test_object_get_size(self, mock_service):
         test_bucket = "test_bucket"
         test_object = "test_object"
         returned_file_size = 1200
 
-        bucket_method = mock_service.return_value.bucket
-        get_blob_method = bucket_method.return_value.get_blob
-        get_blob_method.return_value.size = returned_file_size
+        mock_blob = MagicMock()
+        mock_blob.size = returned_file_size
+        mock_service.return_value = mock_blob
 
         response = self.gcs_hook.get_size(bucket_name=test_bucket, object_name=test_object)
 
         assert response == returned_file_size
 
-    @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+    @mock.patch(GCS_STRING.format("GCSHook._get_blob"))
     def test_object_get_crc32c(self, mock_service):
         test_bucket = "test_bucket"
         test_object = "test_object"
         returned_file_crc32c = "xgdNfQ=="
 
-        bucket_method = mock_service.return_value.bucket
-        get_blob_method = bucket_method.return_value.get_blob
-        get_blob_method.return_value.crc32c = returned_file_crc32c
+        mock_blob = MagicMock()
+        mock_blob.crc32c = returned_file_crc32c
+        mock_service.return_value = mock_blob
 
         response = self.gcs_hook.get_crc32c(bucket_name=test_bucket, object_name=test_object)
 
         assert response == returned_file_crc32c
 
-    @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+    @mock.patch(GCS_STRING.format("GCSHook._get_blob"))
     def test_object_get_md5hash(self, mock_service):
         test_bucket = "test_bucket"
         test_object = "test_object"
         returned_file_md5hash = "leYUJBUWrRtks1UeUFONJQ=="
 
-        bucket_method = mock_service.return_value.bucket
-        get_blob_method = bucket_method.return_value.get_blob
-        get_blob_method.return_value.md5_hash = returned_file_md5hash
+        mock_blob = MagicMock()
+        mock_blob.md5_hash = returned_file_md5hash
+        mock_service.return_value = mock_blob
 
         response = self.gcs_hook.get_md5hash(bucket_name=test_bucket, object_name=test_object)
 
         assert response == returned_file_md5hash
 
-    @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+    @mock.patch(GCS_STRING.format("GCSHook._get_blob"))
     def test_object_get_metadata(self, mock_service):
         test_bucket = "test_bucket"
         test_object = "test_object"
         returned_file_metadata = {"test_metadata_key": "test_metadata_val"}
 
-        bucket_method = mock_service.return_value.bucket
-        get_blob_method = bucket_method.return_value.get_blob
-        get_blob_method.return_value.metadata = returned_file_metadata
+        mock_blob = MagicMock()
+        mock_blob.metadata = returned_file_metadata
+        mock_service.return_value = mock_blob
 
         response = self.gcs_hook.get_metadata(bucket_name=test_bucket, object_name=test_object)
 
@@ -588,7 +588,7 @@ class TestGCSHook:
         get_blob_method = bucket_method.return_value.get_blob
         get_blob_method.return_value = None
 
-        with pytest.raises(ValueError, match=r"Object \((.*?)\) not found in bucket \((.*?)\)"):
+        with pytest.raises(ValueError, match=r"Object \((.*?)\) not found in Bucket \((.*?)\)"):
             self.gcs_hook.get_metadata(bucket_name=test_bucket, object_name=test_object)
 
     @mock.patch("google.cloud.storage.Bucket")
@@ -1509,6 +1509,32 @@ class TestSyncGcsHook:
         mock_rewrite.assert_not_called()
         mock_copy.assert_not_called()
 
+    @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+    def test_object_get_blob(self, mock_service):
+        test_bucket = "test_bucket"
+        test_object = "test_object"
+        mock_blob = mock.MagicMock()
+
+        bucket_method = mock_service.return_value.bucket
+        get_blob_method = bucket_method.return_value.get_blob
+        get_blob_method.return_value = mock_blob
+
+        response = self.gcs_hook._get_blob(bucket_name=test_bucket, object_name=test_object)
+
+        assert response == mock_blob
+
+    @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+    def test_nonexisting_object_get_blob(self, mock_service):
+        test_bucket = "test_bucket"
+        test_object = "test_object"
+
+        bucket_method = mock_service.return_value.bucket
+        get_blob_method = bucket_method.return_value.get_blob
+        get_blob_method.return_value = None
+
+        with pytest.raises(ValueError, match=r"Object \((.*?)\) not found in Bucket \((.*?)\)"):
+            self.gcs_hook._get_blob(bucket_name=test_bucket, object_name=test_object)
+
     def _create_blob(
         self,
         name: str,