You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by sh...@apache.org on 2024/09/27 08:11:15 UTC
(airflow) branch main updated: Refactor ``bucket.get_blob`` calls in ``GCSHook`` to handle validation for non-existent objects. (#42474)
This is an automated email from the ASF dual-hosted git repository.
shahar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b0234cbaf7 Refactor ``bucket.get_blob`` calls in ``GCSHook`` to handle validation for non-existent objects. (#42474)
b0234cbaf7 is described below
commit b0234cbaf73bdf28640b77e2f1c378d5c489b26a
Author: Jason <46...@users.noreply.github.com>
AuthorDate: Fri Sep 27 15:11:08 2024 +0700
Refactor ``bucket.get_blob`` calls in ``GCSHook`` to handle validation for non-existent objects. (#42474)
* Fix implicit exception in GCSToLocalFilesystemOperator when reading non-existing files.
* change method to internal method
* Fix unit test expectations
* Refactor bucket.get_blob calls in GCSHook to handle validation for non-existent objects
---
airflow/providers/google/cloud/hooks/gcs.py | 43 ++++++++++--------
newsfragments/42439.bugfix.rst | 1 +
tests/providers/google/cloud/hooks/test_gcs.py | 60 ++++++++++++++++++--------
3 files changed, 68 insertions(+), 36 deletions(-)
diff --git a/airflow/providers/google/cloud/hooks/gcs.py b/airflow/providers/google/cloud/hooks/gcs.py
index 83118e7079..fb48fcd190 100644
--- a/airflow/providers/google/cloud/hooks/gcs.py
+++ b/airflow/providers/google/cloud/hooks/gcs.py
@@ -59,6 +59,7 @@ if TYPE_CHECKING:
from aiohttp import ClientSession
from google.api_core.retry import Retry
+ from google.cloud.storage.blob import Blob
RT = TypeVar("RT")
@@ -597,11 +598,7 @@ class GCSHook(GoogleBaseHook):
:param object_name: The name of the blob to get updated time from the Google cloud
storage bucket.
"""
- client = self.get_conn()
- bucket = client.bucket(bucket_name)
- blob = bucket.get_blob(blob_name=object_name)
- if blob is None:
- raise ValueError(f"Object ({object_name}) not found in Bucket ({bucket_name})")
+ blob = self._get_blob(bucket_name, object_name)
return blob.updated
def is_updated_after(self, bucket_name: str, object_name: str, ts: datetime) -> bool:
@@ -957,19 +954,35 @@ class GCSHook(GoogleBaseHook):
break
return ids
- def get_size(self, bucket_name: str, object_name: str) -> int:
+ def _get_blob(self, bucket_name: str, object_name: str) -> Blob:
"""
- Get the size of a file in Google Cloud Storage.
+ Get a blob object in Google Cloud Storage.
:param bucket_name: The Google Cloud Storage bucket where the blob_name is.
:param object_name: The name of the object to check in the Google
cloud storage bucket_name.
"""
- self.log.info("Checking the file size of object: %s in bucket_name: %s", object_name, bucket_name)
client = self.get_conn()
bucket = client.bucket(bucket_name)
blob = bucket.get_blob(blob_name=object_name)
+
+ if blob is None:
+ raise ValueError(f"Object ({object_name}) not found in Bucket ({bucket_name})")
+
+ return blob
+
+ def get_size(self, bucket_name: str, object_name: str) -> int:
+ """
+ Get the size of a file in Google Cloud Storage.
+
+ :param bucket_name: The Google Cloud Storage bucket where the blob_name is.
+ :param object_name: The name of the object to check in the Google
+ cloud storage bucket_name.
+
+ """
+ self.log.info("Checking the file size of object: %s in bucket_name: %s", object_name, bucket_name)
+ blob = self._get_blob(bucket_name, object_name)
blob_size = blob.size
self.log.info("The file size of %s is %s bytes.", object_name, blob_size)
return blob_size
@@ -987,9 +1000,7 @@ class GCSHook(GoogleBaseHook):
object_name,
bucket_name,
)
- client = self.get_conn()
- bucket = client.bucket(bucket_name)
- blob = bucket.get_blob(blob_name=object_name)
+ blob = self._get_blob(bucket_name, object_name)
blob_crc32c = blob.crc32c
self.log.info("The crc32c checksum of %s is %s", object_name, blob_crc32c)
return blob_crc32c
@@ -1003,9 +1014,7 @@ class GCSHook(GoogleBaseHook):
storage bucket_name.
"""
self.log.info("Retrieving the MD5 hash of object: %s in bucket: %s", object_name, bucket_name)
- client = self.get_conn()
- bucket = client.bucket(bucket_name)
- blob = bucket.get_blob(blob_name=object_name)
+ blob = self._get_blob(bucket_name, object_name)
blob_md5hash = blob.md5_hash
self.log.info("The md5Hash of %s is %s", object_name, blob_md5hash)
return blob_md5hash
@@ -1019,11 +1028,7 @@ class GCSHook(GoogleBaseHook):
:return: The metadata associated with the object
"""
self.log.info("Retrieving the metadata dict of object (%s) in bucket (%s)", object_name, bucket_name)
- client = self.get_conn()
- bucket = client.bucket(bucket_name)
- blob = bucket.get_blob(blob_name=object_name)
- if blob is None:
- raise ValueError("Object (%s) not found in bucket (%s)", object_name, bucket_name)
+ blob = self._get_blob(bucket_name, object_name)
blob_metadata = blob.metadata
if blob_metadata:
self.log.info("Retrieved metadata of object (%s) with %s fields", object_name, len(blob_metadata))
diff --git a/newsfragments/42439.bugfix.rst b/newsfragments/42439.bugfix.rst
new file mode 100644
index 0000000000..4ecd73eb63
--- /dev/null
+++ b/newsfragments/42439.bugfix.rst
@@ -0,0 +1 @@
+Refactor ``bucket.get_blob`` calls in ``GCSHook`` to handle validation for non-existent objects.
diff --git a/tests/providers/google/cloud/hooks/test_gcs.py b/tests/providers/google/cloud/hooks/test_gcs.py
index 5dee8cbf8c..5d2735834a 100644
--- a/tests/providers/google/cloud/hooks/test_gcs.py
+++ b/tests/providers/google/cloud/hooks/test_gcs.py
@@ -523,57 +523,57 @@ class TestGCSHook:
mock_service.return_value.bucket.return_value.delete.assert_called_once()
assert "Bucket test bucket not exist" in caplog.text
- @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+ @mock.patch(GCS_STRING.format("GCSHook._get_blob"))
def test_object_get_size(self, mock_service):
test_bucket = "test_bucket"
test_object = "test_object"
returned_file_size = 1200
- bucket_method = mock_service.return_value.bucket
- get_blob_method = bucket_method.return_value.get_blob
- get_blob_method.return_value.size = returned_file_size
+ mock_blob = MagicMock()
+ mock_blob.size = returned_file_size
+ mock_service.return_value = mock_blob
response = self.gcs_hook.get_size(bucket_name=test_bucket, object_name=test_object)
assert response == returned_file_size
- @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+ @mock.patch(GCS_STRING.format("GCSHook._get_blob"))
def test_object_get_crc32c(self, mock_service):
test_bucket = "test_bucket"
test_object = "test_object"
returned_file_crc32c = "xgdNfQ=="
- bucket_method = mock_service.return_value.bucket
- get_blob_method = bucket_method.return_value.get_blob
- get_blob_method.return_value.crc32c = returned_file_crc32c
+ mock_blob = MagicMock()
+ mock_blob.crc32c = returned_file_crc32c
+ mock_service.return_value = mock_blob
response = self.gcs_hook.get_crc32c(bucket_name=test_bucket, object_name=test_object)
assert response == returned_file_crc32c
- @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+ @mock.patch(GCS_STRING.format("GCSHook._get_blob"))
def test_object_get_md5hash(self, mock_service):
test_bucket = "test_bucket"
test_object = "test_object"
returned_file_md5hash = "leYUJBUWrRtks1UeUFONJQ=="
- bucket_method = mock_service.return_value.bucket
- get_blob_method = bucket_method.return_value.get_blob
- get_blob_method.return_value.md5_hash = returned_file_md5hash
+ mock_blob = MagicMock()
+ mock_blob.md5_hash = returned_file_md5hash
+ mock_service.return_value = mock_blob
response = self.gcs_hook.get_md5hash(bucket_name=test_bucket, object_name=test_object)
assert response == returned_file_md5hash
- @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+ @mock.patch(GCS_STRING.format("GCSHook._get_blob"))
def test_object_get_metadata(self, mock_service):
test_bucket = "test_bucket"
test_object = "test_object"
returned_file_metadata = {"test_metadata_key": "test_metadata_val"}
- bucket_method = mock_service.return_value.bucket
- get_blob_method = bucket_method.return_value.get_blob
- get_blob_method.return_value.metadata = returned_file_metadata
+ mock_blob = MagicMock()
+ mock_blob.metadata = returned_file_metadata
+ mock_service.return_value = mock_blob
response = self.gcs_hook.get_metadata(bucket_name=test_bucket, object_name=test_object)
@@ -588,7 +588,7 @@ class TestGCSHook:
get_blob_method = bucket_method.return_value.get_blob
get_blob_method.return_value = None
- with pytest.raises(ValueError, match=r"Object \((.*?)\) not found in bucket \((.*?)\)"):
+ with pytest.raises(ValueError, match=r"Object \((.*?)\) not found in Bucket \((.*?)\)"):
self.gcs_hook.get_metadata(bucket_name=test_bucket, object_name=test_object)
@mock.patch("google.cloud.storage.Bucket")
@@ -1509,6 +1509,32 @@ class TestSyncGcsHook:
mock_rewrite.assert_not_called()
mock_copy.assert_not_called()
+ @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+ def test_object_get_blob(self, mock_service):
+ test_bucket = "test_bucket"
+ test_object = "test_object"
+ mock_blob = mock.MagicMock()
+
+ bucket_method = mock_service.return_value.bucket
+ get_blob_method = bucket_method.return_value.get_blob
+ get_blob_method.return_value = mock_blob
+
+ response = self.gcs_hook._get_blob(bucket_name=test_bucket, object_name=test_object)
+
+ assert response == mock_blob
+
+ @mock.patch(GCS_STRING.format("GCSHook.get_conn"))
+ def test_nonexisting_object_get_blob(self, mock_service):
+ test_bucket = "test_bucket"
+ test_object = "test_object"
+
+ bucket_method = mock_service.return_value.bucket
+ get_blob_method = bucket_method.return_value.get_blob
+ get_blob_method.return_value = None
+
+ with pytest.raises(ValueError, match=r"Object \((.*?)\) not found in Bucket \((.*?)\)"):
+ self.gcs_hook._get_blob(bucket_name=test_bucket, object_name=test_object)
+
def _create_blob(
self,
name: str,