You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/06/19 22:08:03 UTC
[airflow] branch main updated: GCSDeleteObjectsOperator empty prefix bug fix (#24353)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e7a1c50d62 GCSDeleteObjectsOperator empty prefix bug fix (#24353)
e7a1c50d62 is described below
commit e7a1c50d62680a521ef90a424b7eff03635081d5
Author: Daniel Lavedonio de Lima <da...@gmail.com>
AuthorDate: Sun Jun 19 19:07:56 2022 -0300
GCSDeleteObjectsOperator empty prefix bug fix (#24353)
---
airflow/providers/google/cloud/operators/gcs.py | 7 +++++--
tests/providers/google/cloud/operators/test_gcs.py | 23 +++++++++++++++++++---
2 files changed, 25 insertions(+), 5 deletions(-)
diff --git a/airflow/providers/google/cloud/operators/gcs.py b/airflow/providers/google/cloud/operators/gcs.py
index bfc2d96919..7c58ee7c5a 100644
--- a/airflow/providers/google/cloud/operators/gcs.py
+++ b/airflow/providers/google/cloud/operators/gcs.py
@@ -305,8 +305,11 @@ class GCSDeleteObjectsOperator(BaseOperator):
self.delegate_to = delegate_to
self.impersonation_chain = impersonation_chain
- if not objects and not prefix:
- raise ValueError("Either object or prefix should be set. Both are None")
+ if objects is None and prefix is None:
+ err_message = "(Task {task_id}) Either object or prefix should be set. Both are None.".format(
+ **kwargs
+ )
+ raise ValueError(err_message)
super().__init__(**kwargs)
diff --git a/tests/providers/google/cloud/operators/test_gcs.py b/tests/providers/google/cloud/operators/test_gcs.py
index 3d6cba0374..23cc280d56 100644
--- a/tests/providers/google/cloud/operators/test_gcs.py
+++ b/tests/providers/google/cloud/operators/test_gcs.py
@@ -38,7 +38,7 @@ TEST_BUCKET = "test-bucket"
TEST_PROJECT = "test-project"
DELIMITER = ".csv"
PREFIX = "TEST"
-MOCK_FILES = ["TEST1.csv", "TEST2.csv", "TEST3.csv"]
+MOCK_FILES = ["TEST1.csv", "TEST2.csv", "TEST3.csv", "OTHERTEST1.csv"]
TEST_OBJECT = "dir1/test-object"
LOCAL_FILE_PATH = "/home/airflow/gcp/test-object"
IMPERSONATION_CHAIN = ["ACCOUNT_1", "ACCOUNT_2", "ACCOUNT_3"]
@@ -108,7 +108,7 @@ class TestGoogleCloudStorageAcl(unittest.TestCase):
)
-class TestGoogleCloudStorageDeleteOperator(unittest.TestCase):
+class TestGCSDeleteObjectsOperator(unittest.TestCase):
@mock.patch("airflow.providers.google.cloud.operators.gcs.GCSHook")
def test_delete_objects(self, mock_hook):
operator = GCSDeleteObjectsOperator(task_id=TASK_ID, bucket_name=TEST_BUCKET, objects=MOCK_FILES[0:2])
@@ -125,7 +125,7 @@ class TestGoogleCloudStorageDeleteOperator(unittest.TestCase):
@mock.patch("airflow.providers.google.cloud.operators.gcs.GCSHook")
def test_delete_prefix(self, mock_hook):
- mock_hook.return_value.list.return_value = MOCK_FILES[1:3]
+ mock_hook.return_value.list.return_value = MOCK_FILES[1:4]
operator = GCSDeleteObjectsOperator(task_id=TASK_ID, bucket_name=TEST_BUCKET, prefix=PREFIX)
operator.execute(None)
@@ -138,6 +138,23 @@ class TestGoogleCloudStorageDeleteOperator(unittest.TestCase):
any_order=True,
)
+ @mock.patch("airflow.providers.google.cloud.operators.gcs.GCSHook")
+ def test_delete_prefix_as_empty_string(self, mock_hook):
+ mock_hook.return_value.list.return_value = MOCK_FILES[0:4]
+ operator = GCSDeleteObjectsOperator(task_id=TASK_ID, bucket_name=TEST_BUCKET, prefix="")
+
+ operator.execute(None)
+ mock_hook.return_value.list.assert_called_once_with(bucket_name=TEST_BUCKET, prefix="")
+ mock_hook.return_value.delete.assert_has_calls(
+ calls=[
+ mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[0]),
+ mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[1]),
+ mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[2]),
+ mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[3]),
+ ],
+ any_order=True,
+ )
+
class TestGoogleCloudStorageListOperator(unittest.TestCase):
@mock.patch("airflow.providers.google.cloud.operators.gcs.GCSHook")