You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/07/03 09:46:02 UTC
(airflow) branch main updated: Revert "Fix `GCSToGCSOperator` behavior difference for moving single object (#40162)" (#40577)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c5c50cc07f Revert "Fix `GCSToGCSOperator` behavior difference for moving single object (#40162)" (#40577)
c5c50cc07f is described below
commit c5c50cc07f6fcd704981139beb54095d8b9938c7
Author: Maksim <ma...@google.com>
AuthorDate: Wed Jul 3 02:45:56 2024 -0700
Revert "Fix `GCSToGCSOperator` behavior difference for moving single object (#40162)" (#40577)
This reverts commit 2f2796f675e238e0b266b8d6e4fdfe5c0b7d1bf3.
---
.../providers/google/cloud/transfers/gcs_to_gcs.py | 18 ++++++---
.../google/cloud/transfers/test_gcs_to_gcs.py | 46 ----------------------
2 files changed, 12 insertions(+), 52 deletions(-)
diff --git a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py
index 7fb12dbf09..0b3d330b65 100644
--- a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py
+++ b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py
@@ -408,9 +408,20 @@ class GCSToGCSOperator(BaseOperator):
msg = f"{prefix} does not exist in bucket {self.source_bucket}"
self.log.warning(msg)
raise AirflowException(msg)
+ if len(objects) == 1 and objects[0][-1] != "/":
+ self._copy_file(hook=hook, source_object=objects[0])
elif len(objects):
self._copy_multiple_objects(hook=hook, source_objects=objects, prefix=prefix)
+ def _copy_file(self, hook, source_object):
+ destination_object = self.destination_object or source_object
+ if self.destination_object and self.destination_object[-1] == "/":
+ file_name = source_object.split("/")[-1]
+ destination_object += file_name
+ self._copy_single_object(
+ hook=hook, source_object=source_object, destination_object=destination_object
+ )
+
def _copy_multiple_objects(self, hook, source_objects, prefix):
# Check whether the prefix is a root directory for all the rest of objects.
_pref = prefix.rstrip("/")
@@ -430,12 +441,7 @@ class GCSToGCSOperator(BaseOperator):
destination_object = source_obj
else:
file_name_postfix = source_obj.replace(base_path, "", 1)
-
- destination_object = (
- self.destination_object.rstrip("/")[0 : self.destination_object.rfind("/")]
- + "/"
- + file_name_postfix
- )
+ destination_object = self.destination_object.rstrip("/") + "/" + file_name_postfix
self._copy_single_object(
hook=hook, source_object=source_obj, destination_object=destination_object
diff --git a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py b/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py
index 4f63326af7..97e16e7366 100644
--- a/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py
+++ b/tests/providers/google/cloud/transfers/test_gcs_to_gcs.py
@@ -742,52 +742,6 @@ class TestGoogleCloudStorageToCloudStorageOperator:
["source/foo.txt"],
["{prefix}/foo.txt"],
),
- (
- ["source/sub1/sub2/sub3/file.txt"],
- "source/",
- None,
- False,
- ["source/sub1/sub2/sub3/file.txt"],
- ["{prefix}/sub1/sub2/sub3/file.txt"],
- ),
- (
- ["source/sub1/sub2/sub3/file.txt", "source/sub1/sub2/sub3/file2.txt"],
- "source/",
- None,
- False,
- ["source/sub1/sub2/sub3/file.txt", "source/sub1/sub2/sub3/file2.txt"],
- ["{prefix}/sub1/sub2/sub3/file.txt", "{prefix}/sub1/sub2/sub3/file2.txt"],
- ),
- (
- [f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt"],
- f"{DESTINATION_OBJECT_PREFIX}",
- None,
- False,
- [f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt"],
- ["{prefix}/sub1/sub2/sub3/file.txt"],
- ),
- (
- [f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt"],
- f"{DESTINATION_OBJECT_PREFIX}/",
- None,
- False,
- [f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt"],
- ["{prefix}/sub1/sub2/sub3/file.txt"],
- ),
- (
- [
- f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt",
- f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file2.txt",
- ],
- f"{DESTINATION_OBJECT_PREFIX}/",
- None,
- False,
- [
- f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file.txt",
- f"{DESTINATION_OBJECT_PREFIX}/sub1/sub2/sub3/file2.txt",
- ],
- ["{prefix}/sub1/sub2/sub3/file.txt", "{prefix}/sub1/sub2/sub3/file2.txt"],
- ),
(
["source/foo.txt", "source/foo.txt.abc", "source/foo.txt/subfolder/file.txt"],
"source/foo.txt",