[arrow-rs] branch master updated: fix(object_store,gcp): test copy_if_not_exist (#3236)

[tu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git


The following commit(s) were added to refs/heads/master by this push:
     new de3828cd7 fix(object_store,gcp): test copy_if_not_exist (#3236)
de3828cd7 is described below

commit de3828cd71a17076147b07a796e4b97bc669648d
Author: Will Jones <wi...@gmail.com>
AuthorDate: Fri Dec 2 02:28:28 2022 -0800

    fix(object_store,gcp): test copy_if_not_exist (#3236)
    
    * fix(object_store,gcp): test copy_if_not_exist
    
    * doc: update GCS testing instructions
    
    * test: move copy test into non-local branch
    
    * Revert CONTENT_LENGTH change
    
    Co-authored-by: Raphael Taylor-Davies <r....@googlemail.com>
---
 object_store/CONTRIBUTING.md |  6 +++---
 object_store/src/gcp/mod.rs  | 35 +++++++++++++++++++++++++++++++----
 2 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/object_store/CONTRIBUTING.md b/object_store/CONTRIBUTING.md
index 4e6b3afe3..efcd5fe34 100644
--- a/object_store/CONTRIBUTING.md
+++ b/object_store/CONTRIBUTING.md
@@ -95,13 +95,13 @@ To test the GCS integration, we use [Fake GCS Server](https://github.com/fsouza/
 Startup the fake server:
 
 ```shell
-docker run -p 4443:4443 fsouza/fake-gcs-server
+docker run -p 4443:4443 fsouza/fake-gcs-server -scheme http
 ```
 
 Configure the account:
 ```shell
-curl --insecure -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "https://localhost:4443/storage/v1/b"
-echo '{"gcs_base_url": "https://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > /tmp/gcs.json
+curl -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b"
+echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > /tmp/gcs.json
 ```
 
 Now run the tests:
diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs
index 41d6696c1..f93cbde3d 100644
--- a/object_store/src/gcp/mod.rs
+++ b/object_store/src/gcp/mod.rs
@@ -123,6 +123,12 @@ enum Error {
 
     #[snafu(display("GCP credential error: {}", source))]
     Credential { source: credential::Error },
+
+    #[snafu(display("Already exists: {}", path))]
+    AlreadyExists {
+        source: crate::client::retry::Error,
+        path: String,
+    },
 }
 
 impl From<Error> for super::Error {
@@ -138,6 +144,10 @@ impl From<Error> for super::Error {
                     source: Box::new(source),
                 }
             }
+            Error::AlreadyExists { source, path } => Self::AlreadyExists {
+                source: Box::new(source),
+                path,
+            },
             _ => Self::Generic {
                 store: "GCS",
                 source: Box::new(err),
@@ -419,8 +429,22 @@ impl GoogleCloudStorageClient {
             .bearer_auth(token)
             .send_retry(&self.retry_config)
             .await
-            .context(CopyRequestSnafu {
-                path: from.as_ref(),
+            .map_err(|err| {
+                if err
+                    .status()
+                    .map(|status| status == reqwest::StatusCode::PRECONDITION_FAILED)
+                    .unwrap_or_else(|| false)
+                {
+                    Error::AlreadyExists {
+                        source: err,
+                        path: to.to_string(),
+                    }
+                } else {
+                    Error::CopyRequest {
+                        source: err,
+                        path: from.to_string(),
+                    }
+                }
             })?;
 
         Ok(())
@@ -880,8 +904,8 @@ mod test {
 
     use crate::{
         tests::{
-            get_nonexistent_object, list_uses_directories_correctly, list_with_delimiter,
-            put_get_delete_list, rename_and_copy, stream_get,
+            copy_if_not_exists, get_nonexistent_object, list_uses_directories_correctly,
+            list_with_delimiter, put_get_delete_list, rename_and_copy, stream_get,
         },
         Error as ObjectStoreError, ObjectStore,
     };
@@ -946,6 +970,9 @@ mod test {
         list_with_delimiter(&integration).await;
         rename_and_copy(&integration).await;
         if integration.client.base_url == default_gcs_base_url() {
+            // Fake GCS server doesn't currently honor ifGenerationMatch
+            // https://github.com/fsouza/fake-gcs-server/issues/994
+            copy_if_not_exists(&integration).await;
             // Fake GCS server does not yet implement XML Multipart uploads
             // https://github.com/fsouza/fake-gcs-server/issues/852
             stream_get(&integration).await;