You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/11/22 11:52:42 UTC
(incubator-opendal) 15/23: Fix gcs
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch fix-stat
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit eb953e63e52f47935d25494e86ced18edb78819d
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed Nov 22 19:09:42 2023 +0800
Fix gcs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/gcs/backend.rs | 32 ++++---
core/src/services/gcs/core.rs | 175 ++++++++++++++++++++++++++++++++++++++
core/src/services/gcs/lister.rs | 177 +--------------------------------------
3 files changed, 198 insertions(+), 186 deletions(-)
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index d528479dc..b3cc960a3 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -30,7 +30,7 @@ use reqsign::GoogleTokenLoader;
use serde::Deserialize;
use serde_json;
-use super::core::GcsCore;
+use super::core::*;
use super::error::parse_error;
use super::lister::GcsLister;
use super::writer::GcsWriter;
@@ -422,21 +422,35 @@ impl Accessor for GcsBackend {
return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
}
+ if path.ends_with('/') {
+ let resp = self
+ .core
+ .gcs_list_objects(path, "", "", Some(1), None)
+ .await?;
+
+ let bs = resp.into_body().bytes().await?;
+ let output: ListResponse =
+ serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?;
+
+ return if !output.items.is_empty() {
+ Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
+ } else {
+ Err(
+ Error::new(ErrorKind::NotFound, "The directory is not found")
+ .with_context("path", path),
+ )
+ };
+ }
+
let resp = self.core.gcs_get_object_metadata(path, &args).await?;
if resp.status().is_success() {
- // read http response body
let slc = resp.into_body().bytes().await?;
let meta: GetObjectJsonResponse =
serde_json::from_slice(&slc).map_err(new_json_deserialize_error)?;
- let mode = if path.ends_with('/') {
- EntryMode::DIR
- } else {
- EntryMode::FILE
- };
- let mut m = Metadata::new(mode);
+ let mut m = Metadata::new(EntryMode::FILE);
m.set_etag(&meta.etag);
m.set_content_md5(&meta.md5_hash);
@@ -453,8 +467,6 @@ impl Accessor for GcsBackend {
m.set_last_modified(parse_datetime_from_rfc3339(&meta.updated)?);
Ok(RpStat::new(m))
- } else if resp.status() == StatusCode::NOT_FOUND && path.ends_with('/') {
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
} else {
Err(parse_error(resp).await?)
}
diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs
index 5720db2cb..df112e504 100644
--- a/core/src/services/gcs/core.rs
+++ b/core/src/services/gcs/core.rs
@@ -37,6 +37,7 @@ use reqsign::GoogleCredentialLoader;
use reqsign::GoogleSigner;
use reqsign::GoogleToken;
use reqsign::GoogleTokenLoader;
+use serde::Deserialize;
use serde_json::json;
use super::uri::percent_encode_path;
@@ -597,3 +598,177 @@ impl GcsCore {
self.send(req).await
}
}
+
+/// Response JSON from GCS list objects API.
+///
+/// refer to https://cloud.google.com/storage/docs/json_api/v1/objects/list for details
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
+pub struct ListResponse {
+ /// The continuation token.
+ ///
+ /// If this is the last page of results, then no continuation token is returned.
+ pub next_page_token: Option<String>,
+ /// Object name prefixes for objects that matched the listing request
+ /// but were excluded from [items] because of a delimiter.
+ pub prefixes: Vec<String>,
+ /// The list of objects, ordered lexicographically by name.
+ pub items: Vec<ListResponseItem>,
+}
+
+#[derive(Default, Debug, Eq, PartialEq, Deserialize)]
+#[serde(default, rename_all = "camelCase")]
+pub struct ListResponseItem {
+ pub name: String,
+ pub size: String,
+ // metadata
+ pub etag: String,
+ pub md5_hash: String,
+ pub updated: String,
+ pub content_type: String,
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_deserialize_list_response() {
+ let content = r#"
+ {
+ "kind": "storage#objects",
+ "prefixes": [
+ "dir/",
+ "test/"
+ ],
+ "items": [
+ {
+ "kind": "storage#object",
+ "id": "example/1.png/1660563214863653",
+ "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png",
+ "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media",
+ "name": "1.png",
+ "bucket": "example",
+ "generation": "1660563214863653",
+ "metageneration": "1",
+ "contentType": "image/png",
+ "storageClass": "STANDARD",
+ "size": "56535",
+ "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==",
+ "crc32c": "j/un9g==",
+ "etag": "CKWasoTgyPkCEAE=",
+ "timeCreated": "2022-08-15T11:33:34.866Z",
+ "updated": "2022-08-15T11:33:34.866Z",
+ "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z"
+ },
+ {
+ "kind": "storage#object",
+ "id": "example/2.png/1660563214883337",
+ "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png",
+ "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media",
+ "name": "2.png",
+ "bucket": "example",
+ "generation": "1660563214883337",
+ "metageneration": "1",
+ "contentType": "image/png",
+ "storageClass": "STANDARD",
+ "size": "45506",
+ "md5Hash": "e6LsGusU7pFJZk+114NV1g==",
+ "crc32c": "L00QAg==",
+ "etag": "CIm0s4TgyPkCEAE=",
+ "timeCreated": "2022-08-15T11:33:34.886Z",
+ "updated": "2022-08-15T11:33:34.886Z",
+ "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z"
+ }
+ ]
+}
+ "#;
+
+ let output: ListResponse =
+ serde_json::from_str(content).expect("JSON deserialize must succeed");
+ assert!(output.next_page_token.is_none());
+ assert_eq!(output.items.len(), 2);
+ assert_eq!(output.items[0].name, "1.png");
+ assert_eq!(output.items[0].size, "56535");
+ assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg==");
+ assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE=");
+ assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z");
+ assert_eq!(output.items[1].name, "2.png");
+ assert_eq!(output.items[1].size, "45506");
+ assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g==");
+ assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE=");
+ assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z");
+ assert_eq!(output.items[1].content_type, "image/png");
+ assert_eq!(output.prefixes, vec!["dir/", "test/"])
+ }
+
+ #[test]
+ fn test_deserialize_list_response_with_next_page_token() {
+ let content = r#"
+ {
+ "kind": "storage#objects",
+ "prefixes": [
+ "dir/",
+ "test/"
+ ],
+ "nextPageToken": "CgYxMC5wbmc=",
+ "items": [
+ {
+ "kind": "storage#object",
+ "id": "example/1.png/1660563214863653",
+ "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png",
+ "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media",
+ "name": "1.png",
+ "bucket": "example",
+ "generation": "1660563214863653",
+ "metageneration": "1",
+ "contentType": "image/png",
+ "storageClass": "STANDARD",
+ "size": "56535",
+ "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==",
+ "crc32c": "j/un9g==",
+ "etag": "CKWasoTgyPkCEAE=",
+ "timeCreated": "2022-08-15T11:33:34.866Z",
+ "updated": "2022-08-15T11:33:34.866Z",
+ "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z"
+ },
+ {
+ "kind": "storage#object",
+ "id": "example/2.png/1660563214883337",
+ "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png",
+ "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media",
+ "name": "2.png",
+ "bucket": "example",
+ "generation": "1660563214883337",
+ "metageneration": "1",
+ "contentType": "image/png",
+ "storageClass": "STANDARD",
+ "size": "45506",
+ "md5Hash": "e6LsGusU7pFJZk+114NV1g==",
+ "crc32c": "L00QAg==",
+ "etag": "CIm0s4TgyPkCEAE=",
+ "timeCreated": "2022-08-15T11:33:34.886Z",
+ "updated": "2022-08-15T11:33:34.886Z",
+ "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z"
+ }
+ ]
+}
+ "#;
+
+ let output: ListResponse =
+ serde_json::from_str(content).expect("JSON deserialize must succeed");
+ assert_eq!(output.next_page_token, Some("CgYxMC5wbmc=".to_string()));
+ assert_eq!(output.items.len(), 2);
+ assert_eq!(output.items[0].name, "1.png");
+ assert_eq!(output.items[0].size, "56535");
+ assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg==");
+ assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE=");
+ assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z");
+ assert_eq!(output.items[1].name, "2.png");
+ assert_eq!(output.items[1].size, "45506");
+ assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g==");
+ assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE=");
+ assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z");
+ assert_eq!(output.prefixes, vec!["dir/", "test/"])
+ }
+}
diff --git a/core/src/services/gcs/lister.rs b/core/src/services/gcs/lister.rs
index 244ca2a93..b9b61f542 100644
--- a/core/src/services/gcs/lister.rs
+++ b/core/src/services/gcs/lister.rs
@@ -18,10 +18,9 @@
use std::sync::Arc;
use async_trait::async_trait;
-use serde::Deserialize;
use serde_json;
-use super::core::GcsCore;
+use super::core::*;
use super::error::parse_error;
use crate::raw::*;
use crate::*;
@@ -137,177 +136,3 @@ impl oio::PageList for GcsLister {
Ok(())
}
}
-
-/// Response JSON from GCS list objects API.
-///
-/// refer to https://cloud.google.com/storage/docs/json_api/v1/objects/list for details
-#[derive(Default, Debug, Deserialize)]
-#[serde(default, rename_all = "camelCase")]
-struct ListResponse {
- /// The continuation token.
- ///
- /// If this is the last page of results, then no continuation token is returned.
- next_page_token: Option<String>,
- /// Object name prefixes for objects that matched the listing request
- /// but were excluded from [items] because of a delimiter.
- prefixes: Vec<String>,
- /// The list of objects, ordered lexicographically by name.
- items: Vec<ListResponseItem>,
-}
-
-#[derive(Default, Debug, Eq, PartialEq, Deserialize)]
-#[serde(default, rename_all = "camelCase")]
-struct ListResponseItem {
- name: String,
- size: String,
- // metadata
- etag: String,
- md5_hash: String,
- updated: String,
- content_type: String,
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_deserialize_list_response() {
- let content = r#"
- {
- "kind": "storage#objects",
- "prefixes": [
- "dir/",
- "test/"
- ],
- "items": [
- {
- "kind": "storage#object",
- "id": "example/1.png/1660563214863653",
- "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png",
- "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media",
- "name": "1.png",
- "bucket": "example",
- "generation": "1660563214863653",
- "metageneration": "1",
- "contentType": "image/png",
- "storageClass": "STANDARD",
- "size": "56535",
- "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==",
- "crc32c": "j/un9g==",
- "etag": "CKWasoTgyPkCEAE=",
- "timeCreated": "2022-08-15T11:33:34.866Z",
- "updated": "2022-08-15T11:33:34.866Z",
- "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z"
- },
- {
- "kind": "storage#object",
- "id": "example/2.png/1660563214883337",
- "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png",
- "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media",
- "name": "2.png",
- "bucket": "example",
- "generation": "1660563214883337",
- "metageneration": "1",
- "contentType": "image/png",
- "storageClass": "STANDARD",
- "size": "45506",
- "md5Hash": "e6LsGusU7pFJZk+114NV1g==",
- "crc32c": "L00QAg==",
- "etag": "CIm0s4TgyPkCEAE=",
- "timeCreated": "2022-08-15T11:33:34.886Z",
- "updated": "2022-08-15T11:33:34.886Z",
- "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z"
- }
- ]
-}
- "#;
-
- let output: ListResponse =
- serde_json::from_str(content).expect("JSON deserialize must succeed");
- assert!(output.next_page_token.is_none());
- assert_eq!(output.items.len(), 2);
- assert_eq!(output.items[0].name, "1.png");
- assert_eq!(output.items[0].size, "56535");
- assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg==");
- assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE=");
- assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z");
- assert_eq!(output.items[1].name, "2.png");
- assert_eq!(output.items[1].size, "45506");
- assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g==");
- assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE=");
- assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z");
- assert_eq!(output.items[1].content_type, "image/png");
- assert_eq!(output.prefixes, vec!["dir/", "test/"])
- }
-
- #[test]
- fn test_deserialize_list_response_with_next_page_token() {
- let content = r#"
- {
- "kind": "storage#objects",
- "prefixes": [
- "dir/",
- "test/"
- ],
- "nextPageToken": "CgYxMC5wbmc=",
- "items": [
- {
- "kind": "storage#object",
- "id": "example/1.png/1660563214863653",
- "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/1.png",
- "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/1.png?generation=1660563214863653&alt=media",
- "name": "1.png",
- "bucket": "example",
- "generation": "1660563214863653",
- "metageneration": "1",
- "contentType": "image/png",
- "storageClass": "STANDARD",
- "size": "56535",
- "md5Hash": "fHcEH1vPwA6eTPqxuasXcg==",
- "crc32c": "j/un9g==",
- "etag": "CKWasoTgyPkCEAE=",
- "timeCreated": "2022-08-15T11:33:34.866Z",
- "updated": "2022-08-15T11:33:34.866Z",
- "timeStorageClassUpdated": "2022-08-15T11:33:34.866Z"
- },
- {
- "kind": "storage#object",
- "id": "example/2.png/1660563214883337",
- "selfLink": "https://www.googleapis.com/storage/v1/b/example/o/2.png",
- "mediaLink": "https://content-storage.googleapis.com/download/storage/v1/b/example/o/2.png?generation=1660563214883337&alt=media",
- "name": "2.png",
- "bucket": "example",
- "generation": "1660563214883337",
- "metageneration": "1",
- "contentType": "image/png",
- "storageClass": "STANDARD",
- "size": "45506",
- "md5Hash": "e6LsGusU7pFJZk+114NV1g==",
- "crc32c": "L00QAg==",
- "etag": "CIm0s4TgyPkCEAE=",
- "timeCreated": "2022-08-15T11:33:34.886Z",
- "updated": "2022-08-15T11:33:34.886Z",
- "timeStorageClassUpdated": "2022-08-15T11:33:34.886Z"
- }
- ]
-}
- "#;
-
- let output: ListResponse =
- serde_json::from_str(content).expect("JSON deserialize must succeed");
- assert_eq!(output.next_page_token, Some("CgYxMC5wbmc=".to_string()));
- assert_eq!(output.items.len(), 2);
- assert_eq!(output.items[0].name, "1.png");
- assert_eq!(output.items[0].size, "56535");
- assert_eq!(output.items[0].md5_hash, "fHcEH1vPwA6eTPqxuasXcg==");
- assert_eq!(output.items[0].etag, "CKWasoTgyPkCEAE=");
- assert_eq!(output.items[0].updated, "2022-08-15T11:33:34.866Z");
- assert_eq!(output.items[1].name, "2.png");
- assert_eq!(output.items[1].size, "45506");
- assert_eq!(output.items[1].md5_hash, "e6LsGusU7pFJZk+114NV1g==");
- assert_eq!(output.items[1].etag, "CIm0s4TgyPkCEAE=");
- assert_eq!(output.items[1].updated, "2022-08-15T11:33:34.886Z");
- assert_eq!(output.prefixes, vec!["dir/", "test/"])
- }
-}