You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/11/22 11:52:33 UTC
(incubator-opendal) 06/23: Implement for s3
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch fix-stat
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 744c5cf23bdc234228826922370f4027116f524d
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Nov 21 17:31:03 2023 +0800
Implement for s3
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/s3/backend.rs | 29 ++++++++--
core/src/services/s3/core.rs | 109 +++++++++++++++++++++++++++++++++++++
core/src/services/s3/lister.rs | 117 +---------------------------------------
3 files changed, 136 insertions(+), 119 deletions(-)
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 56663f522..bd87d726e 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -1102,6 +1102,32 @@ impl Accessor for S3Backend {
return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
}
+ if path.ends_with('/') {
+ let resp = self
+ .core
+ .s3_list_objects(path, "", "", Some(1), None)
+ .await?;
+
+ let status = resp.status();
+
+ return match status {
+ StatusCode::OK => {
+ let bs = resp.into_body().bytes().await?;
+ let output: Output = quick_xml::de::from_reader(bs.reader())
+ .map_err(new_xml_deserialize_error)?;
+ if !output.contents.is_empty() {
+ Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
+ } else {
+ Err(
+ Error::new(ErrorKind::NotFound, "The directory is not found")
+ .with_context("path", path),
+ )
+ }
+ }
+ _ => Err(parse_error(resp).await?),
+ };
+ }
+
let resp = self
.core
.s3_head_object(path, args.if_none_match(), args.if_match())
@@ -1111,9 +1137,6 @@ impl Accessor for S3Backend {
match status {
StatusCode::OK => parse_into_metadata(path, resp.headers()).map(RpStat::new),
- StatusCode::NOT_FOUND if path.ends_with('/') => {
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
- }
_ => Err(parse_error(resp).await?),
}
}
diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs
index 1021399e4..66dbfce72 100644
--- a/core/src/services/s3/core.rs
+++ b/core/src/services/s3/core.rs
@@ -761,6 +761,40 @@ pub struct DeleteObjectsResultError {
pub message: String,
}
+/// Output of ListBucket/ListObjects.
+///
+/// ## Note
+///
+/// Use `Option` in `is_truncated` and `next_continuation_token` to make
+/// the behavior more clear so that we can be compatible to more s3 services.
+///
+/// And enable `serde(default)` so that we can keep going even when some field
+/// is not exist.
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+pub struct Output {
+ pub is_truncated: Option<bool>,
+ pub next_continuation_token: Option<String>,
+ pub common_prefixes: Vec<OutputCommonPrefix>,
+ pub contents: Vec<OutputContent>,
+}
+
+#[derive(Default, Debug, Eq, PartialEq, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub struct OutputContent {
+ pub key: String,
+ pub size: u64,
+ pub last_modified: String,
+ #[serde(rename = "ETag")]
+ pub etag: Option<String>,
+}
+
+#[derive(Default, Debug, Eq, PartialEq, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub struct OutputCommonPrefix {
+ pub prefix: String,
+}
+
#[cfg(test)]
mod tests {
use bytes::Buf;
@@ -892,4 +926,79 @@ mod tests {
assert_eq!(out.error[0].code, "AccessDenied");
assert_eq!(out.error[0].message, "Access Denied");
}
+
+ #[test]
+ fn test_parse_list_output() {
+ let bs = bytes::Bytes::from(
+ r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+ <Name>example-bucket</Name>
+ <Prefix>photos/2006/</Prefix>
+ <KeyCount>3</KeyCount>
+ <MaxKeys>1000</MaxKeys>
+ <Delimiter>/</Delimiter>
+ <IsTruncated>false</IsTruncated>
+ <Contents>
+ <Key>photos/2006</Key>
+ <LastModified>2016-04-30T23:51:29.000Z</LastModified>
+ <ETag>"d41d8cd98f00b204e9800998ecf8427e"</ETag>
+ <Size>56</Size>
+ <StorageClass>STANDARD</StorageClass>
+ </Contents>
+ <Contents>
+ <Key>photos/2007</Key>
+ <LastModified>2016-04-30T23:51:29.000Z</LastModified>
+ <ETag>"d41d8cd98f00b204e9800998ecf8427e"</ETag>
+ <Size>100</Size>
+ <StorageClass>STANDARD</StorageClass>
+ </Contents>
+ <Contents>
+ <Key>photos/2008</Key>
+ <LastModified>2016-05-30T23:51:29.000Z</LastModified>
+ <Size>42</Size>
+ </Contents>
+
+ <CommonPrefixes>
+ <Prefix>photos/2006/February/</Prefix>
+ </CommonPrefixes>
+ <CommonPrefixes>
+ <Prefix>photos/2006/January/</Prefix>
+ </CommonPrefixes>
+</ListBucketResult>"#,
+ );
+
+ let out: Output = quick_xml::de::from_reader(bs.reader()).expect("must success");
+
+ assert!(!out.is_truncated.unwrap());
+ assert!(out.next_continuation_token.is_none());
+ assert_eq!(
+ out.common_prefixes
+ .iter()
+ .map(|v| v.prefix.clone())
+ .collect::<Vec<String>>(),
+ vec!["photos/2006/February/", "photos/2006/January/"]
+ );
+ assert_eq!(
+ out.contents,
+ vec![
+ OutputContent {
+ key: "photos/2006".to_string(),
+ size: 56,
+ etag: Some("\"d41d8cd98f00b204e9800998ecf8427e\"".to_string()),
+ last_modified: "2016-04-30T23:51:29.000Z".to_string(),
+ },
+ OutputContent {
+ key: "photos/2007".to_string(),
+ size: 100,
+ last_modified: "2016-04-30T23:51:29.000Z".to_string(),
+ etag: Some("\"d41d8cd98f00b204e9800998ecf8427e\"".to_string()),
+ },
+ OutputContent {
+ key: "photos/2008".to_string(),
+ size: 42,
+ last_modified: "2016-05-30T23:51:29.000Z".to_string(),
+ etag: None,
+ },
+ ]
+ )
+ }
}
diff --git a/core/src/services/s3/lister.rs b/core/src/services/s3/lister.rs
index 78d5e1ff3..cab972c1f 100644
--- a/core/src/services/s3/lister.rs
+++ b/core/src/services/s3/lister.rs
@@ -20,9 +20,8 @@ use std::sync::Arc;
use async_trait::async_trait;
use bytes::Buf;
use quick_xml::de;
-use serde::Deserialize;
-use super::core::S3Core;
+use super::core::{Output, S3Core};
use super::error::parse_error;
use crate::raw::*;
use crate::EntryMode;
@@ -138,117 +137,3 @@ impl oio::PageList for S3Lister {
Ok(())
}
}
-
-/// Output of ListBucket/ListObjects.
-///
-/// ## Note
-///
-/// Use `Option` in `is_truncated` and `next_continuation_token` to make
-/// the behavior more clear so that we can be compatible to more s3 services.
-///
-/// And enable `serde(default)` so that we can keep going even when some field
-/// is not exist.
-#[derive(Default, Debug, Deserialize)]
-#[serde(default, rename_all = "PascalCase")]
-struct Output {
- is_truncated: Option<bool>,
- next_continuation_token: Option<String>,
- common_prefixes: Vec<OutputCommonPrefix>,
- contents: Vec<OutputContent>,
-}
-
-#[derive(Default, Debug, Eq, PartialEq, Deserialize)]
-#[serde(rename_all = "PascalCase")]
-struct OutputContent {
- key: String,
- size: u64,
- last_modified: String,
- #[serde(rename = "ETag")]
- etag: Option<String>,
-}
-
-#[derive(Default, Debug, Eq, PartialEq, Deserialize)]
-#[serde(rename_all = "PascalCase")]
-struct OutputCommonPrefix {
- prefix: String,
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_parse_list_output() {
- let bs = bytes::Bytes::from(
- r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
- <Name>example-bucket</Name>
- <Prefix>photos/2006/</Prefix>
- <KeyCount>3</KeyCount>
- <MaxKeys>1000</MaxKeys>
- <Delimiter>/</Delimiter>
- <IsTruncated>false</IsTruncated>
- <Contents>
- <Key>photos/2006</Key>
- <LastModified>2016-04-30T23:51:29.000Z</LastModified>
- <ETag>"d41d8cd98f00b204e9800998ecf8427e"</ETag>
- <Size>56</Size>
- <StorageClass>STANDARD</StorageClass>
- </Contents>
- <Contents>
- <Key>photos/2007</Key>
- <LastModified>2016-04-30T23:51:29.000Z</LastModified>
- <ETag>"d41d8cd98f00b204e9800998ecf8427e"</ETag>
- <Size>100</Size>
- <StorageClass>STANDARD</StorageClass>
- </Contents>
- <Contents>
- <Key>photos/2008</Key>
- <LastModified>2016-05-30T23:51:29.000Z</LastModified>
- <Size>42</Size>
- </Contents>
-
- <CommonPrefixes>
- <Prefix>photos/2006/February/</Prefix>
- </CommonPrefixes>
- <CommonPrefixes>
- <Prefix>photos/2006/January/</Prefix>
- </CommonPrefixes>
-</ListBucketResult>"#,
- );
-
- let out: Output = de::from_reader(bs.reader()).expect("must success");
-
- assert!(!out.is_truncated.unwrap());
- assert!(out.next_continuation_token.is_none());
- assert_eq!(
- out.common_prefixes
- .iter()
- .map(|v| v.prefix.clone())
- .collect::<Vec<String>>(),
- vec!["photos/2006/February/", "photos/2006/January/"]
- );
- assert_eq!(
- out.contents,
- vec![
- OutputContent {
- key: "photos/2006".to_string(),
- size: 56,
- etag: Some("\"d41d8cd98f00b204e9800998ecf8427e\"".to_string()),
- last_modified: "2016-04-30T23:51:29.000Z".to_string(),
- },
- OutputContent {
- key: "photos/2007".to_string(),
- size: 100,
- last_modified: "2016-04-30T23:51:29.000Z".to_string(),
- etag: Some("\"d41d8cd98f00b204e9800998ecf8427e\"".to_string()),
- },
- OutputContent {
- key: "photos/2008".to_string(),
- size: 42,
- last_modified: "2016-05-30T23:51:29.000Z".to_string(),
- etag: None,
- },
- ]
- )
- }
-}