You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/04/25 16:26:08 UTC
[incubator-opendal] branch main updated: feat(services/gcs): Add start-after support for list (#2107)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 2554897a feat(services/gcs): Add start-after support for list (#2107)
2554897a is described below
commit 2554897a1e183869399a5fecae019b2728e8d044
Author: Lex Cao <le...@foxmail.com>
AuthorDate: Wed Apr 26 00:26:04 2023 +0800
feat(services/gcs): Add start-after support for list (#2107)
Closes #2106
This PR add new features for GCS:
- add `list_with_start_after` capability
- exclude the inclusive `start_after` itself
---
core/src/services/gcs/backend.rs | 12 ++++++++++--
core/src/services/gcs/core.rs | 7 +++++++
core/src/services/gcs/pager.rs | 29 ++++++++++++++++++++++++++---
core/src/services/s3/pager.rs | 2 ++
4 files changed, 45 insertions(+), 5 deletions(-)
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index 20a051c5..006c3b1b 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -385,6 +385,8 @@ impl Accessor for GcsBackend {
write_without_content_length: true,
list: true,
+ list_with_limit: true,
+ list_with_start_after: true,
scan: true,
copy: true,
@@ -502,14 +504,20 @@ impl Accessor for GcsBackend {
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
Ok((
RpList::default(),
- GcsPager::new(self.core.clone(), path, "/", args.limit()),
+ GcsPager::new(
+ self.core.clone(),
+ path,
+ "/",
+ args.limit(),
+ args.start_after(),
+ ),
))
}
async fn scan(&self, path: &str, args: OpScan) -> Result<(RpScan, Self::Pager)> {
Ok((
RpScan::default(),
- GcsPager::new(self.core.clone(), path, "", args.limit()),
+ GcsPager::new(self.core.clone(), path, "", args.limit(), None),
))
}
}
diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs
index aeb7f622..323cd015 100644
--- a/core/src/services/gcs/core.rs
+++ b/core/src/services/gcs/core.rs
@@ -298,6 +298,7 @@ impl GcsCore {
page_token: &str,
delimiter: &str,
limit: Option<usize>,
+ start_after: Option<String>,
) -> Result<Response<IncomingAsyncBody>> {
let p = build_abs_path(&self.root, path);
@@ -313,6 +314,12 @@ impl GcsCore {
if let Some(limit) = limit {
write!(url, "&maxResults={limit}").expect("write into string must succeed");
}
+ if let Some(start_after) = start_after {
+ let start_after = build_abs_path(&self.root, &start_after);
+ write!(url, "&startOffset={}", percent_encode_path(&start_after))
+ .expect("write into string must succeed");
+ }
+
if !page_token.is_empty() {
// NOTE:
//
diff --git a/core/src/services/gcs/pager.rs b/core/src/services/gcs/pager.rs
index bfa99981..e6cffd67 100644
--- a/core/src/services/gcs/pager.rs
+++ b/core/src/services/gcs/pager.rs
@@ -35,19 +35,30 @@ pub struct GcsPager {
delimiter: String,
limit: Option<usize>,
+ /// Filter results to objects whose names are lexicographically
+ /// **equal to or after** startOffset
+ start_after: Option<String>,
+
page_token: String,
done: bool,
}
impl GcsPager {
/// Generate a new directory walker
- pub fn new(core: Arc<GcsCore>, path: &str, delimiter: &str, limit: Option<usize>) -> Self {
+ pub fn new(
+ core: Arc<GcsCore>,
+ path: &str,
+ delimiter: &str,
+ limit: Option<usize>,
+ start_after: Option<&str>,
+ ) -> Self {
Self {
core,
path: path.to_string(),
delimiter: delimiter.to_string(),
limit,
+ start_after: start_after.map(String::from),
page_token: "".to_string(),
done: false,
@@ -64,7 +75,13 @@ impl oio::Page for GcsPager {
let resp = self
.core
- .gcs_list_objects(&self.path, &self.page_token, &self.delimiter, self.limit)
+ .gcs_list_objects(
+ &self.path,
+ &self.page_token,
+ &self.delimiter,
+ self.limit,
+ self.start_after.clone(),
+ )
.await?;
if !resp.status().is_success() {
@@ -97,6 +114,12 @@ impl oio::Page for GcsPager {
continue;
}
+ // exclude the inclusive start_after itself
+ let path = &build_rel_path(&self.core.root, &object.name);
+ if self.start_after.as_ref() == Some(path) {
+ continue;
+ }
+
let mut meta = Metadata::new(EntryMode::FILE);
// set metadata fields
@@ -113,7 +136,7 @@ impl oio::Page for GcsPager {
meta.set_last_modified(parse_datetime_from_rfc3339(object.updated.as_str())?);
- let de = oio::Entry::new(&build_rel_path(&self.core.root, &object.name), meta);
+ let de = oio::Entry::new(path, meta);
entries.push(de);
}
diff --git a/core/src/services/s3/pager.rs b/core/src/services/s3/pager.rs
index c8e98c16..3c5b15ed 100644
--- a/core/src/services/s3/pager.rs
+++ b/core/src/services/s3/pager.rs
@@ -35,6 +35,8 @@ pub struct S3Pager {
path: String,
delimiter: String,
limit: Option<usize>,
+
+ /// Amazon S3 starts listing **after** this specified key
start_after: Option<String>,
token: String,