You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/04/25 16:26:08 UTC

[incubator-opendal] branch main updated: feat(services/gcs): Add start-after support for list (#2107)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 2554897a feat(services/gcs): Add start-after support for list (#2107)
2554897a is described below

commit 2554897a1e183869399a5fecae019b2728e8d044
Author: Lex Cao <le...@foxmail.com>
AuthorDate: Wed Apr 26 00:26:04 2023 +0800

    feat(services/gcs): Add start-after support for list (#2107)
    
    Closes #2106
    This PR add new features for GCS:
    - add `list_with_start_after` capability
    - exclude the inclusive `start_after` itself
---
 core/src/services/gcs/backend.rs | 12 ++++++++++--
 core/src/services/gcs/core.rs    |  7 +++++++
 core/src/services/gcs/pager.rs   | 29 ++++++++++++++++++++++++++---
 core/src/services/s3/pager.rs    |  2 ++
 4 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index 20a051c5..006c3b1b 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -385,6 +385,8 @@ impl Accessor for GcsBackend {
                 write_without_content_length: true,
 
                 list: true,
+                list_with_limit: true,
+                list_with_start_after: true,
                 scan: true,
                 copy: true,
 
@@ -502,14 +504,20 @@ impl Accessor for GcsBackend {
     async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
         Ok((
             RpList::default(),
-            GcsPager::new(self.core.clone(), path, "/", args.limit()),
+            GcsPager::new(
+                self.core.clone(),
+                path,
+                "/",
+                args.limit(),
+                args.start_after(),
+            ),
         ))
     }
 
     async fn scan(&self, path: &str, args: OpScan) -> Result<(RpScan, Self::Pager)> {
         Ok((
             RpScan::default(),
-            GcsPager::new(self.core.clone(), path, "", args.limit()),
+            GcsPager::new(self.core.clone(), path, "", args.limit(), None),
         ))
     }
 }
diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs
index aeb7f622..323cd015 100644
--- a/core/src/services/gcs/core.rs
+++ b/core/src/services/gcs/core.rs
@@ -298,6 +298,7 @@ impl GcsCore {
         page_token: &str,
         delimiter: &str,
         limit: Option<usize>,
+        start_after: Option<String>,
     ) -> Result<Response<IncomingAsyncBody>> {
         let p = build_abs_path(&self.root, path);
 
@@ -313,6 +314,12 @@ impl GcsCore {
         if let Some(limit) = limit {
             write!(url, "&maxResults={limit}").expect("write into string must succeed");
         }
+        if let Some(start_after) = start_after {
+            let start_after = build_abs_path(&self.root, &start_after);
+            write!(url, "&startOffset={}", percent_encode_path(&start_after))
+                .expect("write into string must succeed");
+        }
+
         if !page_token.is_empty() {
             // NOTE:
             //
diff --git a/core/src/services/gcs/pager.rs b/core/src/services/gcs/pager.rs
index bfa99981..e6cffd67 100644
--- a/core/src/services/gcs/pager.rs
+++ b/core/src/services/gcs/pager.rs
@@ -35,19 +35,30 @@ pub struct GcsPager {
     delimiter: String,
     limit: Option<usize>,
 
+    /// Filter results to objects whose names are lexicographically
+    /// **equal to or after** startOffset
+    start_after: Option<String>,
+
     page_token: String,
     done: bool,
 }
 
 impl GcsPager {
     /// Generate a new directory walker
-    pub fn new(core: Arc<GcsCore>, path: &str, delimiter: &str, limit: Option<usize>) -> Self {
+    pub fn new(
+        core: Arc<GcsCore>,
+        path: &str,
+        delimiter: &str,
+        limit: Option<usize>,
+        start_after: Option<&str>,
+    ) -> Self {
         Self {
             core,
 
             path: path.to_string(),
             delimiter: delimiter.to_string(),
             limit,
+            start_after: start_after.map(String::from),
 
             page_token: "".to_string(),
             done: false,
@@ -64,7 +75,13 @@ impl oio::Page for GcsPager {
 
         let resp = self
             .core
-            .gcs_list_objects(&self.path, &self.page_token, &self.delimiter, self.limit)
+            .gcs_list_objects(
+                &self.path,
+                &self.page_token,
+                &self.delimiter,
+                self.limit,
+                self.start_after.clone(),
+            )
             .await?;
 
         if !resp.status().is_success() {
@@ -97,6 +114,12 @@ impl oio::Page for GcsPager {
                 continue;
             }
 
+            // exclude the inclusive start_after itself
+            let path = &build_rel_path(&self.core.root, &object.name);
+            if self.start_after.as_ref() == Some(path) {
+                continue;
+            }
+
             let mut meta = Metadata::new(EntryMode::FILE);
 
             // set metadata fields
@@ -113,7 +136,7 @@ impl oio::Page for GcsPager {
 
             meta.set_last_modified(parse_datetime_from_rfc3339(object.updated.as_str())?);
 
-            let de = oio::Entry::new(&build_rel_path(&self.core.root, &object.name), meta);
+            let de = oio::Entry::new(path, meta);
 
             entries.push(de);
         }
diff --git a/core/src/services/s3/pager.rs b/core/src/services/s3/pager.rs
index c8e98c16..3c5b15ed 100644
--- a/core/src/services/s3/pager.rs
+++ b/core/src/services/s3/pager.rs
@@ -35,6 +35,8 @@ pub struct S3Pager {
     path: String,
     delimiter: String,
     limit: Option<usize>,
+
+    /// Amazon S3 starts listing **after** this specified key
     start_after: Option<String>,
 
     token: String,