You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/12/26 14:22:43 UTC

(incubator-opendal) branch main updated: fix: Add tests for listing recursively on not supported services (#3826)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new da4f7cc15 fix: Add tests for listing recursively on not supported services (#3826)
da4f7cc15 is described below

commit da4f7cc151623d5ff1c946d1971065b536c135fb
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Dec 26 22:22:36 2023 +0800

    fix: Add tests for listing recursively on not supported services (#3826)
    
    * fix: Add tests for listing recursively on not supported services
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    * Polish lister docs
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    * Fix typos
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    * Fix generated
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    * Fix
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    ---------
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 bindings/nodejs/generated.d.ts               |   1 -
 bindings/nodejs/src/lib.rs                   |   1 -
 core/src/layers/complete.rs                  |  27 ++-
 core/src/types/operator/blocking_operator.rs |  25 ++-
 core/src/types/operator/operator.rs          | 257 ++++++++++++++++++++++-----
 core/tests/behavior/list.rs                  |  38 +++-
 6 files changed, 282 insertions(+), 67 deletions(-)

diff --git a/bindings/nodejs/generated.d.ts b/bindings/nodejs/generated.d.ts
index cb9e5e2fc..ecca49c98 100644
--- a/bindings/nodejs/generated.d.ts
+++ b/bindings/nodejs/generated.d.ts
@@ -430,7 +430,6 @@ export class Operator {
    *   }
    * }
    * ```
-   *
    */
   list(path: string, options?: ListOptions | undefined | null): Promise<Array<Entry>>
   /**
diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs
index f88a4d69f..58457a922 100644
--- a/bindings/nodejs/src/lib.rs
+++ b/bindings/nodejs/src/lib.rs
@@ -425,7 +425,6 @@ impl Operator {
     ///   }
     /// }
     /// ```
-    ///
     #[napi]
     pub async fn list(&self, path: String, options: Option<ListOptions>) -> Result<Vec<Entry>> {
         let mut l = self.0.list_with(&path);
diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs
index d130149af..12a9aa44c 100644
--- a/core/src/layers/complete.rs
+++ b/core/src/layers/complete.rs
@@ -372,8 +372,16 @@ impl<A: Accessor> CompleteAccessor<A> {
             }
             // If recursive is true but service can't list_with_recursive
             (true, false) => {
-                let p = FlatLister::new(self.inner.clone(), path);
-                Ok((RpList::default(), CompleteLister::NeedFlat(p)))
+                // Forward path that ends with /
+                if path.ends_with('/') {
+                    let p = FlatLister::new(self.inner.clone(), path);
+                    Ok((RpList::default(), CompleteLister::NeedFlat(p)))
+                } else {
+                    let parent = get_parent(path);
+                    let p = FlatLister::new(self.inner.clone(), parent);
+                    let p = PrefixLister::new(p, path);
+                    Ok((RpList::default(), CompleteLister::Both(p)))
+                }
             }
             // If recursive and service doesn't support list_with_recursive, we need to handle
             // list prefix by ourselves.
@@ -412,8 +420,16 @@ impl<A: Accessor> CompleteAccessor<A> {
             }
             // If recursive is true but service can't list_with_recursive
             (true, false) => {
-                let p = FlatLister::new(self.inner.clone(), path);
-                Ok((RpList::default(), CompleteLister::NeedFlat(p)))
+                // Forward path that ends with /
+                if path.ends_with('/') {
+                    let p = FlatLister::new(self.inner.clone(), path);
+                    Ok((RpList::default(), CompleteLister::NeedFlat(p)))
+                } else {
+                    let parent = get_parent(path);
+                    let p = FlatLister::new(self.inner.clone(), parent);
+                    let p = PrefixLister::new(p, path);
+                    Ok((RpList::default(), CompleteLister::Both(p)))
+                }
             }
             // If recursive and service doesn't support list_with_recursive, we need to handle
             // list prefix by ourselves.
@@ -733,6 +749,7 @@ pub enum CompleteLister<A: Accessor, P> {
     AlreadyComplete(P),
     NeedFlat(FlatLister<Arc<A>, P>),
     NeedPrefix(PrefixLister<P>),
+    Both(PrefixLister<FlatLister<Arc<A>, P>>),
 }
 
 impl<A, P> oio::List for CompleteLister<A, P>
@@ -747,6 +764,7 @@ where
             AlreadyComplete(p) => p.poll_next(cx),
             NeedFlat(p) => p.poll_next(cx),
             NeedPrefix(p) => p.poll_next(cx),
+            Both(p) => p.poll_next(cx),
         }
     }
 }
@@ -763,6 +781,7 @@ where
             AlreadyComplete(p) => p.next(),
             NeedFlat(p) => p.next(),
             NeedPrefix(p) => p.next(),
+            Both(p) => p.next(),
         }
     }
 }
diff --git a/core/src/types/operator/blocking_operator.rs b/core/src/types/operator/blocking_operator.rs
index 9d15bafe9..fa1a1bcbb 100644
--- a/core/src/types/operator/blocking_operator.rs
+++ b/core/src/types/operator/blocking_operator.rs
@@ -886,17 +886,17 @@ impl BlockingOperator {
         Ok(())
     }
 
-    /// List entries within a given directory.
+    /// List entries that starts with given `path` in parent dir.
     ///
     /// # Notes
     ///
-    /// ## Listing recursively
+    /// ## Recursively List
     ///
     /// This function only read the children of the given directory. To read
     /// all entries recursively, use `BlockingOperator::list_with("path").recursive(true)`
     /// instead.
     ///
-    /// ## Streaming
+    /// ## Streaming List
     ///
     /// This function will read all entries in the given directory. It could
     /// take very long time and consume a lot of memory if the directory
@@ -905,7 +905,7 @@ impl BlockingOperator {
     /// In order to avoid this, you can use [`BlockingOperator::lister`] to list entries in
     /// a streaming way.
     ///
-    /// ## Metadata
+    /// ## Reuse Metadata
     ///
     /// The only metadata that is guaranteed to be available is the `Mode`.
     /// For fetching more metadata, please use [`BlockingOperator::list_with`] and `metakey`.
@@ -937,20 +937,20 @@ impl BlockingOperator {
         self.list_with(path).call()
     }
 
-    /// List entries within a given directory with options.
+    /// List entries that starts with given `path` in parent dir. with options.
     ///
     /// # Notes
     ///
-    /// ## For streaming
+    /// ## Streaming List
     ///
     /// This function will read all entries in the given directory. It could
     /// take very long time and consume a lot of memory if the directory
     /// contains a lot of entries.
     ///
-    /// In order to avoid this, you can use [`Operator::lister`] to list entries in
+    /// In order to avoid this, you can use [`BlockingOperator::lister`] to list entries in
     /// a streaming way.
     ///
-    /// ## Metadata
+    /// ## Reuse Metadata
     ///
     /// The only metadata that is guaranteed to be available is the `Mode`.
     /// For fetching more metadata, please specify the `metakey`.
@@ -1029,15 +1029,14 @@ impl BlockingOperator {
         ))
     }
 
-    /// List entries within a given directory as an iterator.
+    /// List entries that starts with given `path` in parent dir.
     ///
-    /// This function will create a new handle to list entries.
-    ///
-    /// An error will be returned if given path doesn't end with `/`.
+    /// This function will create a new [`BlockingLister`] to list entries. Users can stop listing
+    /// via dropping this [`Lister`].
     ///
     /// # Notes
     ///
-    /// ## Listing recursively
+    /// ## Recursively List
     ///
     /// This function only read the children of the given directory. To read
     /// all entries recursively, use [`BlockingOperator::lister_with`] and `delimiter("")`
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index 9475606c9..8ba9899b8 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -1012,17 +1012,17 @@ impl Operator {
         Ok(())
     }
 
-    /// List entries within a given directory.
+    /// List entries that starts with given `path` in parent dir.
     ///
     /// # Notes
     ///
-    /// ## Listing recursively
+    /// ## Recursively list
     ///
     /// This function only read the children of the given directory. To read
     /// all entries recursively, use `Operator::list_with("path").recursive(true)`
     /// instead.
     ///
-    /// ## Streaming
+    /// ## Streaming list
     ///
     /// This function will read all entries in the given directory. It could
     /// take very long time and consume a lot of memory if the directory
@@ -1031,13 +1031,17 @@ impl Operator {
     /// In order to avoid this, you can use [`Operator::lister`] to list entries in
     /// a streaming way.
     ///
-    /// ## Metadata
+    /// ## Reuse Metadata
     ///
     /// The only metadata that is guaranteed to be available is the `Mode`.
     /// For fetching more metadata, please use [`Operator::list_with`] and `metakey`.
     ///
     /// # Examples
     ///
+    /// ## List entries under a dir
+    ///
+    /// This example will list all entries under the dir `path/to/dir/`.
+    ///
     /// ```no_run
     /// # use anyhow::Result;
     /// use opendal::EntryMode;
@@ -1060,15 +1064,46 @@ impl Operator {
     /// # Ok(())
     /// # }
     /// ```
+    ///
+    /// ## List entries with prefix
+    ///
+    /// This example will list all entries under the prefix `path/to/prefix`.
+    ///
+    /// NOTE: it's possible that the prefix itself is also a dir. In this case, you could get
+    /// `path/to/prefix/`, `path/to/prefix_1` and so on. If you do want to list a dir, please
+    /// make sure the path is end with `/`.
+    ///
+    /// ```no_run
+    /// # use anyhow::Result;
+    /// use opendal::EntryMode;
+    /// use opendal::Metakey;
+    /// use opendal::Operator;
+    /// # #[tokio::main]
+    /// # async fn test(op: Operator) -> Result<()> {
+    /// let mut entries = op.list("path/to/prefix").await?;
+    /// for entry in entries {
+    ///     match entry.metadata().mode() {
+    ///         EntryMode::FILE => {
+    ///             println!("Handling file")
+    ///         }
+    ///         EntryMode::DIR => {
+    ///             println!("Handling dir {}", entry.path())
+    ///         }
+    ///         EntryMode::Unknown => continue,
+    ///     }
+    /// }
+    /// # Ok(())
+    /// # }
+    /// ```
     pub async fn list(&self, path: &str) -> Result<Vec<Entry>> {
         self.list_with(path).await
     }
 
-    /// List entries within a given directory with options.
+    /// List entries that starts with given `path` in parent dir with more options.
     ///
     /// # Notes
     ///
-    /// ## For streaming
+    /// ## Streaming list
     ///
     /// This function will read all entries in the given directory. It could
     /// take very long time and consume a lot of memory if the directory
@@ -1077,16 +1112,97 @@ impl Operator {
     /// In order to avoid this, you can use [`Operator::lister`] to list entries in
     /// a streaming way.
     ///
-    /// ## Metadata
+    /// # Options
     ///
-    /// The only metadata that is guaranteed to be available is the `Mode`.
-    /// For fetching more metadata, please specify the `metakey`.
+    /// ## `start_after`
+    ///
+    /// Specify the specified key to start listing from.
+    ///
+    /// This feature can be used to resume a listing from a previous point.
+    ///
+    /// The following example will resume the list operation from the `breakpoint`.
+    ///
+    /// ```no_run
+    /// # use opendal::Result;
+    /// use opendal::Operator;
+    /// # #[tokio::main]
+    /// # async fn test(op: Operator) -> Result<()> {
+    /// let mut entries = op
+    ///     .list_with("path/to/dir/")
+    ///     .start_after("breakpoint")
+    ///     .await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    ///
+    /// ## `recursive`
+    ///
+    /// Specify whether to list recursively or not.
+    ///
+    /// If `recursive` is set to `true`, we will list all entries recursively. If not, we'll only
+    /// list the entries in the specified dir.
+    ///
+    /// ```no_run
+    /// # use opendal::Result;
+    /// use opendal::Operator;
+    /// # #[tokio::main]
+    /// # async fn test(op: Operator) -> Result<()> {
+    /// let mut entries = op.list_with("path/to/dir/").recursive(true).await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    ///
+    /// ## `metakey`
+    ///
+    /// Specify the metadata that required to be fetched in entries.
+    ///
+    /// If `metakey` is not set, we will fetch only the entry's `mode`. Otherwise, we will retrieve
+    /// the required metadata from storage services. Even if `metakey` is specified, the metadata
+    /// may still be `None`, indicating that the storage service does not supply this information.
+    ///
+    /// Some storage services like `s3` could return more metadata like `content-length` and
+    /// `last-modified`. By using `metakey`, we can fetch those metadata without an extra `stat` call.
+    /// Please pick up the metadata you need to reduce the extra `stat` cost.
+    ///
+    /// This example shows how to list entries with `content-length` and `last-modified` metadata:
+    ///
+    /// ```no_run
+    /// # use anyhow::Result;
+    /// use opendal::EntryMode;
+    /// use opendal::Metakey;
+    /// use opendal::Operator;
+    /// # #[tokio::main]
+    /// # async fn test(op: Operator) -> Result<()> {
+    /// let mut entries = op
+    ///     .list_with("dir/")
+    ///     // Make sure content-length and last-modified been fetched.
+    ///     .metakey(Metakey::ContentLength | Metakey::LastModified)
+    ///     .await?;
+    /// for entry in entries {
+    ///     let meta = entry.metadata();
+    ///     match meta.mode() {
+    ///         EntryMode::FILE => {
+    ///             println!(
+    ///                 "Handling file {} with size {}",
+    ///                 entry.path(),
+    ///                 meta.content_length()
+    ///             )
+    ///         }
+    ///         EntryMode::DIR => {
+    ///             println!("Handling dir {}", entry.path())
+    ///         }
+    ///         EntryMode::Unknown => continue,
+    ///     }
+    /// }
+    /// # Ok(())
+    /// # }
+    /// ```
     ///
     /// # Examples
     ///
-    /// ## List entries with prefix
+    /// ## List all entries recursively
     ///
-    /// This function can also be used to list entries in recursive way.
+    /// This example will list all entries under the dir `path/to/dir/`
     ///
     /// ```no_run
     /// # use anyhow::Result;
@@ -1095,7 +1211,7 @@ impl Operator {
     /// use opendal::Operator;
     /// # #[tokio::main]
     /// # async fn test(op: Operator) -> Result<()> {
-    /// let mut entries = op.list_with("prefix/").recursive(true).await?;
+    /// let mut entries = op.list_with("path/to/dir/").recursive(true).await?;
     /// for entry in entries {
     ///     match entry.metadata().mode() {
     ///         EntryMode::FILE => {
@@ -1111,7 +1227,9 @@ impl Operator {
     /// # }
     /// ```
     ///
-    /// ## List entries with metakey for more metadata
+    /// ## List all entries start with prefix
+    ///
+    /// This example will list all entries starts with prefix `path/to/prefix`
     ///
     /// ```no_run
     /// # use anyhow::Result;
@@ -1120,22 +1238,14 @@ impl Operator {
     /// use opendal::Operator;
     /// # #[tokio::main]
     /// # async fn test(op: Operator) -> Result<()> {
-    /// let mut entries = op
-    ///     .list_with("dir/")
-    ///     .metakey(Metakey::ContentLength | Metakey::LastModified)
-    ///     .await?;
+    /// let mut entries = op.list_with("path/to/prefix").recursive(true).await?;
     /// for entry in entries {
-    ///     let meta = entry.metadata();
-    ///     match meta.mode() {
+    ///     match entry.metadata().mode() {
     ///         EntryMode::FILE => {
-    ///             println!(
-    ///                 "Handling file {} with size {}",
-    ///                 entry.path(),
-    ///                 meta.content_length()
-    ///             )
+    ///             println!("Handling file")
     ///         }
     ///         EntryMode::DIR => {
-    ///             println!("Handling dir {}", entry.path())
+    ///             println!("Handling dir like start a new list via meta.path()")
     ///         }
     ///         EntryMode::Unknown => continue,
     ///     }
@@ -1162,21 +1272,20 @@ impl Operator {
         fut
     }
 
-    /// List entries within a given directory as a stream.
-    ///
-    /// This function will create a new handle to list entries.
+    /// List entries that starts with given `path` in parent dir.
     ///
-    /// An error will be returned if given path doesn't end with `/`.
+    /// This function will create a new [`Lister`] to list entries. Users can stop listing via
+    /// dropping this [`Lister`].
     ///
     /// # Notes
     ///
-    /// ## Listing recursively
+    /// ## Recursively list
     ///
     /// This function only read the children of the given directory. To read
     /// all entries recursively, use [`Operator::lister_with`] and `recursive(true)`
     /// instead.
     ///
-    /// ## Metadata
+    /// ## Reuse Metadata
     ///
     /// The only metadata that is guaranteed to be available is the `Mode`.
     /// For fetching more metadata, please use [`Operator::lister_with`] and `metakey`.
@@ -1211,34 +1320,87 @@ impl Operator {
         self.lister_with(path).await
     }
 
-    /// List entries within a given directory as a stream with options.
+    /// List entries that starts with given `path` in parent dir with options.
     ///
-    /// This function will create a new handle to list entries.
+    /// This function will create a new [`Lister`] to list entries. Users can stop listing via
+    /// dropping this [`Lister`].
     ///
-    /// An error will be returned if given path doesn't end with `/`.
+    /// # Options
     ///
-    /// # Examples
+    /// ## `start_after`
+    ///
+    /// Specify the specified key to start listing from.
+    ///
+    /// This feature can be used to resume a listing from a previous point.
     ///
-    /// ## List current dir
+    /// The following example will resume the list operation from the `breakpoint`.
+    ///
+    /// ```no_run
+    /// # use opendal::Result;
+    /// use opendal::Operator;
+    /// # #[tokio::main]
+    /// # async fn test(op: Operator) -> Result<()> {
+    /// let mut lister = op
+    ///     .lister_with("path/to/dir/")
+    ///     .start_after("breakpoint")
+    ///     .await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    ///
+    /// ## `recursive`
+    ///
+    /// Specify whether to list recursively or not.
+    ///
+    /// If `recursive` is set to `true`, we will list all entries recursively. If not, we'll only
+    /// list the entries in the specified dir.
+    ///
+    /// ```no_run
+    /// # use opendal::Result;
+    /// use opendal::Operator;
+    /// # #[tokio::main]
+    /// # async fn test(op: Operator) -> Result<()> {
+    /// let mut lister = op.lister_with("path/to/dir/").recursive(true).await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    ///
+    /// ## `metakey`
+    ///
+    /// Specify the metadata that required to be fetched in entries.
+    ///
+    /// If `metakey` is not set, we will fetch only the entry's `mode`. Otherwise, we will retrieve
+    /// the required metadata from storage services. Even if `metakey` is specified, the metadata
+    /// may still be `None`, indicating that the storage service does not supply this information.
+    ///
+    /// Some storage services like `s3` could return more metadata like `content-length` and
+    /// `last-modified`. By using `metakey`, we can fetch those metadata without an extra `stat` call.
+    /// Please pick up the metadata you need to reduce the extra `stat` cost.
+    ///
+    /// This example shows how to list entries with `content-length` and `last-modified` metadata:
     ///
     /// ```no_run
     /// # use anyhow::Result;
-    /// # use futures::io;
     /// use futures::TryStreamExt;
     /// use opendal::EntryMode;
     /// use opendal::Metakey;
     /// use opendal::Operator;
     /// # #[tokio::main]
     /// # async fn test(op: Operator) -> Result<()> {
-    /// let mut ds = op
-    ///     .lister_with("path/to/dir/")
-    ///     .limit(10)
-    ///     .start_after("start")
+    /// let mut lister = op
+    ///     .lister_with("dir/")
+    ///     // Make sure content-length and last-modified been fetched.
+    ///     .metakey(Metakey::ContentLength | Metakey::LastModified)
     ///     .await?;
-    /// while let Some(mut entry) = ds.try_next().await? {
-    ///     match entry.metadata().mode() {
+    /// while let Some(mut entry) = lister.try_next().await? {
+    ///     let meta = entry.metadata();
+    ///     match meta.mode() {
     ///         EntryMode::FILE => {
-    ///             println!("Handling file {}", entry.path())
+    ///             println!(
+    ///                 "Handling file {} with size {}",
+    ///                 entry.path(),
+    ///                 meta.content_length()
+    ///             )
     ///         }
     ///         EntryMode::DIR => {
     ///             println!("Handling dir {}", entry.path())
@@ -1250,19 +1412,20 @@ impl Operator {
     /// # }
     /// ```
     ///
+    /// # Examples
+    ///
     /// ## List all files recursively
     ///
     /// ```no_run
     /// # use anyhow::Result;
-    /// # use futures::io;
     /// use futures::TryStreamExt;
     /// use opendal::EntryMode;
     /// use opendal::Metakey;
     /// use opendal::Operator;
     /// # #[tokio::main]
     /// # async fn test(op: Operator) -> Result<()> {
-    /// let mut ds = op.lister_with("path/to/dir/").recursive(true).await?;
-    /// while let Some(mut entry) = ds.try_next().await? {
+    /// let mut lister = op.lister_with("path/to/dir/").recursive(true).await?;
+    /// while let Some(mut entry) = lister.try_next().await? {
     ///     match entry.metadata().mode() {
     ///         EntryMode::FILE => {
     ///             println!("Handling file {}", entry.path())
diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/list.rs
index c29f0e2dc..b421da57f 100644
--- a/core/tests/behavior/list.rs
+++ b/core/tests/behavior/list.rs
@@ -243,15 +243,51 @@ pub async fn test_list_empty_dir(op: Operator) -> Result<()> {
 
     op.create_dir(&dir).await.expect("write must succeed");
 
+    // List "dir/" should return empty object.
     let mut obs = op.lister(&dir).await?;
     let mut objects = HashMap::new();
     while let Some(de) = obs.try_next().await? {
         objects.insert(de.path().to_string(), de);
     }
-    debug!("got objects: {:?}", objects);
+    assert_eq!(objects.len(), 0, "dir should only return empty");
 
+    // List "dir" should return "dir/".
+    let mut obs = op.lister(dir.trim_end_matches('/')).await?;
+    let mut objects = HashMap::new();
+    while let Some(de) = obs.try_next().await? {
+        objects.insert(de.path().to_string(), de);
+    }
+    assert_eq!(objects.len(), 1, "only return the dir itself");
+    assert_eq!(
+        objects[&dir].metadata().mode(),
+        EntryMode::DIR,
+        "given dir should exist and must be dir"
+    );
+
+    // List "dir/" should return empty object.
+    let mut obs = op.lister_with(&dir).recursive(true).await?;
+    let mut objects = HashMap::new();
+    while let Some(de) = obs.try_next().await? {
+        objects.insert(de.path().to_string(), de);
+    }
     assert_eq!(objects.len(), 0, "dir should only return empty");
 
+    // List "dir" should return "dir/".
+    let mut obs = op
+        .lister_with(dir.trim_end_matches('/'))
+        .recursive(true)
+        .await?;
+    let mut objects = HashMap::new();
+    while let Some(de) = obs.try_next().await? {
+        objects.insert(de.path().to_string(), de);
+    }
+    assert_eq!(objects.len(), 1, "only return the dir itself");
+    assert_eq!(
+        objects[&dir].metadata().mode(),
+        EntryMode::DIR,
+        "given dir should exist and must be dir"
+    );
+
     op.delete(&dir).await.expect("delete must succeed");
     Ok(())
 }