You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/12/26 14:22:43 UTC
(incubator-opendal) branch main updated: fix: Add tests for listing recursively on not supported services (#3826)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new da4f7cc15 fix: Add tests for listing recursively on not supported services (#3826)
da4f7cc15 is described below
commit da4f7cc151623d5ff1c946d1971065b536c135fb
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Dec 26 22:22:36 2023 +0800
fix: Add tests for listing recursively on not supported services (#3826)
* fix: Add tests for listing recursively on not supported services
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Polish lister docs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix typos
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix generated
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
bindings/nodejs/generated.d.ts | 1 -
bindings/nodejs/src/lib.rs | 1 -
core/src/layers/complete.rs | 27 ++-
core/src/types/operator/blocking_operator.rs | 25 ++-
core/src/types/operator/operator.rs | 257 ++++++++++++++++++++++-----
core/tests/behavior/list.rs | 38 +++-
6 files changed, 282 insertions(+), 67 deletions(-)
diff --git a/bindings/nodejs/generated.d.ts b/bindings/nodejs/generated.d.ts
index cb9e5e2fc..ecca49c98 100644
--- a/bindings/nodejs/generated.d.ts
+++ b/bindings/nodejs/generated.d.ts
@@ -430,7 +430,6 @@ export class Operator {
* }
* }
* ```
- *
*/
list(path: string, options?: ListOptions | undefined | null): Promise<Array<Entry>>
/**
diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs
index f88a4d69f..58457a922 100644
--- a/bindings/nodejs/src/lib.rs
+++ b/bindings/nodejs/src/lib.rs
@@ -425,7 +425,6 @@ impl Operator {
/// }
/// }
/// ```
- ///
#[napi]
pub async fn list(&self, path: String, options: Option<ListOptions>) -> Result<Vec<Entry>> {
let mut l = self.0.list_with(&path);
diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs
index d130149af..12a9aa44c 100644
--- a/core/src/layers/complete.rs
+++ b/core/src/layers/complete.rs
@@ -372,8 +372,16 @@ impl<A: Accessor> CompleteAccessor<A> {
}
// If recursive is true but service can't list_with_recursive
(true, false) => {
- let p = FlatLister::new(self.inner.clone(), path);
- Ok((RpList::default(), CompleteLister::NeedFlat(p)))
+ // Forward path that ends with /
+ if path.ends_with('/') {
+ let p = FlatLister::new(self.inner.clone(), path);
+ Ok((RpList::default(), CompleteLister::NeedFlat(p)))
+ } else {
+ let parent = get_parent(path);
+ let p = FlatLister::new(self.inner.clone(), parent);
+ let p = PrefixLister::new(p, path);
+ Ok((RpList::default(), CompleteLister::Both(p)))
+ }
}
// If recursive and service doesn't support list_with_recursive, we need to handle
// list prefix by ourselves.
@@ -412,8 +420,16 @@ impl<A: Accessor> CompleteAccessor<A> {
}
// If recursive is true but service can't list_with_recursive
(true, false) => {
- let p = FlatLister::new(self.inner.clone(), path);
- Ok((RpList::default(), CompleteLister::NeedFlat(p)))
+ // Forward path that ends with /
+ if path.ends_with('/') {
+ let p = FlatLister::new(self.inner.clone(), path);
+ Ok((RpList::default(), CompleteLister::NeedFlat(p)))
+ } else {
+ let parent = get_parent(path);
+ let p = FlatLister::new(self.inner.clone(), parent);
+ let p = PrefixLister::new(p, path);
+ Ok((RpList::default(), CompleteLister::Both(p)))
+ }
}
// If recursive and service doesn't support list_with_recursive, we need to handle
// list prefix by ourselves.
@@ -733,6 +749,7 @@ pub enum CompleteLister<A: Accessor, P> {
AlreadyComplete(P),
NeedFlat(FlatLister<Arc<A>, P>),
NeedPrefix(PrefixLister<P>),
+ Both(PrefixLister<FlatLister<Arc<A>, P>>),
}
impl<A, P> oio::List for CompleteLister<A, P>
@@ -747,6 +764,7 @@ where
AlreadyComplete(p) => p.poll_next(cx),
NeedFlat(p) => p.poll_next(cx),
NeedPrefix(p) => p.poll_next(cx),
+ Both(p) => p.poll_next(cx),
}
}
}
@@ -763,6 +781,7 @@ where
AlreadyComplete(p) => p.next(),
NeedFlat(p) => p.next(),
NeedPrefix(p) => p.next(),
+ Both(p) => p.next(),
}
}
}
diff --git a/core/src/types/operator/blocking_operator.rs b/core/src/types/operator/blocking_operator.rs
index 9d15bafe9..fa1a1bcbb 100644
--- a/core/src/types/operator/blocking_operator.rs
+++ b/core/src/types/operator/blocking_operator.rs
@@ -886,17 +886,17 @@ impl BlockingOperator {
Ok(())
}
- /// List entries within a given directory.
+ /// List entries that starts with given `path` in parent dir.
///
/// # Notes
///
- /// ## Listing recursively
+ /// ## Recursively List
///
/// This function only read the children of the given directory. To read
/// all entries recursively, use `BlockingOperator::list_with("path").recursive(true)`
/// instead.
///
- /// ## Streaming
+ /// ## Streaming List
///
/// This function will read all entries in the given directory. It could
/// take very long time and consume a lot of memory if the directory
@@ -905,7 +905,7 @@ impl BlockingOperator {
/// In order to avoid this, you can use [`BlockingOperator::lister`] to list entries in
/// a streaming way.
///
- /// ## Metadata
+ /// ## Reuse Metadata
///
/// The only metadata that is guaranteed to be available is the `Mode`.
/// For fetching more metadata, please use [`BlockingOperator::list_with`] and `metakey`.
@@ -937,20 +937,20 @@ impl BlockingOperator {
self.list_with(path).call()
}
- /// List entries within a given directory with options.
+ /// List entries that starts with given `path` in parent dir. with options.
///
/// # Notes
///
- /// ## For streaming
+ /// ## Streaming List
///
/// This function will read all entries in the given directory. It could
/// take very long time and consume a lot of memory if the directory
/// contains a lot of entries.
///
- /// In order to avoid this, you can use [`Operator::lister`] to list entries in
+ /// In order to avoid this, you can use [`BlockingOperator::lister`] to list entries in
/// a streaming way.
///
- /// ## Metadata
+ /// ## Reuse Metadata
///
/// The only metadata that is guaranteed to be available is the `Mode`.
/// For fetching more metadata, please specify the `metakey`.
@@ -1029,15 +1029,14 @@ impl BlockingOperator {
))
}
- /// List entries within a given directory as an iterator.
+ /// List entries that starts with given `path` in parent dir.
///
- /// This function will create a new handle to list entries.
- ///
- /// An error will be returned if given path doesn't end with `/`.
+ /// This function will create a new [`BlockingLister`] to list entries. Users can stop listing
+ /// via dropping this [`Lister`].
///
/// # Notes
///
- /// ## Listing recursively
+ /// ## Recursively List
///
/// This function only read the children of the given directory. To read
/// all entries recursively, use [`BlockingOperator::lister_with`] and `delimiter("")`
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index 9475606c9..8ba9899b8 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -1012,17 +1012,17 @@ impl Operator {
Ok(())
}
- /// List entries within a given directory.
+ /// List entries that starts with given `path` in parent dir.
///
/// # Notes
///
- /// ## Listing recursively
+ /// ## Recursively list
///
/// This function only read the children of the given directory. To read
/// all entries recursively, use `Operator::list_with("path").recursive(true)`
/// instead.
///
- /// ## Streaming
+ /// ## Streaming list
///
/// This function will read all entries in the given directory. It could
/// take very long time and consume a lot of memory if the directory
@@ -1031,13 +1031,17 @@ impl Operator {
/// In order to avoid this, you can use [`Operator::lister`] to list entries in
/// a streaming way.
///
- /// ## Metadata
+ /// ## Reuse Metadata
///
/// The only metadata that is guaranteed to be available is the `Mode`.
/// For fetching more metadata, please use [`Operator::list_with`] and `metakey`.
///
/// # Examples
///
+ /// ## List entries under a dir
+ ///
+ /// This example will list all entries under the dir `path/to/dir/`.
+ ///
/// ```no_run
/// # use anyhow::Result;
/// use opendal::EntryMode;
@@ -1060,15 +1064,46 @@ impl Operator {
/// # Ok(())
/// # }
/// ```
+ ///
+ /// ## List entries with prefix
+ ///
+ /// This example will list all entries under the prefix `path/to/prefix`.
+ ///
+ /// NOTE: it's possible that the prefix itself is also a dir. In this case, you could get
+ /// `path/to/prefix/`, `path/to/prefix_1` and so on. If you do want to list a dir, please
+ /// make sure the path is end with `/`.
+ ///
+ /// ```no_run
+ /// # use anyhow::Result;
+ /// use opendal::EntryMode;
+ /// use opendal::Metakey;
+ /// use opendal::Operator;
+ /// # #[tokio::main]
+ /// # async fn test(op: Operator) -> Result<()> {
+ /// let mut entries = op.list("path/to/prefix").await?;
+ /// for entry in entries {
+ /// match entry.metadata().mode() {
+ /// EntryMode::FILE => {
+ /// println!("Handling file")
+ /// }
+ /// EntryMode::DIR => {
+ /// println!("Handling dir {}", entry.path())
+ /// }
+ /// EntryMode::Unknown => continue,
+ /// }
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
pub async fn list(&self, path: &str) -> Result<Vec<Entry>> {
self.list_with(path).await
}
- /// List entries within a given directory with options.
+ /// List entries that starts with given `path` in parent dir with more options.
///
/// # Notes
///
- /// ## For streaming
+ /// ## Streaming list
///
/// This function will read all entries in the given directory. It could
/// take very long time and consume a lot of memory if the directory
@@ -1077,16 +1112,97 @@ impl Operator {
/// In order to avoid this, you can use [`Operator::lister`] to list entries in
/// a streaming way.
///
- /// ## Metadata
+ /// # Options
///
- /// The only metadata that is guaranteed to be available is the `Mode`.
- /// For fetching more metadata, please specify the `metakey`.
+ /// ## `start_after`
+ ///
+ /// Specify the specified key to start listing from.
+ ///
+ /// This feature can be used to resume a listing from a previous point.
+ ///
+ /// The following example will resume the list operation from the `breakpoint`.
+ ///
+ /// ```no_run
+ /// # use opendal::Result;
+ /// use opendal::Operator;
+ /// # #[tokio::main]
+ /// # async fn test(op: Operator) -> Result<()> {
+ /// let mut entries = op
+ /// .list_with("path/to/dir/")
+ /// .start_after("breakpoint")
+ /// .await?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// ## `recursive`
+ ///
+ /// Specify whether to list recursively or not.
+ ///
+ /// If `recursive` is set to `true`, we will list all entries recursively. If not, we'll only
+ /// list the entries in the specified dir.
+ ///
+ /// ```no_run
+ /// # use opendal::Result;
+ /// use opendal::Operator;
+ /// # #[tokio::main]
+ /// # async fn test(op: Operator) -> Result<()> {
+ /// let mut entries = op.list_with("path/to/dir/").recursive(true).await?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// ## `metakey`
+ ///
+ /// Specify the metadata that required to be fetched in entries.
+ ///
+ /// If `metakey` is not set, we will fetch only the entry's `mode`. Otherwise, we will retrieve
+ /// the required metadata from storage services. Even if `metakey` is specified, the metadata
+ /// may still be `None`, indicating that the storage service does not supply this information.
+ ///
+ /// Some storage services like `s3` could return more metadata like `content-length` and
+ /// `last-modified`. By using `metakey`, we can fetch those metadata without an extra `stat` call.
+ /// Please pick up the metadata you need to reduce the extra `stat` cost.
+ ///
+ /// This example shows how to list entries with `content-length` and `last-modified` metadata:
+ ///
+ /// ```no_run
+ /// # use anyhow::Result;
+ /// use opendal::EntryMode;
+ /// use opendal::Metakey;
+ /// use opendal::Operator;
+ /// # #[tokio::main]
+ /// # async fn test(op: Operator) -> Result<()> {
+ /// let mut entries = op
+ /// .list_with("dir/")
+ /// // Make sure content-length and last-modified been fetched.
+ /// .metakey(Metakey::ContentLength | Metakey::LastModified)
+ /// .await?;
+ /// for entry in entries {
+ /// let meta = entry.metadata();
+ /// match meta.mode() {
+ /// EntryMode::FILE => {
+ /// println!(
+ /// "Handling file {} with size {}",
+ /// entry.path(),
+ /// meta.content_length()
+ /// )
+ /// }
+ /// EntryMode::DIR => {
+ /// println!("Handling dir {}", entry.path())
+ /// }
+ /// EntryMode::Unknown => continue,
+ /// }
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
///
/// # Examples
///
- /// ## List entries with prefix
+ /// ## List all entries recursively
///
- /// This function can also be used to list entries in recursive way.
+ /// This example will list all entries under the dir `path/to/dir/`
///
/// ```no_run
/// # use anyhow::Result;
@@ -1095,7 +1211,7 @@ impl Operator {
/// use opendal::Operator;
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
- /// let mut entries = op.list_with("prefix/").recursive(true).await?;
+ /// let mut entries = op.list_with("path/to/dir/").recursive(true).await?;
/// for entry in entries {
/// match entry.metadata().mode() {
/// EntryMode::FILE => {
@@ -1111,7 +1227,9 @@ impl Operator {
/// # }
/// ```
///
- /// ## List entries with metakey for more metadata
+ /// ## List all entries start with prefix
+ ///
+ /// This example will list all entries starts with prefix `path/to/prefix`
///
/// ```no_run
/// # use anyhow::Result;
@@ -1120,22 +1238,14 @@ impl Operator {
/// use opendal::Operator;
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
- /// let mut entries = op
- /// .list_with("dir/")
- /// .metakey(Metakey::ContentLength | Metakey::LastModified)
- /// .await?;
+ /// let mut entries = op.list_with("path/to/prefix").recursive(true).await?;
/// for entry in entries {
- /// let meta = entry.metadata();
- /// match meta.mode() {
+ /// match entry.metadata().mode() {
/// EntryMode::FILE => {
- /// println!(
- /// "Handling file {} with size {}",
- /// entry.path(),
- /// meta.content_length()
- /// )
+ /// println!("Handling file")
/// }
/// EntryMode::DIR => {
- /// println!("Handling dir {}", entry.path())
+ /// println!("Handling dir like start a new list via meta.path()")
/// }
/// EntryMode::Unknown => continue,
/// }
@@ -1162,21 +1272,20 @@ impl Operator {
fut
}
- /// List entries within a given directory as a stream.
- ///
- /// This function will create a new handle to list entries.
+ /// List entries that starts with given `path` in parent dir.
///
- /// An error will be returned if given path doesn't end with `/`.
+ /// This function will create a new [`Lister`] to list entries. Users can stop listing via
+ /// dropping this [`Lister`].
///
/// # Notes
///
- /// ## Listing recursively
+ /// ## Recursively list
///
/// This function only read the children of the given directory. To read
/// all entries recursively, use [`Operator::lister_with`] and `recursive(true)`
/// instead.
///
- /// ## Metadata
+ /// ## Reuse Metadata
///
/// The only metadata that is guaranteed to be available is the `Mode`.
/// For fetching more metadata, please use [`Operator::lister_with`] and `metakey`.
@@ -1211,34 +1320,87 @@ impl Operator {
self.lister_with(path).await
}
- /// List entries within a given directory as a stream with options.
+ /// List entries that starts with given `path` in parent dir with options.
///
- /// This function will create a new handle to list entries.
+ /// This function will create a new [`Lister`] to list entries. Users can stop listing via
+ /// dropping this [`Lister`].
///
- /// An error will be returned if given path doesn't end with `/`.
+ /// # Options
///
- /// # Examples
+ /// ## `start_after`
+ ///
+ /// Specify the specified key to start listing from.
+ ///
+ /// This feature can be used to resume a listing from a previous point.
///
- /// ## List current dir
+ /// The following example will resume the list operation from the `breakpoint`.
+ ///
+ /// ```no_run
+ /// # use opendal::Result;
+ /// use opendal::Operator;
+ /// # #[tokio::main]
+ /// # async fn test(op: Operator) -> Result<()> {
+ /// let mut lister = op
+ /// .lister_with("path/to/dir/")
+ /// .start_after("breakpoint")
+ /// .await?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// ## `recursive`
+ ///
+ /// Specify whether to list recursively or not.
+ ///
+ /// If `recursive` is set to `true`, we will list all entries recursively. If not, we'll only
+ /// list the entries in the specified dir.
+ ///
+ /// ```no_run
+ /// # use opendal::Result;
+ /// use opendal::Operator;
+ /// # #[tokio::main]
+ /// # async fn test(op: Operator) -> Result<()> {
+ /// let mut lister = op.lister_with("path/to/dir/").recursive(true).await?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// ## `metakey`
+ ///
+ /// Specify the metadata that required to be fetched in entries.
+ ///
+ /// If `metakey` is not set, we will fetch only the entry's `mode`. Otherwise, we will retrieve
+ /// the required metadata from storage services. Even if `metakey` is specified, the metadata
+ /// may still be `None`, indicating that the storage service does not supply this information.
+ ///
+ /// Some storage services like `s3` could return more metadata like `content-length` and
+ /// `last-modified`. By using `metakey`, we can fetch those metadata without an extra `stat` call.
+ /// Please pick up the metadata you need to reduce the extra `stat` cost.
+ ///
+ /// This example shows how to list entries with `content-length` and `last-modified` metadata:
///
/// ```no_run
/// # use anyhow::Result;
- /// # use futures::io;
/// use futures::TryStreamExt;
/// use opendal::EntryMode;
/// use opendal::Metakey;
/// use opendal::Operator;
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
- /// let mut ds = op
- /// .lister_with("path/to/dir/")
- /// .limit(10)
- /// .start_after("start")
+ /// let mut lister = op
+ /// .lister_with("dir/")
+ /// // Make sure content-length and last-modified been fetched.
+ /// .metakey(Metakey::ContentLength | Metakey::LastModified)
/// .await?;
- /// while let Some(mut entry) = ds.try_next().await? {
- /// match entry.metadata().mode() {
+ /// while let Some(mut entry) = lister.try_next().await? {
+ /// let meta = entry.metadata();
+ /// match meta.mode() {
/// EntryMode::FILE => {
- /// println!("Handling file {}", entry.path())
+ /// println!(
+ /// "Handling file {} with size {}",
+ /// entry.path(),
+ /// meta.content_length()
+ /// )
/// }
/// EntryMode::DIR => {
/// println!("Handling dir {}", entry.path())
@@ -1250,19 +1412,20 @@ impl Operator {
/// # }
/// ```
///
+ /// # Examples
+ ///
/// ## List all files recursively
///
/// ```no_run
/// # use anyhow::Result;
- /// # use futures::io;
/// use futures::TryStreamExt;
/// use opendal::EntryMode;
/// use opendal::Metakey;
/// use opendal::Operator;
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
- /// let mut ds = op.lister_with("path/to/dir/").recursive(true).await?;
- /// while let Some(mut entry) = ds.try_next().await? {
+ /// let mut lister = op.lister_with("path/to/dir/").recursive(true).await?;
+ /// while let Some(mut entry) = lister.try_next().await? {
/// match entry.metadata().mode() {
/// EntryMode::FILE => {
/// println!("Handling file {}", entry.path())
diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/list.rs
index c29f0e2dc..b421da57f 100644
--- a/core/tests/behavior/list.rs
+++ b/core/tests/behavior/list.rs
@@ -243,15 +243,51 @@ pub async fn test_list_empty_dir(op: Operator) -> Result<()> {
op.create_dir(&dir).await.expect("write must succeed");
+ // List "dir/" should return empty object.
let mut obs = op.lister(&dir).await?;
let mut objects = HashMap::new();
while let Some(de) = obs.try_next().await? {
objects.insert(de.path().to_string(), de);
}
- debug!("got objects: {:?}", objects);
+ assert_eq!(objects.len(), 0, "dir should only return empty");
+ // List "dir" should return "dir/".
+ let mut obs = op.lister(dir.trim_end_matches('/')).await?;
+ let mut objects = HashMap::new();
+ while let Some(de) = obs.try_next().await? {
+ objects.insert(de.path().to_string(), de);
+ }
+ assert_eq!(objects.len(), 1, "only return the dir itself");
+ assert_eq!(
+ objects[&dir].metadata().mode(),
+ EntryMode::DIR,
+ "given dir should exist and must be dir"
+ );
+
+ // List "dir/" should return empty object.
+ let mut obs = op.lister_with(&dir).recursive(true).await?;
+ let mut objects = HashMap::new();
+ while let Some(de) = obs.try_next().await? {
+ objects.insert(de.path().to_string(), de);
+ }
assert_eq!(objects.len(), 0, "dir should only return empty");
+ // List "dir" should return "dir/".
+ let mut obs = op
+ .lister_with(dir.trim_end_matches('/'))
+ .recursive(true)
+ .await?;
+ let mut objects = HashMap::new();
+ while let Some(de) = obs.try_next().await? {
+ objects.insert(de.path().to_string(), de);
+ }
+ assert_eq!(objects.len(), 1, "only return the dir itself");
+ assert_eq!(
+ objects[&dir].metadata().mode(),
+ EntryMode::DIR,
+ "given dir should exist and must be dir"
+ );
+
op.delete(&dir).await.expect("delete must succeed");
Ok(())
}