You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/11/10 05:53:28 UTC
(incubator-opendal) branch main updated: feat: Implement RFC-3526: List Recursive (#3556)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new fd3148a01 feat: Implement RFC-3526: List Recursive (#3556)
fd3148a01 is described below
commit fd3148a01e565ff7a3412e94f83fc3b8b9f99271
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Nov 10 13:53:24 2023 +0800
feat: Implement RFC-3526: List Recursive (#3556)
* feat: Implement RFC-3526: List Recursive
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Add upgrade
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix unit test
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix doc test
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
bin/oli/src/commands/cp.rs | 2 +-
bin/oli/src/commands/ls.rs | 2 +-
bindings/c/include/opendal.h | 4 +-
bindings/c/src/operator_info.rs | 8 +-
bindings/c/tests/opinfo.cpp | 2 +-
bindings/haskell/src/lib.rs | 2 +-
bindings/java/src/lib.rs | 4 +-
bindings/nodejs/src/lib.rs | 5 +-
bindings/python/python/opendal/__init__.pyi | 4 +-
bindings/python/src/capability.rs | 8 +-
bindings/python/src/operator.rs | 4 +-
core/src/docs/upgrade.md | 13 ++++
core/src/layers/complete.rs | 105 ++++++++++++--------------
core/src/layers/immutable_index.rs | 26 ++-----
core/src/layers/retry.rs | 4 +-
core/src/raw/adapters/kv/backend.rs | 20 +----
core/src/raw/adapters/typed_kv/backend.rs | 20 +----
core/src/raw/oio/page/into_flat_page.rs | 4 +-
core/src/raw/ops.rs | 41 ++++++----
core/src/services/azblob/backend.rs | 6 +-
core/src/services/azblob/pager.rs | 13 ++--
core/src/services/azdls/backend.rs | 2 +-
core/src/services/azfile/backend.rs | 2 +-
core/src/services/cos/backend.rs | 6 +-
core/src/services/cos/pager.rs | 9 ++-
core/src/services/dbfs/backend.rs | 2 +-
core/src/services/fs/backend.rs | 2 +-
core/src/services/ftp/backend.rs | 2 +-
core/src/services/gcs/backend.rs | 6 +-
core/src/services/gcs/pager.rs | 9 ++-
core/src/services/gdrive/backend.rs | 2 +-
core/src/services/hdfs/backend.rs | 2 +-
core/src/services/ipfs/backend.rs | 2 +-
core/src/services/ipmfs/backend.rs | 2 +-
core/src/services/obs/backend.rs | 6 +-
core/src/services/obs/pager.rs | 10 ++-
core/src/services/onedrive/backend.rs | 2 +-
core/src/services/oss/backend.rs | 6 +-
core/src/services/oss/pager.rs | 9 ++-
core/src/services/s3/backend.rs | 6 +-
core/src/services/s3/pager.rs | 9 ++-
core/src/services/sftp/backend.rs | 2 +-
core/src/services/swift/backend.rs | 8 +-
core/src/services/swift/pager.rs | 7 +-
core/src/services/webdav/backend.rs | 6 +-
core/src/services/webhdfs/backend.rs | 6 +-
core/src/types/capability.rs | 8 +-
core/src/types/operator/blocking_operator.rs | 8 +-
core/src/types/operator/operator.rs | 10 +--
core/src/types/operator/operator_functions.rs | 56 +++++++++++---
core/src/types/operator/operator_futures.rs | 56 +++++++++++---
core/tests/behavior/blocking_list.rs | 2 +-
core/tests/behavior/list.rs | 4 +-
integrations/object_store/src/lib.rs | 2 +-
54 files changed, 308 insertions(+), 260 deletions(-)
diff --git a/bin/oli/src/commands/cp.rs b/bin/oli/src/commands/cp.rs
index 5f17f4fa9..609b3d48b 100644
--- a/bin/oli/src/commands/cp.rs
+++ b/bin/oli/src/commands/cp.rs
@@ -56,7 +56,7 @@ pub async fn main(args: &ArgMatches) -> Result<()> {
}
let dst_root = Path::new(&dst_path);
- let mut ds = src_op.lister_with(&src_path).delimiter("").await?;
+ let mut ds = src_op.lister_with(&src_path).recursive(true).await?;
while let Some(de) = ds.try_next().await? {
let meta = de.metadata();
if meta.mode().is_dir() {
diff --git a/bin/oli/src/commands/ls.rs b/bin/oli/src/commands/ls.rs
index 719b7dcb4..50c60c56e 100644
--- a/bin/oli/src/commands/ls.rs
+++ b/bin/oli/src/commands/ls.rs
@@ -48,7 +48,7 @@ pub async fn main(args: &ArgMatches) -> Result<()> {
return Ok(());
}
- let mut ds = op.lister_with(&path).delimiter("").await?;
+ let mut ds = op.lister_with(&path).recursive(true).await?;
while let Some(de) = ds.try_next().await? {
println!("{}", de.path());
}
diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h
index 3e17f86dc..32c3abfce 100644
--- a/bindings/c/include/opendal.h
+++ b/bindings/c/include/opendal.h
@@ -635,11 +635,11 @@ typedef struct opendal_capability {
/**
* If backend support list with using slash as delimiter.
*/
- bool list_with_delimiter_slash;
+ bool list_without_recursive;
/**
* If backend supports list without delimiter.
*/
- bool list_without_delimiter;
+ bool list_with_recursive;
/**
* If operator supports presign.
*/
diff --git a/bindings/c/src/operator_info.rs b/bindings/c/src/operator_info.rs
index 2a700a6e7..9785700fd 100644
--- a/bindings/c/src/operator_info.rs
+++ b/bindings/c/src/operator_info.rs
@@ -116,9 +116,9 @@ pub struct opendal_capability {
/// If backend supports list with start after.
pub list_with_start_after: bool,
/// If backend support list with using slash as delimiter.
- pub list_with_delimiter_slash: bool,
+ pub list_without_recursive: bool,
/// If backend supports list without delimiter.
- pub list_without_delimiter: bool,
+ pub list_with_recursive: bool,
/// If operator supports presign.
pub presign: bool,
@@ -264,8 +264,8 @@ impl From<core::Capability> for opendal_capability {
list: value.list,
list_with_limit: value.list_with_limit,
list_with_start_after: value.list_with_start_after,
- list_without_delimiter: value.list_without_delimiter,
- list_with_delimiter_slash: value.list_with_delimiter_slash,
+ list_with_recursive: value.list_with_recursive,
+ list_without_recursive: value.list_without_recursive,
presign: value.presign,
presign_read: value.presign_read,
presign_stat: value.presign_stat,
diff --git a/bindings/c/tests/opinfo.cpp b/bindings/c/tests/opinfo.cpp
index 7239b7543..56e0bae3f 100644
--- a/bindings/c/tests/opinfo.cpp
+++ b/bindings/c/tests/opinfo.cpp
@@ -82,7 +82,7 @@ TEST_F(OpendalOperatorInfoTest, CapabilityTest)
EXPECT_TRUE(cap.delete_);
EXPECT_TRUE(cap.list);
- EXPECT_TRUE(cap.list_without_delimiter);
+ EXPECT_TRUE(cap.list_with_recursive);
EXPECT_TRUE(cap.copy);
diff --git a/bindings/haskell/src/lib.rs b/bindings/haskell/src/lib.rs
index 3b0233388..602706e49 100644
--- a/bindings/haskell/src/lib.rs
+++ b/bindings/haskell/src/lib.rs
@@ -540,7 +540,7 @@ pub unsafe extern "C" fn blocking_scan(
}
};
- let res = match op.lister_with(path_str).delimiter("").call() {
+ let res = match op.lister_with(path_str).recursive(true).call() {
Ok(lister) => FFIResult::ok(Box::into_raw(Box::new(lister))),
Err(e) => FFIResult::err_with_source("Failed to scan", e),
};
diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs
index b853cfbcf..b3631d14e 100644
--- a/bindings/java/src/lib.rs
+++ b/bindings/java/src/lib.rs
@@ -183,8 +183,8 @@ fn make_capability<'a>(env: &mut JNIEnv<'a>, cap: Capability) -> Result<JObject<
JValue::Bool(cap.list as jboolean),
JValue::Bool(cap.list_with_limit as jboolean),
JValue::Bool(cap.list_with_start_after as jboolean),
- JValue::Bool(cap.list_with_delimiter_slash as jboolean),
- JValue::Bool(cap.list_without_delimiter as jboolean),
+ JValue::Bool(cap.list_without_recursive as jboolean),
+ JValue::Bool(cap.list_with_recursive as jboolean),
JValue::Bool(cap.presign as jboolean),
JValue::Bool(cap.presign_read as jboolean),
JValue::Bool(cap.presign_stat as jboolean),
diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs
index 3961b9d74..7882be4cb 100644
--- a/bindings/nodejs/src/lib.rs
+++ b/bindings/nodejs/src/lib.rs
@@ -322,7 +322,7 @@ impl Operator {
Ok(Lister(
self.0
.lister_with(&path)
- .delimiter("")
+ .recursive(true)
.await
.map_err(format_napi_error)?,
))
@@ -354,7 +354,7 @@ impl Operator {
self.0
.blocking()
.lister_with(&path)
- .delimiter("")
+ .recursive(true)
.call()
.map_err(format_napi_error)?,
))
@@ -466,7 +466,6 @@ impl Operator {
self.0
.blocking()
.lister_with(&path)
- .delimiter("")
.call()
.map_err(format_napi_error)?,
))
diff --git a/bindings/python/python/opendal/__init__.pyi b/bindings/python/python/opendal/__init__.pyi
index 03f702337..79162f0d9 100644
--- a/bindings/python/python/opendal/__init__.pyi
+++ b/bindings/python/python/opendal/__init__.pyi
@@ -159,8 +159,8 @@ class Capability:
list: bool
list_with_limit: bool
list_with_start_after: bool
- list_with_delimiter_slash: bool
- list_without_delimiter: bool
+ list_without_recursive: bool
+ list_with_recursive: bool
presign: bool
presign_read: bool
diff --git a/bindings/python/src/capability.rs b/bindings/python/src/capability.rs
index 373ea9599..533d016fa 100644
--- a/bindings/python/src/capability.rs
+++ b/bindings/python/src/capability.rs
@@ -97,9 +97,9 @@ pub struct Capability {
/// If backend supports list with start after.
pub list_with_start_after: bool,
/// If backend support list with using slash as delimiter.
- pub list_with_delimiter_slash: bool,
+ pub list_without_recursive: bool,
/// If backend supports list without delimiter.
- pub list_without_delimiter: bool,
+ pub list_with_recursive: bool,
/// If operator supports presign.
pub presign: bool,
@@ -155,8 +155,8 @@ impl Capability {
list: capability.list,
list_with_limit: capability.list_with_limit,
list_with_start_after: capability.list_with_start_after,
- list_with_delimiter_slash: capability.list_with_delimiter_slash,
- list_without_delimiter: capability.list_without_delimiter,
+ list_without_recursive: capability.list_without_recursive,
+ list_with_recursive: capability.list_with_recursive,
presign: capability.presign,
presign_read: capability.presign_read,
presign_stat: capability.presign_stat,
diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs
index 19908bffe..d8e6763af 100644
--- a/bindings/python/src/operator.rs
+++ b/bindings/python/src/operator.rs
@@ -174,7 +174,7 @@ impl Operator {
let l = self
.0
.lister_with(path)
- .delimiter("")
+ .recursive(true)
.call()
.map_err(format_pyerr)?;
Ok(BlockingLister::new(l))
@@ -390,7 +390,7 @@ impl AsyncOperator {
future_into_py(py, async move {
let lister = this
.lister_with(&path)
- .delimiter("")
+ .recursive(true)
.await
.map_err(format_pyerr)?;
let pylister: PyObject = Python::with_gil(|py| AsyncLister::new(lister).into_py(py));
diff --git a/core/src/docs/upgrade.md b/core/src/docs/upgrade.md
index 394c10edc..f8bbece96 100644
--- a/core/src/docs/upgrade.md
+++ b/core/src/docs/upgrade.md
@@ -1,3 +1,16 @@
+# Unreleased
+
+## Public API
+
+After [RFC-3526: List Recursive](crate::docs::rfcs::rfc_3526_list_recursive) landed, we have changed the `list` API to accept `recursive` instead of `delimiter`:
+
+Users will need to change the following usage:
+
+- `op.list_with(path).delimiter("")` -> `op.list_with(path).recursive(true)`
+- `op.list_with(path).delimiter("/")` -> `op.list_with(path).recursive(false)`
+
+`delimiter` other than `""` and `"/"` is not supported anymore.
+
# Upgrade to v0.42
## Public API
diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs
index c5e593a47..5e22c4d1a 100644
--- a/core/src/layers/complete.rs
+++ b/core/src/layers/complete.rs
@@ -103,9 +103,9 @@ use crate::*;
/// Underlying services will return [`Capability`] to indicate the
/// features that returning pagers support.
///
-/// - If both `list_with_delimiter_slash` and `list_without_delimiter`, return directly.
-/// - If only `list_without_delimiter`, with [`oio::to_flat_pager`].
-/// - if only `list_with_delimiter_slash`, with [`oio::to_hierarchy_pager`].
+/// - If both `list_without_recursive` and `list_with_recursive`, return directly.
+/// - If only `list_with_recursive`, with [`oio::to_flat_pager`].
+/// - if only `list_without_recursive`, with [`oio::to_hierarchy_pager`].
/// - If neither not supported, something must be wrong for `list` is true.
///
/// ## Capability Check
@@ -230,39 +230,36 @@ impl<A: Accessor> CompleteAccessor<A> {
return Err(self.new_unsupported_error(Operation::List));
}
- let delimiter = args.delimiter();
+ let recursive = args.recursive();
- if delimiter.is_empty() {
- return if cap.list_without_delimiter {
+ match (
+ recursive,
+ cap.list_with_recursive,
+ cap.list_without_recursive,
+ ) {
+ // - If service can both list_with_recursive and list_without_recursive
+ // - If recursive is true while services can list_with_recursive
+ // - If recursive is false while services can list_without_recursive
+ (_, true, true) | (true, true, _) | (false, _, true) => {
let (rp, p) = self.inner.list(path, args).await?;
Ok((rp, CompletePager::AlreadyComplete(p)))
- } else {
- let p = into_flat_page(
- self.inner.clone(),
- path,
- args.with_delimiter("/").limit().unwrap_or(1000),
- );
+ }
+ // If services can't list_with_recursive nor list_without_recursive.
+ //
+ // It should be a service level bug.
+ (_, false, false) => Err(self.new_unsupported_error(Operation::List)),
+ // If recursive is true but service can't list_with_recursive
+ (true, false, true) => {
+ let p = into_flat_page(self.inner.clone(), path, args.limit().unwrap_or(1000));
Ok((RpList::default(), CompletePager::NeedFlat(p)))
- };
- }
-
- if delimiter == "/" {
- return if cap.list_with_delimiter_slash {
- let (rp, p) = self.inner.list(path, args).await?;
- Ok((rp, CompletePager::AlreadyComplete(p)))
- } else {
- let (_, p) = self.inner.list(path, args.with_delimiter("")).await?;
+ }
+ // If recursive is false but service can't list_without_recursive
+ (false, true, false) => {
+ let (_, p) = self.inner.list(path, args.with_recursive(true)).await?;
let p = into_hierarchy_page(p, path);
Ok((RpList::default(), CompletePager::NeedHierarchy(p)))
- };
+ }
}
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "list with other delimiter is not supported",
- )
- .with_context("service", self.meta.scheme())
- .with_context("delimiter", delimiter))
}
fn complete_blocking_list(
@@ -275,40 +272,37 @@ impl<A: Accessor> CompleteAccessor<A> {
return Err(self.new_unsupported_error(Operation::BlockingList));
}
- let delimiter = args.delimiter();
+ let recursive = args.recursive();
- if delimiter.is_empty() {
- return if cap.list_without_delimiter {
+ match (
+ recursive,
+ cap.list_with_recursive,
+ cap.list_without_recursive,
+ ) {
+ // - If service can both list_with_recursive and list_without_recursive
+ // - If recursive is true while services can list_with_recursive
+ // - If recursive is false while services can list_without_recursive
+ (_, true, true) | (true, true, _) | (false, _, true) => {
let (rp, p) = self.inner.blocking_list(path, args)?;
Ok((rp, CompletePager::AlreadyComplete(p)))
- } else {
- let p = into_flat_page(
- self.inner.clone(),
- path,
- args.with_delimiter("/").limit().unwrap_or(1000),
- );
+ }
+ // If services can't list_with_recursive nor list_without_recursive.
+ //
+ // It should be a service level bug.
+ (_, false, false) => Err(self.new_unsupported_error(Operation::List)),
+ // If recursive is true but service can't list_with_recursive
+ (true, false, true) => {
+ let p = into_flat_page(self.inner.clone(), path, args.limit().unwrap_or(1000));
Ok((RpList::default(), CompletePager::NeedFlat(p)))
- };
- }
-
- if delimiter == "/" {
- return if cap.list_with_delimiter_slash {
- let (rp, p) = self.inner.blocking_list(path, args)?;
- Ok((rp, CompletePager::AlreadyComplete(p)))
- } else {
- let (_, p) = self.inner.blocking_list(path, args.with_delimiter(""))?;
+ }
+ // If recursive is false but service can't list_without_recursive
+ (false, true, false) => {
+ let (_, p) = self.inner.blocking_list(path, args.with_recursive(true))?;
let p: HierarchyPager<<A as Accessor>::BlockingPager> =
into_hierarchy_page(p, path);
Ok((RpList::default(), CompletePager::NeedHierarchy(p)))
- };
+ }
}
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "list with other delimiter is not supported",
- )
- .with_context("service", self.meta.scheme())
- .with_context("delimiter", delimiter))
}
}
@@ -947,6 +941,7 @@ mod tests {
let op = new_test_operator(Capability {
list: true,
+ list_with_recursive: true,
..Default::default()
});
let res = op.list("path/").await;
diff --git a/core/src/layers/immutable_index.rs b/core/src/layers/immutable_index.rs
index 31858211c..0186267cd 100644
--- a/core/src/layers/immutable_index.rs
+++ b/core/src/layers/immutable_index.rs
@@ -153,8 +153,8 @@ impl<A: Accessor> LayeredAccessor for ImmutableIndexAccessor<A> {
let cap = meta.full_capability_mut();
cap.list = true;
- cap.list_with_delimiter_slash = true;
- cap.list_without_delimiter = true;
+ cap.list_with_recursive = true;
+ cap.list_without_recursive = true;
meta
}
@@ -169,15 +169,10 @@ impl<A: Accessor> LayeredAccessor for ImmutableIndexAccessor<A> {
path = ""
}
- let idx = if args.delimiter() == "/" {
- self.children_hierarchy(path)
- } else if args.delimiter().is_empty() {
+ let idx = if args.recursive() {
self.children_flat(path)
} else {
- return Err(Error::new(
- ErrorKind::Unsupported,
- &format!("delimiter {} is not supported", args.delimiter()),
- ));
+ self.children_hierarchy(path)
};
Ok((RpList::default(), ImmutableDir::new(idx)))
@@ -201,15 +196,10 @@ impl<A: Accessor> LayeredAccessor for ImmutableIndexAccessor<A> {
path = ""
}
- let idx = if args.delimiter() == "/" {
- self.children_hierarchy(path)
- } else if args.delimiter().is_empty() {
+ let idx = if args.recursive() {
self.children_flat(path)
} else {
- return Err(Error::new(
- ErrorKind::Unsupported,
- &format!("delimiter {} is not supported", args.delimiter()),
- ));
+ self.children_hierarchy(path)
};
Ok((RpList::default(), ImmutableDir::new(idx)))
@@ -333,7 +323,7 @@ mod tests {
.layer(iil)
.finish();
- let mut ds = op.lister_with("/").delimiter("").await?;
+ let mut ds = op.lister_with("/").recursive(true).await?;
let mut set = HashSet::new();
let mut map = HashMap::new();
while let Some(entry) = ds.try_next().await? {
@@ -435,7 +425,7 @@ mod tests {
.layer(iil)
.finish();
- let mut ds = op.lister_with("/").delimiter("").await?;
+ let mut ds = op.lister_with("/").recursive(true).await?;
let mut map = HashMap::new();
let mut set = HashSet::new();
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index 9ae6c877d..96c6e144a 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -1153,8 +1153,8 @@ mod tests {
am.set_native_capability(Capability {
read: true,
list: true,
- list_with_delimiter_slash: true,
- list_without_delimiter: true,
+ list_without_recursive: true,
+ list_with_recursive: true,
batch: true,
..Default::default()
});
diff --git a/core/src/raw/adapters/kv/backend.rs b/core/src/raw/adapters/kv/backend.rs
index 799dd1be5..a05dee052 100644
--- a/core/src/raw/adapters/kv/backend.rs
+++ b/core/src/raw/adapters/kv/backend.rs
@@ -98,7 +98,7 @@ impl<S: Adapter> Accessor for Backend<S> {
}
if cap.list {
- cap.list_without_delimiter = true;
+ cap.list_with_recursive = true;
}
am.set_native_capability(cap);
@@ -202,14 +202,7 @@ impl<S: Adapter> Accessor for Backend<S> {
Ok(RpDelete::default())
}
- async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
- if !args.delimiter().is_empty() {
- return Err(Error::new(
- ErrorKind::Unsupported,
- "kv doesn't support delimiter",
- ));
- }
-
+ async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Pager)> {
let p = build_abs_path(&self.root, path);
let res = self.kv.scan(&p).await?;
let pager = KvPager::new(&self.root, res);
@@ -217,14 +210,7 @@ impl<S: Adapter> Accessor for Backend<S> {
Ok((RpList::default(), pager))
}
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingPager)> {
- if !args.delimiter().is_empty() {
- return Err(Error::new(
- ErrorKind::Unsupported,
- "kv doesn't support delimiter",
- ));
- }
-
+ fn blocking_list(&self, path: &str, _: OpList) -> Result<(RpList, Self::BlockingPager)> {
let p = build_abs_path(&self.root, path);
let res = self.kv.blocking_scan(&p)?;
let pager = KvPager::new(&self.root, res);
diff --git a/core/src/raw/adapters/typed_kv/backend.rs b/core/src/raw/adapters/typed_kv/backend.rs
index ca872346a..8d2343026 100644
--- a/core/src/raw/adapters/typed_kv/backend.rs
+++ b/core/src/raw/adapters/typed_kv/backend.rs
@@ -94,7 +94,7 @@ impl<S: Adapter> Accessor for Backend<S> {
if kv_cap.scan {
cap.list = true;
- cap.list_without_delimiter = true;
+ cap.list_with_recursive = true;
}
if cap.read && cap.write {
@@ -205,14 +205,7 @@ impl<S: Adapter> Accessor for Backend<S> {
Ok(RpDelete::default())
}
- async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
- if !args.delimiter().is_empty() {
- return Err(Error::new(
- ErrorKind::Unsupported,
- "kv doesn't support delimiter",
- ));
- }
-
+ async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Pager)> {
let p = build_abs_path(&self.root, path);
let res = self.kv.scan(&p).await?;
let pager = KvPager::new(&self.root, res);
@@ -220,14 +213,7 @@ impl<S: Adapter> Accessor for Backend<S> {
Ok((RpList::default(), pager))
}
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingPager)> {
- if !args.delimiter().is_empty() {
- return Err(Error::new(
- ErrorKind::Unsupported,
- "kv doesn't support delimiter",
- ));
- }
-
+ fn blocking_list(&self, path: &str, _: OpList) -> Result<(RpList, Self::BlockingPager)> {
let p = build_abs_path(&self.root, path);
let res = self.kv.blocking_scan(&p)?;
let pager = KvPager::new(&self.root, res);
diff --git a/core/src/raw/oio/page/into_flat_page.rs b/core/src/raw/oio/page/into_flat_page.rs
index 0efdea6be..4af725043 100644
--- a/core/src/raw/oio/page/into_flat_page.rs
+++ b/core/src/raw/oio/page/into_flat_page.rs
@@ -29,7 +29,7 @@ pub fn into_flat_page<A: Accessor, P>(acc: A, path: &str, size: usize) -> FlatPa
{
let meta = acc.info();
debug_assert!(
- meta.full_capability().list_with_delimiter_slash,
+ meta.full_capability().list_without_recursive,
"service doesn't support list hierarchy, it must be a bug"
);
}
@@ -254,7 +254,7 @@ mod tests {
fn info(&self) -> AccessorInfo {
let mut am = AccessorInfo::default();
am.full_capability_mut().list = true;
- am.full_capability_mut().list_with_delimiter_slash = true;
+ am.full_capability_mut().list_without_recursive = true;
am
}
diff --git a/core/src/raw/ops.rs b/core/src/raw/ops.rs
index 0eb3937c6..dbb26227b 100644
--- a/core/src/raw/ops.rs
+++ b/core/src/raw/ops.rs
@@ -71,16 +71,26 @@ impl OpDelete {
#[derive(Debug, Clone)]
pub struct OpList {
/// The limit passed to underlying service to specify the max results
- /// that could return.
+ /// that could return per-request.
+ ///
+ /// Users could use this to control the memory usage of list operation.
limit: Option<usize>,
-
/// The start_after passes to underlying service to specify the specified key
/// to start listing from.
start_after: Option<String>,
-
- /// The delimiter used to for the list operation. Default to be `/`
- delimiter: String,
-
+ /// The recursive is used to control whether the list operation is recursive.
+ ///
+ /// - If `false`, list operation will only list the entries under the given path.
+ /// - If `true`, list operation will list all entries that starts with given path.
+ ///
+ /// Default to `false`.
+ recursive: bool,
+ /// Metakey is used to control which meta should be returned.
+ ///
+ /// Lister will make sure the result for specified meta is **known**:
+ ///
+ /// - `Some(v)` means exist.
+ /// - `None` means services doesn't have this meta.
metakey: FlagSet<Metakey>,
}
@@ -89,7 +99,7 @@ impl Default for OpList {
OpList {
limit: None,
start_after: None,
- delimiter: "/".to_string(),
+ recursive: false,
// By default, we want to know what's the mode of this entry.
metakey: Metakey::Mode.into(),
}
@@ -124,15 +134,20 @@ impl OpList {
self.start_after.as_deref()
}
- /// Change the delimiter. The default delimiter is "/"
- pub fn with_delimiter(mut self, delimiter: &str) -> Self {
- self.delimiter = delimiter.to_string();
+ /// The recursive is used to control whether the list operation is recursive.
+ ///
+ /// - If `false`, list operation will only list the entries under the given path.
+ /// - If `true`, list operation will list all entries that starts with given path.
+ ///
+ /// Default to `false`.
+ pub fn with_recursive(mut self, recursive: bool) -> Self {
+ self.recursive = recursive;
self
}
- /// Get the current delimiter.
- pub fn delimiter(&self) -> &str {
- &self.delimiter
+ /// Get the current recursive.
+ pub fn recursive(&self) -> bool {
+ self.recursive
}
/// Change the metakey of this list operation.
diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs
index 6e46e1b35..6d6f5612d 100644
--- a/core/src/services/azblob/backend.rs
+++ b/core/src/services/azblob/backend.rs
@@ -539,8 +539,8 @@ impl Accessor for AzblobBackend {
copy: true,
list: true,
- list_with_delimiter_slash: true,
- list_without_delimiter: true,
+ list_without_recursive: true,
+ list_with_recursive: true,
presign: self.has_sas_token,
presign_stat: self.has_sas_token,
@@ -654,7 +654,7 @@ impl Accessor for AzblobBackend {
let op = AzblobPager::new(
self.core.clone(),
path.to_string(),
- args.delimiter().to_string(),
+ args.recursive(),
args.limit(),
);
diff --git a/core/src/services/azblob/pager.rs b/core/src/services/azblob/pager.rs
index 540b01644..19c11bf36 100644
--- a/core/src/services/azblob/pager.rs
+++ b/core/src/services/azblob/pager.rs
@@ -31,7 +31,7 @@ pub struct AzblobPager {
core: Arc<AzblobCore>,
path: String,
- delimiter: String,
+ delimiter: &'static str,
limit: Option<usize>,
next_marker: String,
@@ -39,12 +39,9 @@ pub struct AzblobPager {
}
impl AzblobPager {
- pub fn new(
- core: Arc<AzblobCore>,
- path: String,
- delimiter: String,
- limit: Option<usize>,
- ) -> Self {
+ pub fn new(core: Arc<AzblobCore>, path: String, recursive: bool, limit: Option<usize>) -> Self {
+ let delimiter = if recursive { "" } else { "/" };
+
Self {
core,
path,
@@ -66,7 +63,7 @@ impl oio::Page for AzblobPager {
let resp = self
.core
- .azblob_list_blobs(&self.path, &self.next_marker, &self.delimiter, self.limit)
+ .azblob_list_blobs(&self.path, &self.next_marker, self.delimiter, self.limit)
.await?;
if resp.status() != http::StatusCode::OK {
diff --git a/core/src/services/azdls/backend.rs b/core/src/services/azdls/backend.rs
index ad03274a2..909bbd571 100644
--- a/core/src/services/azdls/backend.rs
+++ b/core/src/services/azdls/backend.rs
@@ -255,7 +255,7 @@ impl Accessor for AzdlsBackend {
rename: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs
index 106ec405d..612edb286 100644
--- a/core/src/services/azfile/backend.rs
+++ b/core/src/services/azfile/backend.rs
@@ -271,7 +271,7 @@ impl Accessor for AzfileBackend {
rename: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs
index e5a521718..f03311f45 100644
--- a/core/src/services/cos/backend.rs
+++ b/core/src/services/cos/backend.rs
@@ -289,8 +289,8 @@ impl Accessor for CosBackend {
copy: true,
list: true,
- list_with_delimiter_slash: true,
- list_without_delimiter: true,
+ list_without_recursive: true,
+ list_with_recursive: true,
presign: true,
presign_stat: true,
@@ -424,7 +424,7 @@ impl Accessor for CosBackend {
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
Ok((
RpList::default(),
- CosPager::new(self.core.clone(), path, args.delimiter(), args.limit()),
+ CosPager::new(self.core.clone(), path, args.recursive(), args.limit()),
))
}
}
diff --git a/core/src/services/cos/pager.rs b/core/src/services/cos/pager.rs
index 10784eb22..f2b38d791 100644
--- a/core/src/services/cos/pager.rs
+++ b/core/src/services/cos/pager.rs
@@ -34,7 +34,7 @@ use crate::Result;
pub struct CosPager {
core: Arc<CosCore>,
path: String,
- delimiter: String,
+ delimiter: &'static str,
limit: Option<usize>,
next_marker: String,
@@ -42,11 +42,12 @@ pub struct CosPager {
}
impl CosPager {
- pub fn new(core: Arc<CosCore>, path: &str, delimiter: &str, limit: Option<usize>) -> Self {
+ pub fn new(core: Arc<CosCore>, path: &str, recursive: bool, limit: Option<usize>) -> Self {
+ let delimiter = if recursive { "" } else { "/" };
Self {
core,
path: path.to_string(),
- delimiter: delimiter.to_string(),
+ delimiter,
limit,
next_marker: "".to_string(),
@@ -64,7 +65,7 @@ impl oio::Page for CosPager {
let resp = self
.core
- .cos_list_objects(&self.path, &self.next_marker, &self.delimiter, self.limit)
+ .cos_list_objects(&self.path, &self.next_marker, self.delimiter, self.limit)
.await?;
if resp.status() != http::StatusCode::OK {
diff --git a/core/src/services/dbfs/backend.rs b/core/src/services/dbfs/backend.rs
index be4822309..afebda36b 100644
--- a/core/src/services/dbfs/backend.rs
+++ b/core/src/services/dbfs/backend.rs
@@ -178,7 +178,7 @@ impl Accessor for DbfsBackend {
rename: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs
index 4b0d14207..874acfce4 100644
--- a/core/src/services/fs/backend.rs
+++ b/core/src/services/fs/backend.rs
@@ -269,7 +269,7 @@ impl Accessor for FsBackend {
delete: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
copy: true,
rename: true,
diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs
index e7e4b101f..0b25fd68c 100644
--- a/core/src/services/ftp/backend.rs
+++ b/core/src/services/ftp/backend.rs
@@ -303,7 +303,7 @@ impl Accessor for FtpBackend {
create_dir: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index 0522a8d5c..8fb6eb9a4 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -350,8 +350,8 @@ impl Accessor for GcsBackend {
list: true,
list_with_limit: true,
list_with_start_after: true,
- list_with_delimiter_slash: true,
- list_without_delimiter: true,
+ list_without_recursive: true,
+ list_with_recursive: true,
batch: true,
batch_max_operations: Some(100),
@@ -477,7 +477,7 @@ impl Accessor for GcsBackend {
GcsPager::new(
self.core.clone(),
path,
- args.delimiter(),
+ args.recursive(),
args.limit(),
args.start_after(),
),
diff --git a/core/src/services/gcs/pager.rs b/core/src/services/gcs/pager.rs
index e6cffd675..5c4fdc50d 100644
--- a/core/src/services/gcs/pager.rs
+++ b/core/src/services/gcs/pager.rs
@@ -32,7 +32,7 @@ pub struct GcsPager {
core: Arc<GcsCore>,
path: String,
- delimiter: String,
+ delimiter: &'static str,
limit: Option<usize>,
/// Filter results to objects whose names are lexicographically
@@ -48,15 +48,16 @@ impl GcsPager {
pub fn new(
core: Arc<GcsCore>,
path: &str,
- delimiter: &str,
+ recursive: bool,
limit: Option<usize>,
start_after: Option<&str>,
) -> Self {
+ let delimiter = if recursive { "" } else { "/" };
Self {
core,
path: path.to_string(),
- delimiter: delimiter.to_string(),
+ delimiter,
limit,
start_after: start_after.map(String::from),
@@ -78,7 +79,7 @@ impl oio::Page for GcsPager {
.gcs_list_objects(
&self.path,
&self.page_token,
- &self.delimiter,
+ self.delimiter,
self.limit,
self.start_after.clone(),
)
diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs
index 4e1274d21..029f79a08 100644
--- a/core/src/services/gdrive/backend.rs
+++ b/core/src/services/gdrive/backend.rs
@@ -59,7 +59,7 @@ impl Accessor for GdriveBackend {
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
write: true,
diff --git a/core/src/services/hdfs/backend.rs b/core/src/services/hdfs/backend.rs
index a52d9c284..cbcaea45f 100644
--- a/core/src/services/hdfs/backend.rs
+++ b/core/src/services/hdfs/backend.rs
@@ -182,7 +182,7 @@ impl Accessor for HdfsBackend {
delete: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
blocking: true,
diff --git a/core/src/services/ipfs/backend.rs b/core/src/services/ipfs/backend.rs
index 6a557d6a4..8dcf8610e 100644
--- a/core/src/services/ipfs/backend.rs
+++ b/core/src/services/ipfs/backend.rs
@@ -180,7 +180,7 @@ impl Accessor for IpfsBackend {
read_with_range: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/ipmfs/backend.rs b/core/src/services/ipmfs/backend.rs
index 303b1db8d..ef39c17f0 100644
--- a/core/src/services/ipmfs/backend.rs
+++ b/core/src/services/ipmfs/backend.rs
@@ -84,7 +84,7 @@ impl Accessor for IpmfsBackend {
delete: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs
index 311af30a6..f28363af8 100644
--- a/core/src/services/obs/backend.rs
+++ b/core/src/services/obs/backend.rs
@@ -295,8 +295,8 @@ impl Accessor for ObsBackend {
copy: true,
list: true,
- list_with_delimiter_slash: true,
- list_without_delimiter: true,
+ list_without_recursive: true,
+ list_with_recursive: true,
presign: true,
presign_stat: true,
@@ -430,7 +430,7 @@ impl Accessor for ObsBackend {
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
Ok((
RpList::default(),
- ObsPager::new(self.core.clone(), path, args.delimiter(), args.limit()),
+ ObsPager::new(self.core.clone(), path, args.recursive(), args.limit()),
))
}
}
diff --git a/core/src/services/obs/pager.rs b/core/src/services/obs/pager.rs
index e24f4f6a6..9908d95f7 100644
--- a/core/src/services/obs/pager.rs
+++ b/core/src/services/obs/pager.rs
@@ -34,7 +34,7 @@ use crate::Result;
pub struct ObsPager {
core: Arc<ObsCore>,
path: String,
- delimiter: String,
+ delimiter: &'static str,
limit: Option<usize>,
next_marker: String,
@@ -42,11 +42,13 @@ pub struct ObsPager {
}
impl ObsPager {
- pub fn new(core: Arc<ObsCore>, path: &str, delimiter: &str, limit: Option<usize>) -> Self {
+ pub fn new(core: Arc<ObsCore>, path: &str, recursive: bool, limit: Option<usize>) -> Self {
+ let delimiter = if recursive { "" } else { "/" };
+
Self {
core,
path: path.to_string(),
- delimiter: delimiter.to_string(),
+ delimiter,
limit,
next_marker: "".to_string(),
@@ -64,7 +66,7 @@ impl oio::Page for ObsPager {
let resp = self
.core
- .obs_list_objects(&self.path, &self.next_marker, &self.delimiter, self.limit)
+ .obs_list_objects(&self.path, &self.next_marker, self.delimiter, self.limit)
.await?;
if resp.status() != http::StatusCode::OK {
diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs
index b00efd9bd..8ed92ab4a 100644
--- a/core/src/services/onedrive/backend.rs
+++ b/core/src/services/onedrive/backend.rs
@@ -80,7 +80,7 @@ impl Accessor for OnedriveBackend {
delete: true,
create_dir: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
..Default::default()
});
diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs
index 93323b3a3..8fd43971b 100644
--- a/core/src/services/oss/backend.rs
+++ b/core/src/services/oss/backend.rs
@@ -426,8 +426,8 @@ impl Accessor for OssBackend {
list: true,
list_with_limit: true,
list_with_start_after: true,
- list_with_delimiter_slash: true,
- list_without_delimiter: true,
+ list_without_recursive: true,
+ list_with_recursive: true,
presign: true,
presign_stat: true,
@@ -550,7 +550,7 @@ impl Accessor for OssBackend {
OssPager::new(
self.core.clone(),
path,
- args.delimiter(),
+ args.recursive(),
args.limit(),
args.start_after(),
),
diff --git a/core/src/services/oss/pager.rs b/core/src/services/oss/pager.rs
index fd989dfe3..d865b4ec0 100644
--- a/core/src/services/oss/pager.rs
+++ b/core/src/services/oss/pager.rs
@@ -36,7 +36,7 @@ pub struct OssPager {
core: Arc<OssCore>,
path: String,
- delimiter: String,
+ delimiter: &'static str,
limit: Option<usize>,
/// Filter results to objects whose names are lexicographically
/// **equal to or after** startOffset
@@ -50,14 +50,15 @@ impl OssPager {
pub fn new(
core: Arc<OssCore>,
path: &str,
- delimiter: &str,
+ recursive: bool,
limit: Option<usize>,
start_after: Option<&str>,
) -> Self {
+ let delimiter = if recursive { "" } else { "/" };
Self {
core,
path: path.to_string(),
- delimiter: delimiter.to_string(),
+ delimiter,
limit,
start_after: start_after.map(String::from),
token: None,
@@ -79,7 +80,7 @@ impl oio::Page for OssPager {
.oss_list_object(
&self.path,
self.token.as_deref(),
- &self.delimiter,
+ self.delimiter,
self.limit,
self.start_after.clone(),
)
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 840e05149..49cff44b3 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -1016,8 +1016,8 @@ impl Accessor for S3Backend {
list: true,
list_with_limit: true,
list_with_start_after: true,
- list_without_delimiter: true,
- list_with_delimiter_slash: true,
+ list_with_recursive: true,
+ list_without_recursive: true,
presign: true,
presign_stat: true,
@@ -1139,7 +1139,7 @@ impl Accessor for S3Backend {
S3Pager::new(
self.core.clone(),
path,
- args.delimiter(),
+ args.recursive(),
args.limit(),
args.start_after(),
),
diff --git a/core/src/services/s3/pager.rs b/core/src/services/s3/pager.rs
index 1525aa4ab..55ebec1a3 100644
--- a/core/src/services/s3/pager.rs
+++ b/core/src/services/s3/pager.rs
@@ -33,7 +33,7 @@ pub struct S3Pager {
core: Arc<S3Core>,
path: String,
- delimiter: String,
+ delimiter: &'static str,
limit: Option<usize>,
/// Amazon S3 starts listing **after** this specified key
@@ -47,15 +47,16 @@ impl S3Pager {
pub fn new(
core: Arc<S3Core>,
path: &str,
- delimiter: &str,
+ recursive: bool,
limit: Option<usize>,
start_after: Option<&str>,
) -> Self {
+ let delimiter = if recursive { "" } else { "/" };
Self {
core,
path: path.to_string(),
- delimiter: delimiter.to_string(),
+ delimiter,
limit,
start_after: start_after.map(String::from),
@@ -77,7 +78,7 @@ impl oio::Page for S3Pager {
.s3_list_objects(
&self.path,
&self.token,
- &self.delimiter,
+ self.delimiter,
self.limit,
self.start_after.clone(),
)
diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs
index e778dfe69..7152f0267 100644
--- a/core/src/services/sftp/backend.rs
+++ b/core/src/services/sftp/backend.rs
@@ -268,7 +268,7 @@ impl Accessor for SftpBackend {
list: true,
list_with_limit: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
copy: self.copyable,
rename: true,
diff --git a/core/src/services/swift/backend.rs b/core/src/services/swift/backend.rs
index 080b525c1..50cec342c 100644
--- a/core/src/services/swift/backend.rs
+++ b/core/src/services/swift/backend.rs
@@ -237,7 +237,7 @@ impl Accessor for SwiftBackend {
delete: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
..Default::default()
});
@@ -329,11 +329,7 @@ impl Accessor for SwiftBackend {
}
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
- let op = SwiftPager::new(
- self.core.clone(),
- path.to_string(),
- args.delimiter().to_string(),
- );
+ let op = SwiftPager::new(self.core.clone(), path.to_string(), args.recursive());
Ok((RpList::default(), op))
}
diff --git a/core/src/services/swift/pager.rs b/core/src/services/swift/pager.rs
index ea97e69e7..62ff2304b 100644
--- a/core/src/services/swift/pager.rs
+++ b/core/src/services/swift/pager.rs
@@ -28,12 +28,13 @@ use crate::*;
pub struct SwiftPager {
core: Arc<SwiftCore>,
path: String,
- delimiter: String,
+ delimiter: &'static str,
done: bool,
}
impl SwiftPager {
- pub fn new(core: Arc<SwiftCore>, path: String, delimiter: String) -> Self {
+ pub fn new(core: Arc<SwiftCore>, path: String, recursive: bool) -> Self {
+ let delimiter = if recursive { "" } else { "/" };
Self {
core,
path,
@@ -50,7 +51,7 @@ impl oio::Page for SwiftPager {
return Ok(None);
}
- let response = self.core.swift_list(&self.path, &self.delimiter).await?;
+ let response = self.core.swift_list(&self.path, self.delimiter).await?;
let status_code = response.status();
diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs
index 93e0a70ca..310c1a0fe 100644
--- a/core/src/services/webdav/backend.rs
+++ b/core/src/services/webdav/backend.rs
@@ -248,7 +248,7 @@ impl Accessor for WebdavBackend {
rename: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
..Default::default()
});
@@ -379,10 +379,10 @@ impl Accessor for WebdavBackend {
}
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
- if args.delimiter() != "/" {
+ if args.recursive() {
return Err(Error::new(
ErrorKind::Unsupported,
- "webdav only support delimiter `/`",
+ "webdav doesn't support list with recursive",
));
}
diff --git a/core/src/services/webhdfs/backend.rs b/core/src/services/webhdfs/backend.rs
index 1c6a5a4cf..b80423b6d 100644
--- a/core/src/services/webhdfs/backend.rs
+++ b/core/src/services/webhdfs/backend.rs
@@ -421,7 +421,7 @@ impl Accessor for WebhdfsBackend {
delete: true,
list: true,
- list_with_delimiter_slash: true,
+ list_without_recursive: true,
..Default::default()
});
@@ -541,10 +541,10 @@ impl Accessor for WebhdfsBackend {
}
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {
- if args.delimiter() != "/" {
+ if args.recursive() {
return Err(Error::new(
ErrorKind::Unsupported,
- "WebHDFS only support delimiter `/`",
+ "WebHDFS doesn't support list with recursive",
));
}
diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs
index 8c1a49779..4e85f4106 100644
--- a/core/src/types/capability.rs
+++ b/core/src/types/capability.rs
@@ -120,10 +120,10 @@ pub struct Capability {
pub list_with_limit: bool,
/// If backend supports list with start after.
pub list_with_start_after: bool,
- /// If backend support list with using slash as delimiter.
- pub list_with_delimiter_slash: bool,
- /// If backend supports list without delimiter.
- pub list_without_delimiter: bool,
+ /// If backend supports list with recursive.
+ pub list_with_recursive: bool,
+ /// If backend supports list without recursive.
+ pub list_without_recursive: bool,
/// If operator supports presign.
pub presign: bool,
diff --git a/core/src/types/operator/blocking_operator.rs b/core/src/types/operator/blocking_operator.rs
index e8b8572da..2b1f4a456 100644
--- a/core/src/types/operator/blocking_operator.rs
+++ b/core/src/types/operator/blocking_operator.rs
@@ -797,7 +797,7 @@ impl BlockingOperator {
return self.delete(path);
}
- let obs = self.lister_with(path).delimiter("").call()?;
+ let obs = self.lister_with(path).recursive(true).call()?;
for v in obs {
match v {
@@ -822,7 +822,7 @@ impl BlockingOperator {
/// ## Listing recursively
///
/// This function only read the children of the given directory. To read
- /// all entries recursively, use `BlockingOperator::list_with("path").delimiter("")`
+ /// all entries recursively, use `BlockingOperator::list_with("path").recursive(true)`
/// instead.
///
/// ## Streaming
@@ -896,7 +896,7 @@ impl BlockingOperator {
/// use opendal::EntryMode;
/// use opendal::Metakey;
/// # fn test(op: BlockingOperator) -> Result<()> {
- /// let mut entries = op.list_with("prefix/").delimiter("").call()?;
+ /// let mut entries = op.list_with("prefix/").recursive(true).call()?;
/// for entry in entries {
/// match entry.metadata().mode() {
/// EntryMode::FILE => {
@@ -1066,7 +1066,7 @@ impl BlockingOperator {
/// use opendal::EntryMode;
/// use opendal::Metakey;
/// # fn test(op: BlockingOperator) -> Result<()> {
- /// let mut ds = op.lister_with("path/to/dir/").delimiter("").call()?;
+ /// let mut ds = op.lister_with("path/to/dir/").recursive(true).call()?;
/// for entry in ds {
/// let entry = entry?;
/// match entry.metadata().mode() {
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index 1abb1f027..4461edce8 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -901,7 +901,7 @@ impl Operator {
return self.delete(path).await;
}
- let obs = self.lister_with(path).delimiter("").await?;
+ let obs = self.lister_with(path).recursive(true).await?;
if self.info().full_capability().batch {
let mut obs = obs.try_chunks(self.limit());
@@ -942,7 +942,7 @@ impl Operator {
/// ## Listing recursively
///
/// This function only read the children of the given directory. To read
- /// all entries recursively, use `Operator::list_with("path").delimiter("")`
+ /// all entries recursively, use `Operator::list_with("path").recursive(true)`
/// instead.
///
/// ## Streaming
@@ -1018,7 +1018,7 @@ impl Operator {
/// use opendal::Operator;
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
- /// let mut entries = op.list_with("prefix/").delimiter("").await?;
+ /// let mut entries = op.list_with("prefix/").recursive(true).await?;
/// for entry in entries {
/// match entry.metadata().mode() {
/// EntryMode::FILE => {
@@ -1106,7 +1106,7 @@ impl Operator {
/// ## Listing recursively
///
/// This function only read the children of the given directory. To read
- /// all entries recursively, use [`Operator::lister_with`] and `delimiter("")`
+ /// all entries recursively, use [`Operator::lister_with`] and `recursive(true)`
/// instead.
///
/// ## Metadata
@@ -1194,7 +1194,7 @@ impl Operator {
/// use opendal::Operator;
/// # #[tokio::main]
/// # async fn test(op: Operator) -> Result<()> {
- /// let mut ds = op.lister_with("path/to/dir/").delimiter("").await?;
+ /// let mut ds = op.lister_with("path/to/dir/").recursive(true).await?;
/// while let Some(mut entry) = ds.try_next().await? {
/// match entry.metadata().mode() {
/// EntryMode::FILE => {
diff --git a/core/src/types/operator/operator_functions.rs b/core/src/types/operator/operator_functions.rs
index 367c5b93f..4afa157f0 100644
--- a/core/src/types/operator/operator_functions.rs
+++ b/core/src/types/operator/operator_functions.rs
@@ -217,25 +217,41 @@ impl FunctionDelete {
pub struct FunctionList(pub(crate) OperatorFunction<OpList, Vec<Entry>>);
impl FunctionList {
- /// Change the limit of this list operation.
+ /// The limit passed to underlying service to specify the max results
+ /// that could return per-request.
+ ///
+ /// Users could use this to control the memory usage of list operation.
pub fn limit(mut self, v: usize) -> Self {
self.0 = self.0.map_args(|args| args.with_limit(v));
self
}
- /// Change the start_after of this list operation.
+ /// The start_after passes to underlying service to specify the specified key
+ /// to start listing from.
pub fn start_after(mut self, v: &str) -> Self {
self.0 = self.0.map_args(|args| args.with_start_after(v));
self
}
- /// Change the delimiter. The default delimiter is "/"
- pub fn delimiter(mut self, v: &str) -> Self {
- self.0 = self.0.map_args(|args| args.with_delimiter(v));
+ /// The recursive is used to control whether the list operation is recursive.
+ ///
+ /// - If `false`, list operation will only list the entries under the given path.
+ /// - If `true`, list operation will list all entries that starts with given path.
+ ///
+ /// Default to `false`.
+ pub fn recursive(mut self, v: bool) -> Self {
+ self.0 = self.0.map_args(|args| args.with_recursive(v));
self
}
- /// Change the metakey. The default metakey is `Metakey::Mode`.
+ /// Metakey is used to control which meta should be returned.
+ ///
+ /// Lister will make sure the result for specified meta is **known**:
+ ///
+ /// - `Some(v)` means exist.
+ /// - `None` means services doesn't have this meta.
+ ///
+ /// The default metakey is `Metakey::Mode`.
pub fn metakey(mut self, v: impl Into<FlagSet<Metakey>>) -> Self {
self.0 = self.0.map_args(|args| args.with_metakey(v));
self
@@ -254,25 +270,41 @@ impl FunctionList {
pub struct FunctionLister(pub(crate) OperatorFunction<OpList, BlockingLister>);
impl FunctionLister {
- /// Change the limit of this list operation.
+ /// The limit passed to underlying service to specify the max results
+ /// that could return per-request.
+ ///
+ /// Users could use this to control the memory usage of list operation.
pub fn limit(mut self, v: usize) -> Self {
self.0 = self.0.map_args(|args| args.with_limit(v));
self
}
- /// Change the start_after of this list operation.
+ /// The start_after passes to underlying service to specify the specified key
+ /// to start listing from.
pub fn start_after(mut self, v: &str) -> Self {
self.0 = self.0.map_args(|args| args.with_start_after(v));
self
}
- /// Change the delimiter. The default delimiter is "/"
- pub fn delimiter(mut self, v: &str) -> Self {
- self.0 = self.0.map_args(|args| args.with_delimiter(v));
+ /// The recursive is used to control whether the list operation is recursive.
+ ///
+ /// - If `false`, list operation will only list the entries under the given path.
+ /// - If `true`, list operation will list all entries that starts with given path.
+ ///
+ /// Default to `false`.
+ pub fn recursive(mut self, v: bool) -> Self {
+ self.0 = self.0.map_args(|args| args.with_recursive(v));
self
}
- /// Change the metakey. The default metakey is `Metakey::Mode`.
+ /// Metakey is used to control which meta should be returned.
+ ///
+ /// Lister will make sure the result for specified meta is **known**:
+ ///
+ /// - `Some(v)` means exist.
+ /// - `None` means services doesn't have this meta.
+ ///
+ /// The default metakey is `Metakey::Mode`.
pub fn metakey(mut self, v: impl Into<FlagSet<Metakey>>) -> Self {
self.0 = self.0.map_args(|args| args.with_metakey(v));
self
diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs
index 879601769..b895df19e 100644
--- a/core/src/types/operator/operator_futures.rs
+++ b/core/src/types/operator/operator_futures.rs
@@ -522,25 +522,41 @@ impl Future for FutureDelete {
pub struct FutureList(pub(crate) OperatorFuture<OpList, Vec<Entry>>);
impl FutureList {
- /// Change the limit of this list operation.
+ /// The limit passed to underlying service to specify the max results
+ /// that could return per-request.
+ ///
+ /// Users could use this to control the memory usage of list operation.
pub fn limit(mut self, v: usize) -> Self {
self.0 = self.0.map_args(|args| args.with_limit(v));
self
}
- /// Change the start_after of this list operation.
+ /// The start_after passes to underlying service to specify the specified key
+ /// to start listing from.
pub fn start_after(mut self, v: &str) -> Self {
self.0 = self.0.map_args(|args| args.with_start_after(v));
self
}
- /// Change the delimiter. The default delimiter is "/"
- pub fn delimiter(mut self, v: &str) -> Self {
- self.0 = self.0.map_args(|args| args.with_delimiter(v));
+ /// The recursive is used to control whether the list operation is recursive.
+ ///
+ /// - If `false`, list operation will only list the entries under the given path.
+ /// - If `true`, list operation will list all entries that starts with given path.
+ ///
+ /// Default to `false`.
+ pub fn recursive(mut self, v: bool) -> Self {
+ self.0 = self.0.map_args(|args| args.with_recursive(v));
self
}
- /// Change the metakey. The default metakey is `Metakey::Mode`.
+ /// Metakey is used to control which meta should be returned.
+ ///
+ /// Lister will make sure the result for specified meta is **known**:
+ ///
+ /// - `Some(v)` means exist.
+ /// - `None` means services doesn't have this meta.
+ ///
+ /// The default metakey is `Metakey::Mode`.
pub fn metakey(mut self, v: impl Into<FlagSet<Metakey>>) -> Self {
self.0 = self.0.map_args(|args| args.with_metakey(v));
self
@@ -561,25 +577,41 @@ impl Future for FutureList {
pub struct FutureLister(pub(crate) OperatorFuture<OpList, Lister>);
impl FutureLister {
- /// Change the limit of this list operation.
+ /// The limit passed to underlying service to specify the max results
+ /// that could return per-request.
+ ///
+ /// Users could use this to control the memory usage of list operation.
pub fn limit(mut self, v: usize) -> Self {
self.0 = self.0.map_args(|args| args.with_limit(v));
self
}
- /// Change the start_after of this list operation.
+ /// The start_after passes to underlying service to specify the specified key
+ /// to start listing from.
pub fn start_after(mut self, v: &str) -> Self {
self.0 = self.0.map_args(|args| args.with_start_after(v));
self
}
- /// Change the delimiter. The default delimiter is "/"
- pub fn delimiter(mut self, v: &str) -> Self {
- self.0 = self.0.map_args(|args| args.with_delimiter(v));
+ /// The recursive is used to control whether the list operation is recursive.
+ ///
+ /// - If `false`, list operation will only list the entries under the given path.
+ /// - If `true`, list operation will list all entries that starts with given path.
+ ///
+ /// Default to `false`.
+ pub fn recursive(mut self, v: bool) -> Self {
+ self.0 = self.0.map_args(|args| args.with_recursive(v));
self
}
- /// Change the metakey. The default metakey is `Metakey::Mode`.
+ /// Metakey is used to control which meta should be returned.
+ ///
+ /// Lister will make sure the result for specified meta is **known**:
+ ///
+ /// - `Some(v)` means exist.
+ /// - `None` means services doesn't have this meta.
+ ///
+ /// The default metakey is `Metakey::Mode`.
pub fn metakey(mut self, v: impl Into<FlagSet<Metakey>>) -> Self {
self.0 = self.0.map_args(|args| args.with_metakey(v));
self
diff --git a/core/tests/behavior/blocking_list.rs b/core/tests/behavior/blocking_list.rs
index e53b8a1de..174f9907b 100644
--- a/core/tests/behavior/blocking_list.rs
+++ b/core/tests/behavior/blocking_list.rs
@@ -191,7 +191,7 @@ pub fn test_blocking_scan(op: BlockingOperator) -> Result<()> {
let w = op
.lister_with(&format!("{parent}/x/"))
- .delimiter("")
+ .recursive(true)
.call()?;
let actual = w
.collect::<Vec<_>>()
diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/list.rs
index a65fb126b..64911f8fa 100644
--- a/core/tests/behavior/list.rs
+++ b/core/tests/behavior/list.rs
@@ -366,7 +366,7 @@ pub async fn test_list_with_start_after(op: Operator) -> Result<()> {
}
pub async fn test_scan_root(op: Operator) -> Result<()> {
- let w = op.lister_with("").delimiter("").await?;
+ let w = op.lister_with("").recursive(true).await?;
let actual = w
.try_collect::<Vec<_>>()
.await?
@@ -396,7 +396,7 @@ pub async fn test_scan(op: Operator) -> Result<()> {
let w = op
.lister_with(&format!("{parent}/x/"))
- .delimiter("")
+ .recursive(true)
.await?;
let actual = w
.try_collect::<Vec<_>>()
diff --git a/integrations/object_store/src/lib.rs b/integrations/object_store/src/lib.rs
index 017335228..9ffd8277a 100644
--- a/integrations/object_store/src/lib.rs
+++ b/integrations/object_store/src/lib.rs
@@ -151,7 +151,7 @@ impl ObjectStore for OpendalStore {
.inner
.lister_with(&path)
.metakey(Metakey::ContentLength | Metakey::LastModified)
- .delimiter("")
+ .recursive(true)
.await
.map_err(|err| format_object_store_error(err, &path))?;