You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/08/14 07:18:45 UTC
[incubator-opendal] 02/02: Implement list for blocking operator
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch list-with-metakey-for-blocking
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 57c96f1f6ac360a2c3a6d70239c2e55b0f354af1
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Aug 14 15:18:31 2023 +0800
Implement list for blocking operator
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/types/operator/blocking_operator.rs | 153 ++++++++++++++++++++++++++
core/src/types/operator/operator_functions.rs | 37 +++++++
2 files changed, 190 insertions(+)
diff --git a/core/src/types/operator/blocking_operator.rs b/core/src/types/operator/blocking_operator.rs
index 143592762..c964bf78a 100644
--- a/core/src/types/operator/blocking_operator.rs
+++ b/core/src/types/operator/blocking_operator.rs
@@ -711,6 +711,159 @@ impl BlockingOperator {
Ok(())
}
+ /// List entries within a given directory.
+ ///
+ /// # Notes
+ ///
+ /// ## Listing recursively
+ ///
+ /// This function only read the children of the given directory. To read
+ /// all entries recursively, use `BlockingOperator::list_with("path").delimiter("")`
+ /// instead.
+ ///
+ /// ## Streaming
+ ///
+ /// This function will read all entries in the given directory. It could
+ /// take very long time and consume a lot of memory if the directory
+ /// contains a lot of entries.
+ ///
+ /// In order to avoid this, you can use [`BlockingOperator::lister`] to list entries in
+ /// a streaming way.
+ ///
+ /// ## Metadata
+ ///
+ /// The only metadata that is guaranteed to be available is the `Mode`.
+ /// For fetching more metadata, please use [`BlockingOperator::list_with`] and `metakey`.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use anyhow::Result;
+ /// use opendal::EntryMode;
+ /// use opendal::Metakey;
+ /// use opendal::BlockingOperator;
+ /// # fn test(op: BlockingOperator) -> Result<()> {
+ /// let mut entries = op.list("path/to/dir/")?;
+ /// for entry in entries {
+ /// match entry.metadata().mode() {
+ /// EntryMode::FILE => {
+ /// println!("Handling file")
+ /// }
+ /// EntryMode::DIR => {
+ /// println!("Handling dir {}", entry.path())
+ /// }
+ /// EntryMode::Unknown => continue,
+ /// }
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn list(&self, path: &str) -> Result<Vec<Entry>> {
+ self.list_with(path).call()
+ }
+
+ /// List entries within a given directory with options.
+ ///
+ /// # Notes
+ ///
+ /// ## For streaming
+ ///
+ /// This function will read all entries in the given directory. It could
+ /// take very long time and consume a lot of memory if the directory
+ /// contains a lot of entries.
+ ///
+ /// In order to avoid this, you can use [`Operator::lister`] to list entries in
+ /// a streaming way.
+ ///
+ /// ## Metadata
+ ///
+ /// The only metadata that is guaranteed to be available is the `Mode`.
+ /// For fetching more metadata, please specify the `metakey`.
+ ///
+ /// # Examples
+ ///
+ /// ## List entries with prefix
+ ///
+ /// This function can also be used to list entries in recursive way.
+ ///
+ /// ```no_run
+ /// # use anyhow::Result;
+ /// use opendal::EntryMode;
+ /// use opendal::Metakey;
+ /// use opendal::BlockingOperator;
+ /// # fn test(op: BlockingOperator) -> Result<()> {
+ /// let mut entries = op.list_with("prefix/").delimiter("").call()?;
+ /// for entry in entries {
+ /// match entry.metadata().mode() {
+ /// EntryMode::FILE => {
+ /// println!("Handling file")
+ /// }
+ /// EntryMode::DIR => {
+ /// println!("Handling dir like start a new list via meta.path()")
+ /// }
+ /// EntryMode::Unknown => continue,
+ /// }
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// ## List entries with metakey for more metadata
+ ///
+ /// ```no_run
+ /// # use anyhow::Result;
+ /// use opendal::EntryMode;
+ /// use opendal::Metakey;
+ /// use opendal::BlockingOperator;
+ /// # fn test(op: BlockingOperator) -> Result<()> {
+ /// let mut entries = op
+ /// .list_with("dir/")
+ /// .metakey(Metakey::ContentLength | Metakey::LastModified)
+ /// .call()?;
+ /// for entry in entries {
+ /// let meta = entry.metadata();
+ /// match meta.mode() {
+ /// EntryMode::FILE => {
+ /// println!(
+ /// "Handling file {} with size {}",
+ /// entry.path(),
+ /// meta.content_length()
+ /// )
+ /// }
+ /// EntryMode::DIR => {
+ /// println!("Handling dir {}", entry.path())
+ /// }
+ /// EntryMode::Unknown => continue,
+ /// }
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn list_with(&self, path: &str) -> FunctionList {
+ let path = normalize_path(path);
+
+ FunctionList(OperatorFunction::new(
+ self.inner().clone(),
+ path,
+ OpList::default(),
+ |inner, path, args| {
+ if !validate_path(&path, EntryMode::FILE) {
+ return Err(
+ Error::new(ErrorKind::IsADirectory, "write path is a directory")
+ .with_operation("BlockingOperator::write_with")
+ .with_context("service", inner.info().scheme().into_static())
+ .with_context("path", &path),
+ );
+ }
+
+ let (_, pager) = inner.blocking_list(&path, args)?;
+ let lister = BlockingLister::new(pager);
+
+ lister.collect()
+ },
+ ))
+ }
+
/// List entries within a given directory as an iterator.
///
/// This function will create a new handle to list entries.
diff --git a/core/src/types/operator/operator_functions.rs b/core/src/types/operator/operator_functions.rs
index 5734950e0..2164bcafb 100644
--- a/core/src/types/operator/operator_functions.rs
+++ b/core/src/types/operator/operator_functions.rs
@@ -116,6 +116,43 @@ impl FunctionDelete {
}
}
+/// Function that generated by [`BlockingOperator::list_with`].
+///
+/// Users can add more options by public functions provided by this struct.
+pub struct FunctionList(pub(crate) OperatorFunction<OpList, Vec<Entry>>);
+
+impl FunctionList {
+ /// Change the limit of this list operation.
+ pub fn limit(mut self, v: usize) -> Self {
+ self.0 = self.0.map_args(|args| args.with_limit(v));
+ self
+ }
+
+ /// Change the start_after of this list operation.
+ pub fn start_after(mut self, v: &str) -> Self {
+ self.0 = self.0.map_args(|args| args.with_start_after(v));
+ self
+ }
+
+ /// Change the delimiter. The default delimiter is "/"
+ pub fn delimiter(mut self, v: &str) -> Self {
+ self.0 = self.0.map_args(|args| args.with_delimiter(v));
+ self
+ }
+
+ /// Change the metakey. The default metakey is `Metakey::Mode`.
+ pub fn metakey(mut self, v: impl Into<FlagSet<Metakey>>) -> Self {
+ self.0 = self.0.map_args(|args| args.with_metakey(v));
+ self
+ }
+
+ /// Call the function to consume all the input and generate a
+ /// result.
+ pub fn call(self) -> Result<Vec<Entry>> {
+ self.0.call()
+ }
+}
+
/// Function that generated by [`BlockingOperator::lister_with`].
///
/// Users can add more options by public functions provided by this struct.