You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/11/13 15:36:41 UTC
(opendal) 01/01: refactor: Remove metakey concept
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch remove-metakey
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit db0fd6195e75cfb0f347f23be13064b83b33b41e
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed Nov 13 23:36:23 2024 +0800
refactor: Remove metakey concept
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
bin/oay/src/services/s3/service.rs | 2 -
bindings/java/src/error.rs | 2 +-
bindings/java/src/lib.rs | 85 ++++--------
bindings/java/src/operator_input_stream.rs | 2 +-
core/src/layers/complete.rs | 14 +-
core/src/layers/mime_guess.rs | 5 +-
core/src/raw/ops.rs | 23 ----
core/src/services/fs/backend.rs | 8 +-
core/src/services/fs/lister.rs | 75 ++--------
core/src/services/webdav/core.rs | 2 +-
core/src/types/entry.rs | 120 ----------------
core/src/types/list.rs | 190 ++++----------------------
core/src/types/metadata.rs | 167 ----------------------
core/src/types/mod.rs | 1 -
core/src/types/operator/blocking_operator.rs | 103 --------------
core/src/types/operator/operator.rs | 155 ---------------------
core/src/types/operator/operator_functions.rs | 28 ----
core/src/types/operator/operator_futures.rs | 43 ------
core/tests/behavior/async_list.rs | 126 -----------------
core/tests/behavior/blocking_list.rs | 91 ------------
integrations/cloud_filter/src/lib.rs | 3 +-
integrations/compat/Cargo.toml | 2 +-
integrations/fuse3/src/file_system.rs | 2 -
integrations/object_store/src/store.rs | 16 +--
integrations/object_store/src/utils.rs | 16 +--
25 files changed, 80 insertions(+), 1201 deletions(-)
diff --git a/bin/oay/src/services/s3/service.rs b/bin/oay/src/services/s3/service.rs
index b30bae284..db40cafb7 100644
--- a/bin/oay/src/services/s3/service.rs
+++ b/bin/oay/src/services/s3/service.rs
@@ -26,7 +26,6 @@ use axum::routing::get;
use axum::Router;
use chrono::SecondsFormat;
use futures_util::StreamExt;
-use opendal::Metakey;
use opendal::Operator;
use serde::Deserialize;
use serde::Serialize;
@@ -100,7 +99,6 @@ async fn handle_list_objects(
.op
.lister_with(¶ms.prefix)
.start_after(¶ms.start_after)
- .metakey(Metakey::Mode | Metakey::LastModified | Metakey::Etag | Metakey::ContentLength)
.await?
.chunks(1000);
diff --git a/bindings/java/src/error.rs b/bindings/java/src/error.rs
index 598050463..eb5cf30d8 100644
--- a/bindings/java/src/error.rs
+++ b/bindings/java/src/error.rs
@@ -83,7 +83,7 @@ impl From<opendal::Error> for Error {
impl From<jni::errors::Error> for Error {
fn from(err: jni::errors::Error) -> Self {
- opendal::Error::new(ErrorKind::Unexpected, &err.to_string()).into()
+ opendal::Error::new(ErrorKind::Unexpected, err.to_string()).into()
}
}
diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs
index a7a67334b..47113ae74 100644
--- a/bindings/java/src/lib.rs
+++ b/bindings/java/src/lib.rs
@@ -28,7 +28,6 @@ use opendal::Capability;
use opendal::Entry;
use opendal::EntryMode;
use opendal::Metadata;
-use opendal::Metakey;
use opendal::OperatorInfo;
mod async_operator;
@@ -51,7 +50,7 @@ fn make_presigned_request<'a>(env: &mut JNIEnv<'a>, req: PresignedRequest) -> Re
for (k, v) in req.header().iter() {
let key = k.to_string();
let value = v.to_str().map_err(|err| {
- opendal::Error::new(opendal::ErrorKind::Unexpected, &err.to_string())
+ opendal::Error::new(opendal::ErrorKind::Unexpected, err.to_string())
})?;
map.insert(key, value.to_owned());
}
@@ -143,72 +142,36 @@ fn make_metadata<'a>(env: &mut JNIEnv<'a>, metadata: Metadata) -> Result<JObject
EntryMode::Unknown => 2,
};
- let metakey = metadata.metakey();
-
- let contains_metakey = |k| metakey.contains(k) || metakey.contains(Metakey::Complete);
-
- let last_modified = if contains_metakey(Metakey::LastModified) {
- metadata.last_modified().map_or_else(
- || Ok::<JObject<'_>, error::Error>(JObject::null()),
- |v| {
- Ok(env
- .call_static_method(
- "java/time/Instant",
- "ofEpochSecond",
- "(JJ)Ljava/time/Instant;",
- &[
- JValue::Long(v.timestamp()),
- JValue::Long(v.timestamp_subsec_nanos() as jlong),
- ],
- )?
- .l()?)
- },
- )?
- } else {
- JObject::null()
- };
+ let last_modified = metadata.last_modified().map_or_else(
+ || Ok::<JObject<'_>, error::Error>(JObject::null()),
+ |v| {
+ Ok(env
+ .call_static_method(
+ "java/time/Instant",
+ "ofEpochSecond",
+ "(JJ)Ljava/time/Instant;",
+ &[
+ JValue::Long(v.timestamp()),
+ JValue::Long(v.timestamp_subsec_nanos() as jlong),
+ ],
+ )?
+ .l()?)
+ },
+ )?;
- let cache_control = if contains_metakey(Metakey::CacheControl) {
- convert::string_to_jstring(env, metadata.cache_control())?
- } else {
- JObject::null()
- };
+ let cache_control = convert::string_to_jstring(env, metadata.cache_control())?;
- let content_disposition = if contains_metakey(Metakey::ContentDisposition) {
- convert::string_to_jstring(env, metadata.content_disposition())?
- } else {
- JObject::null()
- };
+ let content_disposition = convert::string_to_jstring(env, metadata.content_disposition())?;
- let content_md5 = if contains_metakey(Metakey::ContentMd5) {
- convert::string_to_jstring(env, metadata.content_md5())?
- } else {
- JObject::null()
- };
+ let content_md5 = convert::string_to_jstring(env, metadata.content_md5())?;
- let content_type = if contains_metakey(Metakey::ContentType) {
- convert::string_to_jstring(env, metadata.content_type())?
- } else {
- JObject::null()
- };
+ let content_type = convert::string_to_jstring(env, metadata.content_type())?;
- let etag = if contains_metakey(Metakey::Etag) {
- convert::string_to_jstring(env, metadata.etag())?
- } else {
- JObject::null()
- };
+ let etag = convert::string_to_jstring(env, metadata.etag())?;
- let version = if contains_metakey(Metakey::Version) {
- convert::string_to_jstring(env, metadata.version())?
- } else {
- JObject::null()
- };
+ let version = convert::string_to_jstring(env, metadata.version())?;
- let content_length = if contains_metakey(Metakey::ContentLength) {
- metadata.content_length() as jlong
- } else {
- -1
- };
+ let content_length = metadata.content_length() as jlong;
let result = env
.new_object(
diff --git a/bindings/java/src/operator_input_stream.rs b/bindings/java/src/operator_input_stream.rs
index 7af8f31a3..105d15154 100644
--- a/bindings/java/src/operator_input_stream.rs
+++ b/bindings/java/src/operator_input_stream.rs
@@ -87,7 +87,7 @@ fn intern_read_next_bytes(
match reader
.next()
.transpose()
- .map_err(|err| opendal::Error::new(opendal::ErrorKind::Unexpected, &err.to_string()))?
+ .map_err(|err| opendal::Error::new(opendal::ErrorKind::Unexpected, err.to_string()))?
{
None => Ok(JObject::null().into_raw()),
Some(content) => {
diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs
index ad26e5b7b..2d1787638 100644
--- a/core/src/layers/complete.rs
+++ b/core/src/layers/complete.rs
@@ -209,12 +209,7 @@ impl<A: Access> CompleteAccessor<A> {
}
// Forward to underlying storage directly since we don't know how to handle stat dir.
- self.inner.stat(path, args).await.map(|v| {
- v.map_metadata(|m| {
- let bit = m.metakey();
- m.with_metakey(bit | Metakey::Complete)
- })
- })
+ self.inner.stat(path, args).await
}
fn complete_blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
@@ -258,12 +253,7 @@ impl<A: Access> CompleteAccessor<A> {
}
// Forward to underlying storage directly since we don't know how to handle stat dir.
- self.inner.blocking_stat(path, args).map(|v| {
- v.map_metadata(|m| {
- let bit = m.metakey();
- m.with_metakey(bit | Metakey::Complete)
- })
- })
+ self.inner.blocking_stat(path, args)
}
async fn complete_list(
diff --git a/core/src/layers/mime_guess.rs b/core/src/layers/mime_guess.rs
index 7a4bc08b8..1a4f460e1 100644
--- a/core/src/layers/mime_guess.rs
+++ b/core/src/layers/mime_guess.rs
@@ -161,7 +161,6 @@ impl<A: Access> LayeredAccess for MimeGuessAccessor<A> {
mod tests {
use super::*;
use crate::services::Memory;
- use crate::Metakey;
use crate::Operator;
const DATA: &str = "<html>test</html>";
@@ -196,7 +195,7 @@ mod tests {
Some(CUSTOM)
);
- let entries = op.list_with("").metakey(Metakey::Complete).await.unwrap();
+ let entries = op.list_with("").await.unwrap();
assert_eq!(entries[0].metadata().content_type(), Some(HTML));
assert_eq!(entries[1].metadata().content_type(), None);
assert_eq!(entries[2].metadata().content_type(), Some(CUSTOM));
@@ -222,7 +221,7 @@ mod tests {
.unwrap();
assert_eq!(op.stat("test2.html").unwrap().content_type(), Some(CUSTOM));
- let entries = op.list_with("").metakey(Metakey::Complete).call().unwrap();
+ let entries = op.list_with("").call().unwrap();
assert_eq!(entries[0].metadata().content_type(), Some(HTML));
assert_eq!(entries[1].metadata().content_type(), None);
assert_eq!(entries[2].metadata().content_type(), Some(CUSTOM));
diff --git a/core/src/raw/ops.rs b/core/src/raw/ops.rs
index 5875160b0..69615e0c9 100644
--- a/core/src/raw/ops.rs
+++ b/core/src/raw/ops.rs
@@ -24,7 +24,6 @@ use std::time::Duration;
use crate::raw::*;
use crate::*;
-use flagset::FlagSet;
/// Args for `create` operation.
///
@@ -85,13 +84,6 @@ pub struct OpList {
///
/// Default to `false`.
recursive: bool,
- /// Metakey is used to control which meta should be returned.
- ///
- /// Lister will make sure the result for specified meta is **known**:
- ///
- /// - `Some(v)` means exist.
- /// - `None` means services doesn't have this meta.
- metakey: FlagSet<Metakey>,
/// The concurrent of stat operations inside list operation.
/// Users could use this to control the number of concurrent stat operation when metadata is unknown.
///
@@ -115,8 +107,6 @@ impl Default for OpList {
limit: None,
start_after: None,
recursive: false,
- // By default, we want to know what's the mode of this entry.
- metakey: Metakey::Mode.into(),
concurrent: 1,
version: false,
}
@@ -167,19 +157,6 @@ impl OpList {
self.recursive
}
- /// Change the metakey of this list operation.
- ///
- /// The default metakey is `Metakey::Mode`.
- pub fn with_metakey(mut self, metakey: impl Into<FlagSet<Metakey>>) -> Self {
- self.metakey = metakey.into();
- self
- }
-
- /// Get the current metakey.
- pub fn metakey(&self) -> FlagSet<Metakey> {
- self.metakey
- }
-
/// Change the concurrent of this list operation.
///
/// The default concurrent is 1.
diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs
index a3c40ead1..2572361e3 100644
--- a/core/src/services/fs/backend.rs
+++ b/core/src/services/fs/backend.rs
@@ -342,7 +342,7 @@ impl Access for FsBackend {
}
}
- async fn list(&self, path: &str, arg: OpList) -> Result<(RpList, Self::Lister)> {
+ async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Lister)> {
let p = self.core.root.join(path.trim_end_matches('/'));
let f = match tokio::fs::read_dir(&p).await {
@@ -356,7 +356,7 @@ impl Access for FsBackend {
}
};
- let rd = FsLister::new(&self.core.root, path, f, arg);
+ let rd = FsLister::new(&self.core.root, path, f);
Ok((RpList::default(), Some(rd)))
}
@@ -511,7 +511,7 @@ impl Access for FsBackend {
}
}
- fn blocking_list(&self, path: &str, arg: OpList) -> Result<(RpList, Self::BlockingLister)> {
+ fn blocking_list(&self, path: &str, _: OpList) -> Result<(RpList, Self::BlockingLister)> {
let p = self.core.root.join(path.trim_end_matches('/'));
let f = match std::fs::read_dir(p) {
@@ -525,7 +525,7 @@ impl Access for FsBackend {
}
};
- let rd = FsLister::new(&self.core.root, path, f, arg);
+ let rd = FsLister::new(&self.core.root, path, f);
Ok((RpList::default(), Some(rd)))
}
diff --git a/core/src/services/fs/lister.rs b/core/src/services/fs/lister.rs
index adafd2c9d..fe9e02499 100644
--- a/core/src/services/fs/lister.rs
+++ b/core/src/services/fs/lister.rs
@@ -21,7 +21,6 @@ use std::path::PathBuf;
use crate::raw::*;
use crate::EntryMode;
use crate::Metadata;
-use crate::Metakey;
use crate::Result;
pub struct FsLister<P> {
@@ -30,17 +29,14 @@ pub struct FsLister<P> {
current_path: Option<String>,
rd: P,
-
- op: OpList,
}
impl<P> FsLister<P> {
- pub fn new(root: &Path, path: &str, rd: P, arg: OpList) -> Self {
+ pub fn new(root: &Path, path: &str, rd: P) -> Self {
Self {
root: root.to_owned(),
current_path: Some(path.to_string()),
rd,
- op: arg,
}
}
}
@@ -71,38 +67,15 @@ impl oio::List for FsLister<tokio::fs::ReadDir> {
.replace('\\', "/"),
);
- let default_meta = self.op.metakey() == Metakey::Mode;
-
- let metadata = if default_meta {
- let ft = de.file_type().await.map_err(new_std_io_error)?;
- if ft.is_file() {
- Metadata::new(EntryMode::FILE)
- } else if ft.is_dir() {
- Metadata::new(EntryMode::DIR)
- } else {
- Metadata::new(EntryMode::Unknown)
- }
- } else {
- let fs_meta = de.metadata().await.map_err(new_std_io_error)?;
- let mut meta = if fs_meta.file_type().is_file() {
- Metadata::new(EntryMode::FILE)
- } else if fs_meta.file_type().is_dir() {
- Metadata::new(EntryMode::DIR)
- } else {
- Metadata::new(EntryMode::Unknown)
- };
- meta.set_content_length(fs_meta.len());
- meta.set_last_modified(fs_meta.modified().map_err(new_std_io_error)?.into());
- meta
- };
-
- let entry = if metadata.is_dir() {
+ let ft = de.file_type().await.map_err(new_std_io_error)?;
+ let entry = if ft.is_dir() {
// Make sure we are returning the correct path.
- oio::Entry::new(&format!("{rel_path}/"), metadata)
+ oio::Entry::new(&format!("{rel_path}/"), Metadata::new(EntryMode::DIR))
+ } else if ft.is_file() {
+ oio::Entry::new(&rel_path, Metadata::new(EntryMode::FILE))
} else {
- oio::Entry::new(&rel_path, metadata)
+ oio::Entry::new(&rel_path, Metadata::new(EntryMode::Unknown))
};
-
Ok(Some(entry))
}
}
@@ -129,36 +102,14 @@ impl oio::BlockingList for FsLister<std::fs::ReadDir> {
.replace('\\', "/"),
);
- let default_meta = self.op.metakey() == Metakey::Mode;
-
- let metadata = if default_meta {
- let ft = de.file_type().map_err(new_std_io_error)?;
- if ft.is_file() {
- Metadata::new(EntryMode::FILE)
- } else if ft.is_dir() {
- Metadata::new(EntryMode::DIR)
- } else {
- Metadata::new(EntryMode::Unknown)
- }
- } else {
- let fs_meta = de.metadata().map_err(new_std_io_error)?;
- let mut meta = if fs_meta.file_type().is_file() {
- Metadata::new(EntryMode::FILE)
- } else if fs_meta.file_type().is_dir() {
- Metadata::new(EntryMode::DIR)
- } else {
- Metadata::new(EntryMode::Unknown)
- };
- meta.set_content_length(fs_meta.len());
- meta.set_last_modified(fs_meta.modified().map_err(new_std_io_error)?.into());
- meta
- };
-
- let entry = if metadata.is_dir() {
+ let ft = de.file_type().map_err(new_std_io_error)?;
+ let entry = if ft.is_dir() {
// Make sure we are returning the correct path.
- oio::Entry::new(&format!("{rel_path}/"), metadata)
+ oio::Entry::new(&format!("{rel_path}/"), Metadata::new(EntryMode::DIR))
+ } else if ft.is_file() {
+ oio::Entry::new(&rel_path, Metadata::new(EntryMode::FILE))
} else {
- oio::Entry::new(&rel_path, metadata)
+ oio::Entry::new(&rel_path, Metadata::new(EntryMode::Unknown))
};
Ok(Some(entry))
diff --git a/core/src/services/webdav/core.rs b/core/src/services/webdav/core.rs
index 41026454e..71efd0a96 100644
--- a/core/src/services/webdav/core.rs
+++ b/core/src/services/webdav/core.rs
@@ -412,7 +412,7 @@ pub fn parse_propstat(propstat: &Propstat) -> Result<Metadata> {
m.set_last_modified(parse_datetime_from_rfc2822(getlastmodified)?);
// the storage services have returned all the properties
- Ok(m.with_metakey(Metakey::Complete))
+ Ok(m)
}
#[derive(Deserialize, Debug, PartialEq, Eq, Clone, Default)]
diff --git a/core/src/types/entry.rs b/core/src/types/entry.rs
index 5e699deaf..9fe451080 100644
--- a/core/src/types/entry.rs
+++ b/core/src/types/entry.rs
@@ -19,44 +19,6 @@ use crate::raw::*;
use crate::*;
/// Entry returned by [`Lister`] or [`BlockingLister`] to represent a path and it's relative metadata.
-///
-/// # Notes
-///
-/// Entry returned by [`Lister`] or [`BlockingLister`] may carry some already known metadata.
-/// Lister by default only make sure that `Mode` is fetched. To make sure the entry contains
-/// metadata you want, please use `list_with` or `lister_with` and `metakey`.
-///
-/// For example:
-///
-/// ```no_run
-/// # use anyhow::Result;
-/// use opendal::EntryMode;
-/// use opendal::Metakey;
-/// use opendal::Operator;
-/// # async fn test(op: Operator) -> Result<()> {
-/// let mut entries = op
-/// .list_with("dir/")
-/// .metakey(Metakey::ContentLength | Metakey::LastModified)
-/// .await?;
-/// for entry in entries {
-/// let meta = entry.metadata();
-/// match meta.mode() {
-/// EntryMode::FILE => {
-/// println!(
-/// "Handling file {} with size {}",
-/// entry.path(),
-/// meta.content_length()
-/// )
-/// }
-/// EntryMode::DIR => {
-/// println!("Handling dir {}", entry.path())
-/// }
-/// EntryMode::Unknown => continue,
-/// }
-/// }
-/// # Ok(())
-/// # }
-/// ```
#[derive(Clone, Debug)]
pub struct Entry {
/// Path of this entry.
@@ -96,93 +58,11 @@ impl Entry {
}
/// Fetch metadata of this entry.
- ///
- /// # Notes
- ///
- /// Metadata only guaranteed to have results of `metakey` (which default to `Metakey::Mode`).
- ///
- /// - `Some(T)` means the metadata is valid.
- /// - `None` means the metadata is not provided by services.
- ///
- /// Visiting a metadata that not covered by `metakey` could result in panic.
- ///
- /// # Examples
- ///
- /// Please use `metakey` to specify the metadata you want, for example:
- ///
- /// ```no_run
- /// # use anyhow::Result;
- /// use opendal::EntryMode;
- /// use opendal::Metakey;
- /// use opendal::Operator;
- /// # async fn test(op: Operator) -> Result<()> {
- /// let mut entries = op
- /// .list_with("dir/")
- /// .metakey(Metakey::ContentLength | Metakey::LastModified)
- /// .await?;
- /// for entry in entries {
- /// let meta = entry.metadata();
- /// match meta.mode() {
- /// EntryMode::FILE => {
- /// println!(
- /// "Handling file {} with size {}",
- /// entry.path(),
- /// meta.content_length()
- /// )
- /// }
- /// EntryMode::DIR => {
- /// println!("Handling dir {}", entry.path())
- /// }
- /// EntryMode::Unknown => continue,
- /// }
- /// }
- /// # Ok(())
- /// # }
- /// ```
pub fn metadata(&self) -> &Metadata {
&self.metadata
}
/// Consume this entry to get it's path and metadata.
- ///
- /// # Notes
- ///
- /// Metadata only guaranteed to have results of `metakey` (which default to `Metakey::Mode`).
- ///
- /// - `Some(T)` means the metadata is valid.
- /// - `None` means the metadata is not provided by services.
- ///
- /// Visiting a metadata that not covered by `metakey` could result in panic.
- ///
- /// # Examples
- ///
- /// Please use `metakey` to specify the metadata you want, for example:
- ///
- /// ```no_run
- /// # use anyhow::Result;
- /// use opendal::EntryMode;
- /// use opendal::Metakey;
- /// use opendal::Operator;
- /// # async fn test(op: Operator) -> Result<()> {
- /// let mut entries = op
- /// .list_with("dir/")
- /// .metakey(Metakey::ContentLength | Metakey::LastModified)
- /// .await?;
- /// for entry in entries {
- /// let (path, meta) = entry.into_parts();
- /// match meta.mode() {
- /// EntryMode::FILE => {
- /// println!("Handling file {} with size {}", path, meta.content_length())
- /// }
- /// EntryMode::DIR => {
- /// println!("Handling dir {}", path)
- /// }
- /// EntryMode::Unknown => continue,
- /// }
- /// }
- /// # Ok(())
- /// # }
- /// ```
pub fn into_parts(self) -> (String, Metadata) {
(self.path, self.metadata)
}
diff --git a/core/src/types/list.rs b/core/src/types/list.rs
index 8b3bb341b..e53781dc1 100644
--- a/core/src/types/list.rs
+++ b/core/src/types/list.rs
@@ -15,16 +15,12 @@
// specific language governing permissions and limitations
// under the License.
-use std::cmp;
-use std::future::Future;
use std::pin::Pin;
use std::task::ready;
use std::task::Context;
use std::task::Poll;
-use flagset::FlagSet;
use futures::Stream;
-use futures::StreamExt;
use crate::raw::*;
use crate::*;
@@ -32,93 +28,15 @@ use crate::*;
/// Lister is designed to list entries at given path in an asynchronous
/// manner.
///
-/// Users can construct Lister by [`Operator::lister`] or [`Operator::lister_with`], and can use `metakey` along with list.
-/// For example, suppose you need to access `content_length`, you can bring the corresponding field in metakey when listing:
-/// `op.list_with("dir/").metakey(Metakey::ContentLength).await?;`.
-///
/// - Lister implements `Stream<Item = Result<Entry>>`.
/// - Lister will return `None` if there is no more entries or error has been returned.
pub struct Lister {
- acc: Accessor,
lister: Option<oio::Lister>,
- /// required_metakey is the metakey required by users.
- required_metakey: FlagSet<Metakey>,
fut: Option<BoxedStaticFuture<(oio::Lister, Result<Option<oio::Entry>>)>>,
-
- /// tasks is used to store tasks that are run in concurrent.
- ///
- /// TODO: maybe we should move logic inside?
- tasks: ConcurrentFutures<StatTask>,
errored: bool,
}
-/// StatTask is used to store the task that is run in concurrent.
-///
-/// # Note for clippy
-///
-/// Clippy will raise error for this enum like the following:
-///
-/// ```shell
-/// error: large size difference between variants
-/// --> core/src/types/list.rs:64:1
-/// |
-/// 64 | / enum StatTask {
-/// 65 | | /// BoxFuture is used to store the join handle of spawned task.
-/// 66 | | Handle(BoxFuture<(String, Result<RpStat>)>),
-/// | | -------------------------------------------- the second-largest variant contains at least 0 bytes
-/// 67 | | /// KnownEntry is used to store the entry that already contains the required metakey.
-/// 68 | | KnownEntry(Option<Entry>),
-/// | | ------------------------- the largest variant contains at least 264 bytes
-/// 69 | | }
-/// | |_^ the entire enum is at least 0 bytes
-/// |
-/// = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#large_enum_variant
-/// = note: `-D clippy::large-enum-variant` implied by `-D warnings`
-/// = help: to override `-D warnings` add `#[allow(clippy::large_enum_variant)]`
-/// help: consider boxing the large fields to reduce the total size of the enum
-/// |
-/// 68 | KnownEntry(Box<Option<Entry>>),
-/// | ~~~~~~~~~~~~~~~~~~
-/// ```
-/// But this lint is wrong since it doesn't take the generic param JoinHandle into account. In fact, they have exactly
-/// the same size:
-///
-/// ```rust
-/// use std::mem::size_of;
-///
-/// use opendal::Entry;
-/// use opendal::Result;
-///
-/// assert_eq!(304, size_of::<(String, Result<opendal::raw::RpStat>)>());
-/// assert_eq!(304, size_of::<Option<Entry>>());
-/// ```
-///
-/// So let's ignore this lint:
-#[allow(clippy::large_enum_variant)]
-enum StatTask {
- /// Stating is used to store the join handle of spawned task.
- ///
- /// TODO: Replace with static future type after rust supported.
- Stating(BoxedStaticFuture<(String, Result<Metadata>)>),
- /// Known is used to store the entry that already contains the required metakey.
- Known(Option<(String, Metadata)>),
-}
-
-impl Future for StatTask {
- type Output = (String, Result<Metadata>);
-
- fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
- match self.get_mut() {
- StatTask::Stating(fut) => Pin::new(fut).poll(cx),
- StatTask::Known(entry) => {
- let (path, metadata) = entry.take().expect("entry should not be None");
- Poll::Ready((path, Ok(metadata)))
- }
- }
- }
-}
-
/// # Safety
///
/// Lister will only be accessed by `&mut Self`
@@ -127,18 +45,12 @@ unsafe impl Sync for Lister {}
impl Lister {
/// Create a new lister.
pub(crate) async fn create(acc: Accessor, path: &str, args: OpList) -> Result<Self> {
- let required_metakey = args.metakey();
- let concurrent = cmp::max(1, args.concurrent());
-
let (_, lister) = acc.list(path, args).await?;
Ok(Self {
- acc,
lister: Some(lister),
- required_metakey,
fut: None,
- tasks: ConcurrentFutures::new(concurrent),
errored: false,
})
}
@@ -153,60 +65,33 @@ impl Stream for Lister {
return Poll::Ready(None);
}
- // Trying to pull more tasks if there are more space.
- if self.tasks.has_remaining() {
- // Building future if we have a lister available.
- if let Some(mut lister) = self.lister.take() {
- let fut = async move {
- let res = lister.next_dyn().await;
- (lister, res)
- };
- self.fut = Some(Box::pin(fut));
- }
+ if let Some(mut lister) = self.lister.take() {
+ let fut = async move {
+ let res = lister.next_dyn().await;
+ (lister, res)
+ };
+ self.fut = Some(Box::pin(fut));
+ }
- if let Some(fut) = self.fut.as_mut() {
- if let Poll::Ready((lister, entry)) = fut.as_mut().poll(cx) {
- self.lister = Some(lister);
- self.fut = None;
+ if let Some(fut) = self.fut.as_mut() {
+ let (lister, entry) = ready!(fut.as_mut().poll(cx));
+ self.lister = Some(lister);
+ self.fut = None;
- match entry {
- Ok(Some(oe)) => {
- let (path, metadata) = oe.into_entry().into_parts();
- if metadata.contains_metakey(self.required_metakey) {
- self.tasks
- .push_back(StatTask::Known(Some((path, metadata))));
- } else {
- let acc = self.acc.clone();
- let fut = async move {
- let res = acc.stat(&path, OpStat::default()).await;
- (path, res.map(|rp| rp.into_metadata()))
- };
- self.tasks.push_back(StatTask::Stating(Box::pin(fut)));
- }
- }
- Ok(None) => {
- self.lister = None;
- }
- Err(err) => {
- self.errored = true;
- return Poll::Ready(Some(Err(err)));
- }
- }
+ return match entry {
+ Ok(Some(oe)) => Poll::Ready(Some(Ok(oe.into_entry()))),
+ Ok(None) => {
+ self.lister = None;
+ Poll::Ready(None)
}
- }
- }
-
- // Try to poll tasks
- if let Some((path, rp)) = ready!(self.tasks.poll_next_unpin(cx)) {
- let metadata = rp?;
- return Poll::Ready(Some(Ok(Entry::new(path, metadata))));
+ Err(err) => {
+ self.errored = true;
+ Poll::Ready(Some(Err(err)))
+ }
+ };
}
- if self.lister.is_some() || self.fut.is_some() {
- Poll::Pending
- } else {
- Poll::Ready(None)
- }
+ Poll::Ready(None)
}
}
@@ -218,10 +103,6 @@ impl Stream for Lister {
/// - Lister implements `Iterator<Item = Result<Entry>>`.
/// - Lister will return `None` if there is no more entries or error has been returned.
pub struct BlockingLister {
- acc: Accessor,
- /// required_metakey is the metakey required by users.
- required_metakey: FlagSet<Metakey>,
-
lister: oio::BlockingLister,
errored: bool,
}
@@ -234,20 +115,15 @@ unsafe impl Sync for BlockingLister {}
impl BlockingLister {
/// Create a new lister.
pub(crate) fn create(acc: Accessor, path: &str, args: OpList) -> Result<Self> {
- let required_metakey = args.metakey();
let (_, lister) = acc.blocking_list(path, args)?;
Ok(Self {
- acc,
- required_metakey,
-
lister,
errored: false,
})
}
}
-/// TODO: we can implement next_chunk.
impl Iterator for BlockingLister {
type Item = Result<Entry>;
@@ -257,28 +133,14 @@ impl Iterator for BlockingLister {
return None;
}
- let entry = match self.lister.next() {
- Ok(Some(entry)) => entry,
- Ok(None) => return None,
+ match self.lister.next() {
+ Ok(Some(entry)) => Some(Ok(entry.into_entry())),
+ Ok(None) => None,
Err(err) => {
self.errored = true;
- return Some(Err(err));
+ Some(Err(err))
}
- };
-
- let (path, metadata) = entry.into_entry().into_parts();
- if metadata.contains_metakey(self.required_metakey) {
- return Some(Ok(Entry::new(path, metadata)));
}
-
- let metadata = match self.acc.blocking_stat(&path, OpStat::default()) {
- Ok(rp) => rp.into_metadata(),
- Err(err) => {
- self.errored = true;
- return Some(Err(err));
- }
- };
- Some(Ok(Entry::new(path, metadata)))
}
}
diff --git a/core/src/types/metadata.rs b/core/src/types/metadata.rs
index 48bfe3cd6..7314f25db 100644
--- a/core/src/types/metadata.rs
+++ b/core/src/types/metadata.rs
@@ -18,8 +18,6 @@
use std::collections::HashMap;
use chrono::prelude::*;
-use flagset::flags;
-use flagset::FlagSet;
use crate::raw::*;
use crate::*;
@@ -33,9 +31,6 @@ use crate::*;
/// a.k.a., `Entry`'s content length could be `None`.
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Metadata {
- /// metakey stores current key store.
- metakey: FlagSet<Metakey>,
-
mode: EntryMode,
cache_control: Option<String>,
@@ -54,16 +49,7 @@ pub struct Metadata {
impl Metadata {
/// Create a new metadata
pub fn new(mode: EntryMode) -> Self {
- // Mode is required to be set for metadata.
- let mut metakey: FlagSet<Metakey> = Metakey::Mode.into();
- // If mode is dir, we should always mark it as complete.
- if mode.is_dir() {
- metakey |= Metakey::Complete
- }
-
Self {
- metakey,
-
mode,
cache_control: None,
@@ -79,38 +65,8 @@ impl Metadata {
}
}
- /// Get the metakey from metadata.
- ///
- /// This value describes which metadata has been set.
- pub fn metakey(&self) -> FlagSet<Metakey> {
- self.metakey
- }
-
- /// Set metakey with given.
- pub(crate) fn with_metakey(mut self, metakey: impl Into<FlagSet<Metakey>>) -> Self {
- self.metakey = metakey.into();
- self
- }
-
- /// Check if the metadata already contains given metakey.
- pub(crate) fn contains_metakey(&self, metakey: impl Into<FlagSet<Metakey>>) -> bool {
- let input_metakey = metakey.into();
-
- // If meta already contains complete, we don't need to check.
- if self.metakey.contains(Metakey::Complete) {
- return true;
- }
-
- self.metakey.contains(input_metakey)
- }
-
/// mode represent this entry's mode.
pub fn mode(&self) -> EntryMode {
- debug_assert!(
- self.metakey.contains(Metakey::Mode) || self.metakey.contains(Metakey::Complete),
- "visiting not set metadata: mode, maybe a bug"
- );
-
self.mode
}
@@ -127,14 +83,12 @@ impl Metadata {
/// Set mode for entry.
pub fn set_mode(&mut self, v: EntryMode) -> &mut Self {
self.mode = v;
- self.metakey |= Metakey::Mode;
self
}
/// Set mode for entry.
pub fn with_mode(mut self, v: EntryMode) -> Self {
self.mode = v;
- self.metakey |= Metakey::Mode;
self
}
@@ -145,13 +99,6 @@ impl Metadata {
/// This value is only available when calling on result of `stat` or `list` with
/// [`Metakey::CacheControl`], otherwise this method returns `None`.
pub fn cache_control(&self) -> Option<&str> {
- #[cfg(feature = "tests")]
- debug_assert!(
- self.metakey.contains(Metakey::CacheControl)
- || self.metakey.contains(Metakey::Complete),
- "visiting not set metadata: cache_control, maybe a bug"
- );
-
self.cache_control.as_deref()
}
@@ -161,7 +108,6 @@ impl Metadata {
/// Refer to [MDN Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) for more information.
pub fn set_cache_control(&mut self, v: &str) -> &mut Self {
self.cache_control = Some(v.to_string());
- self.metakey |= Metakey::CacheControl;
self
}
@@ -171,7 +117,6 @@ impl Metadata {
/// Refer to [MDN Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) for more information.
pub fn with_cache_control(mut self, v: String) -> Self {
self.cache_control = Some(v);
- self.metakey |= Metakey::CacheControl;
self
}
@@ -185,26 +130,18 @@ impl Metadata {
/// This value is only available when calling on result of `stat` or `list` with
/// [`Metakey::ContentLength`], otherwise it will panic.
pub fn content_length(&self) -> u64 {
- debug_assert!(
- self.metakey.contains(Metakey::ContentLength)
- || self.metakey.contains(Metakey::Complete),
- "visiting not set metadata: content_length, maybe a bug"
- );
-
self.content_length.unwrap_or_default()
}
/// Set content length of this entry.
pub fn set_content_length(&mut self, v: u64) -> &mut Self {
self.content_length = Some(v);
- self.metakey |= Metakey::ContentLength;
self
}
/// Set content length of this entry.
pub fn with_content_length(mut self, v: u64) -> Self {
self.content_length = Some(v);
- self.metakey |= Metakey::ContentLength;
self
}
@@ -218,12 +155,6 @@ impl Metadata {
/// This value is only available when calling on result of `stat` or `list` with
/// [`Metakey::ContentMd5`], otherwise this method returns `None`.
pub fn content_md5(&self) -> Option<&str> {
- #[cfg(feature = "tests")]
- debug_assert!(
- self.metakey.contains(Metakey::ContentMd5) || self.metakey.contains(Metakey::Complete),
- "visiting not set metadata: content_md5, maybe a bug"
- );
-
self.content_md5.as_deref()
}
@@ -233,7 +164,6 @@ impl Metadata {
/// And removed by [RFC 7231](https://www.rfc-editor.org/rfc/rfc7231).
pub fn set_content_md5(&mut self, v: &str) -> &mut Self {
self.content_md5 = Some(v.to_string());
- self.metakey |= Metakey::ContentMd5;
self
}
@@ -243,7 +173,6 @@ impl Metadata {
/// And removed by [RFC 7231](https://www.rfc-editor.org/rfc/rfc7231).
pub fn with_content_md5(mut self, v: String) -> Self {
self.content_md5 = Some(v);
- self.metakey |= Metakey::ContentMd5;
self
}
@@ -254,12 +183,6 @@ impl Metadata {
/// This value is only available when calling on result of `stat` or `list` with
/// [`Metakey::ContentType`], otherwise this method returns `None`.
pub fn content_type(&self) -> Option<&str> {
- #[cfg(feature = "tests")]
- debug_assert!(
- self.metakey.contains(Metakey::ContentType) || self.metakey.contains(Metakey::Complete),
- "visiting not set metadata: content_type, maybe a bug"
- );
-
self.content_type.as_deref()
}
@@ -268,7 +191,6 @@ impl Metadata {
/// Content Type is defined by [RFC 9110](https://httpwg.org/specs/rfc9110.html#field.content-type).
pub fn set_content_type(&mut self, v: &str) -> &mut Self {
self.content_type = Some(v.to_string());
- self.metakey |= Metakey::ContentType;
self
}
@@ -277,7 +199,6 @@ impl Metadata {
/// Content Type is defined by [RFC 9110](https://httpwg.org/specs/rfc9110.html#field.content-type).
pub fn with_content_type(mut self, v: String) -> Self {
self.content_type = Some(v);
- self.metakey |= Metakey::ContentType;
self
}
@@ -288,13 +209,6 @@ impl Metadata {
/// This value is only available when calling on result of `stat` or `list` with
/// [`Metakey::ContentRange`], otherwise this method returns `None`.
pub fn content_range(&self) -> Option<BytesContentRange> {
- #[cfg(feature = "tests")]
- debug_assert!(
- self.metakey.contains(Metakey::ContentRange)
- || self.metakey.contains(Metakey::Complete),
- "visiting not set metadata: content_range, maybe a bug"
- );
-
self.content_range
}
@@ -303,7 +217,6 @@ impl Metadata {
/// Content Range is defined by [RFC 9110](https://httpwg.org/specs/rfc9110.html#field.content-range).
pub fn set_content_range(&mut self, v: BytesContentRange) -> &mut Self {
self.content_range = Some(v);
- self.metakey |= Metakey::ContentRange;
self
}
@@ -312,7 +225,6 @@ impl Metadata {
/// Content Range is defined by [RFC 9110](https://httpwg.org/specs/rfc9110.html#field.content-range).
pub fn with_content_range(mut self, v: BytesContentRange) -> Self {
self.content_range = Some(v);
- self.metakey |= Metakey::ContentRange;
self
}
@@ -326,13 +238,6 @@ impl Metadata {
/// This value is only available when calling on result of `stat` or `list` with
/// [`Metakey::LastModified`], otherwise this method returns `None`.
pub fn last_modified(&self) -> Option<DateTime<Utc>> {
- #[cfg(feature = "tests")]
- debug_assert!(
- self.metakey.contains(Metakey::LastModified)
- || self.metakey.contains(Metakey::Complete),
- "visiting not set metadata: last_modified, maybe a bug"
- );
-
self.last_modified
}
@@ -342,7 +247,6 @@ impl Metadata {
/// Refer to [MDN Last-Modified](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified) for more information.
pub fn set_last_modified(&mut self, v: DateTime<Utc>) -> &mut Self {
self.last_modified = Some(v);
- self.metakey |= Metakey::LastModified;
self
}
@@ -352,7 +256,6 @@ impl Metadata {
/// Refer to [MDN Last-Modified](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified) for more information.
pub fn with_last_modified(mut self, v: DateTime<Utc>) -> Self {
self.last_modified = Some(v);
- self.metakey |= Metakey::LastModified;
self
}
@@ -371,12 +274,6 @@ impl Metadata {
/// This value is only available when calling on result of `stat` or `list` with
/// [`Metakey::Etag`], otherwise this method returns `None`.
pub fn etag(&self) -> Option<&str> {
- #[cfg(feature = "tests")]
- debug_assert!(
- self.metakey.contains(Metakey::Etag) || self.metakey.contains(Metakey::Complete),
- "visiting not set metadata: etag, maybe a bug"
- );
-
self.etag.as_deref()
}
@@ -393,7 +290,6 @@ impl Metadata {
/// `"` is part of etag, don't trim it before setting.
pub fn set_etag(&mut self, v: &str) -> &mut Self {
self.etag = Some(v.to_string());
- self.metakey |= Metakey::Etag;
self
}
@@ -410,7 +306,6 @@ impl Metadata {
/// `"` is part of etag, don't trim it before setting.
pub fn with_etag(mut self, v: String) -> Self {
self.etag = Some(v);
- self.metakey |= Metakey::Etag;
self
}
@@ -429,13 +324,6 @@ impl Metadata {
/// This value is only available when calling on result of `stat` or `list` with
/// [`Metakey::ContentDisposition`], otherwise this method returns `None`.
pub fn content_disposition(&self) -> Option<&str> {
- #[cfg(feature = "tests")]
- debug_assert!(
- self.metakey.contains(Metakey::ContentDisposition)
- || self.metakey.contains(Metakey::Complete),
- "visiting not set metadata: content_disposition, maybe a bug"
- );
-
self.content_disposition.as_deref()
}
@@ -452,7 +340,6 @@ impl Metadata {
/// - "attachment; filename=\"filename.jpg\""
pub fn with_content_disposition(mut self, v: String) -> Self {
self.content_disposition = Some(v);
- self.metakey |= Metakey::ContentDisposition;
self
}
@@ -469,7 +356,6 @@ impl Metadata {
/// - "attachment; filename=\"filename.jpg\""
pub fn set_content_disposition(&mut self, v: &str) -> &mut Self {
self.content_disposition = Some(v.to_string());
- self.metakey |= Metakey::ContentDisposition;
self
}
@@ -482,12 +368,6 @@ impl Metadata {
/// This value is only available when calling on result of `stat` or `list` with
/// [`Metakey::Version`], otherwise this method returns `None`.
pub fn version(&self) -> Option<&str> {
- #[cfg(feature = "tests")]
- debug_assert!(
- self.metakey.contains(Metakey::Version) || self.metakey.contains(Metakey::Complete),
- "visiting not set metadata: version, maybe a bug"
- );
-
self.version.as_deref()
}
@@ -498,7 +378,6 @@ impl Metadata {
/// This field may come out from the version control system, like object versioning in AWS S3.
pub fn with_version(mut self, v: String) -> Self {
self.version = Some(v);
- self.metakey |= Metakey::Version;
self
}
@@ -509,7 +388,6 @@ impl Metadata {
/// This field may come out from the version control system, like object versioning in AWS S3.
pub fn set_version(&mut self, v: &str) -> &mut Self {
self.version = Some(v.to_string());
- self.metakey |= Metakey::Version;
self
}
@@ -524,51 +402,6 @@ impl Metadata {
/// Set user defined metadata of this entry
pub fn with_user_metadata(&mut self, data: HashMap<String, String>) -> &mut Self {
self.user_metadata = Some(data);
- self.metakey |= Metakey::UserMetaData;
self
}
}
-
-flags! {
- /// Metakey describes the metadata keys that can be stored
- /// or queried.
- ///
- /// ## For store
- ///
- /// Internally, we will store a flag set of Metakey to check
- /// whether we have set some key already.
- ///
- /// ## For query
- ///
- /// At user side, we will allow user to query the metadata. If
- /// the meta has been stored, we will return directly. If no, we will
- /// call `stat` internally to fetch the metadata.
- pub enum Metakey: u64 {
- /// The special metadata key that used to mark this entry
- /// already contains all metadata.
- Complete,
-
- /// Key for mode.
- Mode,
- /// Key for cache control.
- CacheControl,
- /// Key for content disposition.
- ContentDisposition,
- /// Key for content length.
- ContentLength,
- /// Key for content md5.
- ContentMd5,
- /// Key for content range.
- ContentRange,
- /// Key for content type.
- ContentType,
- /// Key for etag.
- Etag,
- /// Key for last modified.
- LastModified,
- /// Key for version.
- Version,
- /// Key for user metadata
- UserMetaData,
- }
-}
diff --git a/core/src/types/mod.rs b/core/src/types/mod.rs
index 602cf8c55..448a5f430 100644
--- a/core/src/types/mod.rs
+++ b/core/src/types/mod.rs
@@ -26,7 +26,6 @@ pub use entry::Entry;
mod metadata;
pub use metadata::Metadata;
-pub use metadata::Metakey;
mod read;
pub use read::*;
diff --git a/core/src/types/operator/blocking_operator.rs b/core/src/types/operator/blocking_operator.rs
index 38a95357f..9ea08fb81 100644
--- a/core/src/types/operator/blocking_operator.rs
+++ b/core/src/types/operator/blocking_operator.rs
@@ -133,12 +133,6 @@ impl BlockingOperator {
impl BlockingOperator {
/// Get given path's metadata.
///
- /// # Notes
- ///
- /// For fetch metadata of entries returned by [`BlockingLister`], it's better to
- /// use [`BlockingOperator::list_with`] and [`BlockingOperator::lister_with`] with `metakey`
- /// query like `Metakey::ContentLength | Metakey::LastModified` so that we can avoid extra requests.
- ///
/// # Behavior
///
/// ## Services that support `create_dir`
@@ -190,13 +184,6 @@ impl BlockingOperator {
/// Get given path's metadata with extra options.
///
- /// # Notes
- ///
- /// For fetch metadata of entries returned by [`Lister`], it's better to use
- /// [`Operator::list_with`] and [`Operator::lister_with`] with `metakey` query like
- /// `Metakey::ContentLength | Metakey::LastModified`
- /// so that we can avoid extra requests.
- ///
/// # Behavior
///
/// ## Services that support `create_dir`
@@ -383,7 +370,6 @@ impl BlockingOperator {
/// # use anyhow::Result;
/// use opendal::BlockingOperator;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// # fn test(op: BlockingOperator) -> Result<()> {
/// let bs = op.read_with("path/to/file").range(0..10).call()?;
/// # Ok(())
@@ -439,7 +425,6 @@ impl BlockingOperator {
/// # use anyhow::Result;
/// use opendal::BlockingOperator;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// # fn test(op: BlockingOperator) -> Result<()> {
/// let r = op
/// .reader_with("path/to/file")
@@ -695,7 +680,6 @@ impl BlockingOperator {
/// # use anyhow::Result;
/// use opendal::BlockingOperator;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// # fn test(op: BlockingOperator) -> Result<()> {
/// let mut w = op.writer_with("path/to/file").call()?;
/// w.write(vec![0; 4096])?;
@@ -903,18 +887,12 @@ impl BlockingOperator {
/// In order to avoid this, you can use [`BlockingOperator::lister`] to list entries in
/// a streaming way.
///
- /// ## Reuse Metadata
- ///
- /// The only metadata that is guaranteed to be available is the `Mode`.
- /// For fetching more metadata, please use [`BlockingOperator::list_with`] and `metakey`.
- ///
/// # Examples
///
/// ```no_run
/// # use anyhow::Result;
/// use opendal::BlockingOperator;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// # fn test(op: BlockingOperator) -> Result<()> {
/// let mut entries = op.list("path/to/dir/")?;
/// for entry in entries {
@@ -948,11 +926,6 @@ impl BlockingOperator {
/// In order to avoid this, you can use [`BlockingOperator::lister`] to list entries in
/// a streaming way.
///
- /// ## Reuse Metadata
- ///
- /// The only metadata that is guaranteed to be available is the `Mode`.
- /// For fetching more metadata, please specify the `metakey`.
- ///
/// # Examples
///
/// ## List entries with prefix
@@ -963,7 +936,6 @@ impl BlockingOperator {
/// # use anyhow::Result;
/// use opendal::BlockingOperator;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// # fn test(op: BlockingOperator) -> Result<()> {
/// let mut entries = op.list_with("prefix/").recursive(true).call()?;
/// for entry in entries {
@@ -980,38 +952,6 @@ impl BlockingOperator {
/// # Ok(())
/// # }
/// ```
- ///
- /// ## List entries with metakey for more metadata
- ///
- /// ```no_run
- /// # use anyhow::Result;
- /// use opendal::BlockingOperator;
- /// use opendal::EntryMode;
- /// use opendal::Metakey;
- /// # fn test(op: BlockingOperator) -> Result<()> {
- /// let mut entries = op
- /// .list_with("dir/")
- /// .metakey(Metakey::ContentLength | Metakey::LastModified)
- /// .call()?;
- /// for entry in entries {
- /// let meta = entry.metadata();
- /// match meta.mode() {
- /// EntryMode::FILE => {
- /// println!(
- /// "Handling file {} with size {}",
- /// entry.path(),
- /// meta.content_length()
- /// )
- /// }
- /// EntryMode::DIR => {
- /// println!("Handling dir {}", entry.path())
- /// }
- /// EntryMode::Unknown => continue,
- /// }
- /// }
- /// # Ok(())
- /// # }
- /// ```
pub fn list_with(&self, path: &str) -> FunctionList {
let path = normalize_path(path);
@@ -1040,11 +980,6 @@ impl BlockingOperator {
/// all entries recursively, use [`BlockingOperator::lister_with`] and `delimiter("")`
/// instead.
///
- /// ## Metadata
- ///
- /// The only metadata that is guaranteed to be available is the `Mode`.
- /// For fetching more metadata, please use [`BlockingOperator::lister_with`] and `metakey`.
- ///
/// # Examples
///
/// ```no_run
@@ -1053,7 +988,6 @@ impl BlockingOperator {
/// use futures::TryStreamExt;
/// use opendal::BlockingOperator;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// # fn test(op: BlockingOperator) -> Result<()> {
/// let mut ds = op.lister("path/to/dir/")?;
/// for de in ds {
@@ -1091,7 +1025,6 @@ impl BlockingOperator {
/// use futures::TryStreamExt;
/// use opendal::BlockingOperator;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// # fn test(op: BlockingOperator) -> Result<()> {
/// let mut ds = op
/// .lister_with("path/to/dir/")
@@ -1122,7 +1055,6 @@ impl BlockingOperator {
/// use futures::TryStreamExt;
/// use opendal::BlockingOperator;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// # fn test(op: BlockingOperator) -> Result<()> {
/// let mut ds = op.lister_with("path/to/dir/").recursive(true).call()?;
/// for entry in ds {
@@ -1140,41 +1072,6 @@ impl BlockingOperator {
/// # Ok(())
/// # }
/// ```
- ///
- /// ## List files with required metadata
- ///
- /// ```no_run
- /// # use anyhow::Result;
- /// # use futures::io;
- /// use futures::TryStreamExt;
- /// use opendal::BlockingOperator;
- /// use opendal::EntryMode;
- /// use opendal::Metakey;
- /// # fn test(op: BlockingOperator) -> Result<()> {
- /// let mut ds = op
- /// .lister_with("path/to/dir/")
- /// .metakey(Metakey::ContentLength | Metakey::LastModified)
- /// .call()?;
- /// for entry in ds {
- /// let entry = entry?;
- /// let meta = entry.metadata();
- /// match meta.mode() {
- /// EntryMode::FILE => {
- /// println!(
- /// "Handling file {} with size {}",
- /// entry.path(),
- /// meta.content_length()
- /// )
- /// }
- /// EntryMode::DIR => {
- /// println!("Handling dir {}", entry.path())
- /// }
- /// EntryMode::Unknown => continue,
- /// }
- /// }
- /// # Ok(())
- /// # }
- /// ```
pub fn lister_with(&self, path: &str) -> FunctionLister {
let path = normalize_path(path);
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index 84fb6e0aa..fc8b31d25 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -180,12 +180,6 @@ impl Operator {
/// [`Operator::stat`] is a wrapper of [`Operator::stat_with`] without any options. To use extra
/// options like `if_match` and `if_none_match`, please use [`Operator::stat_with`] instead.
///
- /// ## Reuse Metadata
- ///
- /// For fetch metadata of entries returned by [`Lister`], it's better to use
- /// [`Operator::list_with`] and [`Operator::lister_with`] with `metakey` query like
- /// `Metakey::ContentLength | Metakey::LastModified` so that we can avoid extra stat requests.
- ///
/// # Examples
///
/// ## Check if file exists
@@ -211,14 +205,6 @@ impl Operator {
/// Get given path's metadata with extra options.
///
- /// # Notes
- ///
- /// ## Reuse Metadata
- ///
- /// For fetch metadata of entries returned by [`Lister`], it's better to use
- /// [`Operator::list_with`] and [`Operator::lister_with`] with `metakey` query like
- /// `Metakey::ContentLength | Metakey::LastModified` so that we can avoid extra requests.
- ///
/// # Options
///
/// ## `if_match`
@@ -1546,11 +1532,6 @@ impl Operator {
/// In order to avoid this, you can use [`Operator::lister`] to list entries in
/// a streaming way.
///
- /// ## Reuse Metadata
- ///
- /// The only metadata that is guaranteed to be available is the `Mode`.
- /// For fetching more metadata, please use [`Operator::list_with`] and `metakey`.
- ///
/// # Examples
///
/// ## List entries under a dir
@@ -1560,7 +1541,6 @@ impl Operator {
/// ```no_run
/// # use anyhow::Result;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// use opendal::Operator;
/// # async fn test(op: Operator) -> Result<()> {
/// let mut entries = op.list("path/to/dir/").await?;
@@ -1590,7 +1570,6 @@ impl Operator {
/// ```no_run
/// # use anyhow::Result;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// use opendal::Operator;
/// # async fn test(op: Operator) -> Result<()> {
/// let mut entries = op.list("path/to/prefix").await?;
@@ -1663,51 +1642,6 @@ impl Operator {
/// # }
/// ```
///
- /// ## `metakey`
- ///
- /// Specify the metadata that required to be fetched in entries.
- ///
- /// If `metakey` is not set, we will fetch only the entry's `mode`. Otherwise, we will retrieve
- /// the required metadata from storage services. Even if `metakey` is specified, the metadata
- /// may still be `None`, indicating that the storage service does not supply this information.
- ///
- /// Some storage services like `s3` could return more metadata like `content-length` and
- /// `last-modified`. By using `metakey`, we can fetch those metadata without an extra `stat` call.
- /// Please pick up the metadata you need to reduce the extra `stat` cost.
- ///
- /// This example shows how to list entries with `content-length` and `last-modified` metadata:
- ///
- /// ```no_run
- /// # use anyhow::Result;
- /// use opendal::EntryMode;
- /// use opendal::Metakey;
- /// use opendal::Operator;
- /// # async fn test(op: Operator) -> Result<()> {
- /// let mut entries = op
- /// .list_with("dir/")
- /// // Make sure content-length and last-modified been fetched.
- /// .metakey(Metakey::ContentLength | Metakey::LastModified)
- /// .await?;
- /// for entry in entries {
- /// let meta = entry.metadata();
- /// match meta.mode() {
- /// EntryMode::FILE => {
- /// println!(
- /// "Handling file {} with size {}",
- /// entry.path(),
- /// meta.content_length()
- /// )
- /// }
- /// EntryMode::DIR => {
- /// println!("Handling dir {}", entry.path())
- /// }
- /// EntryMode::Unknown => continue,
- /// }
- /// }
- /// # Ok(())
- /// # }
- /// ```
- ///
/// # Examples
///
/// ## List all entries recursively
@@ -1717,7 +1651,6 @@ impl Operator {
/// ```no_run
/// # use anyhow::Result;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// use opendal::Operator;
/// # async fn test(op: Operator) -> Result<()> {
/// let mut entries = op.list_with("path/to/dir/").recursive(true).await?;
@@ -1743,7 +1676,6 @@ impl Operator {
/// ```no_run
/// # use anyhow::Result;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// use opendal::Operator;
/// # async fn test(op: Operator) -> Result<()> {
/// let mut entries = op.list_with("path/to/prefix").recursive(true).await?;
@@ -1789,11 +1721,6 @@ impl Operator {
/// all entries recursively, use [`Operator::lister_with`] and `recursive(true)`
/// instead.
///
- /// ## Reuse Metadata
- ///
- /// The only metadata that is guaranteed to be available is the `Mode`.
- /// For fetching more metadata, please use [`Operator::lister_with`] and `metakey`.
- ///
/// # Examples
///
/// ```no_run
@@ -1801,7 +1728,6 @@ impl Operator {
/// # use futures::io;
/// use futures::TryStreamExt;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// use opendal::Operator;
/// # async fn test(op: Operator) -> Result<()> {
/// let mut ds = op.lister("path/to/dir/").await?;
@@ -1866,52 +1792,6 @@ impl Operator {
/// # }
/// ```
///
- /// ## `metakey`
- ///
- /// Specify the metadata that required to be fetched in entries.
- ///
- /// If `metakey` is not set, we will fetch only the entry's `mode`. Otherwise, we will retrieve
- /// the required metadata from storage services. Even if `metakey` is specified, the metadata
- /// may still be `None`, indicating that the storage service does not supply this information.
- ///
- /// Some storage services like `s3` could return more metadata like `content-length` and
- /// `last-modified`. By using `metakey`, we can fetch those metadata without an extra `stat` call.
- /// Please pick up the metadata you need to reduce the extra `stat` cost.
- ///
- /// This example shows how to list entries with `content-length` and `last-modified` metadata:
- ///
- /// ```no_run
- /// # use anyhow::Result;
- /// use futures::TryStreamExt;
- /// use opendal::EntryMode;
- /// use opendal::Metakey;
- /// use opendal::Operator;
- /// # async fn test(op: Operator) -> Result<()> {
- /// let mut lister = op
- /// .lister_with("dir/")
- /// // Make sure content-length and last-modified been fetched.
- /// .metakey(Metakey::ContentLength | Metakey::LastModified)
- /// .await?;
- /// while let Some(mut entry) = lister.try_next().await? {
- /// let meta = entry.metadata();
- /// match meta.mode() {
- /// EntryMode::FILE => {
- /// println!(
- /// "Handling file {} with size {}",
- /// entry.path(),
- /// meta.content_length()
- /// )
- /// }
- /// EntryMode::DIR => {
- /// println!("Handling dir {}", entry.path())
- /// }
- /// EntryMode::Unknown => continue,
- /// }
- /// }
- /// # Ok(())
- /// # }
- /// ```
- ///
/// # Examples
///
/// ## List all files recursively
@@ -1920,7 +1800,6 @@ impl Operator {
/// # use anyhow::Result;
/// use futures::TryStreamExt;
/// use opendal::EntryMode;
- /// use opendal::Metakey;
/// use opendal::Operator;
/// # async fn test(op: Operator) -> Result<()> {
/// let mut lister = op.lister_with("path/to/dir/").recursive(true).await?;
@@ -1938,40 +1817,6 @@ impl Operator {
/// # Ok(())
/// # }
/// ```
- ///
- /// ## List files with required metadata
- ///
- /// ```no_run
- /// # use anyhow::Result;
- /// # use futures::io;
- /// use futures::TryStreamExt;
- /// use opendal::EntryMode;
- /// use opendal::Metakey;
- /// use opendal::Operator;
- /// # async fn test(op: Operator) -> Result<()> {
- /// let mut ds = op
- /// .lister_with("path/to/dir/")
- /// .metakey(Metakey::ContentLength | Metakey::LastModified)
- /// .await?;
- /// while let Some(mut entry) = ds.try_next().await? {
- /// let meta = entry.metadata();
- /// match meta.mode() {
- /// EntryMode::FILE => {
- /// println!(
- /// "Handling file {} with size {}",
- /// entry.path(),
- /// meta.content_length()
- /// )
- /// }
- /// EntryMode::DIR => {
- /// println!("Handling dir {}", entry.path())
- /// }
- /// EntryMode::Unknown => continue,
- /// }
- /// }
- /// # Ok(())
- /// # }
- /// ```
pub fn lister_with(&self, path: &str) -> FutureLister<impl Future<Output = Result<Lister>>> {
let path = normalize_path(path);
diff --git a/core/src/types/operator/operator_functions.rs b/core/src/types/operator/operator_functions.rs
index dc5209af3..cdfcc555d 100644
--- a/core/src/types/operator/operator_functions.rs
+++ b/core/src/types/operator/operator_functions.rs
@@ -21,8 +21,6 @@
use std::ops::RangeBounds;
-use flagset::FlagSet;
-
use crate::raw::*;
use crate::*;
@@ -277,19 +275,6 @@ impl FunctionList {
self
}
- /// Metakey is used to control which meta should be returned.
- ///
- /// Lister will make sure the result for specified meta is **known**:
- ///
- /// - `Some(v)` means exist.
- /// - `None` means services doesn't have this meta.
- ///
- /// The default metakey is `Metakey::Mode`.
- pub fn metakey(mut self, v: impl Into<FlagSet<Metakey>>) -> Self {
- self.0 = self.0.map_args(|args| args.with_metakey(v));
- self
- }
-
/// Call the function to consume all the input and generate a
/// result.
pub fn call(self) -> Result<Vec<Entry>> {
@@ -330,19 +315,6 @@ impl FunctionLister {
self
}
- /// Metakey is used to control which meta should be returned.
- ///
- /// Lister will make sure the result for specified meta is **known**:
- ///
- /// - `Some(v)` means exist.
- /// - `None` means services doesn't have this meta.
- ///
- /// The default metakey is `Metakey::Mode`.
- pub fn metakey(mut self, v: impl Into<FlagSet<Metakey>>) -> Self {
- self.0 = self.0.map_args(|args| args.with_metakey(v));
- self
- }
-
/// Call the function to consume all the input and generate a
/// result.
pub fn call(self) -> Result<BlockingLister> {
diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs
index 17f1cb77c..53d89f594 100644
--- a/core/src/types/operator/operator_futures.rs
+++ b/core/src/types/operator/operator_futures.rs
@@ -24,7 +24,6 @@ use std::future::IntoFuture;
use std::ops::RangeBounds;
use std::time::Duration;
-use flagset::FlagSet;
use futures::Future;
use crate::raw::*;
@@ -465,27 +464,6 @@ impl<F: Future<Output = Result<Vec<Entry>>>> FutureList<F> {
self.map(|args| args.with_recursive(v))
}
- /// Metakey is used to control which meta should be returned.
- ///
- /// Lister will make sure the result for specified meta is **known**:
- ///
- /// - `Some(v)` means exist.
- /// - `None` means services doesn't have this meta.
- ///
- /// The default metakey is `Metakey::Mode`.
- pub fn metakey(self, v: impl Into<FlagSet<Metakey>>) -> Self {
- self.map(|args| args.with_metakey(v))
- }
-
- /// Concurrent is used to control the number of concurrent stat requests.
- ///
- /// If concurrent is set to <=1, the lister will perform stat requests sequentially.
- ///
- /// The default concurrent is 1.
- pub fn concurrent(self, v: usize) -> Self {
- self.map(|args| args.with_concurrent(v))
- }
-
/// The version is used to control whether the object versions should be returned.
///
/// - If `false`, list operation will not return with object versions
@@ -528,27 +506,6 @@ impl<F: Future<Output = Result<Lister>>> FutureLister<F> {
self.map(|args| args.with_recursive(v))
}
- /// Metakey is used to control which meta should be returned.
- ///
- /// Lister will make sure the result for specified meta is **known**:
- ///
- /// - `Some(v)` means exist.
- /// - `None` means services doesn't have this meta.
- ///
- /// The default metakey is `Metakey::Mode`.
- pub fn metakey(self, v: impl Into<FlagSet<Metakey>>) -> Self {
- self.map(|args| args.with_metakey(v))
- }
-
- /// Concurrent is used to control the number of concurrent stat requests.
- ///
- /// If concurrent is set to <=1, the lister will perform stat requests sequentially.
- ///
- /// The default concurrent is 1.
- pub fn concurrent(self, v: usize) -> Self {
- self.map(|args| args.with_concurrent(v))
- }
-
/// The version is used to control whether the object versions should be returned.
///
/// - If `false`, list operation will not return with object versions
diff --git a/core/tests/behavior/async_list.rs b/core/tests/behavior/async_list.rs
index 69157cdc1..9e9367b1a 100644
--- a/core/tests/behavior/async_list.rs
+++ b/core/tests/behavior/async_list.rs
@@ -34,8 +34,6 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
op,
test_check,
test_list_dir,
- test_list_dir_with_metakey,
- test_list_dir_with_metakey_complete,
test_list_prefix,
test_list_rich_dir,
test_list_empty_dir,
@@ -94,95 +92,6 @@ pub async fn test_list_dir(op: Operator) -> Result<()> {
Ok(())
}
-/// List dir with metakey
-pub async fn test_list_dir_with_metakey(op: Operator) -> Result<()> {
- let parent = uuid::Uuid::new_v4().to_string();
- let path = format!("{parent}/{}", uuid::Uuid::new_v4());
- debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes(op.info().full_capability());
-
- op.write(&path, content).await.expect("write must succeed");
-
- let mut obs = op
- .lister_with(&format!("{parent}/"))
- .metakey(
- Metakey::Mode
- | Metakey::CacheControl
- | Metakey::ContentDisposition
- | Metakey::ContentLength
- | Metakey::ContentMd5
- | Metakey::ContentRange
- | Metakey::ContentType
- | Metakey::Etag
- | Metakey::LastModified
- | Metakey::Version,
- )
- .await?;
- let mut found = false;
- while let Some(de) = obs.try_next().await? {
- let meta = de.metadata();
- if de.path() == path {
- assert_eq!(meta.mode(), EntryMode::FILE);
- assert_eq!(meta.content_length(), size as u64);
-
- // We don't care about the value, we just to check there is no panic.
- let _ = meta.cache_control();
- let _ = meta.content_disposition();
- let _ = meta.content_md5();
- let _ = meta.content_range();
- let _ = meta.content_type();
- let _ = meta.etag();
- let _ = meta.last_modified();
- let _ = meta.version();
-
- found = true
- }
- }
- assert!(found, "file should be found in list");
-
- op.delete(&path).await.expect("delete must succeed");
- Ok(())
-}
-
-/// List dir with metakey complete
-pub async fn test_list_dir_with_metakey_complete(op: Operator) -> Result<()> {
- let parent = uuid::Uuid::new_v4().to_string();
- let path = format!("{parent}/{}", uuid::Uuid::new_v4());
- debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes(op.info().full_capability());
-
- op.write(&path, content).await.expect("write must succeed");
-
- let mut obs = op
- .lister_with(&format!("{parent}/"))
- .metakey(Metakey::Complete)
- .await?;
- let mut found = false;
- while let Some(de) = obs.try_next().await? {
- let meta = de.metadata();
- if de.path() == path {
- assert_eq!(meta.mode(), EntryMode::FILE);
- assert_eq!(meta.content_length(), size as u64);
-
- // We don't care about the value, we just to check there is no panic.
- let _ = meta.cache_control();
- let _ = meta.content_disposition();
- let _ = meta.content_md5();
- let _ = meta.content_range();
- let _ = meta.content_type();
- let _ = meta.etag();
- let _ = meta.last_modified();
- let _ = meta.version();
-
- found = true
- }
- }
- assert!(found, "file should be found in list");
-
- op.delete(&path).await.expect("delete must succeed");
- Ok(())
-}
-
/// List prefix should return newly created file.
pub async fn test_list_prefix(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
@@ -228,23 +137,6 @@ pub async fn test_list_rich_dir(op: Operator) -> Result<()> {
assert_eq!(actual, expected);
- // List concurrently.
- let mut objects = op
- .lister_with(parent)
- .limit(5)
- .concurrent(5)
- .metakey(Metakey::Complete)
- .await?;
- let mut actual = vec![];
- while let Some(o) = objects.try_next().await? {
- let path = o.path().to_string();
- actual.push(path)
- }
- expected.sort_unstable();
- actual.sort_unstable();
-
- assert_eq!(actual, expected);
-
op.remove_all(parent).await?;
Ok(())
}
@@ -752,24 +644,6 @@ pub async fn test_list_with_version_and_limit(op: Operator) -> Result<()> {
assert_eq!(actual, expected);
- // List concurrently.
- let mut objects = op
- .lister_with(parent)
- .version(true)
- .limit(5)
- .concurrent(5)
- .metakey(Metakey::Complete)
- .await?;
- let mut actual = vec![];
- while let Some(o) = objects.try_next().await? {
- let path = o.path().to_string();
- actual.push(path)
- }
- expected.sort_unstable();
- actual.sort_unstable();
-
- assert_eq!(actual, expected);
-
op.remove_all(parent).await?;
Ok(())
}
diff --git a/core/tests/behavior/blocking_list.rs b/core/tests/behavior/blocking_list.rs
index 225d19cad..80866b8a7 100644
--- a/core/tests/behavior/blocking_list.rs
+++ b/core/tests/behavior/blocking_list.rs
@@ -29,8 +29,6 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
tests.extend(blocking_trials!(
op,
test_blocking_list_dir,
- test_blocking_list_dir_with_metakey,
- test_blocking_list_dir_with_metakey_complete,
test_blocking_list_non_exist_dir,
test_blocking_list_dir_with_recursive,
test_blocking_list_dir_with_recursive_no_trailing_slash,
@@ -68,95 +66,6 @@ pub fn test_blocking_list_dir(op: BlockingOperator) -> Result<()> {
Ok(())
}
-/// List dir with metakey
-pub fn test_blocking_list_dir_with_metakey(op: BlockingOperator) -> Result<()> {
- let parent = uuid::Uuid::new_v4().to_string();
- let path = format!("{parent}/{}", uuid::Uuid::new_v4());
- debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes(op.info().full_capability());
-
- op.write(&path, content).expect("write must succeed");
-
- let mut obs = op
- .lister_with(&format!("{parent}/"))
- .metakey(
- Metakey::Mode
- | Metakey::CacheControl
- | Metakey::ContentDisposition
- | Metakey::ContentLength
- | Metakey::ContentMd5
- | Metakey::ContentRange
- | Metakey::ContentType
- | Metakey::Etag
- | Metakey::LastModified
- | Metakey::Version,
- )
- .call()?;
- let mut found = false;
- while let Some(de) = obs.next().transpose()? {
- let meta = de.metadata();
- if de.path() == path {
- assert_eq!(meta.mode(), EntryMode::FILE);
- assert_eq!(meta.content_length(), size as u64);
-
- // We don't care about the value, we just to check there is no panic.
- let _ = meta.cache_control();
- let _ = meta.content_disposition();
- let _ = meta.content_md5();
- let _ = meta.content_range();
- let _ = meta.content_type();
- let _ = meta.etag();
- let _ = meta.last_modified();
- let _ = meta.version();
-
- found = true
- }
- }
- assert!(found, "file should be found in list");
-
- op.delete(&path).expect("delete must succeed");
- Ok(())
-}
-
-/// List dir with metakey complete
-pub fn test_blocking_list_dir_with_metakey_complete(op: BlockingOperator) -> Result<()> {
- let parent = uuid::Uuid::new_v4().to_string();
- let path = format!("{parent}/{}", uuid::Uuid::new_v4());
- debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes(op.info().full_capability());
-
- op.write(&path, content).expect("write must succeed");
-
- let mut obs = op
- .lister_with(&format!("{parent}/"))
- .metakey(Metakey::Complete)
- .call()?;
- let mut found = false;
- while let Some(de) = obs.next().transpose()? {
- let meta = de.metadata();
- if de.path() == path {
- assert_eq!(meta.mode(), EntryMode::FILE);
- assert_eq!(meta.content_length(), size as u64);
-
- // We don't care about the value, we just to check there is no panic.
- let _ = meta.cache_control();
- let _ = meta.content_disposition();
- let _ = meta.content_md5();
- let _ = meta.content_range();
- let _ = meta.content_type();
- let _ = meta.etag();
- let _ = meta.last_modified();
- let _ = meta.version();
-
- found = true
- }
- }
- assert!(found, "file should be found in list");
-
- op.delete(&path).expect("delete must succeed");
- Ok(())
-}
-
/// List non exist dir should return nothing.
pub fn test_blocking_list_non_exist_dir(op: BlockingOperator) -> Result<()> {
let dir = format!("{}/", uuid::Uuid::new_v4());
diff --git a/integrations/cloud_filter/src/lib.rs b/integrations/cloud_filter/src/lib.rs
index eb5518b6e..24ace0324 100644
--- a/integrations/cloud_filter/src/lib.rs
+++ b/integrations/cloud_filter/src/lib.rs
@@ -101,7 +101,7 @@ use cloud_filter::{
};
use file::FileBlob;
use futures::StreamExt;
-use opendal::{Entry, Metakey, Operator};
+use opendal::{Entry, Operator};
const BUF_SIZE: usize = 65536;
@@ -209,7 +209,6 @@ impl Filter for CloudFilter {
let mut entries = self
.op
.lister_with(&remote_path.to_string_lossy().replace('\\', "/"))
- .metakey(Metakey::LastModified | Metakey::ContentLength)
.await
.map_err(|e| {
log::warn!("failed to list files: {}", e);
diff --git a/integrations/compat/Cargo.toml b/integrations/compat/Cargo.toml
index 2fe51ad15..fc4007827 100644
--- a/integrations/compat/Cargo.toml
+++ b/integrations/compat/Cargo.toml
@@ -36,7 +36,7 @@ v0_50_to_v0_49 = ["dep:opendal_v0_49", "dep:opendal_v0_50"]
[dependencies]
async-trait = "0.1"
opendal_v0_49 = { package = "opendal", version = "0.49", optional = true }
-opendal_v0_50 = { package = "opendal", version = "0.50", optional = true, path = "../../core" }
+opendal_v0_50 = { package = "opendal", version = "0.50", optional = true }
[dev-dependencies]
tokio = { version = "1.41", features = ["full"] }
diff --git a/integrations/fuse3/src/file_system.rs b/integrations/fuse3/src/file_system.rs
index 188557871..2811149b2 100644
--- a/integrations/fuse3/src/file_system.rs
+++ b/integrations/fuse3/src/file_system.rs
@@ -33,7 +33,6 @@ use opendal::raw::normalize_path;
use opendal::EntryMode;
use opendal::ErrorKind;
use opendal::Metadata;
-use opendal::Metakey;
use opendal::Operator;
use sharded_slab::Slab;
use tokio::sync::Mutex;
@@ -717,7 +716,6 @@ impl PathFilesystem for Filesystem {
let children = self
.op
.lister_with(&path)
- .metakey(Metakey::ContentLength | Metakey::LastModified | Metakey::Mode)
.await
.map_err(opendal_error2errno)?
.filter_map(move |entry| {
diff --git a/integrations/object_store/src/store.rs b/integrations/object_store/src/store.rs
index 1339f1bb7..c5ba9a675 100644
--- a/integrations/object_store/src/store.rs
+++ b/integrations/object_store/src/store.rs
@@ -39,8 +39,8 @@ use object_store::PutOptions;
use object_store::PutPayload;
use object_store::PutResult;
use object_store::{GetOptions, UploadPart};
+use opendal::Buffer;
use opendal::Writer;
-use opendal::{Buffer, Metakey};
use opendal::{Operator, OperatorInfo};
use tokio::sync::{Mutex, Notify};
@@ -103,16 +103,6 @@ impl OpendalStore {
inner: op,
}
}
-
- /// The metakey that requested by object_store, should align with its meta.
- #[inline]
- fn metakey() -> flagset::FlagSet<Metakey> {
- Metakey::Mode
- | Metakey::LastModified
- | Metakey::ContentLength
- | Metakey::Etag
- | Metakey::Version
- }
}
impl Debug for OpendalStore {
@@ -308,7 +298,6 @@ impl ObjectStore for OpendalStore {
let stream = self
.inner
.lister_with(&path)
- .metakey(Self::metakey())
.recursive(true)
.await
.map_err(|err| format_object_store_error(err, &path))?;
@@ -338,7 +327,6 @@ impl ObjectStore for OpendalStore {
self.inner
.lister_with(&path)
.start_after(offset.as_ref())
- .metakey(Self::metakey())
.recursive(true)
.into_future()
.into_send()
@@ -350,7 +338,6 @@ impl ObjectStore for OpendalStore {
} else {
self.inner
.lister_with(&path)
- .metakey(Self::metakey())
.recursive(true)
.into_future()
.into_send()
@@ -372,7 +359,6 @@ impl ObjectStore for OpendalStore {
let mut stream = self
.inner
.lister_with(&path)
- .metakey(Self::metakey())
.into_future()
.into_send()
.await
diff --git a/integrations/object_store/src/utils.rs b/integrations/object_store/src/utils.rs
index 5bfdc7d23..5eb4a10f7 100644
--- a/integrations/object_store/src/utils.rs
+++ b/integrations/object_store/src/utils.rs
@@ -17,7 +17,7 @@
use futures::Stream;
use object_store::ObjectMeta;
-use opendal::{Entry, Metadata, Metakey};
+use opendal::{Entry, Metadata};
use std::future::IntoFuture;
/// Conditionally add the `Send` marker trait for the wrapped type.
@@ -51,22 +51,12 @@ pub fn format_object_store_error(err: opendal::Error, path: &str) -> object_stor
/// Format `opendal::Metadata` to `object_store::ObjectMeta`.
pub fn format_object_meta(path: &str, meta: &Metadata) -> ObjectMeta {
- let version = match meta.metakey().contains(Metakey::Version) {
- true => meta.version().map(|x| x.to_string()),
- false => None,
- };
-
- let e_tag = match meta.metakey().contains(Metakey::Etag) {
- true => meta.etag().map(|x| x.to_string()),
- false => None,
- };
-
ObjectMeta {
location: path.into(),
last_modified: meta.last_modified().unwrap_or_default(),
size: meta.content_length() as usize,
- e_tag,
- version,
+ e_tag: meta.etag().map(|x| x.to_string()),
+ version: meta.version().map(|x| x.to_string()),
}
}