You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/01/03 07:36:16 UTC
(opendal) branch fix-object-store-without-metakey created (now 04d2184f7)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a change to branch fix-object-store-without-metakey
in repository https://gitbox.apache.org/repos/asf/opendal.git
at 04d2184f7 fix(integration/object_store): object_store requires metadata in list
This branch includes the following new commits:
new 04d2184f7 fix(integration/object_store): object_store requires metadata in list
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opendal) 01/01: fix(integration/object_store): object_store requires metadata in list
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch fix-object-store-without-metakey
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 04d2184f76e670c6f5dcb26e624ea08823e989a3
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri Jan 3 15:35:24 2025 +0800
fix(integration/object_store): object_store requires metadata in list
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
integrations/object_store/src/store.rs | 64 ++++++++++++++++++++++------------
integrations/object_store/src/utils.rs | 12 +------
2 files changed, 43 insertions(+), 33 deletions(-)
diff --git a/integrations/object_store/src/store.rs b/integrations/object_store/src/store.rs
index c5ba9a675..8b64dd35f 100644
--- a/integrations/object_store/src/store.rs
+++ b/integrations/object_store/src/store.rs
@@ -323,32 +323,52 @@ impl ObjectStore for OpendalStore {
let offset = offset.clone();
let fut = async move {
- let fut = if self.inner.info().full_capability().list_with_start_after {
- self.inner
- .lister_with(&path)
- .start_after(offset.as_ref())
- .recursive(true)
- .into_future()
- .into_send()
- .await
- .map_err(|err| format_object_store_error(err, &path))?
- .then(try_format_object_meta)
- .into_send()
- .boxed()
+ let list_with_start_after = self.inner.info().full_capability().list_with_start_after;
+ let mut fut = self.inner.lister_with(&path).recursive(true);
+
+ // Use native start_after support if possible.
+ if list_with_start_after {
+ fut = fut.start_after(offset.as_ref());
+ }
+
+ let lister = fut
+ .await
+ .map_err(|err| format_object_store_error(err, &path))?
+ .then(move |entry| {
+ let path = path.clone();
+
+ async move {
+ let entry = entry.map_err(|err| format_object_store_error(err, &path))?;
+ let (path, metadata) = entry.into_parts();
+
+ // If it's a dir or last_modified is present, we can use it directly.
+ if metadata.is_dir() || metadata.last_modified().is_some() {
+ let object_meta = format_object_meta(&path, &metadata);
+ return Ok(object_meta);
+ }
+
+ let metadata = self
+ .inner
+ .stat(&path)
+ .await
+ .map_err(|err| format_object_store_error(err, &path))?;
+ let object_meta = format_object_meta(&path, &metadata);
+ Ok::<_, object_store::Error>(object_meta)
+ }
+ })
+ .into_send()
+ .boxed();
+
+ let stream = if list_with_start_after {
+ lister
} else {
- self.inner
- .lister_with(&path)
- .recursive(true)
- .into_future()
- .into_send()
- .await
- .map_err(|err| format_object_store_error(err, &path))?
- .try_filter(move |entry| futures::future::ready(entry.path() > offset.as_ref()))
- .then(try_format_object_meta)
+ lister
+ .try_filter(move |entry| futures::future::ready(entry.location > offset))
.into_send()
.boxed()
};
- Ok::<_, object_store::Error>(fut)
+
+ Ok::<_, object_store::Error>(stream)
};
fut.into_stream().into_send().try_flatten().boxed()
diff --git a/integrations/object_store/src/utils.rs b/integrations/object_store/src/utils.rs
index 5eb4a10f7..63357402b 100644
--- a/integrations/object_store/src/utils.rs
+++ b/integrations/object_store/src/utils.rs
@@ -17,7 +17,7 @@
use futures::Stream;
use object_store::ObjectMeta;
-use opendal::{Entry, Metadata};
+use opendal::Metadata;
use std::future::IntoFuture;
/// Conditionally add the `Send` marker trait for the wrapped type.
@@ -60,16 +60,6 @@ pub fn format_object_meta(path: &str, meta: &Metadata) -> ObjectMeta {
}
}
-/// Try to format `opendal::Entry` to `object_store::ObjectMeta`.
-pub async fn try_format_object_meta(
- res: object_store::Result<Entry, opendal::Error>,
-) -> object_store::Result<ObjectMeta> {
- let entry = res.map_err(|err| format_object_store_error(err, ""))?;
- let meta = entry.metadata();
-
- Ok(format_object_meta(entry.path(), meta))
-}
-
/// Make given future `Send`.
pub trait IntoSendFuture {
type Output;