You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/06/16 07:13:13 UTC
(opendal) branch main updated: refactor!: simplify stateful operation factories (#7746)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 33eec87b3 refactor!: simplify stateful operation factories (#7746)
33eec87b3 is described below
commit 33eec87b36f7f34892add108024b820c71536d26
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Jun 16 15:13:08 2026 +0800
refactor!: simplify stateful operation factories (#7746)
* refactor!: simplify stateful operation factories
* fix: handle stateful operation ci regressions
---
core/Cargo.lock | 1 +
core/core/src/docs/internals/accessor.rs | 4 +-
core/core/src/layers/capability_override.rs | 37 +--
core/core/src/layers/complete.rs | 52 ++--
core/core/src/layers/correctness_check.rs | 87 ++----
core/core/src/layers/error_context.rs | 42 +--
core/core/src/layers/simulate.rs | 133 ++++-----
core/core/src/raw/accessor.rs | 143 +++-------
core/core/src/raw/enum_utils.rs | 2 +-
core/core/src/raw/oio/copy/api.rs | 116 +++++++-
core/core/src/raw/oio/list/flat_list.rs | 3 +-
core/core/src/services/memory/backend.rs | 51 ++--
core/core/src/types/context/read.rs | 30 +-
core/core/src/types/context/write.rs | 7 +-
core/core/src/types/copy.rs | 4 +-
core/core/src/types/delete/deleter.rs | 4 +-
core/core/src/types/list.rs | 4 +-
core/core/src/types/operator/operator.rs | 19 +-
core/core/src/types/read/buffer_stream.rs | 8 +-
core/core/src/types/read/futures_async_reader.rs | 40 ++-
core/core/src/types/read/futures_bytes_stream.rs | 23 +-
core/core/src/types/read/reader.rs | 10 +-
core/core/src/types/write/buffer_sink.rs | 2 +-
core/core/src/types/write/futures_async_writer.rs | 2 +-
core/core/src/types/write/futures_bytes_sink.rs | 2 +-
core/core/src/types/write/writer.rs | 2 +-
core/layers/async-backtrace/src/lib.rs | 48 +---
core/layers/await-tree/src/lib.rs | 55 +---
core/layers/capability-check/src/lib.rs | 68 ++---
core/layers/chaos/src/lib.rs | 44 +--
core/layers/concurrent-limit/src/lib.rs | 247 +++++++++--------
core/layers/dtrace/src/lib.rs | 39 +--
core/layers/fastrace/src/lib.rs | 89 ++----
core/layers/foyer/src/full.rs | 21 +-
core/layers/foyer/src/lib.rs | 94 ++-----
core/layers/hotpath/src/lib.rs | 52 +---
core/layers/immutable-index/src/lib.rs | 64 ++---
core/layers/logging/src/lib.rs | 67 ++---
core/layers/mime-guess/src/lib.rs | 37 +--
core/layers/observe-metrics-common/src/lib.rs | 84 +++---
core/layers/oteltrace/src/lib.rs | 47 +---
core/layers/retry/src/lib.rs | 135 +++------
core/layers/route/src/lib.rs | 106 +++----
core/layers/tail-cut/src/lib.rs | 86 ++----
core/layers/throttle/src/lib.rs | 39 +--
core/layers/timeout/src/lib.rs | 128 +++------
core/layers/tracing/src/lib.rs | 64 +----
core/services/aliyun-drive/src/backend.rs | 159 ++++-------
core/services/aliyun-drive/src/lister.rs | 36 ++-
core/services/aliyun-drive/src/writer.rs | 77 ++++++
core/services/alluxio/src/backend.rs | 65 ++---
core/services/azblob/src/backend.rs | 72 ++---
core/services/azdls/Cargo.toml | 1 +
core/services/azdls/src/backend.rs | 82 +++---
core/services/azdls/src/writer.rs | 52 +++-
core/services/azfile/src/backend.rs | 66 ++---
core/services/azfile/src/writer.rs | 10 +
core/services/b2/src/backend.rs | 102 +++----
core/services/cacache/src/backend.rs | 61 ++--
core/services/cloudflare-kv/src/backend.rs | 71 ++---
core/services/compfs/src/backend.rs | 322 +++++++++++++---------
core/services/cos/src/backend.rs | 81 +++---
core/services/d1/src/backend.rs | 67 ++---
core/services/dashmap/src/backend.rs | 67 ++---
core/services/dbfs/src/backend.rs | 64 ++---
core/services/dropbox/src/backend.rs | 113 ++++----
core/services/etcd/src/backend.rs | 67 ++---
core/services/etcd/src/lister.rs | 34 +++
core/services/foundationdb/src/backend.rs | 64 ++---
core/services/foyer/src/backend.rs | 58 ++--
core/services/fs/src/backend.rs | 224 +++++++++------
core/services/ftp/src/backend.rs | 223 +++++++++------
core/services/gcs/src/backend.rs | 72 ++---
core/services/gdrive/src/backend.rs | 132 ++++-----
core/services/gdrive/src/writer.rs | 12 +-
core/services/ghac/src/backend.rs | 67 ++---
core/services/ghac/src/writer.rs | 54 ++++
core/services/github/src/backend.rs | 65 ++---
core/services/goosefs/src/backend.rs | 63 ++---
core/services/gridfs/src/backend.rs | 58 ++--
core/services/hdfs-native/src/backend.rs | 186 ++++++++-----
core/services/hdfs/src/backend.rs | 130 +++++----
core/services/hf/src/backend.rs | 72 ++---
core/services/hf/src/writer.rs | 46 ++++
core/services/http/src/backend.rs | 41 +--
core/services/ipfs/src/backend.rs | 47 ++--
core/services/ipmfs/src/backend.rs | 76 ++---
core/services/koofr/src/backend.rs | 89 +++---
core/services/lakefs/src/backend.rs | 96 +++----
core/services/memcached/src/backend.rs | 61 ++--
core/services/mini_moka/src/backend.rs | 58 ++--
core/services/moka/src/backend.rs | 64 ++---
core/services/mongodb/src/backend.rs | 58 ++--
core/services/monoiofs/src/backend.rs | 245 +++++++++-------
core/services/mysql/src/backend.rs | 70 ++---
core/services/mysql/src/lister.rs | 34 +++
core/services/obs/src/backend.rs | 81 +++---
core/services/onedrive/src/backend.rs | 97 +++----
core/services/opfs/src/backend.rs | 74 ++---
core/services/opfs/src/lister.rs | 33 ++-
core/services/opfs/src/writer.rs | 57 +++-
core/services/oss/src/backend.rs | 82 +++---
core/services/pcloud/src/backend.rs | 86 +++---
core/services/persy/src/backend.rs | 58 ++--
core/services/postgresql/src/backend.rs | 61 ++--
core/services/redb/src/backend.rs | 58 ++--
core/services/redis/src/backend.rs | 58 ++--
core/services/rocksdb/src/backend.rs | 61 ++--
core/services/s3/src/backend.rs | 72 ++---
core/services/seafile/src/backend.rs | 65 ++---
core/services/sftp/src/backend.rs | 275 ++++++++++--------
core/services/sled/src/backend.rs | 61 ++--
core/services/sqlite/src/backend.rs | 68 ++---
core/services/surrealdb/src/backend.rs | 61 ++--
core/services/swift/src/backend.rs | 83 +++---
core/services/tikv/src/backend.rs | 58 ++--
core/services/tos/src/backend.rs | 69 ++---
core/services/upyun/src/backend.rs | 83 +++---
core/services/vercel-artifacts/src/backend.rs | 65 ++---
core/services/vercel-blob/src/backend.rs | 88 +++---
core/services/webdav/src/backend.rs | 129 ++++-----
core/services/webdav/src/writer.rs | 7 +
core/services/webhdfs/src/backend.rs | 65 ++---
core/services/yandex-disk/src/backend.rs | 89 +++---
integrations/object_store/src/service/lister.rs | 2 +-
integrations/object_store/src/service/mod.rs | 79 ++----
integrations/object_store/src/store.rs | 30 +-
127 files changed, 4003 insertions(+), 4828 deletions(-)
diff --git a/core/Cargo.lock b/core/Cargo.lock
index 4b0ce9b90..92d4abd60 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -6818,6 +6818,7 @@ dependencies = [
"bytes",
"http 1.4.1",
"log",
+ "mea",
"opendal-core",
"opendal-service-azure-common",
"quick-xml",
diff --git a/core/core/src/docs/internals/accessor.rs b/core/core/src/docs/internals/accessor.rs
index 3a1218e56..633e7dbe7 100644
--- a/core/core/src/docs/internals/accessor.rs
+++ b/core/core/src/docs/internals/accessor.rs
@@ -41,7 +41,7 @@
//! ctx: &OperationContext,
//! path: &str,
//! args: OpRead,
-//! ) -> impl Future<Output = Result<(RpRead, Self::Reader)>> + MaybeSend;
+//! ) -> impl Future<Output = Result<Self::Reader>> + MaybeSend;
//! }
//! ```
//!
@@ -307,7 +307,7 @@
//! }
//! }
//!
-//! async fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+//! fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
//! gagaga!()
//! }
//! }
diff --git a/core/core/src/layers/capability_override.rs b/core/core/src/layers/capability_override.rs
index 8b91d9914..42138aa42 100644
--- a/core/core/src/layers/capability_override.rs
+++ b/core/core/src/layers/capability_override.rs
@@ -141,46 +141,31 @@ impl Service for CapabilityOverrideService {
self.inner.stat(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- self.inner.read(ctx, path, args).await
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ self.inner.read(ctx, path, args)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- self.inner.write(ctx, path, args).await
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ self.inner.write(ctx, path, args)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner.delete(ctx).await
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- self.inner.list(ctx, path, args).await
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.inner.list(ctx, path, args)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.inner.copy(ctx, from, to, args, opts).await
+ ) -> Result<Self::Copier> {
+ self.inner.copy(ctx, from, to, args, opts)
}
async fn rename(
diff --git a/core/core/src/layers/complete.rs b/core/core/src/layers/complete.rs
index 6a3e8f075..a2c84e160 100644
--- a/core/core/src/layers/complete.rs
+++ b/core/core/src/layers/complete.rs
@@ -79,37 +79,26 @@ impl Service for CompleteService {
self.inner.create_dir(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, reader) = self.inner.read(ctx, path, args).await?;
- let reader = CompleteReader::new(reader);
- Ok((rp, reader))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let reader = self.inner.read(ctx, path, args)?;
+ Ok(CompleteReader::new(reader))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let append = args.append();
- let (rp, w) = self.inner.write(ctx, path, args).await?;
- Ok((rp, CompleteWriter::new(w, append)))
+ let w = self.inner.write(ctx, path, args)?;
+ Ok(CompleteWriter::new(w, append))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.inner.copy(ctx, from, to, args, opts).await
+ ) -> Result<Self::Copier> {
+ self.inner.copy(ctx, from, to, args, opts)
}
async fn rename(
@@ -126,19 +115,13 @@ impl Service for CompleteService {
self.inner.stat(ctx, path, args).await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner.delete(ctx).await
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, lister) = self.inner.list(ctx, path, args).await?;
- let lister = CompleteLister::new(ctx.clone(), self.inner.clone(), lister);
- Ok((rp, lister))
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let lister = self.inner.list(ctx, path, args)?;
+ Ok(CompleteLister::new(ctx.clone(), self.inner.clone(), lister))
}
async fn presign(
@@ -454,7 +437,12 @@ mod tests {
}
async fn read(&self, _: BytesRange) -> Result<(RpRead, Buffer)> {
- Ok((RpRead::default(), self.buffer.clone()))
+ Ok((
+ RpRead::new(
+ Metadata::new(EntryMode::FILE).with_content_length(self.buffer.len() as u64),
+ ),
+ self.buffer.clone(),
+ ))
}
}
diff --git a/core/core/src/layers/correctness_check.rs b/core/core/src/layers/correctness_check.rs
index e1a0cabbd..609dfd5f6 100644
--- a/core/core/src/layers/correctness_check.rs
+++ b/core/core/src/layers/correctness_check.rs
@@ -93,12 +93,7 @@ impl Service for CorrectnessService {
self.inner.capability()
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
let capability = self.capability();
let scheme = self.info().scheme();
if !capability.read_with_version && args.version().is_some() {
@@ -129,15 +124,10 @@ impl Service for CorrectnessService {
));
}
- self.inner.read(ctx, path, args).await
+ self.inner.read(ctx, path, args)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let capability = self.capability();
let scheme = self.info().scheme();
if args.append() && !capability.write_can_append {
@@ -164,7 +154,7 @@ impl Service for CorrectnessService {
}
}
- self.inner.write(ctx, path, args).await
+ self.inner.write(ctx, path, args)
}
async fn stat(&self, ctx: &OperationContext, path: &str, args: OpStat) -> Result<RpStat> {
@@ -201,21 +191,20 @@ impl Service for CorrectnessService {
self.inner.stat(ctx, path, args).await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner.delete(ctx).await.map(|(rp, deleter)| {
- let deleter = CheckWrapper::new(deleter, self.info().scheme(), self.capability());
- (rp, deleter)
- })
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner
+ .delete(ctx)
+ .map(|deleter| CheckWrapper::new(deleter, self.info().scheme(), self.capability()))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
let capability = self.capability();
let scheme = self.info().scheme();
if args.if_not_exists() && !capability.copy_with_if_not_exists {
@@ -236,16 +225,11 @@ impl Service for CorrectnessService {
));
}
- self.inner.copy(ctx, from, to, args, opts).await
+ self.inner.copy(ctx, from, to, args, opts)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- self.inner.list(ctx, path, args).await
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.inner.list(ctx, path, args)
}
async fn create_dir(
@@ -369,48 +353,30 @@ mod tests {
Ok(RpStat::new(Metadata::new(EntryMode::Unknown)))
}
- async fn read(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::new(Metadata::new(EntryMode::FILE).with_content_length(0)),
- MockReader,
- ))
+ fn read(&self, _ctx: &OperationContext, _: &str, _: OpRead) -> Result<Self::Reader> {
+ Ok(MockReader)
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- Ok((RpWrite::new(), MockWriter))
+ fn write(&self, _ctx: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
+ Ok(MockWriter)
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- Ok((RpList::default(), ()))
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
+ Ok(())
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- Ok((RpDelete::default(), MockDeleter))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ Ok(MockDeleter)
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
@@ -443,13 +409,16 @@ mod tests {
impl oio::Read for MockReader {
async fn open(&self, _: BytesRange) -> Result<(RpRead, Box<dyn oio::ReadStreamDyn>)> {
Ok((
- RpRead::default(),
+ RpRead::new(Metadata::new(EntryMode::FILE).with_content_length(0)),
Box::new(Buffer::new()) as Box<dyn oio::ReadStreamDyn>,
))
}
async fn read(&self, _: BytesRange) -> Result<(RpRead, Buffer)> {
- Ok((RpRead::default(), Buffer::new()))
+ Ok((
+ RpRead::new(Metadata::new(EntryMode::FILE).with_content_length(0)),
+ Buffer::new(),
+ ))
}
}
diff --git a/core/core/src/layers/error_context.rs b/core/core/src/layers/error_context.rs
index 0b55d35dd..761112816 100644
--- a/core/core/src/layers/error_context.rs
+++ b/core/core/src/layers/error_context.rs
@@ -98,16 +98,10 @@ impl Service for ErrorContextService {
})
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
self.inner
.read(ctx, path, args)
- .await
- .map(|(rp, r)| (rp, ErrorContextWrapper::new(self.info().scheme(), path, r)))
+ .map(|r| ErrorContextWrapper::new(self.info().scheme(), path, r))
.map_err(|err| {
err.with_operation(Operation::Read)
.with_context("service", self.info().scheme())
@@ -115,16 +109,10 @@ impl Service for ErrorContextService {
})
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
self.inner
.write(ctx, path, args)
- .await
- .map(|(rp, w)| (rp, ErrorContextWrapper::new(self.info().scheme(), path, w)))
+ .map(|w| ErrorContextWrapper::new(self.info().scheme(), path, w))
.map_err(|err| {
err.with_operation(Operation::Write)
.with_context("service", self.info().scheme())
@@ -132,18 +120,17 @@ impl Service for ErrorContextService {
})
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
self.inner
.copy(ctx, from, to, args, opts)
- .await
- .map(|(rp, p)| (rp, ErrorContextWrapper::new(self.info().scheme(), to, p)))
+ .map(|p| ErrorContextWrapper::new(self.info().scheme(), to, p))
.map_err(|err| {
err.with_operation(Operation::Copy)
.with_context("service", self.info().scheme())
@@ -175,27 +162,20 @@ impl Service for ErrorContextService {
})
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
self.inner
.delete(ctx)
- .await
- .map(|(rp, w)| (rp, ErrorContextWrapper::new(self.info().scheme(), "", w)))
+ .map(|w| ErrorContextWrapper::new(self.info().scheme(), "", w))
.map_err(|err| {
err.with_operation(Operation::Delete)
.with_context("service", self.info().scheme())
})
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
self.inner
.list(ctx, path, args)
- .await
- .map(|(rp, p)| (rp, ErrorContextWrapper::new(self.info().scheme(), path, p)))
+ .map(|p| ErrorContextWrapper::new(self.info().scheme(), path, p))
.map_err(|err| {
err.with_operation(Operation::List)
.with_context("service", self.info().scheme())
diff --git a/core/core/src/layers/simulate.rs b/core/core/src/layers/simulate.rs
index dfef4bd64..3dfc091b7 100644
--- a/core/core/src/layers/simulate.rs
+++ b/core/core/src/layers/simulate.rs
@@ -133,7 +133,7 @@ impl SimulateService {
}
if capability.write_can_empty && capability.list {
- let (_, mut w) = self.srv.write(ctx, path, OpWrite::default()).await?;
+ let mut w = self.srv.write(ctx, path, OpWrite::default())?;
oio::Write::close(&mut w).await?;
return Ok(RpCreateDir::default());
}
@@ -172,14 +172,11 @@ impl SimulateService {
}
if self.config.stat_dir && capability.list_with_recursive {
- let (_, mut l) = self
- .srv
- .list(
- ctx,
- path,
- OpList::default().with_recursive(true).with_limit(1),
- )
- .await?;
+ let mut l = self.srv.list(
+ ctx,
+ path,
+ OpList::default().with_recursive(true).with_limit(1),
+ )?;
return if l.next().await?.is_some() {
Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
@@ -195,59 +192,59 @@ impl SimulateService {
self.srv.stat(ctx, path, args).await
}
- async fn simulate_list(
+ fn simulate_list(
&self,
ctx: &OperationContext,
path: &str,
args: OpList,
- ) -> Result<(RpList, SimulateLister)> {
+ ) -> Result<SimulateLister> {
let cap = self.srv.capability();
let recursive = args.recursive();
let forward = args;
- let (rp, lister) = match (
+ let lister = match (
recursive,
cap.list_with_recursive,
self.config.list_recursive,
) {
// Backend supports recursive list, forward directly.
(_, true, _) => {
- let (rp, p) = self.srv.list(ctx, path, forward).await?;
- (rp, SimulateLister::One(p))
+ let p = self.srv.list(ctx, path, forward)?;
+ SimulateLister::One(p)
}
// Simulate recursive via flat list when enabled.
(true, false, true) => {
if path.ends_with('/') {
let p = ServicerFlatLister::new(ctx.clone(), self.srv.clone(), path);
- (RpList::default(), SimulateLister::Two(p))
+ SimulateLister::Two(p)
} else {
let parent = get_parent(path);
let p = ServicerFlatLister::new(ctx.clone(), self.srv.clone(), parent);
let p = PrefixLister::new(p, path);
- (RpList::default(), SimulateLister::Four(p))
+ SimulateLister::Four(p)
}
}
// Recursive requested but simulation disabled; rely on backend and propagate errors.
(true, false, false) => {
- let (rp, p) = self.srv.list(ctx, path, forward).await?;
- (rp, SimulateLister::One(p))
+ let p = self.srv.list(ctx, path, forward)?;
+ SimulateLister::One(p)
}
// Non-recursive list: keep existing prefix handling semantics.
(false, false, _) => {
if path.ends_with('/') {
- let (rp, p) = self.srv.list(ctx, path, forward).await?;
- (rp, SimulateLister::One(p))
+ let p = self.srv.list(ctx, path, forward)?;
+ SimulateLister::One(p)
} else {
let parent = get_parent(path);
- let (rp, p) = self.srv.list(ctx, parent, forward).await?;
+ let p = self.srv.list(ctx, parent, forward)?;
let p = PrefixLister::new(p, path);
- (rp, SimulateLister::Three(p))
+ SimulateLister::Three(p)
}
}
};
- Ok((rp, lister))
+ Ok(lister)
}
async fn simulate_delete_with_recursive(
@@ -266,9 +263,7 @@ impl SimulateService {
let non_recursive = args.clone().with_recursive(false);
- let (_rp, mut lister) = self
- .simulate_list(ctx, path, OpList::new().with_recursive(true))
- .await?;
+ let mut lister = self.simulate_list(ctx, path, OpList::new().with_recursive(true))?;
while let Some(entry) = lister.next().await? {
let entry = entry.into_entry();
@@ -307,16 +302,11 @@ impl Service for SimulateService {
self.simulate_create_dir(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
let capability = self.srv.capability();
let simulate_read_with_suffix =
self.config.read_with_suffix && capability.read && !capability.read_with_suffix;
- let (rp, reader) = self.srv.read(ctx, path, args.clone()).await?;
+ let reader = self.srv.read(ctx, path, args.clone())?;
let reader = SimulateReader::new(
ctx.clone(),
self.srv.clone(),
@@ -325,27 +315,22 @@ impl Service for SimulateService {
reader,
simulate_read_with_suffix,
);
- Ok((rp, reader))
+ Ok(reader)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- self.srv.write(ctx, path, args).await
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ self.srv.write(ctx, path, args)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.srv.copy(ctx, from, to, args, opts).await
+ ) -> Result<Self::Copier> {
+ self.srv.copy(ctx, from, to, args, opts)
}
async fn rename(
@@ -362,21 +347,18 @@ impl Service for SimulateService {
self.simulate_stat(ctx, path, args).await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, deleter) = self.srv.delete(ctx).await?;
- Ok((
- rp,
- SimulateDeleter::new(ctx.clone(), self.srv.clone(), self.config.clone(), deleter),
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let deleter = self.srv.delete(ctx)?;
+ Ok(SimulateDeleter::new(
+ ctx.clone(),
+ self.srv.clone(),
+ self.config.clone(),
+ deleter,
))
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- self.simulate_list(ctx, path, args).await
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.simulate_list(ctx, path, args)
}
async fn presign(
@@ -491,7 +473,7 @@ impl oio::List for ServicerFlatLister {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
loop {
if let Some(de) = self.next_dir.take() {
- let (_, mut l) = match self.srv.list(&self.ctx, de.path(), OpList::new()).await {
+ let mut l = match self.srv.list(&self.ctx, de.path(), OpList::new()) {
Ok(v) => v,
Err(e) if e.kind() == ErrorKind::PermissionDenied => {
log::warn!(
@@ -677,57 +659,42 @@ mod tests {
))
}
- async fn read(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, _ctx: &OperationContext, _: &str, _: OpRead) -> Result<Self::Reader> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _ctx: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
@@ -762,12 +729,18 @@ mod tests {
impl oio::Read for MockReader {
async fn open(&self, range: BytesRange) -> Result<(RpRead, Box<dyn oio::ReadStreamDyn>)> {
*self.observed_range.lock().expect("mutex must not poison") = Some(range);
- Ok((RpRead::default(), Box::new(Buffer::new())))
+ Ok((
+ RpRead::new(Metadata::new(EntryMode::FILE).with_content_length(0)),
+ Box::new(Buffer::new()),
+ ))
}
async fn read(&self, range: BytesRange) -> Result<(RpRead, Buffer)> {
*self.observed_range.lock().expect("mutex must not poison") = Some(range);
- Ok((RpRead::default(), Buffer::new()))
+ Ok((
+ RpRead::new(Metadata::new(EntryMode::FILE).with_content_length(0)),
+ Buffer::new(),
+ ))
}
}
diff --git a/core/core/src/raw/accessor.rs b/core/core/src/raw/accessor.rs
index c567e065d..f68f6f821 100644
--- a/core/core/src/raw/accessor.rs
+++ b/core/core/src/raw/accessor.rs
@@ -217,12 +217,7 @@ pub trait Service: Send + Sync + Debug + Unpin + 'static {
///
/// - `path` is a normalized file path.
/// - Range I/O is handled by the returned reader.
- fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> impl Future<Output = Result<(RpRead, Self::Reader)>> + MaybeSend;
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader>;
/// Invoke the `write` operation on the specified path.
///
@@ -231,12 +226,7 @@ pub trait Service: Send + Sync + Debug + Unpin + 'static {
/// # Behavior
///
/// - `path` is a normalized file path.
- fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> impl Future<Output = Result<(RpWrite, Self::Writer)>> + MaybeSend;
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer>;
/// Invoke the `delete` operation.
///
@@ -246,10 +236,7 @@ pub trait Service: Send + Sync + Debug + Unpin + 'static {
///
/// - The returned deleter handles one or more delete requests.
/// - Deleting a missing path should succeed.
- fn delete(
- &self,
- ctx: &OperationContext,
- ) -> impl Future<Output = Result<(RpDelete, Self::Deleter)>> + MaybeSend;
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter>;
/// Invoke the `list` operation on the specified path.
///
@@ -259,12 +246,7 @@ pub trait Service: Send + Sync + Debug + Unpin + 'static {
///
/// - `path` is a normalized directory path or prefix.
/// - Listing a non-existing directory should return an empty stream.
- fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> impl Future<Output = Result<(RpList, Self::Lister)>> + MaybeSend;
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister>;
/// Invoke the `copy` operation on the specified `from` path and `to` path.
///
@@ -281,7 +263,7 @@ pub trait Service: Send + Sync + Debug + Unpin + 'static {
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> impl Future<Output = Result<(RpCopy, Self::Copier)>> + MaybeSend;
+ ) -> Result<Self::Copier>;
/// Invoke the `rename` operation on the specified `from` path and `to` path.
///
@@ -340,7 +322,7 @@ pub trait ServiceDyn: Send + Sync + Debug + Unpin + 'static {
ctx: &'a OperationContext,
path: &'a str,
args: OpRead,
- ) -> BoxedFuture<'a, Result<(RpRead, oio::Reader)>>;
+ ) -> Result<oio::Reader>;
/// Dyn version of [`Service::write`].
fn write_dyn<'a>(
@@ -348,13 +330,10 @@ pub trait ServiceDyn: Send + Sync + Debug + Unpin + 'static {
ctx: &'a OperationContext,
path: &'a str,
args: OpWrite,
- ) -> BoxedFuture<'a, Result<(RpWrite, oio::Writer)>>;
+ ) -> Result<oio::Writer>;
/// Dyn version of [`Service::delete`].
- fn delete_dyn<'a>(
- &'a self,
- ctx: &'a OperationContext,
- ) -> BoxedFuture<'a, Result<(RpDelete, oio::Deleter)>>;
+ fn delete_dyn<'a>(&'a self, ctx: &'a OperationContext) -> Result<oio::Deleter>;
/// Dyn version of [`Service::list`].
fn list_dyn<'a>(
@@ -362,7 +341,7 @@ pub trait ServiceDyn: Send + Sync + Debug + Unpin + 'static {
ctx: &'a OperationContext,
path: &'a str,
args: OpList,
- ) -> BoxedFuture<'a, Result<(RpList, oio::Lister)>>;
+ ) -> Result<oio::Lister>;
/// Dyn version of [`Service::copy`].
fn copy_dyn<'a>(
@@ -372,7 +351,7 @@ pub trait ServiceDyn: Send + Sync + Debug + Unpin + 'static {
to: &'a str,
args: OpCopy,
opts: OpCopier,
- ) -> BoxedFuture<'a, Result<(RpCopy, oio::Copier)>>;
+ ) -> Result<oio::Copier>;
/// Dyn version of [`Service::rename`].
fn rename_dyn<'a>(
@@ -427,11 +406,8 @@ impl<S: Service + ?Sized> ServiceDyn for S {
ctx: &'a OperationContext,
path: &'a str,
args: OpRead,
- ) -> BoxedFuture<'a, Result<(RpRead, oio::Reader)>> {
- Box::pin(async move {
- let (rp, reader) = self.read(ctx, path, args).await?;
- Ok((rp, Box::new(reader) as oio::Reader))
- })
+ ) -> Result<oio::Reader> {
+ Ok(Box::new(self.read(ctx, path, args)?) as oio::Reader)
}
fn write_dyn<'a>(
@@ -439,21 +415,12 @@ impl<S: Service + ?Sized> ServiceDyn for S {
ctx: &'a OperationContext,
path: &'a str,
args: OpWrite,
- ) -> BoxedFuture<'a, Result<(RpWrite, oio::Writer)>> {
- Box::pin(async move {
- let (rp, writer) = self.write(ctx, path, args).await?;
- Ok((rp, Box::new(writer) as oio::Writer))
- })
+ ) -> Result<oio::Writer> {
+ Ok(Box::new(self.write(ctx, path, args)?) as oio::Writer)
}
- fn delete_dyn<'a>(
- &'a self,
- ctx: &'a OperationContext,
- ) -> BoxedFuture<'a, Result<(RpDelete, oio::Deleter)>> {
- Box::pin(async move {
- let (rp, deleter) = self.delete(ctx).await?;
- Ok((rp, Box::new(deleter) as oio::Deleter))
- })
+ fn delete_dyn<'a>(&'a self, ctx: &'a OperationContext) -> Result<oio::Deleter> {
+ Ok(Box::new(self.delete(ctx)?) as oio::Deleter)
}
fn list_dyn<'a>(
@@ -461,11 +428,8 @@ impl<S: Service + ?Sized> ServiceDyn for S {
ctx: &'a OperationContext,
path: &'a str,
args: OpList,
- ) -> BoxedFuture<'a, Result<(RpList, oio::Lister)>> {
- Box::pin(async move {
- let (rp, lister) = self.list(ctx, path, args).await?;
- Ok((rp, Box::new(lister) as oio::Lister))
- })
+ ) -> Result<oio::Lister> {
+ Ok(Box::new(self.list(ctx, path, args)?) as oio::Lister)
}
fn copy_dyn<'a>(
@@ -475,11 +439,8 @@ impl<S: Service + ?Sized> ServiceDyn for S {
to: &'a str,
args: OpCopy,
opts: OpCopier,
- ) -> BoxedFuture<'a, Result<(RpCopy, oio::Copier)>> {
- Box::pin(async move {
- let (rp, copier) = self.copy(ctx, from, to, args, opts).await?;
- Ok((rp, Box::new(copier) as oio::Copier))
- })
+ ) -> Result<oio::Copier> {
+ Ok(Box::new(self.copy(ctx, from, to, args, opts)?) as oio::Copier)
}
fn rename_dyn<'a>(
@@ -531,46 +492,31 @@ impl<T: ServiceDyn + ?Sized> Service for Arc<T> {
self.as_ref().stat_dyn(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, oio::Reader)> {
- self.as_ref().read_dyn(ctx, path, args).await
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<oio::Reader> {
+ self.as_ref().read_dyn(ctx, path, args)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, oio::Writer)> {
- self.as_ref().write_dyn(ctx, path, args).await
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<oio::Writer> {
+ self.as_ref().write_dyn(ctx, path, args)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, oio::Deleter)> {
- self.as_ref().delete_dyn(ctx).await
+ fn delete(&self, ctx: &OperationContext) -> Result<oio::Deleter> {
+ self.as_ref().delete_dyn(ctx)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, oio::Lister)> {
- self.as_ref().list_dyn(ctx, path, args).await
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<oio::Lister> {
+ self.as_ref().list_dyn(ctx, path, args)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, oio::Copier)> {
- self.as_ref().copy_dyn(ctx, from, to, args, opts).await
+ ) -> Result<oio::Copier> {
+ self.as_ref().copy_dyn(ctx, from, to, args, opts)
}
async fn rename(
@@ -628,57 +574,42 @@ impl Service for () {
))
}
- async fn read(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, _: &OperationContext, _: &str, _: OpRead) -> Result<Self::Reader> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/core/src/raw/enum_utils.rs b/core/core/src/raw/enum_utils.rs
index c80cc5eb9..ce3f8b8b2 100644
--- a/core/core/src/raw/enum_utils.rs
+++ b/core/core/src/raw/enum_utils.rs
@@ -20,7 +20,7 @@
//!
//! ```txt
//! impl Service for S3Backend {
-//! fn write(...) -> Result<(RpWrite, oio::Writer)>;
+//! fn write(...) -> Result<oio::Writer>;
//! }
//! ```
//!
diff --git a/core/core/src/raw/oio/copy/api.rs b/core/core/src/raw/oio/copy/api.rs
index 0bd81aff8..24f75aa1a 100644
--- a/core/core/src/raw/oio/copy/api.rs
+++ b/core/core/src/raw/oio/copy/api.rs
@@ -55,8 +55,10 @@ impl Copy for () {
/// OneShotCopier drives a single asynchronous copy step.
pub struct OneShotCopier {
+ factory: Option<Box<dyn FnMut() -> BoxedStaticFuture<Result<Metadata>>>>,
fut: Option<BoxedStaticFuture<Result<Metadata>>>,
meta: Option<Metadata>,
+ consumed: bool,
}
/// # Safety
@@ -73,16 +75,39 @@ impl OneShotCopier {
/// Create a new one-shot copier.
pub fn new(fut: impl Future<Output = Result<Metadata>> + MaybeSend + 'static) -> Self {
Self {
+ factory: None,
fut: Some(Box::pin(fut)),
meta: None,
+ consumed: false,
+ }
+ }
+
+ /// Create a new one-shot copier with a future factory.
+ ///
+ /// The factory will be called again if the previous future fails, allowing
+ /// upper layers like retry to rerun the one-shot copy body.
+ pub fn new_with<F, Fut>(mut factory: F) -> Self
+ where
+ F: FnMut() -> Fut + 'static,
+ Fut: Future<Output = Result<Metadata>> + MaybeSend + 'static,
+ {
+ Self {
+ factory: Some(Box::new(move || {
+ Box::pin(factory()) as BoxedStaticFuture<Result<Metadata>>
+ })),
+ fut: None,
+ meta: None,
+ consumed: false,
}
}
/// Create a one-shot copier that has already completed.
pub fn completed() -> Self {
Self {
+ factory: None,
fut: None,
meta: Some(Metadata::default()),
+ consumed: true,
}
}
}
@@ -97,16 +122,55 @@ impl Copy for OneShotCopier {
}
async fn close(&mut self) -> Result<Metadata> {
+ if let Some(meta) = self.meta.clone() {
+ return Ok(meta);
+ }
+
+ if self.consumed {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "one-shot copier has already been consumed",
+ )
+ .set_persistent());
+ }
+
+ if self.fut.is_none() {
+ if let Some(factory) = self.factory.as_mut() {
+ self.fut = Some(factory());
+ }
+ }
+
if let Some(fut) = self.fut.take() {
- self.meta = Some(fut.await?);
+ match fut.await {
+ Ok(meta) => {
+ self.consumed = true;
+ self.meta = Some(meta.clone());
+ return Ok(meta);
+ }
+ Err(err) => {
+ if self.factory.is_none() {
+ self.consumed = true;
+ return Err(err.set_persistent());
+ }
+
+ return Err(err);
+ }
+ }
}
- Ok(self.meta.clone().unwrap_or_default())
+ self.consumed = true;
+ Err(Error::new(
+ ErrorKind::Unexpected,
+ "one-shot copier has no future to drive",
+ )
+ .set_persistent())
}
async fn abort(&mut self) -> Result<()> {
+ self.factory = None;
self.fut = None;
self.meta = None;
+ self.consumed = true;
Ok(())
}
}
@@ -150,3 +214,51 @@ impl<T: CopyDyn + ?Sized> Copy for Box<T> {
self.deref_mut().abort_dyn().await
}
}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::Arc;
+ use std::sync::atomic::AtomicUsize;
+ use std::sync::atomic::Ordering;
+
+ use super::*;
+
+ #[tokio::test]
+ async fn one_shot_copier_rebuilds_future_after_error() {
+ let attempts = Arc::new(AtomicUsize::new(0));
+ let retry_attempts = attempts.clone();
+
+ let mut copier = OneShotCopier::new_with(move || {
+ let attempts = retry_attempts.clone();
+
+ async move {
+ if attempts.fetch_add(1, Ordering::Relaxed) == 0 {
+ return Err(
+ Error::new(ErrorKind::Unexpected, "temporary copy failure").set_temporary()
+ );
+ }
+
+ Ok(Metadata::new(EntryMode::FILE))
+ }
+ });
+
+ assert!(copier.close().await.unwrap_err().is_temporary());
+ assert_eq!(copier.close().await.unwrap().mode(), EntryMode::FILE);
+ assert_eq!(attempts.load(Ordering::Relaxed), 2);
+ }
+
+ #[tokio::test]
+ async fn one_shot_copier_does_not_succeed_after_consumed_error() {
+ let mut copier = OneShotCopier::new(async {
+ Err(Error::new(ErrorKind::Unexpected, "copy failure").set_temporary())
+ });
+
+ let err = copier.close().await.unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::Unexpected);
+ assert!(err.is_persistent());
+
+ let err = copier.close().await.unwrap_err();
+ assert_eq!(err.kind(), ErrorKind::Unexpected);
+ assert!(err.is_persistent());
+ }
+}
diff --git a/core/core/src/raw/oio/list/flat_list.rs b/core/core/src/raw/oio/list/flat_list.rs
index 42c8042bd..f7f06747b 100644
--- a/core/core/src/raw/oio/list/flat_list.rs
+++ b/core/core/src/raw/oio/list/flat_list.rs
@@ -89,11 +89,10 @@ impl<S: Service> oio::List for FlatLister<S> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
loop {
if let Some(de) = self.next_dir.take() {
- let (_, mut l) = match self
+ let mut l = match self
.service
.as_ref()
.list(&self.ctx, de.path(), OpList::new())
- .await
{
Ok(v) => v,
Err(e) if e.kind() == ErrorKind::PermissionDenied => {
diff --git a/core/core/src/services/memory/backend.rs b/core/core/src/services/memory/backend.rs
index 030bb24c7..02f894973 100644
--- a/core/core/src/services/memory/backend.rs
+++ b/core/core/src/services/memory/backend.rs
@@ -141,60 +141,43 @@ impl Service for MemoryBackend {
}
}
- async fn read(
- &self,
- _: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(MemoryReader::new(self.clone(), path, args)),
- ))
+ fn read(&self, _ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ Ok(oio::StreamReader::new(MemoryReader::new(
+ self.clone(),
+ path,
+ args,
+ )))
}
- async fn write(
- &self,
- _: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let p = build_abs_path(&self.root, path);
- Ok((
- RpWrite::new(),
- MemoryWriter::new(self.core.clone(), p, args),
- ))
+ Ok(MemoryWriter::new(self.core.clone(), p, args))
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(MemoryDeleter::new(self.core.clone(), self.root.clone())),
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ Ok(oio::OneShotDeleter::new(MemoryDeleter::new(
+ self.core.clone(),
+ self.root.clone(),
+ )))
}
- async fn list(
- &self,
- _: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
let p = build_abs_path(&self.root, path);
let keys = self.core.scan(&p)?;
let lister = MemoryLister::new(&self.root, keys);
let lister = oio::HierarchyLister::new(lister, path, args.recursive());
- Ok((RpList::default(), lister))
+ Ok(lister)
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/core/src/types/context/read.rs b/core/core/src/types/context/read.rs
index 23509b43b..f14ed168f 100644
--- a/core/core/src/types/context/read.rs
+++ b/core/core/src/types/context/read.rs
@@ -251,23 +251,23 @@ mod tests {
use super::*;
- async fn new_read_context(
+ fn new_read_context(
ctx: OperationContext,
srv: Servicer,
path: &str,
options: crate::raw::OpReader,
) -> crate::Result<ReadContext> {
- new_read_context_with_args(ctx, srv, path, crate::raw::OpRead::new(), options).await
+ new_read_context_with_args(ctx, srv, path, crate::raw::OpRead::new(), options)
}
- async fn new_read_context_with_args(
+ fn new_read_context_with_args(
ctx: OperationContext,
srv: Servicer,
path: &str,
args: crate::raw::OpRead,
options: crate::raw::OpReader,
) -> crate::Result<ReadContext> {
- let (_, reader) = srv.read(&ctx, path, args.clone()).await?;
+ let reader = srv.read(&ctx, path, args.clone())?;
Ok(ReadContext::new(
ctx,
srv,
@@ -289,8 +289,12 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx =
- Arc::new(new_read_context(ctx, srv, "test", OpReader::new().with_chunk(3)).await?);
+ let ctx = Arc::new(new_read_context(
+ ctx,
+ srv,
+ "test",
+ OpReader::new().with_chunk(3),
+ )?);
let mut generator = ReadGenerator::new(ctx, BytesRange::new(0, Some(10)));
let mut readers = vec![];
while let Some(r) = generator.next_reader().await? {
@@ -312,8 +316,12 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx =
- Arc::new(new_read_context(ctx, srv, "test", OpReader::new().with_chunk(3)).await?);
+ let ctx = Arc::new(new_read_context(
+ ctx,
+ srv,
+ "test",
+ OpReader::new().with_chunk(3),
+ )?);
let mut generator = ReadGenerator::new(ctx, BytesRange::new(0, None));
let mut readers = vec![];
while let Some(r) = generator.next_reader().await? {
@@ -331,7 +339,7 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = new_read_context(ctx, srv, "test", OpReader::new()).await?;
+ let ctx = new_read_context(ctx, srv, "test", OpReader::new())?;
let result = ctx
.parse_into_range(BytesRange::new(u64::MAX, Some(1)))
@@ -355,7 +363,7 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = new_read_context_with_args(ctx, srv, "test", args, options).await?;
+ let ctx = new_read_context_with_args(ctx, srv, "test", args, options)?;
let range = ctx.parse_into_range(10..).await?;
@@ -374,7 +382,7 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = new_read_context_with_args(ctx, srv, "test", args, options).await?;
+ let ctx = new_read_context_with_args(ctx, srv, "test", args, options)?;
let range = ctx.parse_into_range(BytesRange::suffix(10)).await?;
diff --git a/core/core/src/types/context/write.rs b/core/core/src/types/context/write.rs
index bd23dfb9e..1bdfded45 100644
--- a/core/core/src/types/context/write.rs
+++ b/core/core/src/types/context/write.rs
@@ -111,12 +111,9 @@ pub struct WriteGenerator<W> {
impl WriteGenerator<oio::Writer> {
/// Create a new exact buf writer.
- pub async fn create(ctx: Arc<WriteContext>) -> Result<Self> {
+ pub fn create(ctx: Arc<WriteContext>) -> Result<Self> {
let (chunk_size, exact) = ctx.calculate_chunk_size();
- let (_, w) = ctx
- .srv
- .write(&ctx.ctx, ctx.path(), ctx.args().clone())
- .await?;
+ let w = ctx.srv.write(&ctx.ctx, ctx.path(), ctx.args().clone())?;
Ok(Self {
w,
diff --git a/core/core/src/types/copy.rs b/core/core/src/types/copy.rs
index aca78a731..9fe488d37 100644
--- a/core/core/src/types/copy.rs
+++ b/core/core/src/types/copy.rs
@@ -55,7 +55,7 @@ unsafe impl Sync for Copier {}
impl Copier {
/// Create a new copier.
- pub(crate) async fn create(
+ pub(crate) fn create(
ctx: OperationContext,
srv: Servicer,
from: &str,
@@ -63,7 +63,7 @@ impl Copier {
args: OpCopy,
opts: OpCopier,
) -> Result<Self> {
- let (_, copier) = srv.copy(&ctx, from, to, args, opts).await?;
+ let copier = srv.copy(&ctx, from, to, args, opts)?;
Ok(Self {
copier: Some(copier),
diff --git a/core/core/src/types/delete/deleter.rs b/core/core/src/types/delete/deleter.rs
index ca8c3e0bc..bf83df06f 100644
--- a/core/core/src/types/delete/deleter.rs
+++ b/core/core/src/types/delete/deleter.rs
@@ -86,8 +86,8 @@ pub struct Deleter {
}
impl Deleter {
- pub(crate) async fn create(ctx: OperationContext, srv: Servicer) -> Result<Self> {
- let (_, deleter) = srv.delete(&ctx).await?;
+ pub(crate) fn create(ctx: OperationContext, srv: Servicer) -> Result<Self> {
+ let deleter = srv.delete(&ctx)?;
Ok(Self { deleter })
}
diff --git a/core/core/src/types/list.rs b/core/core/src/types/list.rs
index 1d12fc64a..ac8f9ede9 100644
--- a/core/core/src/types/list.rs
+++ b/core/core/src/types/list.rs
@@ -44,13 +44,13 @@ unsafe impl Sync for Lister {}
impl Lister {
/// Create a new lister.
- pub(crate) async fn create(
+ pub(crate) fn create(
ctx: OperationContext,
srv: Servicer,
path: &str,
args: OpList,
) -> Result<Self> {
- let (_, lister) = srv.list(&ctx, path, args).await?;
+ let lister = srv.list(&ctx, path, args)?;
Ok(Self {
lister: Some(lister),
diff --git a/core/core/src/types/operator/operator.rs b/core/core/src/types/operator/operator.rs
index 94e389c05..3bb8658d3 100644
--- a/core/core/src/types/operator/operator.rs
+++ b/core/core/src/types/operator/operator.rs
@@ -638,7 +638,7 @@ impl Operator {
}
let (range, args, opts) = opts.into();
- let (_, reader) = srv.read(&ctx, &path, args.clone()).await?;
+ let reader = srv.read(&ctx, &path, args.clone())?;
let read_context = ReadContext::new(ctx, srv, path, args, opts, reader);
let r = Reader::new(read_context);
let buf = r.read(range).await?;
@@ -749,7 +749,7 @@ impl Operator {
}
let (args, opts) = options.into();
- let (_, reader) = srv.read(&ctx, &path, args.clone()).await?;
+ let reader = srv.read(&ctx, &path, args.clone())?;
let read_context = ReadContext::new(ctx, srv, path, args, opts, reader);
Ok(Reader::new(read_context))
}
@@ -1335,7 +1335,7 @@ impl Operator {
}
let (args, opts) = opts.into();
- Copier::create(ctx, srv, &from, &to, args, opts).await
+ std::future::ready(Copier::create(ctx, srv, &from, &to, args, opts)).await
}
/// Rename a file from `from` to `to`.
@@ -1488,7 +1488,7 @@ impl Operator {
path: String,
opts: options::DeleteOptions,
) -> Result<()> {
- let (_, mut deleter) = srv.delete(&ctx).await?;
+ let mut deleter = srv.delete(&ctx)?;
let args = opts.into();
deleter.delete_dyn(&path, args).await?;
deleter.close_dyn().await?;
@@ -1573,7 +1573,11 @@ impl Operator {
///
/// Users can have more control over the deletion process by using [`Deleter`] directly.
pub async fn deleter(&self) -> Result<Deleter> {
- Deleter::create(self.context().clone(), self.service().clone()).await
+ std::future::ready(Deleter::create(
+ self.context().clone(),
+ self.service().clone(),
+ ))
+ .await
}
/// Remove the path and all nested dirs and files recursively.
@@ -1772,7 +1776,7 @@ impl Operator {
opts: options::ListOptions,
) -> Result<Vec<Entry>> {
let args = opts.into();
- let lister = Lister::create(ctx, srv, &path, args).await?;
+ let lister = Lister::create(ctx, srv, &path, args)?;
lister.try_collect().await
}
@@ -1911,8 +1915,7 @@ impl Operator {
opts: options::ListOptions,
) -> Result<Lister> {
let args = opts.into();
- let lister = Lister::create(ctx, srv, &path, args).await?;
- Ok(lister)
+ std::future::ready(Lister::create(ctx, srv, &path, args)).await
}
}
diff --git a/core/core/src/types/read/buffer_stream.rs b/core/core/src/types/read/buffer_stream.rs
index d7d71796c..82be5380f 100644
--- a/core/core/src/types/read/buffer_stream.rs
+++ b/core/core/src/types/read/buffer_stream.rs
@@ -378,14 +378,14 @@ mod tests {
use super::*;
- async fn new_read_context(
+ fn new_read_context(
ctx: OperationContext,
srv: Servicer,
path: &str,
options: crate::raw::OpReader,
) -> crate::Result<ReadContext> {
let args = crate::raw::OpRead::new();
- let (_, reader) = srv.read(&ctx, path, args.clone()).await?;
+ let reader = srv.read(&ctx, path, args.clone())?;
Ok(ReadContext::new(
ctx,
srv,
@@ -401,7 +401,7 @@ mod tests {
let op = Operator::via_iter(services::MEMORY_SCHEME, [])?;
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new()).await?);
+ let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new())?);
let v = BufferStream::create(ctx, 4..8).await?;
let _: Box<dyn Unpin + MaybeSend + 'static> = Box::new(v);
@@ -420,7 +420,7 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new()).await?);
+ let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new())?);
let s = BufferStream::create(ctx, 4..8).await?;
let bufs: Vec<_> = s.try_collect().await.unwrap();
diff --git a/core/core/src/types/read/futures_async_reader.rs b/core/core/src/types/read/futures_async_reader.rs
index 279d40929..cfb41f0b5 100644
--- a/core/core/src/types/read/futures_async_reader.rs
+++ b/core/core/src/types/read/futures_async_reader.rs
@@ -190,14 +190,14 @@ mod tests {
use super::*;
- async fn new_read_context(
+ fn new_read_context(
ctx: OperationContext,
srv: Servicer,
path: &str,
options: crate::raw::OpReader,
) -> crate::Result<ReadContext> {
let args = crate::raw::OpRead::new();
- let (_, reader) = srv.read(&ctx, path, args.clone()).await?;
+ let reader = srv.read(&ctx, path, args.clone())?;
Ok(ReadContext::new(
ctx,
srv,
@@ -213,7 +213,7 @@ mod tests {
let op = Operator::via_iter(services::MEMORY_SCHEME, [])?;
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new()).await?);
+ let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new())?);
let v = FuturesAsyncReader::new(ctx, 4..8);
@@ -232,7 +232,7 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new()).await?);
+ let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new())?);
let mut fr = FuturesAsyncReader::new(ctx, 4..8);
let mut bs = vec![];
@@ -259,15 +259,12 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = Arc::new(
- new_read_context(
- ctx,
- srv,
- "test",
- OpReader::new().with_concurrent(3).with_chunk(1),
- )
- .await?,
- );
+ let ctx = Arc::new(new_read_context(
+ ctx,
+ srv,
+ "test",
+ OpReader::new().with_concurrent(3).with_chunk(1),
+ )?);
let mut fr = FuturesAsyncReader::new(ctx, 4..8);
let mut bs = vec![];
@@ -294,15 +291,12 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = Arc::new(
- new_read_context(
- ctx,
- srv,
- "test",
- OpReader::new().with_concurrent(3).with_chunk(1),
- )
- .await?,
- );
+ let ctx = Arc::new(new_read_context(
+ ctx,
+ srv,
+ "test",
+ OpReader::new().with_concurrent(3).with_chunk(1),
+ )?);
let mut fr = FuturesAsyncReader::new(ctx, 4..8);
let chunk = fr.fill_buf().await.unwrap();
@@ -326,7 +320,7 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new()).await?);
+ let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new())?);
// Range 4..8, in total 4 bytes of logical data.
let mut fr = FuturesAsyncReader::new(ctx, 4..8);
diff --git a/core/core/src/types/read/futures_bytes_stream.rs b/core/core/src/types/read/futures_bytes_stream.rs
index cf030caf9..703f0fc4d 100644
--- a/core/core/src/types/read/futures_bytes_stream.rs
+++ b/core/core/src/types/read/futures_bytes_stream.rs
@@ -95,14 +95,14 @@ mod tests {
use super::*;
- async fn new_read_context(
+ fn new_read_context(
ctx: OperationContext,
srv: Servicer,
path: &str,
options: crate::raw::OpReader,
) -> crate::Result<ReadContext> {
let args = crate::raw::OpRead::new();
- let (_, reader) = srv.read(&ctx, path, args.clone()).await?;
+ let reader = srv.read(&ctx, path, args.clone())?;
Ok(ReadContext::new(
ctx,
srv,
@@ -118,7 +118,7 @@ mod tests {
let op = Operator::via_iter(services::MEMORY_SCHEME, [])?;
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new()).await?);
+ let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new())?);
let v = FuturesBytesStream::new(ctx, 4..8).await?;
let _: Box<dyn Unpin + MaybeSend + Sync + 'static> = Box::new(v);
@@ -137,7 +137,7 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new()).await?);
+ let ctx = Arc::new(new_read_context(ctx, srv, "test", OpReader::new())?);
let s = FuturesBytesStream::new(ctx, 4..8).await?;
let bufs: Vec<Bytes> = s.try_collect().await.unwrap();
@@ -158,15 +158,12 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = Arc::new(
- new_read_context(
- ctx,
- srv,
- "test",
- OpReader::new().with_concurrent(3).with_chunk(1),
- )
- .await?,
- );
+ let ctx = Arc::new(new_read_context(
+ ctx,
+ srv,
+ "test",
+ OpReader::new().with_concurrent(3).with_chunk(1),
+ )?);
let s = FuturesBytesStream::new(ctx, 4..8).await?;
let bufs: Vec<Bytes> = s.try_collect().await.unwrap();
diff --git a/core/core/src/types/read/reader.rs b/core/core/src/types/read/reader.rs
index aec26db4a..9fcfa691a 100644
--- a/core/core/src/types/read/reader.rs
+++ b/core/core/src/types/read/reader.rs
@@ -552,14 +552,14 @@ mod tests {
use crate::services;
use crate::*;
- async fn new_read_context(
+ fn new_read_context(
ctx: OperationContext,
srv: Servicer,
path: &str,
options: crate::raw::OpReader,
) -> crate::Result<ReadContext> {
let args = crate::raw::OpRead::new();
- let (_, reader) = srv.read(&ctx, path, args.clone()).await?;
+ let reader = srv.read(&ctx, path, args.clone())?;
Ok(ReadContext::new(
ctx,
srv,
@@ -612,7 +612,9 @@ mod tests {
}
Ok((
- RpRead::default(),
+ RpRead::new(
+ Metadata::new(EntryMode::FILE).with_content_length(self.content.len() as u64),
+ ),
Buffer::from(self.content.slice(start..end)),
))
}
@@ -645,7 +647,7 @@ mod tests {
let ctx = op.context().clone();
let srv = op.service().clone();
- let ctx = new_read_context(ctx, srv, "test", OpReader::new()).await?;
+ let ctx = new_read_context(ctx, srv, "test", OpReader::new())?;
let _: Box<dyn Unpin + MaybeSend + Sync + 'static> = Box::new(Reader::new(ctx));
diff --git a/core/core/src/types/write/buffer_sink.rs b/core/core/src/types/write/buffer_sink.rs
index 55bec881f..8f91c7178 100644
--- a/core/core/src/types/write/buffer_sink.rs
+++ b/core/core/src/types/write/buffer_sink.rs
@@ -181,7 +181,7 @@ mod tests {
OpWrite::new(),
OpWriter::new().with_chunk(1),
));
- let write_gen = WriteGenerator::create(ctx).await.unwrap();
+ let write_gen = WriteGenerator::create(ctx).unwrap();
let v = BufferSink::new(write_gen);
diff --git a/core/core/src/types/write/futures_async_writer.rs b/core/core/src/types/write/futures_async_writer.rs
index c3adf51ce..2e92386bd 100644
--- a/core/core/src/types/write/futures_async_writer.rs
+++ b/core/core/src/types/write/futures_async_writer.rs
@@ -129,7 +129,7 @@ mod tests {
OpWrite::new(),
OpWriter::new().with_chunk(1),
));
- let write_gen = WriteGenerator::create(ctx).await.unwrap();
+ let write_gen = WriteGenerator::create(ctx).unwrap();
let v = FuturesAsyncWriter::new(write_gen);
diff --git a/core/core/src/types/write/futures_bytes_sink.rs b/core/core/src/types/write/futures_bytes_sink.rs
index 4467768db..b1ecd8053 100644
--- a/core/core/src/types/write/futures_bytes_sink.rs
+++ b/core/core/src/types/write/futures_bytes_sink.rs
@@ -96,7 +96,7 @@ mod tests {
OpWrite::new(),
OpWriter::new().with_chunk(1),
));
- let write_gen = WriteGenerator::create(ctx).await.unwrap();
+ let write_gen = WriteGenerator::create(ctx).unwrap();
let v = FuturesBytesSink::new(write_gen);
diff --git a/core/core/src/types/write/writer.rs b/core/core/src/types/write/writer.rs
index fb44c3ede..80dbd3153 100644
--- a/core/core/src/types/write/writer.rs
+++ b/core/core/src/types/write/writer.rs
@@ -107,7 +107,7 @@ impl Writer {
/// Create a new writer from an `oio::Writer`.
pub(crate) async fn new(ctx: WriteContext) -> Result<Self> {
let ctx = Arc::new(ctx);
- let inner = WriteGenerator::create(ctx.clone()).await?;
+ let inner = std::future::ready(WriteGenerator::create(ctx.clone())).await?;
Ok(Self { _ctx: ctx, inner })
}
diff --git a/core/layers/async-backtrace/src/lib.rs b/core/layers/async-backtrace/src/lib.rs
index 7507511cd..bfaadaf30 100644
--- a/core/layers/async-backtrace/src/lib.rs
+++ b/core/layers/async-backtrace/src/lib.rs
@@ -100,42 +100,27 @@ impl Service for AsyncBacktraceAccessor {
self.inner.create_dir(ctx, path, args).await
}
- #[async_backtrace::framed]
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
self.inner
.read(ctx, path, args)
- .await
- .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
+ .map(AsyncBacktraceWrapper::new)
}
- #[async_backtrace::framed]
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
self.inner
.write(ctx, path, args)
- .await
- .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
+ .map(AsyncBacktraceWrapper::new)
}
- #[async_backtrace::framed]
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.inner.copy(ctx, from, to, args, opts).await
+ ) -> Result<Self::Copier> {
+ self.inner.copy(ctx, from, to, args, opts)
}
#[async_backtrace::framed]
@@ -154,25 +139,14 @@ impl Service for AsyncBacktraceAccessor {
self.inner.stat(ctx, path, args).await
}
- #[async_backtrace::framed]
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner
- .delete(ctx)
- .await
- .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx).map(AsyncBacktraceWrapper::new)
}
- #[async_backtrace::framed]
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
self.inner
.list(ctx, path, args)
- .await
- .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
+ .map(AsyncBacktraceWrapper::new)
}
#[async_backtrace::framed]
diff --git a/core/layers/await-tree/src/lib.rs b/core/layers/await-tree/src/lib.rs
index 4ec489bfa..46430c41f 100644
--- a/core/layers/await-tree/src/lib.rs
+++ b/core/layers/await-tree/src/lib.rs
@@ -104,45 +104,25 @@ impl Service for AwaitTreeAccessor {
.await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- self.inner
- .read(ctx, path, args)
- .instrument_await(format!("opendal::{}", Operation::Read))
- .await
- .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ self.inner.read(ctx, path, args).map(AwaitTreeWrapper::new)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- self.inner
- .write(ctx, path, args)
- .instrument_await(format!("opendal::{}", Operation::Write))
- .await
- .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ self.inner.write(ctx, path, args).map(AwaitTreeWrapper::new)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
self.inner
.copy(ctx, from, to, args, opts)
- .instrument_await(format!("opendal::{}", Operation::Copy))
- .await
- .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
+ .map(AwaitTreeWrapper::new)
}
async fn rename(
@@ -165,25 +145,12 @@ impl Service for AwaitTreeAccessor {
.await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner
- .delete(ctx)
- .instrument_await(format!("opendal::{}", Operation::Delete))
- .await
- .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx).map(AwaitTreeWrapper::new)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- self.inner
- .list(ctx, path, args)
- .instrument_await(format!("opendal::{}", Operation::List))
- .await
- .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.inner.list(ctx, path, args).map(AwaitTreeWrapper::new)
}
async fn presign(
diff --git a/core/layers/capability-check/src/lib.rs b/core/layers/capability-check/src/lib.rs
index 019150e52..4e0d7d5b2 100644
--- a/core/layers/capability-check/src/lib.rs
+++ b/core/layers/capability-check/src/lib.rs
@@ -123,21 +123,11 @@ impl Service for CapabilityCheckService {
self.inner.stat(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- self.inner.read(ctx, path, args).await
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ self.inner.read(ctx, path, args)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let capability = self.capability();
let info = self.info();
if !capability.write_with_content_type && args.content_type().is_some() {
@@ -162,17 +152,17 @@ impl Service for CapabilityCheckService {
));
}
- self.inner.write(ctx, path, args).await
+ self.inner.write(ctx, path, args)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
let capability = self.capability();
let info = self.info();
if args.if_not_exists() && !capability.copy_with_if_not_exists {
@@ -193,26 +183,21 @@ impl Service for CapabilityCheckService {
));
}
- self.inner.copy(ctx, from, to, args, opts).await
+ self.inner.copy(ctx, from, to, args, opts)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner.delete(ctx).await
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
let capability = self.capability();
if !capability.list_with_versions && args.versions() {
let info = self.info();
return Err(new_unsupported_error(&info, Operation::List, "version"));
}
- self.inner.list(ctx, path, args).await
+ self.inner.list(ctx, path, args)
}
async fn rename(
@@ -278,51 +263,36 @@ mod tests {
))
}
- async fn read(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, _ctx: &OperationContext, _: &str, _: OpRead) -> Result<Self::Reader> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- Ok((RpWrite::new(), ()))
+ fn write(&self, _ctx: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
+ Ok(())
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- Ok((RpList {}, ()))
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
+ Ok(())
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/layers/chaos/src/lib.rs b/core/layers/chaos/src/lib.rs
index ca2afb236..e2560b3dd 100644
--- a/core/layers/chaos/src/lib.rs
+++ b/core/layers/chaos/src/lib.rs
@@ -135,51 +135,33 @@ impl Service for ChaosService {
self.inner.stat(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- self.inner.read(ctx, path, args).await.map(|(rp, r)| {
- (
- rp,
- ChaosReader::new(r, Arc::clone(&self.rng), self.error_ratio),
- )
- })
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ self.inner
+ .read(ctx, path, args)
+ .map(|r| ChaosReader::new(r, Arc::clone(&self.rng), self.error_ratio))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- self.inner.write(ctx, path, args).await
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ self.inner.write(ctx, path, args)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.inner.copy(ctx, from, to, args, opts).await
+ ) -> Result<Self::Copier> {
+ self.inner.copy(ctx, from, to, args, opts)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- self.inner.list(ctx, path, args).await
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.inner.list(ctx, path, args)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner.delete(ctx).await
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx)
}
async fn rename(
diff --git a/core/layers/concurrent-limit/src/lib.rs b/core/layers/concurrent-limit/src/lib.rs
index f96665500..132896b6c 100644
--- a/core/layers/concurrent-limit/src/lib.rs
+++ b/core/layers/concurrent-limit/src/lib.rs
@@ -256,10 +256,10 @@ where
S::Permit: Send + Sync + 'static + Unpin,
{
type Reader = ConcurrentLimitReader<oio::Reader, S>;
- type Writer = ConcurrentLimitWrapper<oio::Writer, S::Permit>;
- type Lister = ConcurrentLimitWrapper<oio::Lister, S::Permit>;
- type Deleter = ConcurrentLimitWrapper<oio::Deleter, S::Permit>;
- type Copier = ConcurrentLimitWrapper<oio::Copier, S::Permit>;
+ type Writer = ConcurrentLimitWrapper<oio::Writer, S>;
+ type Lister = ConcurrentLimitWrapper<oio::Lister, S>;
+ type Deleter = ConcurrentLimitWrapper<oio::Deleter, S>;
+ type Copier = ConcurrentLimitWrapper<oio::Copier, S>;
fn info(&self) -> ServiceInfo {
self.inner.info()
@@ -279,44 +279,29 @@ where
self.inner.create_dir(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
self.inner
.read(ctx, path, args)
- .await
- .map(|(rp, r)| (rp, ConcurrentLimitReader::new(r, self.semaphore.clone())))
+ .map(|r| ConcurrentLimitReader::new(r, self.semaphore.clone()))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let permit = self.semaphore.acquire().await;
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
self.inner
.write(ctx, path, args)
- .await
- .map(|(rp, w)| (rp, ConcurrentLimitWrapper::new(w, permit)))
+ .map(|w| ConcurrentLimitWrapper::new(w, self.semaphore.clone()))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let permit = self.semaphore.acquire().await;
+ ) -> Result<Self::Copier> {
self.inner
.copy(ctx, from, to, args, opts)
- .await
- .map(|(rp, c)| (rp, ConcurrentLimitWrapper::new(c, permit)))
+ .map(|c| ConcurrentLimitWrapper::new(c, self.semaphore.clone()))
}
async fn rename(
@@ -335,25 +320,16 @@ where
self.inner.stat(ctx, path, args).await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let permit = self.semaphore.acquire().await;
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
self.inner
.delete(ctx)
- .await
- .map(|(rp, w)| (rp, ConcurrentLimitWrapper::new(w, permit)))
+ .map(|w| ConcurrentLimitWrapper::new(w, self.semaphore.clone()))
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let permit = self.semaphore.acquire().await;
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
self.inner
.list(ctx, path, args)
- .await
- .map(|(rp, s)| (rp, ConcurrentLimitWrapper::new(s, permit)))
+ .map(|s| ConcurrentLimitWrapper::new(s, self.semaphore.clone()))
}
async fn presign(
@@ -388,7 +364,11 @@ where
let (rp, stream) = self.inner.open(range).await?;
Ok((
rp,
- Box::new(ConcurrentLimitWrapper::new(stream, permit)) as Box<dyn oio::ReadStreamDyn>,
+ Box::new(ConcurrentLimitWrapper::new_with_permit(
+ stream,
+ self.semaphore.clone(),
+ permit,
+ )) as Box<dyn oio::ReadStreamDyn>,
))
}
@@ -399,31 +379,52 @@ where
}
#[doc(hidden)]
-pub struct ConcurrentLimitWrapper<R, P> {
+pub struct ConcurrentLimitWrapper<R, S: ConcurrentLimitSemaphore> {
inner: R,
-
+ semaphore: S,
// Hold this permit until the wrapped operation body is dropped.
- _permit: P,
+ permit: Option<S::Permit>,
}
-impl<R, P> ConcurrentLimitWrapper<R, P> {
- fn new(inner: R, permit: P) -> Self {
+impl<R, S: ConcurrentLimitSemaphore> ConcurrentLimitWrapper<R, S> {
+ fn new(inner: R, semaphore: S) -> Self {
+ Self {
+ inner,
+ semaphore,
+ permit: None,
+ }
+ }
+
+ fn new_with_permit(inner: R, semaphore: S, permit: S::Permit) -> Self {
Self {
inner,
- _permit: permit,
+ semaphore,
+ permit: Some(permit),
+ }
+ }
+
+ async fn acquire(&mut self) {
+ if self.permit.is_none() {
+ self.permit = Some(self.semaphore.acquire().await);
}
}
}
-impl<R: oio::ReadStream, P: Send + Sync + 'static + Unpin> oio::ReadStream
- for ConcurrentLimitWrapper<R, P>
+impl<R: oio::ReadStream, S: ConcurrentLimitSemaphore> oio::ReadStream
+ for ConcurrentLimitWrapper<R, S>
+where
+ S::Permit: Send + Sync + 'static + Unpin,
{
async fn read(&mut self) -> Result<Buffer> {
+ self.acquire().await;
self.inner.read().await
}
}
-impl<R: oio::Read, P: Send + Sync + 'static + Unpin> oio::Read for ConcurrentLimitWrapper<R, P> {
+impl<R: oio::Read, S: ConcurrentLimitSemaphore> oio::Read for ConcurrentLimitWrapper<R, S>
+where
+ S::Permit: Send + Sync + 'static + Unpin,
+{
async fn open(&self, range: BytesRange) -> Result<(RpRead, Box<dyn oio::ReadStreamDyn>)> {
self.inner.open(range).await
}
@@ -433,48 +434,67 @@ impl<R: oio::Read, P: Send + Sync + 'static + Unpin> oio::Read for ConcurrentLim
}
}
-impl<R: oio::Write, P: Send + Sync + 'static + Unpin> oio::Write for ConcurrentLimitWrapper<R, P> {
+impl<R: oio::Write, S: ConcurrentLimitSemaphore> oio::Write for ConcurrentLimitWrapper<R, S>
+where
+ S::Permit: Send + Sync + 'static + Unpin,
+{
async fn write(&mut self, bs: Buffer) -> Result<()> {
+ self.acquire().await;
self.inner.write(bs).await
}
async fn close(&mut self) -> Result<Metadata> {
+ self.acquire().await;
self.inner.close().await
}
async fn abort(&mut self) -> Result<()> {
+ self.acquire().await;
self.inner.abort().await
}
}
-impl<R: oio::List, P: Send + Sync + 'static + Unpin> oio::List for ConcurrentLimitWrapper<R, P> {
+impl<R: oio::List, S: ConcurrentLimitSemaphore> oio::List for ConcurrentLimitWrapper<R, S>
+where
+ S::Permit: Send + Sync + 'static + Unpin,
+{
async fn next(&mut self) -> Result<Option<oio::Entry>> {
+ self.acquire().await;
self.inner.next().await
}
}
-impl<R: oio::Delete, P: Send + Sync + 'static + Unpin> oio::Delete
- for ConcurrentLimitWrapper<R, P>
+impl<R: oio::Delete, S: ConcurrentLimitSemaphore> oio::Delete for ConcurrentLimitWrapper<R, S>
+where
+ S::Permit: Send + Sync + 'static + Unpin,
{
async fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
+ self.acquire().await;
self.inner.delete(path, args).await
}
async fn close(&mut self) -> Result<()> {
+ self.acquire().await;
self.inner.close().await
}
}
-impl<C: oio::Copy, P: Send + Sync + 'static + Unpin> oio::Copy for ConcurrentLimitWrapper<C, P> {
+impl<C: oio::Copy, S: ConcurrentLimitSemaphore> oio::Copy for ConcurrentLimitWrapper<C, S>
+where
+ S::Permit: Send + Sync + 'static + Unpin,
+{
async fn next(&mut self) -> Result<Option<usize>> {
+ self.acquire().await;
self.inner.next().await
}
async fn close(&mut self) -> Result<Metadata> {
+ self.acquire().await;
self.inner.close().await
}
async fn abort(&mut self) -> Result<()> {
+ self.acquire().await;
self.inner.abort().await
}
}
@@ -484,6 +504,7 @@ mod tests {
use super::*;
use opendal_core::Operator;
use opendal_core::services;
+ use std::future::pending;
use std::sync::Arc;
use std::time::Duration;
use tokio::time::timeout;
@@ -559,58 +580,43 @@ mod tests {
))
}
- async fn read(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, _ctx: &OperationContext, _: &str, _: OpRead) -> Result<Self::Reader> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- Ok((RpCopy::default(), ()))
+ ) -> Result<Self::Copier> {
+ Ok(())
}
async fn rename(
@@ -674,6 +680,23 @@ mod tests {
#[tokio::test]
async fn operation_semaphore_held_until_copier_dropped() {
+ #[derive(Debug)]
+ struct PendingCopier;
+
+ impl oio::Copy for PendingCopier {
+ async fn next(&mut self) -> Result<Option<usize>> {
+ pending().await
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ pending().await
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ Ok(())
+ }
+ }
+
#[derive(Clone, Debug)]
struct CopierBackend {
info: ServiceInfo,
@@ -685,7 +708,7 @@ mod tests {
type Writer = ();
type Lister = ();
type Deleter = ();
- type Copier = ();
+ type Copier = PendingCopier;
fn info(&self) -> ServiceInfo {
self.info.clone()
@@ -707,58 +730,43 @@ mod tests {
))
}
- async fn read(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, _ctx: &OperationContext, _: &str, _: OpRead) -> Result<Self::Reader> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- Ok((RpCopy::default(), ()))
+ ) -> Result<Self::Copier> {
+ Ok(PendingCopier)
}
async fn stat(&self, _: &OperationContext, _: &str, _: OpStat) -> Result<RpStat> {
@@ -804,12 +812,15 @@ mod tests {
}))
.layer(layer);
- let copier = timeout(Duration::from_millis(50), op.copier("from", "to"))
+ let mut copier = timeout(Duration::from_millis(50), op.copier("from", "to"))
.await
.expect("copier setup should not block")
.expect("copier should be created");
- // The permit is held by the live copier, so concurrent operations
+ let copy = timeout(Duration::from_millis(50), copier.next()).await;
+ assert!(copy.is_err(), "copy body should remain pending");
+
+ // The permit is held by the active copy body, so concurrent operations
// must time out until the copier is dropped.
let blocked = timeout(Duration::from_millis(50), op.stat("any")).await;
assert!(
@@ -910,16 +921,18 @@ mod tests {
))
}
- async fn read(
+ fn read(
&self,
ctx: &OperationContext,
path: &str,
args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(HttpReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ ) -> Result<Self::Reader> {
+ Ok(oio::StreamReader::new(HttpReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}
async fn stat(&self, _: &OperationContext, _: &str, _: OpStat) -> Result<RpStat> {
@@ -928,45 +941,35 @@ mod tests {
))
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/layers/dtrace/src/lib.rs b/core/layers/dtrace/src/lib.rs
index 95e71e921..c4678b80c 100644
--- a/core/layers/dtrace/src/lib.rs
+++ b/core/layers/dtrace/src/lib.rs
@@ -181,52 +181,40 @@ impl Service for DTraceService {
result
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
let c_path = CString::new(path).unwrap();
probe_lazy!(opendal, read_start, c_path.as_ptr());
let result = self
.inner
.read(ctx, path, args)
- .await
- .map(|(rp, r)| (rp, DtraceLayerWrapper::new(r, path)));
+ .map(|r| DtraceLayerWrapper::new(r, path));
probe_lazy!(opendal, read_end, c_path.as_ptr());
result
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let c_path = CString::new(path).unwrap();
probe_lazy!(opendal, write_start, c_path.as_ptr());
let result = self
.inner
.write(ctx, path, args)
- .await
- .map(|(rp, r)| (rp, DtraceLayerWrapper::new(r, path)));
+ .map(|r| DtraceLayerWrapper::new(r, path));
probe_lazy!(opendal, write_end, c_path.as_ptr());
result
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
let c_from = CString::new(from).unwrap();
probe_lazy!(opendal, copy_start, c_from.as_ptr());
- let result = self.inner.copy(ctx, from, to, args, opts).await;
+ let result = self.inner.copy(ctx, from, to, args, opts);
probe_lazy!(opendal, copy_end, c_from.as_ptr());
result
}
@@ -249,19 +237,14 @@ impl Service for DTraceService {
result
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner.delete(ctx).await
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
let c_path = CString::new(path).unwrap();
probe_lazy!(opendal, list_start, c_path.as_ptr());
- let result = self.inner.list(ctx, path, args).await;
+ let result = self.inner.list(ctx, path, args);
probe_lazy!(opendal, list_end, c_path.as_ptr());
result
}
diff --git a/core/layers/fastrace/src/lib.rs b/core/layers/fastrace/src/lib.rs
index 63458dcf8..bc6c8fa98 100644
--- a/core/layers/fastrace/src/lib.rs
+++ b/core/layers/fastrace/src/lib.rs
@@ -160,63 +160,41 @@ impl Service for FastraceAccessor {
self.inner.create_dir(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
let _guard = Span::enter_with_local_parent(Operation::Read.into_static());
- self.inner.read(ctx, path, args).await.map(|(rp, r)| {
- (
- rp,
- FastraceWrapper::new(
- Span::enter_with_local_parent(Operation::Read.into_static()),
- r,
- ),
+ self.inner.read(ctx, path, args).map(|r| {
+ FastraceWrapper::new(
+ Span::enter_with_local_parent(Operation::Read.into_static()),
+ r,
)
})
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let _guard = Span::enter_with_local_parent(Operation::Write.into_static());
- self.inner.write(ctx, path, args).await.map(|(rp, r)| {
- (
- rp,
- FastraceWrapper::new(
- Span::enter_with_local_parent(Operation::Write.into_static()),
- r,
- ),
+ self.inner.write(ctx, path, args).map(|r| {
+ FastraceWrapper::new(
+ Span::enter_with_local_parent(Operation::Write.into_static()),
+ r,
)
})
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
let _guard = Span::enter_with_local_parent(Operation::Copy.into_static());
- self.inner
- .copy(ctx, from, to, args, opts)
- .await
- .map(|(rp, c)| {
- (
- rp,
- FastraceWrapper::new(
- Span::enter_with_local_parent(Operation::Copy.into_static()),
- c,
- ),
- )
- })
+ self.inner.copy(ctx, from, to, args, opts).map(|c| {
+ FastraceWrapper::new(
+ Span::enter_with_local_parent(Operation::Copy.into_static()),
+ c,
+ )
+ })
}
async fn rename(
@@ -235,33 +213,22 @@ impl Service for FastraceAccessor {
self.inner.stat(ctx, path, args).await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
let _guard = Span::enter_with_local_parent(Operation::Delete.into_static());
- self.inner.delete(ctx).await.map(|(rp, r)| {
- (
- rp,
- FastraceWrapper::new(
- Span::enter_with_local_parent(Operation::Delete.into_static()),
- r,
- ),
+ self.inner.delete(ctx).map(|r| {
+ FastraceWrapper::new(
+ Span::enter_with_local_parent(Operation::Delete.into_static()),
+ r,
)
})
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
let _guard = Span::enter_with_local_parent(Operation::List.into_static());
- self.inner.list(ctx, path, args).await.map(|(rp, s)| {
- (
- rp,
- FastraceWrapper::new(
- Span::enter_with_local_parent(Operation::List.into_static()),
- s,
- ),
+ self.inner.list(ctx, path, args).map(|s| {
+ FastraceWrapper::new(
+ Span::enter_with_local_parent(Operation::List.into_static()),
+ s,
)
})
}
diff --git a/core/layers/foyer/src/full.rs b/core/layers/foyer/src/full.rs
index f683adab8..bdcb39a8a 100644
--- a/core/layers/foyer/src/full.rs
+++ b/core/layers/foyer/src/full.rs
@@ -85,25 +85,19 @@ impl FullReader {
&self,
range: BytesRange,
) -> Result<(RpRead, Box<dyn oio::ReadStreamDyn>)> {
- let (rp, reader) = self
+ let reader = self
.inner
.srv
- .read(&self.inner.ctx, &self.path, self.args.clone())
- .await?;
- let (rp_open, stream) = reader.open(range).await?;
- let rp = rp_open.into_metadata().map(RpRead::new).unwrap_or(rp);
- Ok((rp, stream))
+ .read(&self.inner.ctx, &self.path, self.args.clone())?;
+ reader.open(range).await
}
async fn fallback_read(&self, range: BytesRange) -> Result<(RpRead, Buffer)> {
- let (rp, reader) = self
+ let reader = self
.inner
.srv
- .read(&self.inner.ctx, &self.path, self.args.clone())
- .await?;
- let (rp_read, buffer) = reader.read(range).await?;
- let rp = rp_read.into_metadata().map(RpRead::new).unwrap_or(rp);
- Ok((rp, buffer))
+ .read(&self.inner.ctx, &self.path, self.args.clone())?;
+ reader.read(range).await
}
async fn read_full_object(&self) -> Result<Option<Buffer>> {
@@ -144,10 +138,9 @@ impl FullReader {
}
// fetch the ENTIRE object from remote.
- let (_, reader) = inner
+ let reader = inner
.srv
.read(&inner.ctx, &path_clone, OpRead::default())
- .await
.map_err(FetchError::from_error)?;
let (_, mut stream) = reader
.open(BytesRange::new(0, None))
diff --git a/core/layers/foyer/src/lib.rs b/core/layers/foyer/src/lib.rs
index 0eb37beab..f51649764 100644
--- a/core/layers/foyer/src/lib.rs
+++ b/core/layers/foyer/src/lib.rs
@@ -212,60 +212,42 @@ impl Service for FoyerService {
self.inner.capability()
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- full::FullReader::new(
- self.operation_inner(ctx),
- self.size_limit.clone(),
- path.to_string(),
- args,
- ),
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ Ok(full::FullReader::new(
+ self.operation_inner(ctx),
+ self.size_limit.clone(),
+ path.to_string(),
+ args,
))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let inner = self.operation_inner(ctx);
let size_limit = self.size_limit.clone();
let path = path.to_string();
- let (rp, w) = inner.srv.write(&inner.ctx, &path, args).await?;
- Ok((rp, Writer::new(w, path, inner, size_limit)))
+ let w = inner.srv.write(&inner.ctx, &path, args)?;
+ Ok(Writer::new(w, path, inner, size_limit))
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
let inner = self.operation_inner(ctx);
- let (rp, d) = inner.srv.delete(&inner.ctx).await?;
- Ok((rp, Deleter::new(d, inner)))
+ let d = inner.srv.delete(&inner.ctx)?;
+ Ok(Deleter::new(d, inner))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.inner.copy(ctx, from, to, args, opts).await
+ ) -> Result<Self::Copier> {
+ self.inner.copy(ctx, from, to, args, opts)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- self.inner.list(ctx, path, args).await
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.inner.list(ctx, path, args)
}
async fn create_dir(
@@ -449,59 +431,41 @@ mod tests {
Ok(RpStat::new(self.state.metadata()))
}
- async fn read(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- Ok((
- self.state.rp_read(),
- MockReadReader {
- state: self.state.clone(),
- },
- ))
+ fn read(&self, _ctx: &OperationContext, _: &str, _: OpRead) -> Result<Self::Reader> {
+ Ok(MockReadReader {
+ state: self.state.clone(),
+ })
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _ctx: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
@@ -543,7 +507,7 @@ mod tests {
.apply_service(source);
let ctx = service_context(&service);
- let (_, reader) = service.read(&ctx, "test", OpRead::default()).await.unwrap();
+ let reader = service.read(&ctx, "test", OpRead::default()).unwrap();
let (_, mut stream) = reader.open(BytesRange::new(0, None)).await.unwrap();
let buffer = stream.read_all().await.unwrap();
@@ -563,7 +527,7 @@ mod tests {
.apply_service(source);
let ctx = service_context(&service);
- let (_, reader) = service.read(&ctx, "test", OpRead::default()).await.unwrap();
+ let reader = service.read(&ctx, "test", OpRead::default()).unwrap();
let (_, mut stream) = reader.open(BytesRange::from(0_u64..2)).await.unwrap();
let buffer = stream.read_all().await.unwrap();
@@ -572,7 +536,7 @@ mod tests {
assert_eq!(state.open_calls.load(Ordering::Relaxed), 1);
assert_eq!(state.read_calls.load(Ordering::Relaxed), 0);
- let (_, reader) = service.read(&ctx, "test", OpRead::default()).await.unwrap();
+ let reader = service.read(&ctx, "test", OpRead::default()).unwrap();
let (_, mut stream) = reader.open(BytesRange::from(4_u64..7)).await.unwrap();
let buffer = stream.read_all().await.unwrap();
diff --git a/core/layers/hotpath/src/lib.rs b/core/layers/hotpath/src/lib.rs
index a9ec268ef..370398e16 100644
--- a/core/layers/hotpath/src/lib.rs
+++ b/core/layers/hotpath/src/lib.rs
@@ -32,12 +32,8 @@ use opendal_core::*;
const LABEL_CREATE_DIR: &str = "opendal.create_dir";
const LABEL_READ: &str = "opendal.read";
-const LABEL_WRITE: &str = "opendal.write";
-const LABEL_COPY: &str = "opendal.copy";
const LABEL_RENAME: &str = "opendal.rename";
const LABEL_STAT: &str = "opendal.stat";
-const LABEL_DELETE: &str = "opendal.delete";
-const LABEL_LIST: &str = "opendal.list";
const LABEL_PRESIGN: &str = "opendal.presign";
const LABEL_READER_READ: &str = "opendal.reader.read";
@@ -135,39 +131,25 @@ impl Service for HotpathAccessor {
hotpath::measure_async(LABEL_CREATE_DIR, self.inner.create_dir(ctx, path, args)).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, reader) =
- hotpath::measure_async(LABEL_READ, self.inner.read(ctx, path, args)).await?;
- Ok((rp, HotpathWrapper::new(reader)))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ self.inner.read(ctx, path, args).map(HotpathWrapper::new)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, writer) =
- hotpath::measure_async(LABEL_WRITE, self.inner.write(ctx, path, args)).await?;
- Ok((rp, HotpathWrapper::new(writer)))
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ self.inner.write(ctx, path, args).map(HotpathWrapper::new)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let (rp, copier) =
- hotpath::measure_async(LABEL_COPY, self.inner.copy(ctx, from, to, args, opts)).await?;
- Ok((rp, HotpathWrapper::new(copier)))
+ ) -> Result<Self::Copier> {
+ self.inner
+ .copy(ctx, from, to, args, opts)
+ .map(HotpathWrapper::new)
}
async fn rename(
@@ -184,20 +166,12 @@ impl Service for HotpathAccessor {
hotpath::measure_async(LABEL_STAT, self.inner.stat(ctx, path, args)).await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, deleter) = hotpath::measure_async(LABEL_DELETE, self.inner.delete(ctx)).await?;
- Ok((rp, HotpathWrapper::new(deleter)))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx).map(HotpathWrapper::new)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, lister) =
- hotpath::measure_async(LABEL_LIST, self.inner.list(ctx, path, args)).await?;
- Ok((rp, HotpathWrapper::new(lister)))
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.inner.list(ctx, path, args).map(HotpathWrapper::new)
}
async fn presign(
diff --git a/core/layers/immutable-index/src/lib.rs b/core/layers/immutable-index/src/lib.rs
index 643406773..b4ae177d0 100644
--- a/core/layers/immutable-index/src/lib.rs
+++ b/core/layers/immutable-index/src/lib.rs
@@ -181,41 +181,26 @@ impl Service for ImmutableIndexService {
self.inner.stat(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- self.inner.read(ctx, path, args).await
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ self.inner.read(ctx, path, args)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- self.inner.write(ctx, path, args).await
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ self.inner.write(ctx, path, args)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.inner.copy(ctx, from, to, args, opts).await
+ ) -> Result<Self::Copier> {
+ self.inner.copy(ctx, from, to, args, opts)
}
- async fn list(
- &self,
- _: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
let mut path = path;
if path == "/" {
path = ""
@@ -227,11 +212,11 @@ impl Service for ImmutableIndexService {
self.children_hierarchy(path)
};
- Ok((RpList::default(), ImmutableDir::new(idx)))
+ Ok(ImmutableDir::new(idx))
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner.delete(ctx).await
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx)
}
async fn rename(
@@ -338,57 +323,42 @@ mod tests {
))
}
- async fn read(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, _ctx: &OperationContext, _: &str, _: OpRead) -> Result<Self::Reader> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _ctx: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/layers/logging/src/lib.rs b/core/layers/logging/src/lib.rs
index bc894e009..738fab433 100644
--- a/core/layers/logging/src/lib.rs
+++ b/core/layers/logging/src/lib.rs
@@ -292,17 +292,11 @@ impl<I: LoggingInterceptor> Service for LoggingService<I> {
result
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
self.log_start(Operation::Read, &[("path", path)]);
self.inner
.read(ctx, path, args)
- .await
- .map(|(rp, r)| {
+ .map(|r| {
self.logger.log(
&self.info,
Operation::Read,
@@ -310,10 +304,7 @@ impl<I: LoggingInterceptor> Service for LoggingService<I> {
"created reader",
None,
);
- (
- rp,
- LoggingReader::new(self.info.clone(), self.logger.clone(), path, r),
- )
+ LoggingReader::new(self.info.clone(), self.logger.clone(), path, r)
})
.inspect_err(|err| {
self.logger.log(
@@ -326,17 +317,11 @@ impl<I: LoggingInterceptor> Service for LoggingService<I> {
})
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
self.log_start(Operation::Write, &[("path", path)]);
self.inner
.write(ctx, path, args)
- .await
- .map(|(rp, w)| {
+ .map(|w| {
self.logger.log(
&self.info,
Operation::Write,
@@ -344,10 +329,7 @@ impl<I: LoggingInterceptor> Service for LoggingService<I> {
"created writer",
None,
);
- (
- rp,
- LoggingWriter::new(self.info.clone(), self.logger.clone(), path, w),
- )
+ LoggingWriter::new(self.info.clone(), self.logger.clone(), path, w)
})
.inspect_err(|err| {
self.logger.log(
@@ -367,18 +349,14 @@ impl<I: LoggingInterceptor> Service for LoggingService<I> {
result
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
self.log_start(Operation::Delete, &[]);
self.inner
.delete(ctx)
- .await
- .map(|(rp, d)| {
+ .map(|d| {
self.logger
.log(&self.info, Operation::Delete, &[], "finished", None);
- (
- rp,
- LoggingDeleter::new(self.info.clone(), self.logger.clone(), d),
- )
+ LoggingDeleter::new(self.info.clone(), self.logger.clone(), d)
})
.inspect_err(|err| {
self.logger
@@ -386,19 +364,18 @@ impl<I: LoggingInterceptor> Service for LoggingService<I> {
})
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
self.log_start(Operation::Copy, &[("from", from), ("to", to)]);
self.inner
.copy(ctx, from, to, args, opts)
- .await
- .map(|(rp, c)| {
+ .map(|c| {
self.logger.log(
&self.info,
Operation::Copy,
@@ -406,10 +383,7 @@ impl<I: LoggingInterceptor> Service for LoggingService<I> {
"created copier",
None,
);
- (
- rp,
- LoggingCopier::new(self.info.clone(), self.logger.clone(), from, to, c),
- )
+ LoggingCopier::new(self.info.clone(), self.logger.clone(), from, to, c)
})
.inspect_err(|err| {
self.logger.log(
@@ -439,17 +413,11 @@ impl<I: LoggingInterceptor> Service for LoggingService<I> {
result
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
self.log_start(Operation::List, &[("path", path)]);
self.inner
.list(ctx, path, args)
- .await
- .map(|(rp, v)| {
+ .map(|v| {
self.logger.log(
&self.info,
Operation::List,
@@ -457,10 +425,7 @@ impl<I: LoggingInterceptor> Service for LoggingService<I> {
"created lister",
None,
);
- (
- rp,
- LoggingLister::new(self.info.clone(), self.logger.clone(), path, v),
- )
+ LoggingLister::new(self.info.clone(), self.logger.clone(), path, v)
})
.inspect_err(|err| {
self.logger.log(
diff --git a/core/layers/mime-guess/src/lib.rs b/core/layers/mime-guess/src/lib.rs
index 336ade115..a5dd1fe35 100644
--- a/core/layers/mime-guess/src/lib.rs
+++ b/core/layers/mime-guess/src/lib.rs
@@ -144,33 +144,23 @@ impl Service for MimeGuessAccessor {
self.0.create_dir(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- self.0.read(ctx, path, args).await
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ self.0.read(ctx, path, args)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- self.0.write(ctx, path, opwrite_with_mime(path, args)).await
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ self.0.write(ctx, path, opwrite_with_mime(path, args))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.0.copy(ctx, from, to, args, opts).await
+ ) -> Result<Self::Copier> {
+ self.0.copy(ctx, from, to, args, opts)
}
async fn stat(&self, ctx: &OperationContext, path: &str, args: OpStat) -> Result<RpStat> {
@@ -190,17 +180,12 @@ impl Service for MimeGuessAccessor {
self.0.rename(ctx, from, to, args).await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.0.delete(ctx).await
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.0.delete(ctx)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- self.0.list(ctx, path, args).await
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.0.list(ctx, path, args)
}
async fn presign(
diff --git a/core/layers/observe-metrics-common/src/lib.rs b/core/layers/observe-metrics-common/src/lib.rs
index 9548395b8..f0a393b68 100644
--- a/core/layers/observe-metrics-common/src/lib.rs
+++ b/core/layers/observe-metrics-common/src/lib.rs
@@ -772,12 +772,7 @@ impl<I: MetricsIntercept> Service for MetricsAccessor<I> {
res
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
let labels = MetricLabels::new(self.info.clone(), Operation::Read.into_static());
let start = Instant::now();
@@ -787,13 +782,10 @@ impl<I: MetricsIntercept> Service for MetricsAccessor<I> {
let mut guard =
ExecutingGuard::new_operation(self.interceptor.clone(), labels.clone(), start);
- match self.inner.read(ctx, path, args).await {
- Ok((rp, reader)) => {
+ match self.inner.read(ctx, path, args) {
+ Ok(reader) => {
guard.complete();
- Ok((
- rp,
- MetricsReader::new(reader, self.interceptor.clone(), labels),
- ))
+ Ok(MetricsReader::new(reader, self.interceptor.clone(), labels))
}
Err(err) => {
self.interceptor.observe(
@@ -806,12 +798,7 @@ impl<I: MetricsIntercept> Service for MetricsAccessor<I> {
}
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let labels = MetricLabels::new(self.info.clone(), Operation::Write.into_static());
let start = Instant::now();
@@ -821,12 +808,14 @@ impl<I: MetricsIntercept> Service for MetricsAccessor<I> {
let mut guard =
ExecutingGuard::new_operation(self.interceptor.clone(), labels.clone(), start);
- match self.inner.write(ctx, path, args).await {
- Ok((rp, writer)) => {
+ match self.inner.write(ctx, path, args) {
+ Ok(writer) => {
guard.defuse();
- Ok((
- rp,
- MetricsWrapper::new(writer, self.interceptor.clone(), labels, start),
+ Ok(MetricsWrapper::new(
+ writer,
+ self.interceptor.clone(),
+ labels,
+ start,
))
}
Err(err) => {
@@ -840,14 +829,14 @@ impl<I: MetricsIntercept> Service for MetricsAccessor<I> {
}
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
let labels = MetricLabels::new(self.info.clone(), Operation::Copy.into_static());
let start = Instant::now();
@@ -857,12 +846,14 @@ impl<I: MetricsIntercept> Service for MetricsAccessor<I> {
let mut guard =
ExecutingGuard::new_operation(self.interceptor.clone(), labels.clone(), start);
- match self.inner.copy(ctx, from, to, args, opts.clone()).await {
- Ok((rp, copier)) => {
+ match self.inner.copy(ctx, from, to, args, opts.clone()) {
+ Ok(copier) => {
guard.defuse();
- Ok((
- rp,
- MetricsWrapper::new(copier, self.interceptor.clone(), labels, start),
+ Ok(MetricsWrapper::new(
+ copier,
+ self.interceptor.clone(),
+ labels,
+ start,
))
}
Err(err) => {
@@ -944,7 +935,7 @@ impl<I: MetricsIntercept> Service for MetricsAccessor<I> {
res
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
let labels = MetricLabels::new(self.info.clone(), Operation::Delete.into_static());
let start = Instant::now();
@@ -954,12 +945,14 @@ impl<I: MetricsIntercept> Service for MetricsAccessor<I> {
let mut guard =
ExecutingGuard::new_operation(self.interceptor.clone(), labels.clone(), start);
- match self.inner.delete(ctx).await {
- Ok((rp, deleter)) => {
+ match self.inner.delete(ctx) {
+ Ok(deleter) => {
guard.defuse();
- Ok((
- rp,
- MetricsWrapper::new(deleter, self.interceptor.clone(), labels, start),
+ Ok(MetricsWrapper::new(
+ deleter,
+ self.interceptor.clone(),
+ labels,
+ start,
))
}
Err(err) => {
@@ -973,12 +966,7 @@ impl<I: MetricsIntercept> Service for MetricsAccessor<I> {
}
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
let labels = MetricLabels::new(self.info.clone(), Operation::List.into_static());
let start = Instant::now();
@@ -988,12 +976,14 @@ impl<I: MetricsIntercept> Service for MetricsAccessor<I> {
let mut guard =
ExecutingGuard::new_operation(self.interceptor.clone(), labels.clone(), start);
- match self.inner.list(ctx, path, args).await {
- Ok((rp, lister)) => {
+ match self.inner.list(ctx, path, args) {
+ Ok(lister) => {
guard.defuse();
- Ok((
- rp,
- MetricsWrapper::new(lister, self.interceptor.clone(), labels, start),
+ Ok(MetricsWrapper::new(
+ lister,
+ self.interceptor.clone(),
+ labels,
+ start,
))
}
Err(err) => {
diff --git a/core/layers/oteltrace/src/lib.rs b/core/layers/oteltrace/src/lib.rs
index 24e5f9328..7c3ecad5a 100644
--- a/core/layers/oteltrace/src/lib.rs
+++ b/core/layers/oteltrace/src/lib.rs
@@ -112,12 +112,7 @@ impl Service for OtelTraceService {
.await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
let tracer = global::tracer("opendal");
let mut span = tracer.start("read");
span.set_attribute(KeyValue::new("path", path.to_string()));
@@ -125,17 +120,10 @@ impl Service for OtelTraceService {
let cx = TraceContext::current_with_span(span);
self.inner
.read(ctx, path, args)
- .with_context(cx.clone())
- .await
- .map(|(rp, r)| (rp, OtelTraceWrapper::new(cx, r)))
+ .map(|r| OtelTraceWrapper::new(cx, r))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let tracer = global::tracer("opendal");
let mut span = tracer.start("write");
span.set_attribute(KeyValue::new("path", path.to_string()));
@@ -143,29 +131,25 @@ impl Service for OtelTraceService {
let cx = TraceContext::current_with_span(span);
self.inner
.write(ctx, path, args)
- .with_context(cx.clone())
- .await
- .map(|(rp, r)| (rp, OtelTraceWrapper::new(cx, r)))
+ .map(|r| OtelTraceWrapper::new(cx, r))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
let tracer = global::tracer("opendal");
let mut span = tracer.start("copy");
span.set_attribute(KeyValue::new("from", from.to_string()));
span.set_attribute(KeyValue::new("to", to.to_string()));
span.set_attribute(KeyValue::new("args", format!("{args:?}")));
let cx = TraceContext::current_with_span(span);
- self.inner
- .copy(ctx, from, to, args, opts)
- .with_context(cx)
- .await
+ let _guard = cx.attach();
+ self.inner.copy(ctx, from, to, args, opts)
}
async fn rename(
@@ -196,16 +180,11 @@ impl Service for OtelTraceService {
self.inner.stat(ctx, path, args).with_context(cx).await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner.delete(ctx).await
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
let tracer = global::tracer("opendal");
let mut span = tracer.start("list");
span.set_attribute(KeyValue::new("path", path.to_string()));
@@ -213,9 +192,7 @@ impl Service for OtelTraceService {
let cx = TraceContext::current_with_span(span);
self.inner
.list(ctx, path, args)
- .with_context(cx.clone())
- .await
- .map(|(rp, s)| (rp, OtelTraceWrapper::new(cx, s)))
+ .map(|s| OtelTraceWrapper::new(cx, s))
}
async fn presign(
diff --git a/core/layers/retry/src/lib.rs b/core/layers/retry/src/lib.rs
index bbcdb5558..416fe1d67 100644
--- a/core/layers/retry/src/lib.rs
+++ b/core/layers/retry/src/lib.rs
@@ -23,6 +23,7 @@
use std::fmt::Debug;
use std::sync::Arc;
+use backon::BlockingRetryable;
use backon::ExponentialBuilder;
use backon::Retryable;
use opendal_core::raw::*;
@@ -330,14 +331,9 @@ impl<I: RetryInterceptor> Service for RetryService<I> {
.map_err(|err| err.set_persistent())
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
let mut attempt: u32 = 0;
- let (rp, reader) = { || self.inner.read(ctx, path, args.clone()) }
+ let reader = { || self.inner.read(ctx, path, args.clone()) }
.retry(self.builder)
.when(|e| e.is_temporary())
.notify(|err, dur| {
@@ -349,23 +345,15 @@ impl<I: RetryInterceptor> Service for RetryService<I> {
attempt,
})
})
- .await
+ .call()
.map_err(|err| err.set_persistent())?;
- Ok((
- rp,
- RetryReader::new(reader, self.notify.clone(), self.builder),
- ))
+ Ok(RetryReader::new(reader, self.notify.clone(), self.builder))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let mut attempt: u32 = 0;
- let (rp, writer) = { || self.inner.write(ctx, path, args.clone()) }
+ let writer = { || self.inner.write(ctx, path, args.clone()) }
.retry(self.builder)
.when(|e| e.is_temporary())
.notify(|err, dur| {
@@ -377,13 +365,10 @@ impl<I: RetryInterceptor> Service for RetryService<I> {
attempt,
})
})
- .await
+ .call()
.map_err(|err| err.set_persistent())?;
- Ok((
- rp,
- RetryWrapper::new(writer, self.notify.clone(), self.builder),
- ))
+ Ok(RetryWrapper::new(writer, self.notify.clone(), self.builder))
}
async fn stat(&self, ctx: &OperationContext, path: &str, args: OpStat) -> Result<RpStat> {
@@ -404,9 +389,9 @@ impl<I: RetryInterceptor> Service for RetryService<I> {
.map_err(|err| err.set_persistent())
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
let mut attempt: u32 = 0;
- let (rp, deleter) = { || self.inner.delete(ctx) }
+ let deleter = { || self.inner.delete(ctx) }
.retry(self.builder)
.when(|e| e.is_temporary())
.notify(|err, dur| {
@@ -418,25 +403,26 @@ impl<I: RetryInterceptor> Service for RetryService<I> {
attempt,
})
})
- .await
+ .call()
.map_err(|err| err.set_persistent())?;
- Ok((
- rp,
- RetryWrapper::new(deleter, self.notify.clone(), self.builder),
+ Ok(RetryWrapper::new(
+ deleter,
+ self.notify.clone(),
+ self.builder,
))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
let mut attempt: u32 = 0;
- let (rp, copier) = { || self.inner.copy(ctx, from, to, args.clone(), opts.clone()) }
+ let copier = { || self.inner.copy(ctx, from, to, args.clone(), opts.clone()) }
.retry(self.builder)
.when(|e| e.is_temporary())
.notify(|err, dur| {
@@ -448,13 +434,10 @@ impl<I: RetryInterceptor> Service for RetryService<I> {
attempt,
})
})
- .await
+ .call()
.map_err(|err| err.set_persistent())?;
- Ok((
- rp,
- RetryWrapper::new(copier, self.notify.clone(), self.builder),
- ))
+ Ok(RetryWrapper::new(copier, self.notify.clone(), self.builder))
}
async fn rename(
@@ -481,14 +464,9 @@ impl<I: RetryInterceptor> Service for RetryService<I> {
.map_err(|err| err.set_persistent())
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
let mut attempt: u32 = 0;
- let (rp, lister) = { || self.inner.list(ctx, path, args.clone()) }
+ let lister = { || self.inner.list(ctx, path, args.clone()) }
.retry(self.builder)
.when(|e| e.is_temporary())
.notify(|err, dur| {
@@ -500,13 +478,10 @@ impl<I: RetryInterceptor> Service for RetryService<I> {
attempt,
})
})
- .await
+ .call()
.map_err(|err| err.set_persistent())?;
- Ok((
- rp,
- RetryWrapper::new(lister, self.notify.clone(), self.builder),
- ))
+ Ok(RetryWrapper::new(lister, self.notify.clone(), self.builder))
}
async fn presign(
@@ -1154,61 +1129,41 @@ mod tests {
))
}
- async fn read(
- &self,
- _: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(MockReader::new(self.clone(), path, args)),
- ))
+ fn read(&self, _: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ Ok(oio::StreamReader::new(MockReader::new(
+ self.clone(),
+ path,
+ args,
+ )))
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- Ok((
- RpDelete::default(),
- MockDeleter {
- size: 0,
- attempt: self.attempt.clone(),
- },
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ Ok(MockDeleter {
+ size: 0,
+ attempt: self.attempt.clone(),
+ })
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- Ok((RpWrite::new(), MockWriter {}))
+ fn write(&self, _ctx: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
+ Ok(MockWriter {})
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
let lister = MockLister::default();
- Ok((RpList::default(), lister))
+ Ok(lister)
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- Ok((
- RpCopy::default(),
- MockCopier {
- attempt: self.attempt.clone(),
- },
- ))
+ ) -> Result<Self::Copier> {
+ Ok(MockCopier {
+ attempt: self.attempt.clone(),
+ })
}
async fn rename(
diff --git a/core/layers/route/src/lib.rs b/core/layers/route/src/lib.rs
index b64922e2a..71cd7683d 100644
--- a/core/layers/route/src/lib.rs
+++ b/core/layers/route/src/lib.rs
@@ -187,43 +187,31 @@ impl Service for RouteAccessor {
}
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, oio::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<oio::Reader> {
match self.select(path) {
- RouteSelected::Default(srv) => srv.read(ctx, path, args).await,
- RouteSelected::Target(target) => target.srv.read(&target.ctx, path, args).await,
+ RouteSelected::Default(srv) => srv.read(ctx, path, args),
+ RouteSelected::Target(target) => target.srv.read(&target.ctx, path, args),
}
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, oio::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<oio::Writer> {
match self.select(path) {
- RouteSelected::Default(srv) => srv.write(ctx, path, args).await,
- RouteSelected::Target(target) => target.srv.write(&target.ctx, path, args).await,
+ RouteSelected::Default(srv) => srv.write(ctx, path, args),
+ RouteSelected::Target(target) => target.srv.write(&target.ctx, path, args),
}
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, oio::Copier)> {
+ ) -> Result<oio::Copier> {
match self.select(from) {
- RouteSelected::Default(srv) => srv.copy(ctx, from, to, args, opts).await,
- RouteSelected::Target(target) => {
- target.srv.copy(&target.ctx, from, to, args, opts).await
- }
+ RouteSelected::Default(srv) => srv.copy(ctx, from, to, args, opts),
+ RouteSelected::Target(target) => target.srv.copy(&target.ctx, from, to, args, opts),
}
}
@@ -247,26 +235,18 @@ impl Service for RouteAccessor {
}
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, oio::Deleter)> {
- Ok((
- RpDelete::default(),
- Box::new(RouteDeleter::new(
- self.inner.clone(),
- self.router.clone(),
- ctx.clone(),
- )) as oio::Deleter,
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<oio::Deleter> {
+ Ok(Box::new(RouteDeleter::new(
+ self.inner.clone(),
+ self.router.clone(),
+ ctx.clone(),
+ )) as oio::Deleter)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, oio::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<oio::Lister> {
match self.select(path) {
- RouteSelected::Default(srv) => srv.list(ctx, path, args).await,
- RouteSelected::Target(target) => target.srv.list(&target.ctx, path, args).await,
+ RouteSelected::Default(srv) => srv.list(ctx, path, args),
+ RouteSelected::Target(target) => target.srv.list(&target.ctx, path, args),
}
}
@@ -309,7 +289,7 @@ impl RouteDeleter {
match key {
RouteKey::Default => {
if self.default_deleter.is_none() {
- let (_, deleter) = self.default.delete(&self.ctx).await?;
+ let deleter = self.default.delete(&self.ctx)?;
self.default_deleter = Some(deleter);
}
Ok(self
@@ -320,7 +300,7 @@ impl RouteDeleter {
RouteKey::Target(idx) => {
if idx >= self.target_deleters.len() {
if self.default_deleter.is_none() {
- let (_, deleter) = self.default.delete(&self.ctx).await?;
+ let deleter = self.default.delete(&self.ctx)?;
self.default_deleter = Some(deleter);
}
return Ok(self
@@ -330,14 +310,14 @@ impl RouteDeleter {
}
if self.target_deleters[idx].is_none() {
let Some(target) = self.router.target(idx) else {
- let (_, deleter) = self.default.delete(&self.ctx).await?;
+ let deleter = self.default.delete(&self.ctx)?;
self.default_deleter = Some(deleter);
return Ok(self
.default_deleter
.as_mut()
.expect("default deleter must exist"));
};
- let (_, deleter) = target.srv.delete(&target.ctx).await?;
+ let deleter = target.srv.delete(&target.ctx)?;
self.target_deleters[idx] = Some(deleter);
}
Ok(self.target_deleters[idx]
@@ -489,65 +469,47 @@ mod tests {
}
}
- async fn read(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, _ctx: &OperationContext, _: &str, _: OpRead) -> Result<Self::Reader> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn write(
- &self,
- _: &OperationContext,
- path: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- Ok((
- RpWrite::default(),
- MockWriter {
- paths: self.paths.clone(),
- path: path.to_string(),
- },
- ))
+ fn write(&self, _: &OperationContext, path: &str, _: OpWrite) -> Result<Self::Writer> {
+ Ok(MockWriter {
+ paths: self.paths.clone(),
+ path: path.to_string(),
+ })
}
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
from: &str,
to: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
if !self.paths.lock().unwrap().contains(from) {
return Err(Error::new(ErrorKind::NotFound, "source not found"));
}
self.paths.lock().unwrap().insert(to.to_string());
- Ok((RpCopy::default(), ()))
+ Ok(())
}
async fn rename(
diff --git a/core/layers/tail-cut/src/lib.rs b/core/layers/tail-cut/src/lib.rs
index 2d25548a2..c283d3749 100644
--- a/core/layers/tail-cut/src/lib.rs
+++ b/core/layers/tail-cut/src/lib.rs
@@ -396,58 +396,29 @@ impl Service for TailCutService {
.await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- self.with_deadline(Operation::Read, None, self.inner.read(ctx, path, args))
- .await
- .map(|(rp, r)| {
- (
- rp,
- TailCutWrapper::new(r, None, self.config.clone(), self.stats.clone()),
- )
- })
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ self.inner
+ .read(ctx, path, args)
+ .map(|r| TailCutWrapper::new(r, None, self.config.clone(), self.stats.clone()))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- self.with_deadline(Operation::Write, None, self.inner.write(ctx, path, args))
- .await
- .map(|(rp, w)| {
- (
- rp,
- TailCutWrapper::new(w, None, self.config.clone(), self.stats.clone()),
- )
- })
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ self.inner
+ .write(ctx, path, args)
+ .map(|w| TailCutWrapper::new(w, None, self.config.clone(), self.stats.clone()))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.with_deadline(
- Operation::Copy,
- None,
- self.inner.copy(ctx, from, to, args, opts),
- )
- .await
- .map(|(rp, c)| {
- (
- rp,
- TailCutWrapper::new(c, None, self.config.clone(), self.stats.clone()),
- )
- })
+ ) -> Result<Self::Copier> {
+ self.inner
+ .copy(ctx, from, to, args, opts)
+ .map(|c| TailCutWrapper::new(c, None, self.config.clone(), self.stats.clone()))
}
async fn rename(
@@ -470,31 +441,16 @@ impl Service for TailCutService {
.await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.with_deadline(Operation::Delete, None, self.inner.delete(ctx))
- .await
- .map(|(rp, d)| {
- (
- rp,
- TailCutWrapper::new(d, None, self.config.clone(), self.stats.clone()),
- )
- })
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner
+ .delete(ctx)
+ .map(|d| TailCutWrapper::new(d, None, self.config.clone(), self.stats.clone()))
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- self.with_deadline(Operation::List, None, self.inner.list(ctx, path, args))
- .await
- .map(|(rp, l)| {
- (
- rp,
- TailCutWrapper::new(l, None, self.config.clone(), self.stats.clone()),
- )
- })
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.inner
+ .list(ctx, path, args)
+ .map(|l| TailCutWrapper::new(l, None, self.config.clone(), self.stats.clone()))
}
async fn presign(
diff --git a/core/layers/throttle/src/lib.rs b/core/layers/throttle/src/lib.rs
index 839f1809c..ebb981e19 100644
--- a/core/layers/throttle/src/lib.rs
+++ b/core/layers/throttle/src/lib.rs
@@ -142,47 +142,35 @@ impl Service for ThrottleAccessor {
self.inner.stat(ctx, path, args).await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
let limiter = self.rate_limiter.clone();
self.inner
.read(ctx, path, args)
- .await
- .map(|(rp, r)| (rp, ThrottleWrapper::new(r, limiter)))
+ .map(|r| ThrottleWrapper::new(r, limiter))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let limiter = self.rate_limiter.clone();
self.inner
.write(ctx, path, args)
- .await
- .map(|(rp, w)| (rp, ThrottleWrapper::new(w, limiter)))
+ .map(|w| ThrottleWrapper::new(w, limiter))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.inner.copy(ctx, from, to, args, opts).await
+ ) -> Result<Self::Copier> {
+ self.inner.copy(ctx, from, to, args, opts)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.inner.delete(ctx).await
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner.delete(ctx)
}
async fn rename(
@@ -195,13 +183,8 @@ impl Service for ThrottleAccessor {
self.inner.rename(ctx, from, to, args).await
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- self.inner.list(ctx, path, args).await
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.inner.list(ctx, path, args)
}
async fn presign(
diff --git a/core/layers/timeout/src/lib.rs b/core/layers/timeout/src/lib.rs
index 710b277c1..3d55ca642 100644
--- a/core/layers/timeout/src/lib.rs
+++ b/core/layers/timeout/src/lib.rs
@@ -38,7 +38,7 @@ use opendal_core::*;
/// `TimeoutLayer` applies two timeout budgets:
///
/// - `timeout` bounds control operations such as `stat`, `create_dir`, `rename`,
-/// `copy`, `delete`, and `presign`.
+/// and `presign`.
/// - `io_timeout` bounds operations that open IO bodies, such as `read`, `write`,
/// and `list`, and every method call on returned readers, writers, listers,
/// deleters, and copiers.
@@ -195,21 +195,6 @@ impl TimeoutService {
.set_temporary()
})?
}
-
- async fn io_timeout<F: Future<Output = Result<T>>, T>(
- &self,
- op: Operation,
- fut: F,
- ) -> Result<T> {
- tokio::time::timeout(self.io_timeout, fut)
- .await
- .map_err(|_| {
- Error::new(ErrorKind::Unexpected, "io timeout reached")
- .with_operation(op)
- .with_context("timeout", self.io_timeout.as_secs_f64().to_string())
- .set_temporary()
- })?
- }
}
impl Service for TimeoutService {
@@ -237,39 +222,29 @@ impl Service for TimeoutService {
.await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- self.io_timeout(Operation::Read, self.inner.read(ctx, path, args))
- .await
- .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.io_timeout)))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ self.inner
+ .read(ctx, path, args)
+ .map(|r| TimeoutWrapper::new(r, self.io_timeout))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- self.io_timeout(Operation::Write, self.inner.write(ctx, path, args))
- .await
- .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.io_timeout)))
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ self.inner
+ .write(ctx, path, args)
+ .map(|r| TimeoutWrapper::new(r, self.io_timeout))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- self.timeout(Operation::Copy, self.inner.copy(ctx, from, to, args, opts))
- .await
- .map(|(rp, c)| (rp, TimeoutWrapper::new(c, self.io_timeout)))
+ ) -> Result<Self::Copier> {
+ self.inner
+ .copy(ctx, from, to, args, opts)
+ .map(|c| TimeoutWrapper::new(c, self.io_timeout))
}
async fn rename(
@@ -288,21 +263,16 @@ impl Service for TimeoutService {
.await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- self.timeout(Operation::Delete, self.inner.delete(ctx))
- .await
- .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.io_timeout)))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ self.inner
+ .delete(ctx)
+ .map(|r| TimeoutWrapper::new(r, self.io_timeout))
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- self.io_timeout(Operation::List, self.inner.list(ctx, path, args))
- .await
- .map(|(rp, r)| (rp, TimeoutWrapper::new(r, self.io_timeout)))
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ self.inner
+ .list(ctx, path, args)
+ .map(|r| TimeoutWrapper::new(r, self.io_timeout))
}
async fn presign(
@@ -453,7 +423,6 @@ mod tests {
use std::future::pending;
use futures::StreamExt;
- use tokio::time::sleep;
use tokio::time::timeout;
use super::*;
@@ -465,7 +434,7 @@ mod tests {
type Reader = MockReader;
type Writer = ();
type Lister = MockLister;
- type Deleter = ();
+ type Deleter = MockDeleter;
type Copier = MockCopier;
fn info(&self) -> ServiceInfo {
@@ -502,55 +471,35 @@ mod tests {
}
/// Return a reader whose operations never complete.
- async fn read(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::new(Metadata::new(EntryMode::FILE).with_content_length(0)),
- MockReader,
- ))
+ fn read(&self, _ctx: &OperationContext, _: &str, _: OpRead) -> Result<Self::Reader> {
+ Ok(MockReader)
}
- async fn write(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _ctx: &OperationContext, _: &str, _: OpWrite) -> Result<Self::Writer> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- /// Never return, so the service-level timeout can fire.
- async fn delete(&self, _: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- sleep(Duration::from_secs(u64::MAX)).await;
-
- Ok((RpDelete::default(), ()))
+ /// Return a deleter whose operations never complete.
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ Ok(MockDeleter)
}
- async fn list(
- &self,
- _: &OperationContext,
- _: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- Ok((RpList::default(), MockLister))
+ fn list(&self, _ctx: &OperationContext, _: &str, _: OpList) -> Result<Self::Lister> {
+ Ok(MockLister)
}
- async fn copy(
+ fn copy(
&self,
_: &OperationContext,
_: &str,
_: &str,
_: OpCopy,
_: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- Ok((RpCopy::default(), MockCopier))
+ ) -> Result<Self::Copier> {
+ Ok(MockCopier)
}
async fn rename(
@@ -627,10 +576,10 @@ mod tests {
}
#[tokio::test]
- async fn test_operation_timeout() {
+ async fn test_delete_timeout() {
let srv = MockService;
let op = Operator::from_inner(Arc::new(srv))
- .layer(TimeoutLayer::default().with_timeout(Duration::from_secs(1)));
+ .layer(TimeoutLayer::default().with_io_timeout(Duration::from_secs(1)));
let fut = async {
let res = op.delete("test").await;
@@ -699,9 +648,8 @@ mod tests {
.with_io_timeout(Duration::from_millis(100))
.apply_service(Arc::new(MockService));
let ctx = OperationContext::new(HttpClient::default(), Executor::default());
- let (_, mut copier) = service
+ let mut copier = service
.copy(&ctx, "f", "t", OpCopy::default(), OpCopier::default())
- .await
.unwrap();
let err = copier.next().await.unwrap_err();
@@ -718,7 +666,7 @@ mod tests {
let service = timeout_layer.apply_service(Arc::new(MockService));
let ctx = OperationContext::new(HttpClient::default(), Executor::default());
- let (_, mut lister) = service.list(&ctx, "test", OpList::default()).await.unwrap();
+ let mut lister = service.list(&ctx, "test", OpList::default()).unwrap();
let res = lister.next().await;
assert!(res.is_err());
diff --git a/core/layers/tracing/src/lib.rs b/core/layers/tracing/src/lib.rs
index fed847c4f..8f0576150 100644
--- a/core/layers/tracing/src/lib.rs
+++ b/core/layers/tracing/src/lib.rs
@@ -224,52 +224,31 @@ impl Service for TracingService {
.await
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
let span = span!(Level::DEBUG, "read", path, ?args);
-
- let (rp, r) = self
- .inner
+ self.inner
.read(ctx, path, args)
- .instrument(span.clone())
- .await?;
- Ok((rp, TracingWrapper::new(span, r)))
+ .map(|r| TracingWrapper::new(span, r))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
let span = span!(Level::DEBUG, "write", path, ?args);
-
- let (rp, r) = self
- .inner
+ self.inner
.write(ctx, path, args)
- .instrument(span.clone())
- .await?;
-
- Ok((rp, TracingWrapper::new(span, r)))
+ .map(|r| TracingWrapper::new(span, r))
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
let span = span!(Level::DEBUG, "copy", from, to, ?args, ?opts);
- self.inner
- .copy(ctx, from, to, args, opts)
- .instrument(span)
- .await
+ let _guard = span.enter();
+ self.inner.copy(ctx, from, to, args, opts)
}
async fn rename(
@@ -291,29 +270,16 @@ impl Service for TracingService {
self.inner.stat(ctx, path, args).instrument(span).await
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
let span = span!(Level::DEBUG, "delete");
-
- let (rp, r) = self.inner.delete(ctx).instrument(span.clone()).await?;
-
- Ok((rp, TracingWrapper::new(span, r)))
+ self.inner.delete(ctx).map(|r| TracingWrapper::new(span, r))
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
let span = span!(Level::DEBUG, "list", path, ?args);
-
- let (rp, r) = self
- .inner
+ self.inner
.list(ctx, path, args)
- .instrument(span.clone())
- .await?;
-
- Ok((rp, TracingWrapper::new(span, r)))
+ .map(|r| TracingWrapper::new(span, r))
}
async fn presign(
diff --git a/core/services/aliyun-drive/src/backend.rs b/core/services/aliyun-drive/src/backend.rs
index 2b6d702c2..56466f83f 100644
--- a/core/services/aliyun-drive/src/backend.rs
+++ b/core/services/aliyun-drive/src/backend.rs
@@ -30,8 +30,7 @@ use super::core::*;
use super::deleter::AliyunDriveDeleter;
use super::error::parse_error;
use super::lister::AliyunDriveLister;
-use super::lister::AliyunDriveParent;
-use super::writer::AliyunDriveWriter;
+use super::writer::AliyunDriveLazyWriter;
use opendal_core::raw::*;
use opendal_core::*;
@@ -234,10 +233,10 @@ impl oio::StreamRead for AliyunDriveReader {
impl Service for AliyunDriveBackend {
type Reader = oio::StreamReader<AliyunDriveReader>;
- type Writer = AliyunDriveWriter;
+ type Writer = AliyunDriveLazyWriter;
type Lister = oio::PageLister<AliyunDriveLister>;
type Deleter = oio::OneShotDeleter<AliyunDriveDeleter>;
- type Copier = ();
+ type Copier = oio::OneShotCopier;
fn info(&self) -> ServiceInfo {
self.core.info.clone()
@@ -297,60 +296,62 @@ impl Service for AliyunDriveBackend {
Ok(RpRename::default())
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let (rp, output): (_, ()) = {
+ ) -> Result<Self::Copier> {
+ let core = self.core.clone();
+ let ctx = ctx.clone();
+ let from = from.to_string();
+ let to = to.to_string();
+
+ Ok(oio::OneShotCopier::new(async move {
if from == to {
- Ok((RpCopy::default(), ()))
+ Ok(Metadata::default())
} else {
- let res = self.core.get_by_path(ctx, from).await?;
+ let res = core.get_by_path(&ctx, &from).await?;
let file: AliyunDriveFile =
serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
// copy can overwrite.
- match self.core.get_by_path(ctx, to).await {
+ match core.get_by_path(&ctx, &to).await {
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => Err(err)?,
Ok(res) => {
let file: AliyunDriveFile = serde_json::from_reader(res.reader())
.map_err(new_json_serialize_error)?;
- self.core.delete_path(ctx, &file.file_id).await?;
+ core.delete_path(&ctx, &file.file_id).await?;
}
}
// there is no direct copy in AliyunDrive.
// so we need to copy the path first and then rename it.
- let parent_path = get_parent(to);
- let parent_file_id = self.core.ensure_dir_exists(ctx, parent_path).await?;
+ let parent_path = get_parent(&to);
+ let parent_file_id = core.ensure_dir_exists(&ctx, parent_path).await?;
// if from and to are going to be placed in the same folder,
// copy_path will fail as we cannot change the name during this action.
// it has to be auto renamed.
let auto_rename = file.parent_file_id == parent_file_id;
- let res = self
- .core
- .copy_path(ctx, &file.file_id, &parent_file_id, auto_rename)
+ let res = core
+ .copy_path(&ctx, &file.file_id, &parent_file_id, auto_rename)
.await?;
let file: CopyResponse =
serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
let file_id = file.file_id;
- let from_name = get_basename(from);
- let to_name = get_basename(to);
+ let from_name = get_basename(&from);
+ let to_name = get_basename(&to);
if from_name != to_name {
- self.core.update_path(ctx, &file_id, to_name).await?;
+ core.update_path(&ctx, &file_id, to_name).await?;
}
- Ok((RpCopy::default(), ()))
+ Ok(Metadata::default())
}
- }?;
-
- Ok((rp, output))
+ }))
}
async fn stat(&self, ctx: &OperationContext, path: &str, _args: OpStat) -> Result<RpStat> {
@@ -382,100 +383,52 @@ impl Service for AliyunDriveBackend {
Ok(RpStat::new(meta))
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<AliyunDriveReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(AliyunDriveReader::new(
- self.clone(),
- ctx.clone(),
- path,
- args,
- )),
- ))
- }?;
-
- Ok((rp, output))
- }
-
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<AliyunDriveDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(AliyunDriveDeleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<AliyunDriveReader> = {
+ Ok(oio::StreamReader::new(AliyunDriveReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<AliyunDriveLister>) = {
- let parent = match self.core.get_by_path(ctx, path).await {
- Err(err) if err.kind() == ErrorKind::NotFound => None,
- Err(err) => return Err(err),
- Ok(res) => {
- let file: AliyunDriveFile =
- serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
- Some(AliyunDriveParent {
- file_id: file.file_id,
- path: path.to_string(),
- updated_at: file.updated_at,
- })
- }
- };
-
- let l = AliyunDriveLister::new(self.core.clone(), ctx.clone(), parent, args.limit());
-
- Ok((RpList::default(), oio::PageLister::new(l)))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<AliyunDriveDeleter> = {
+ Ok(oio::OneShotDeleter::new(AliyunDriveDeleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, AliyunDriveWriter) = {
- let parent_path = get_parent(path);
- let parent_file_id = self.core.ensure_dir_exists(ctx, parent_path).await?;
-
- // write can overwrite
- match self.core.get_by_path(ctx, path).await {
- Err(err) if err.kind() == ErrorKind::NotFound => {}
- Err(err) => return Err(err),
- Ok(res) => {
- let file: AliyunDriveFile =
- serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
- self.core.delete_path(ctx, &file.file_id).await?;
- }
- };
-
- let writer = AliyunDriveWriter::new(
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<AliyunDriveLister> = {
+ let l = AliyunDriveLister::new_with_path(
self.core.clone(),
ctx.clone(),
- &parent_file_id,
- get_basename(path),
- args,
+ path.to_string(),
+ args.limit(),
);
- Ok((RpWrite::default(), writer))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
+ }
+
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ Ok(AliyunDriveLazyWriter::new(
+ self.core.clone(),
+ ctx.clone(),
+ path.to_string(),
+ args,
+ ))
}
async fn presign(
diff --git a/core/services/aliyun-drive/src/lister.rs b/core/services/aliyun-drive/src/lister.rs
index ccc7aebd0..e99addc26 100644
--- a/core/services/aliyun-drive/src/lister.rs
+++ b/core/services/aliyun-drive/src/lister.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
use self::oio::Entry;
use super::core::AliyunDriveCore;
+use super::core::AliyunDriveFile;
use super::core::AliyunDriveFileList;
use bytes::Buf;
use opendal_core::EntryMode;
@@ -33,9 +34,11 @@ pub struct AliyunDriveLister {
ctx: OperationContext,
parent: Option<AliyunDriveParent>,
+ path: Option<String>,
limit: Option<usize>,
}
+#[derive(Clone)]
pub struct AliyunDriveParent {
pub file_id: String,
pub path: String,
@@ -43,24 +46,49 @@ pub struct AliyunDriveParent {
}
impl AliyunDriveLister {
- pub fn new(
+ pub fn new_with_path(
core: Arc<AliyunDriveCore>,
ctx: OperationContext,
- parent: Option<AliyunDriveParent>,
+ path: String,
limit: Option<usize>,
) -> Self {
AliyunDriveLister {
core,
ctx,
- parent,
+ parent: None,
+ path: Some(path),
limit,
}
}
+
+ async fn parent(&self) -> Result<Option<AliyunDriveParent>> {
+ if let Some(parent) = &self.parent {
+ return Ok(Some(parent.clone()));
+ }
+
+ let Some(path) = &self.path else {
+ return Ok(None);
+ };
+
+ match self.core.get_by_path(&self.ctx, path).await {
+ Err(err) if err.kind() == ErrorKind::NotFound => Ok(None),
+ Err(err) => Err(err),
+ Ok(res) => {
+ let file: AliyunDriveFile =
+ serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
+ Ok(Some(AliyunDriveParent {
+ file_id: file.file_id,
+ path: path.to_string(),
+ updated_at: file.updated_at,
+ }))
+ }
+ }
+ }
}
impl oio::PageList for AliyunDriveLister {
async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
- let Some(parent) = &self.parent else {
+ let Some(parent) = self.parent().await? else {
ctx.done = true;
return Ok(());
};
diff --git a/core/services/aliyun-drive/src/writer.rs b/core/services/aliyun-drive/src/writer.rs
index 3c7ddf505..8a557c5d6 100644
--- a/core/services/aliyun-drive/src/writer.rs
+++ b/core/services/aliyun-drive/src/writer.rs
@@ -20,6 +20,7 @@ use std::sync::Arc;
use bytes::Buf;
use super::core::AliyunDriveCore;
+use super::core::AliyunDriveFile;
use super::core::CheckNameMode;
use super::core::CreateResponse;
use super::core::CreateType;
@@ -39,6 +40,82 @@ pub struct AliyunDriveWriter {
part_number: usize,
}
+pub struct AliyunDriveLazyWriter {
+ core: Arc<AliyunDriveCore>,
+ ctx: OperationContext,
+ path: String,
+ args: OpWrite,
+ inner: Option<AliyunDriveWriter>,
+}
+
+impl AliyunDriveLazyWriter {
+ pub fn new(
+ core: Arc<AliyunDriveCore>,
+ ctx: OperationContext,
+ path: String,
+ args: OpWrite,
+ ) -> Self {
+ Self {
+ core,
+ ctx,
+ path,
+ args,
+ inner: None,
+ }
+ }
+
+ async fn inner(&mut self) -> Result<&mut AliyunDriveWriter> {
+ if self.inner.is_none() {
+ let parent_path = get_parent(&self.path);
+ let parent_file_id = self.core.ensure_dir_exists(&self.ctx, parent_path).await?;
+
+ // write can overwrite
+ match self.core.get_by_path(&self.ctx, &self.path).await {
+ Err(err) if err.kind() == ErrorKind::NotFound => {}
+ Err(err) => return Err(err),
+ Ok(res) => {
+ let file: AliyunDriveFile =
+ serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
+ self.core.delete_path(&self.ctx, &file.file_id).await?;
+ }
+ };
+
+ self.inner = Some(AliyunDriveWriter::new(
+ self.core.clone(),
+ self.ctx.clone(),
+ &parent_file_id,
+ get_basename(&self.path),
+ self.args.clone(),
+ ));
+ }
+
+ Ok(self
+ .inner
+ .as_mut()
+ .expect("aliyun drive writer must be initialized"))
+ }
+}
+
+impl oio::Write for AliyunDriveLazyWriter {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ self.inner().await?.write(bs).await
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ match &mut self.inner {
+ Some(w) => w.close().await,
+ None => Ok(Metadata::default()),
+ }
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ match &mut self.inner {
+ Some(w) => w.abort().await,
+ None => Ok(()),
+ }
+ }
+}
+
impl AliyunDriveWriter {
pub fn new(
core: Arc<AliyunDriveCore>,
diff --git a/core/services/alluxio/src/backend.rs b/core/services/alluxio/src/backend.rs
index bca80e425..cf1999bfd 100644
--- a/core/services/alluxio/src/backend.rs
+++ b/core/services/alluxio/src/backend.rs
@@ -195,29 +195,21 @@ impl Service for AlluxioBackend {
Ok(RpStat::new(file_info.try_into()?))
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<AlluxioReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(AlluxioReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<AlluxioReader> = {
+ Ok(oio::StreamReader::new(AlluxioReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, AlluxioWriters) = {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: AlluxioWriters = {
let w = AlluxioWriter::new(
self.core.clone(),
ctx.clone(),
@@ -225,45 +217,40 @@ impl Service for AlluxioBackend {
path.to_string(),
);
- Ok((RpWrite::default(), w))
+ Ok(w)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<AlluxioDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(AlluxioDeleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<AlluxioDeleter> = {
+ Ok(oio::OneShotDeleter::new(AlluxioDeleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<AlluxioLister>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, _args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<AlluxioLister> = {
let l = AlluxioLister::new(self.core.clone(), ctx.clone(), path);
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/azblob/src/backend.rs b/core/services/azblob/src/backend.rs
index 71d3d3e47..7a59e04d7 100644
--- a/core/services/azblob/src/backend.rs
+++ b/core/services/azblob/src/backend.rs
@@ -555,29 +555,21 @@ impl Service for AzblobBackend {
_ => Err(parse_error(resp)),
}
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<AzblobReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(AzblobReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<AzblobReader> = {
+ Ok(oio::StreamReader::new(AzblobReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, AzblobWriters) = {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: AzblobWriters = {
let w = AzblobWriter::new(
self.core.clone(),
ctx.clone(),
@@ -594,33 +586,25 @@ impl Service for AzblobBackend {
))
};
- Ok((RpWrite::default(), w))
+ Ok(w)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::BatchDeleter<AzblobDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::BatchDeleter::new(
- AzblobDeleter::new(self.core.clone(), ctx.clone()),
- self.core.capability.delete_max_size,
- ),
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::BatchDeleter<AzblobDeleter> = {
+ Ok(oio::BatchDeleter::new(
+ AzblobDeleter::new(self.core.clone(), ctx.clone()),
+ self.core.capability.delete_max_size,
))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<AzblobLister>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<AzblobLister> = {
let l = AzblobLister::new(
self.core.clone(),
ctx.clone(),
@@ -629,26 +613,26 @@ impl Service for AzblobBackend {
args.limit(),
);
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let (rp, output): (_, AzblobCopiers) = {
+ ) -> Result<Self::Copier> {
+ let output: AzblobCopiers = {
let copier = new_azblob_copier(self.core.clone(), ctx, from, to, args, opts)?;
- Ok((RpCopy::default(), copier))
+ Ok(copier)
}?;
- Ok((rp, output))
+ Ok(output)
}
async fn rename(
diff --git a/core/services/azdls/Cargo.toml b/core/services/azdls/Cargo.toml
index 8098ce15c..82cafb875 100644
--- a/core/services/azdls/Cargo.toml
+++ b/core/services/azdls/Cargo.toml
@@ -35,6 +35,7 @@ base64 = { workspace = true }
bytes = { workspace = true }
http = { workspace = true }
log = { workspace = true }
+mea = { workspace = true }
opendal-core = { path = "../../core", version = "0.57.0", default-features = false }
opendal-service-azure-common = { path = "../azure-common", version = "0.57.0" }
quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
diff --git a/core/services/azdls/src/backend.rs b/core/services/azdls/src/backend.rs
index 46f4a0708..1d5ef97ca 100644
--- a/core/services/azdls/src/backend.rs
+++ b/core/services/azdls/src/backend.rs
@@ -38,6 +38,7 @@ use super::core::DIRECTORY;
use super::deleter::AzdlsDeleter;
use super::error::parse_error;
use super::lister::AzdlsLister;
+use super::writer::AzdlsLazyPositionWriter;
use super::writer::AzdlsWriter;
use super::writer::AzdlsWriters;
use opendal_core::raw::*;
@@ -457,29 +458,21 @@ impl Service for AzdlsBackend {
let metadata = self.core.azdls_stat_metadata(ctx, path).await?;
Ok(RpStat::new(metadata))
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<AzdlsReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(AzdlsReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<AzdlsReader> = {
+ Ok(oio::StreamReader::new(AzdlsReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, AzdlsWriters) = {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: AzdlsWriters = {
if args.append() {
let w = AzdlsWriter::new(
self.core.clone(),
@@ -487,44 +480,39 @@ impl Service for AzdlsBackend {
args.clone(),
path.to_string(),
);
- Ok((
- RpWrite::default(),
- AzdlsWriters::Two(oio::AppendWriter::new(w)),
- ))
+ Ok(AzdlsWriters::Two(oio::AppendWriter::new(w)))
} else {
- let w = AzdlsWriter::create(
+ let w = AzdlsWriter::new(
self.core.clone(),
ctx.clone(),
args.clone(),
path.to_string(),
- )
- .await?;
- let w = oio::PositionWriter::new(ctx.executor().clone(), w, args.concurrent());
- Ok((RpWrite::default(), AzdlsWriters::One(w)))
+ );
+ let w = oio::PositionWriter::new(
+ ctx.executor().clone(),
+ AzdlsLazyPositionWriter::new(w),
+ args.concurrent(),
+ );
+ Ok(AzdlsWriters::One(w))
}
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<AzdlsDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(AzdlsDeleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<AzdlsDeleter> = {
+ Ok(oio::OneShotDeleter::new(AzdlsDeleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<AzdlsLister>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<AzdlsLister> = {
let l = AzdlsLister::new(
self.core.clone(),
ctx.clone(),
@@ -532,20 +520,20 @@ impl Service for AzdlsBackend {
args.limit(),
);
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/azdls/src/writer.rs b/core/services/azdls/src/writer.rs
index 166402b8c..b971001fa 100644
--- a/core/services/azdls/src/writer.rs
+++ b/core/services/azdls/src/writer.rs
@@ -18,6 +18,7 @@
use std::sync::Arc;
use http::StatusCode;
+use mea::once::OnceCell;
use super::core::AzdlsCore;
use super::core::FILE;
@@ -27,7 +28,29 @@ use opendal_core::raw::*;
use opendal_core::*;
/// Writer type for azdls: non-append uses PositionWriter, append uses AppendWriter.
-pub type AzdlsWriters = TwoWays<oio::PositionWriter<AzdlsWriter>, oio::AppendWriter<AzdlsWriter>>;
+pub type AzdlsWriters =
+ TwoWays<oio::PositionWriter<AzdlsLazyPositionWriter>, oio::AppendWriter<AzdlsWriter>>;
+
+pub struct AzdlsLazyPositionWriter {
+ inner: AzdlsWriter,
+ created: OnceCell<()>,
+}
+
+impl AzdlsLazyPositionWriter {
+ pub fn new(inner: AzdlsWriter) -> Self {
+ Self {
+ inner,
+ created: OnceCell::new(),
+ }
+ }
+
+ async fn ensure_created(&self) -> Result<()> {
+ self.created
+ .get_or_try_init(async || self.inner.create_if_needed().await)
+ .await?;
+ Ok(())
+ }
+}
#[derive(Clone)]
pub struct AzdlsWriter {
@@ -47,17 +70,6 @@ impl AzdlsWriter {
}
}
- pub async fn create(
- core: Arc<AzdlsCore>,
- ctx: OperationContext,
- op: OpWrite,
- path: String,
- ) -> Result<Self> {
- let writer = Self::new(core, ctx, op, path);
- writer.create_if_needed().await?;
- Ok(writer)
- }
-
async fn create_if_needed(&self) -> Result<()> {
let resp = self
.core
@@ -130,6 +142,22 @@ impl oio::PositionWrite for AzdlsWriter {
}
}
+impl oio::PositionWrite for AzdlsLazyPositionWriter {
+ async fn write_all_at(&self, offset: u64, buf: Buffer) -> Result<()> {
+ self.ensure_created().await?;
+ self.inner.write_all_at(offset, buf).await
+ }
+
+ async fn close(&self, size: u64) -> Result<Metadata> {
+ self.ensure_created().await?;
+ self.inner.close(size).await
+ }
+
+ async fn abort(&self) -> Result<()> {
+ self.inner.abort().await
+ }
+}
+
impl oio::AppendWrite for AzdlsWriter {
async fn offset(&self) -> Result<u64> {
let resp = self
diff --git a/core/services/azfile/src/backend.rs b/core/services/azfile/src/backend.rs
index 92e464029..3382ec3b6 100644
--- a/core/services/azfile/src/backend.rs
+++ b/core/services/azfile/src/backend.rs
@@ -378,30 +378,21 @@ impl Service for AzfileBackend {
_ => Err(parse_error(resp)),
}
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<AzfileReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(AzfileReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<AzfileReader> = {
+ Ok(oio::StreamReader::new(AzfileReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, AzfileWriters) = {
- self.core.ensure_parent_dir_exists(ctx, path).await?;
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: AzfileWriters = {
let w = AzfileWriter::new(
self.core.clone(),
ctx.clone(),
@@ -413,30 +404,25 @@ impl Service for AzfileBackend {
} else {
AzfileWriters::One(oio::OneShotWriter::new(w))
};
- Ok((RpWrite::default(), w))
+ Ok(w)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<AzfileDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(AzfileDeleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<AzfileDeleter> = {
+ Ok(oio::OneShotDeleter::new(AzfileDeleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<AzfileLister>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<AzfileLister> = {
let l = AzfileLister::new(
self.core.clone(),
ctx.clone(),
@@ -444,20 +430,20 @@ impl Service for AzfileBackend {
args.limit(),
);
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/azfile/src/writer.rs b/core/services/azfile/src/writer.rs
index a822d4a36..0a72f4a97 100644
--- a/core/services/azfile/src/writer.rs
+++ b/core/services/azfile/src/writer.rs
@@ -43,6 +43,12 @@ impl AzfileWriter {
}
}
+ async fn ensure_parent_dir_exists(&self) -> Result<()> {
+ self.core
+ .ensure_parent_dir_exists(&self.ctx, &self.path)
+ .await
+ }
+
fn parse_metadata(headers: &http::HeaderMap) -> Result<Metadata> {
let mut metadata = Metadata::default();
@@ -60,6 +66,8 @@ impl AzfileWriter {
impl oio::OneShotWrite for AzfileWriter {
async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+ self.ensure_parent_dir_exists().await?;
+
let size = bs.len();
let resp = self
.core
@@ -90,6 +98,8 @@ impl oio::OneShotWrite for AzfileWriter {
impl oio::AppendWrite for AzfileWriter {
async fn offset(&self) -> Result<u64> {
+ self.ensure_parent_dir_exists().await?;
+
let resp = self
.core
.azfile_get_file_properties(&self.ctx, &self.path)
diff --git a/core/services/b2/src/backend.rs b/core/services/b2/src/backend.rs
index 0e9b2a744..0f8ff5274 100644
--- a/core/services/b2/src/backend.rs
+++ b/core/services/b2/src/backend.rs
@@ -269,7 +269,7 @@ impl Service for B2Backend {
type Writer = B2Writers;
type Lister = oio::PageLister<B2Lister>;
type Deleter = oio::OneShotDeleter<B2Deleter>;
- type Copier = ();
+ type Copier = oio::OneShotCopier;
fn info(&self) -> ServiceInfo {
self.core.info.clone()
@@ -305,102 +305,88 @@ impl Service for B2Backend {
let meta = parse_file_info(&file_info);
Ok(RpStat::new(meta))
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<B2Reader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(B2Reader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<B2Reader> = {
+ Ok(oio::StreamReader::new(B2Reader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, B2Writers) = {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: B2Writers = {
let concurrent = args.concurrent();
let writer = B2Writer::new(self.core.clone(), ctx.clone(), path, args);
let w = oio::MultipartWriter::new(ctx.executor().clone(), writer, concurrent);
- Ok((RpWrite::default(), w))
+ Ok(w)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<B2Deleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(B2Deleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<B2Deleter> = {
+ Ok(oio::OneShotDeleter::new(B2Deleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<B2Lister>) = {
- Ok((
- RpList::default(),
- oio::PageLister::new(B2Lister::new(
- self.core.clone(),
- ctx.clone(),
- path,
- args.recursive(),
- args.limit(),
- args.start_after(),
- )),
- ))
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<B2Lister> = {
+ Ok(oio::PageLister::new(B2Lister::new(
+ self.core.clone(),
+ ctx.clone(),
+ path,
+ args.recursive(),
+ args.limit(),
+ args.start_after(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let (rp, output): (_, ()) = {
- let file_info = self.core.get_file_info(ctx, from, None).await?;
-
+ ) -> Result<Self::Copier> {
+ let core = self.core.clone();
+ let ctx = ctx.clone();
+ let from = from.to_string();
+ let to = to.to_string();
+
+ Ok(oio::OneShotCopier::new(async move {
+ let file_info = core.get_file_info(&ctx, &from, None).await?;
let source_file_id = file_info.file_id;
let Some(source_file_id) = source_file_id else {
return Err(Error::new(ErrorKind::IsADirectory, "is a directory"));
};
- let resp = self.core.copy_file(ctx, source_file_id, to).await?;
+ let resp = core.copy_file(&ctx, source_file_id, &to).await?;
let status = resp.status();
match status {
- StatusCode::OK => Ok((RpCopy::default(), ())),
+ StatusCode::OK => Ok(Metadata::default()),
_ => Err(parse_error(resp)),
}
- }?;
-
- Ok((rp, output))
+ }))
}
async fn rename(
diff --git a/core/services/cacache/src/backend.rs b/core/services/cacache/src/backend.rs
index 02c503a0c..3b4dfb95a 100644
--- a/core/services/cacache/src/backend.rs
+++ b/core/services/cacache/src/backend.rs
@@ -161,69 +161,50 @@ impl Service for CacacheBackend {
None => Err(Error::new(ErrorKind::NotFound, "entry not found")),
}
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<CacacheReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(CacacheReader::new(self.clone(), path, args)),
- ))
+ fn read(&self, _ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<CacacheReader> = {
+ Ok(oio::StreamReader::new(CacacheReader::new(
+ self.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, CacacheWriter) = {
- Ok((
- RpWrite::new(),
- CacacheWriter::new(self.core.clone(), path.to_string()),
- ))
- }?;
+ fn write(&self, _ctx: &OperationContext, path: &str, _: OpWrite) -> Result<Self::Writer> {
+ let output: CacacheWriter =
+ { Ok(CacacheWriter::new(self.core.clone(), path.to_string())) }?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<CacacheDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(CacacheDeleter::new(self.core.clone())),
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<CacacheDeleter> = {
+ Ok(oio::OneShotDeleter::new(CacacheDeleter::new(
+ self.core.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- _path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _path: &str, _args: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/cloudflare-kv/src/backend.rs b/core/services/cloudflare-kv/src/backend.rs
index 101a620c9..817f13db4 100644
--- a/core/services/cloudflare-kv/src/backend.rs
+++ b/core/services/cloudflare-kv/src/backend.rs
@@ -479,66 +479,45 @@ impl Service for CloudflareKvBackend {
Ok(RpStat::new(meta))
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<CloudflareKvReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(CloudflareKvReader::new(
- self.clone(),
- ctx.clone(),
- path,
- args,
- )),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<CloudflareKvReader> = {
+ Ok(oio::StreamReader::new(CloudflareKvReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, oio::OneShotWriter<CloudflareWriter>) = {
+ fn write(&self, ctx: &OperationContext, path: &str, _: OpWrite) -> Result<Self::Writer> {
+ let output: oio::OneShotWriter<CloudflareWriter> = {
let path = build_abs_path(&self.core.info.root(), path);
let writer = CloudflareWriter::new(self.core.clone(), ctx.clone(), path);
let w = oio::OneShotWriter::new(writer);
- Ok((RpWrite::default(), w))
+ Ok(w)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::BatchDeleter<CloudflareKvDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::BatchDeleter::new(
- CloudflareKvDeleter::new(self.core.clone(), ctx.clone()),
- self.core.capability.delete_max_size,
- ),
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::BatchDeleter<CloudflareKvDeleter> = {
+ Ok(oio::BatchDeleter::new(
+ CloudflareKvDeleter::new(self.core.clone(), ctx.clone()),
+ self.core.capability.delete_max_size,
))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<CloudflareKvLister>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<CloudflareKvLister> = {
let path = build_abs_path(&self.core.info.root(), path);
let limit = match args.limit() {
@@ -561,20 +540,20 @@ impl Service for CloudflareKvBackend {
Some(limit),
);
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/compfs/src/backend.rs b/core/services/compfs/src/backend.rs
index c44e12a7c..d0f70b41a 100644
--- a/core/services/compfs/src/backend.rs
+++ b/core/services/compfs/src/backend.rs
@@ -16,6 +16,7 @@
// under the License.
use std::io::Cursor;
+use std::path::PathBuf;
use std::sync::Arc;
use compio::buf::IntoInner;
@@ -158,12 +159,158 @@ impl oio::PositionRead for CompfsReader {
}
}
+pub struct CompfsLazyReader {
+ core: Arc<CompfsCore>,
+ path: PathBuf,
+}
+
+impl CompfsLazyReader {
+ fn new(core: Arc<CompfsCore>, path: PathBuf) -> Self {
+ Self { core, path }
+ }
+
+ async fn reader(&self) -> Result<oio::PositionReader<CompfsReader>> {
+ let path = self.path.clone();
+ let file = self
+ .core
+ .exec(move || async move {
+ let file = compio::fs::OpenOptions::new()
+ .read(true)
+ .open(&path)
+ .await?;
+ Ok(file)
+ })
+ .await?;
+
+ Ok(oio::PositionReader::new(CompfsReader::new(
+ self.core.clone(),
+ file,
+ )))
+ }
+}
+
+impl oio::Read for CompfsLazyReader {
+ async fn open(&self, range: BytesRange) -> Result<(RpRead, Box<dyn oio::ReadStreamDyn>)> {
+ self.reader().await?.open(range).await
+ }
+
+ async fn read(&self, range: BytesRange) -> Result<(RpRead, Buffer)> {
+ self.reader().await?.read(range).await
+ }
+}
+
+pub struct CompfsLazyWriter {
+ core: Arc<CompfsCore>,
+ path: PathBuf,
+ args: OpWrite,
+ inner: Option<CompfsWriter>,
+}
+
+impl CompfsLazyWriter {
+ fn new(core: Arc<CompfsCore>, path: PathBuf, args: OpWrite) -> Self {
+ Self {
+ core,
+ path,
+ args,
+ inner: None,
+ }
+ }
+
+ async fn inner(&mut self) -> Result<&mut CompfsWriter> {
+ if self.inner.is_none() {
+ let path = self.path.clone();
+ let append = self.args.append();
+ let file = self
+ .core
+ .exec(move || async move {
+ if let Some(parent) = path.parent() {
+ compio::fs::create_dir_all(parent).await?;
+ }
+ let file = compio::fs::OpenOptions::new()
+ .create(true)
+ .write(true)
+ .truncate(!append)
+ .open(path)
+ .await?;
+ let mut file = Cursor::new(file);
+ if append {
+ let len = file.get_ref().metadata().await?.len();
+ file.set_position(len);
+ }
+ Ok(file)
+ })
+ .await?;
+
+ self.inner = Some(CompfsWriter::new(self.core.clone(), file));
+ }
+
+ Ok(self.inner.as_mut().expect("writer must be initialized"))
+ }
+}
+
+impl oio::Write for CompfsLazyWriter {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ self.inner().await?.write(bs).await
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ self.inner().await?.close().await
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ self.inner().await?.abort().await
+ }
+}
+
+pub struct CompfsLazyLister {
+ core: Arc<CompfsCore>,
+ path: PathBuf,
+ inner: Option<Option<CompfsLister>>,
+}
+
+impl CompfsLazyLister {
+ fn new(core: Arc<CompfsCore>, path: PathBuf) -> Self {
+ Self {
+ core,
+ path,
+ inner: None,
+ }
+ }
+}
+
+impl oio::List for CompfsLazyLister {
+ async fn next(&mut self) -> Result<Option<oio::Entry>> {
+ if self.inner.is_none() {
+ let path = self.path.clone();
+ self.inner = Some(
+ match self
+ .core
+ .exec_blocking({
+ let path = path.clone();
+ move || std::fs::read_dir(path)
+ })
+ .await?
+ {
+ Ok(read_dir) => Some(CompfsLister::new(self.core.clone(), &path, read_dir)),
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
+ Err(e) => return Err(new_std_io_error(e)),
+ },
+ );
+ }
+
+ match self.inner.as_mut().expect("lister must be initialized") {
+ Some(lister) => lister.next().await,
+ None => Ok(None),
+ }
+ }
+}
+
impl Service for CompfsBackend {
- type Reader = oio::PositionReader<CompfsReader>;
- type Writer = CompfsWriter;
- type Lister = Option<CompfsLister>;
+ type Reader = CompfsLazyReader;
+ type Writer = CompfsLazyWriter;
+ type Lister = CompfsLazyLister;
type Deleter = oio::OneShotDeleter<CompfsDeleter>;
- type Copier = ();
+ type Copier = oio::OneShotCopier;
fn info(&self) -> ServiceInfo {
self.core.info.clone()
@@ -209,53 +356,48 @@ impl Service for CompfsBackend {
Ok(RpStat::new(ret))
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<CompfsDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(CompfsDeleter::new(self.core.clone())),
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<CompfsDeleter> = {
+ Ok(oio::OneShotDeleter::new(CompfsDeleter::new(
+ self.core.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
from: &str,
to: &str,
_: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let (rp, output): (_, ()) = {
- let from = self.core.prepare_path(from)?;
- let to = self.core.prepare_path(to)?;
-
- self.core
- .exec(move || async move {
- let from = OpenOptions::new().read(true).open(from).await?;
- if let Some(parent) = to.parent() {
- compio::fs::create_dir_all(parent).await?;
- }
- let to = OpenOptions::new()
- .write(true)
- .create(true)
- .truncate(true)
- .open(to)
- .await?;
-
- let (mut from, mut to) = (Cursor::new(from), Cursor::new(to));
- compio::io::copy(&mut from, &mut to).await?;
+ ) -> Result<Self::Copier> {
+ let core = self.core.clone();
+ let from = self.core.prepare_path(from)?;
+ let to = self.core.prepare_path(to)?;
- Ok(())
- })
- .await?;
+ Ok(oio::OneShotCopier::new(async move {
+ core.exec(move || async move {
+ let from = OpenOptions::new().read(true).open(from).await?;
+ if let Some(parent) = to.parent() {
+ compio::fs::create_dir_all(parent).await?;
+ }
+ let to = OpenOptions::new()
+ .write(true)
+ .create(true)
+ .truncate(true)
+ .open(to)
+ .await?;
- Ok((RpCopy::default(), ()))
- }?;
+ let (mut from, mut to) = (Cursor::new(from), Cursor::new(to));
+ compio::io::copy(&mut from, &mut to).await?;
- Ok((rp, output))
+ Ok(Metadata::default())
+ })
+ .await
+ }))
}
async fn rename(
@@ -279,98 +421,26 @@ impl Service for CompfsBackend {
Ok(RpRename::default())
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::PositionReader<CompfsReader>) = {
- let path = self.core.prepare_path(path)?;
- let file = self
- .core
- .exec(move || async move {
- let file = compio::fs::OpenOptions::new()
- .read(true)
- .open(&path)
- .await?;
- Ok(file)
- })
- .await?;
-
- Ok((
- RpRead::default(),
- oio::PositionReader::new(CompfsReader::new(self.core.clone(), file)),
- ))
- }?;
-
- Ok((rp, output))
+ fn read(&self, _ctx: &OperationContext, path: &str, _: OpRead) -> Result<Self::Reader> {
+ Ok(CompfsLazyReader::new(
+ self.core.clone(),
+ self.core.prepare_path(path)?,
+ ))
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, CompfsWriter) = {
- let path = self.core.prepare_path(path)?;
- let append = args.append();
- let file = self
- .core
- .exec(move || async move {
- if let Some(parent) = path.parent() {
- compio::fs::create_dir_all(parent).await?;
- }
- let file = compio::fs::OpenOptions::new()
- .create(true)
- .write(true)
- .truncate(!append)
- .open(path)
- .await?;
- let mut file = Cursor::new(file);
- if append {
- let len = file.get_ref().metadata().await?.len();
- file.set_position(len);
- }
- Ok(file)
- })
- .await?;
-
- let w = CompfsWriter::new(self.core.clone(), file);
- Ok((RpWrite::new(), w))
- }?;
-
- Ok((rp, output))
+ fn write(&self, _ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ Ok(CompfsLazyWriter::new(
+ self.core.clone(),
+ self.core.prepare_path(path)?,
+ args,
+ ))
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, Option<CompfsLister>) = {
- let path = self.core.prepare_path(path)?;
-
- match self
- .core
- .exec_blocking({
- let path = path.clone();
- move || std::fs::read_dir(path)
- })
- .await?
- {
- Ok(read_dir) => {
- let lister = CompfsLister::new(self.core.clone(), &path, read_dir);
- Ok((RpList::default(), Some(lister)))
- }
- Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok((RpList::default(), None)),
- Err(e) => Err(new_std_io_error(e)),
- }
- }?;
-
- Ok((rp, output))
+ fn list(&self, _ctx: &OperationContext, path: &str, _: OpList) -> Result<Self::Lister> {
+ Ok(CompfsLazyLister::new(
+ self.core.clone(),
+ self.core.prepare_path(path)?,
+ ))
}
async fn presign(
diff --git a/core/services/cos/src/backend.rs b/core/services/cos/src/backend.rs
index 2cde28d89..f03263ef5 100644
--- a/core/services/cos/src/backend.rs
+++ b/core/services/cos/src/backend.rs
@@ -359,7 +359,7 @@ impl Service for CosBackend {
type Writer = CosWriters;
type Lister = CosListers;
type Deleter = oio::OneShotDeleter<CosDeleter>;
- type Copier = ();
+ type Copier = oio::OneShotCopier;
fn info(&self) -> ServiceInfo {
self.core.info.clone()
@@ -407,29 +407,21 @@ impl Service for CosBackend {
_ => Err(parse_error(resp)),
}
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<CosReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(CosReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<CosReader> = {
+ Ok(oio::StreamReader::new(CosReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, CosWriters) = {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: CosWriters = {
let writer = CosWriter::new(self.core.clone(), ctx.clone(), path, args.clone());
let w = if args.append() {
@@ -442,30 +434,25 @@ impl Service for CosBackend {
))
};
- Ok((RpWrite::default(), w))
+ Ok(w)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<CosDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(CosDeleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<CosDeleter> = {
+ Ok(oio::OneShotDeleter::new(CosDeleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, CosListers) = {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: CosListers = {
let l = if args.versions() || args.deleted() {
TwoWays::Two(oio::PageLister::new(CosObjectVersionsLister::new(
self.core.clone(),
@@ -483,32 +470,34 @@ impl Service for CosBackend {
)))
};
- Ok((RpList::default(), l))
+ Ok(l)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let (rp, output): (_, ()) = {
- let resp = self.core.cos_copy_object(ctx, from, to, &args).await?;
+ ) -> Result<Self::Copier> {
+ let core = self.core.clone();
+ let ctx = ctx.clone();
+ let from = from.to_string();
+ let to = to.to_string();
+ Ok(oio::OneShotCopier::new(async move {
+ let resp = core.cos_copy_object(&ctx, &from, &to, &args).await?;
let status = resp.status();
match status {
- StatusCode::OK => Ok((RpCopy::default(), ())),
+ StatusCode::OK => Ok(Metadata::default()),
_ => Err(parse_error(resp)),
}
- }?;
-
- Ok((rp, output))
+ }))
}
async fn rename(
diff --git a/core/services/d1/src/backend.rs b/core/services/d1/src/backend.rs
index 27c1e84af..01769dd3a 100644
--- a/core/services/d1/src/backend.rs
+++ b/core/services/d1/src/backend.rs
@@ -294,74 +294,55 @@ impl Service for D1Backend {
}
}
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<D1Reader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(D1Reader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<D1Reader> = {
+ Ok(oio::StreamReader::new(D1Reader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, D1Writer) = {
+ fn write(&self, ctx: &OperationContext, path: &str, _: OpWrite) -> Result<Self::Writer> {
+ let output: D1Writer = {
let p = build_abs_path(&self.root, path);
- Ok((
- RpWrite::new(),
- D1Writer::new(self.core.clone(), ctx.clone(), p),
- ))
+ Ok(D1Writer::new(self.core.clone(), ctx.clone(), p))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<D1Deleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(D1Deleter::new(
- self.core.clone(),
- ctx.clone(),
- self.root.clone(),
- )),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<D1Deleter> = {
+ Ok(oio::OneShotDeleter::new(D1Deleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ self.root.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- _path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _path: &str, _args: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/dashmap/src/backend.rs b/core/services/dashmap/src/backend.rs
index cc7326e65..8d128063b 100644
--- a/core/services/dashmap/src/backend.rs
+++ b/core/services/dashmap/src/backend.rs
@@ -194,73 +194,56 @@ impl Service for DashmapBackend {
}
}
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<DashmapReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(DashmapReader::new(self.clone(), path, args)),
- ))
+ fn read(&self, _ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<DashmapReader> = {
+ Ok(oio::StreamReader::new(DashmapReader::new(
+ self.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, DashmapWriter) = {
+ fn write(&self, _ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: DashmapWriter = {
let p = build_abs_path(&self.root, path);
- Ok((
- RpWrite::new(),
- DashmapWriter::new(self.core.clone(), p, args),
- ))
+ Ok(DashmapWriter::new(self.core.clone(), p, args))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<DashmapDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(DashmapDeleter::new(self.core.clone(), self.root.clone())),
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<DashmapDeleter> = {
+ Ok(oio::OneShotDeleter::new(DashmapDeleter::new(
+ self.core.clone(),
+ self.root.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::HierarchyLister<DashmapLister>) = {
+ fn list(&self, _ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::HierarchyLister<DashmapLister> = {
let lister = DashmapLister::new(self.core.clone(), self.root.clone(), path.to_string());
let lister = oio::HierarchyLister::new(lister, path, args.recursive());
- Ok((RpList::default(), lister))
+ Ok(lister)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/dbfs/src/backend.rs b/core/services/dbfs/src/backend.rs
index 1a631b4db..2bbc91483 100644
--- a/core/services/dbfs/src/backend.rs
+++ b/core/services/dbfs/src/backend.rs
@@ -204,63 +204,45 @@ impl Service for DbfsBackend {
}
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- _path: &str,
- _args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
+ fn read(&self, _ctx: &OperationContext, _path: &str, _args: OpRead) -> Result<Self::Reader> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, oio::OneShotWriter<DbfsWriter>) = {
- Ok((
- RpWrite::default(),
- oio::OneShotWriter::new(DbfsWriter::new(
- self.core.clone(),
- ctx.clone(),
- args,
- path.to_string(),
- )),
- ))
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: oio::OneShotWriter<DbfsWriter> = {
+ Ok(oio::OneShotWriter::new(DbfsWriter::new(
+ self.core.clone(),
+ ctx.clone(),
+ args,
+ path.to_string(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<DbfsDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(DbfsDeleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<DbfsDeleter> = {
+ Ok(oio::OneShotDeleter::new(DbfsDeleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<DbfsLister>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, _args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<DbfsLister> = {
let l = DbfsLister::new(self.core.clone(), ctx.clone(), path.to_string());
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
async fn rename(
@@ -282,14 +264,14 @@ impl Service for DbfsBackend {
}
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/dropbox/src/backend.rs b/core/services/dropbox/src/backend.rs
index 7d9cbef37..3c73a7e0c 100644
--- a/core/services/dropbox/src/backend.rs
+++ b/core/services/dropbox/src/backend.rs
@@ -85,7 +85,7 @@ impl Service for DropboxBackend {
type Writer = oio::OneShotWriter<DropboxWriter>;
type Lister = oio::PageLister<DropboxLister>;
type Deleter = oio::OneShotDeleter<DropboxDeleter>;
- type Copier = ();
+ type Copier = oio::OneShotCopier;
fn info(&self) -> ServiceInfo {
self.core.info.clone()
@@ -159,102 +159,85 @@ impl Service for DropboxBackend {
_ => Err(parse_error(resp)),
}
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<DropboxReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(DropboxReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<DropboxReader> = {
+ Ok(oio::StreamReader::new(DropboxReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, oio::OneShotWriter<DropboxWriter>) = {
- Ok((
- RpWrite::default(),
- oio::OneShotWriter::new(DropboxWriter::new(
- self.core.clone(),
- ctx.clone(),
- args,
- String::from(path),
- )),
- ))
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: oio::OneShotWriter<DropboxWriter> = {
+ Ok(oio::OneShotWriter::new(DropboxWriter::new(
+ self.core.clone(),
+ ctx.clone(),
+ args,
+ String::from(path),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<DropboxDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(DropboxDeleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<DropboxDeleter> = {
+ Ok(oio::OneShotDeleter::new(DropboxDeleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<DropboxLister>) = {
- Ok((
- RpList::default(),
- oio::PageLister::new(DropboxLister::new(
- self.core.clone(),
- ctx.clone(),
- path.to_string(),
- args.recursive(),
- args.limit(),
- )),
- ))
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<DropboxLister> = {
+ Ok(oio::PageLister::new(DropboxLister::new(
+ self.core.clone(),
+ ctx.clone(),
+ path.to_string(),
+ args.recursive(),
+ args.limit(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
_: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let (rp, output): (_, ()) = {
- let resp = self.core.dropbox_copy(ctx, from, to).await?;
-
+ ) -> Result<Self::Copier> {
+ let core = self.core.clone();
+ let ctx = ctx.clone();
+ let from = from.to_string();
+ let to = to.to_string();
+
+ Ok(oio::OneShotCopier::new(async move {
+ let resp = core.dropbox_copy(&ctx, &from, &to).await?;
let status = resp.status();
match status {
- StatusCode::OK => Ok((RpCopy::default(), ())),
+ StatusCode::OK => Ok(Metadata::default()),
_ => {
let err = parse_error(resp);
match err.kind() {
- ErrorKind::NotFound => Ok((RpCopy::default(), ())),
+ ErrorKind::NotFound => Ok(Metadata::default()),
_ => Err(err),
}
}
}
- }?;
-
- Ok((rp, output))
+ }))
}
async fn rename(
diff --git a/core/services/etcd/src/backend.rs b/core/services/etcd/src/backend.rs
index 9b2d697fe..aa8b6d4e1 100644
--- a/core/services/etcd/src/backend.rs
+++ b/core/services/etcd/src/backend.rs
@@ -27,7 +27,7 @@ use super::config::EtcdConfig;
use super::core::EtcdCore;
use super::core::constants::DEFAULT_ETCD_ENDPOINTS;
use super::deleter::EtcdDeleter;
-use super::lister::EtcdLister;
+use super::lister::EtcdLazyLister;
use super::writer::EtcdWriter;
use opendal_core::raw::*;
use opendal_core::*;
@@ -242,7 +242,7 @@ impl oio::StreamRead for EtcdReader {
impl Service for EtcdBackend {
type Reader = oio::StreamReader<EtcdReader>;
type Writer = EtcdWriter;
- type Lister = oio::HierarchyLister<EtcdLister>;
+ type Lister = oio::HierarchyLister<EtcdLazyLister>;
type Deleter = oio::OneShotDeleter<EtcdDeleter>;
type Copier = ();
@@ -306,77 +306,62 @@ impl Service for EtcdBackend {
}
}
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- op: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<EtcdReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(EtcdReader::new(self.clone(), path, op)),
- ))
+ fn read(&self, _ctx: &OperationContext, path: &str, op: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<EtcdReader> = {
+ Ok(oio::StreamReader::new(EtcdReader::new(
+ self.clone(),
+ path,
+ op,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _op: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, EtcdWriter) = {
+ fn write(&self, _ctx: &OperationContext, path: &str, _op: OpWrite) -> Result<Self::Writer> {
+ let output: EtcdWriter = {
let abs_path = build_abs_path(&self.info.root(), path);
let writer = EtcdWriter::new(self.core.clone(), abs_path);
- Ok((RpWrite::new(), writer))
+ Ok(writer)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<EtcdDeleter>) = {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<EtcdDeleter> = {
let deleter = oio::OneShotDeleter::new(EtcdDeleter::new(
self.core.clone(),
self.info.root().to_string(),
));
- Ok((RpDelete::default(), deleter))
+ Ok(deleter)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::HierarchyLister<EtcdLister>) = {
- let lister = EtcdLister::new(
+ fn list(&self, _ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::HierarchyLister<EtcdLazyLister> = {
+ let lister = EtcdLazyLister::new(
self.core.clone(),
self.info.root().to_string(),
path.to_string(),
- )
- .await?;
+ );
let lister = oio::HierarchyLister::new(lister, path, args.recursive());
- Ok((RpList::default(), lister))
+ Ok(lister)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/etcd/src/lister.rs b/core/services/etcd/src/lister.rs
index 667a6e2c5..1c967a72e 100644
--- a/core/services/etcd/src/lister.rs
+++ b/core/services/etcd/src/lister.rs
@@ -77,3 +77,37 @@ impl oio::List for EtcdLister {
Ok(None)
}
}
+
+pub struct EtcdLazyLister {
+ core: Arc<EtcdCore>,
+ root: String,
+ path: String,
+ inner: Option<EtcdLister>,
+}
+
+impl EtcdLazyLister {
+ pub fn new(core: Arc<EtcdCore>, root: String, path: String) -> Self {
+ Self {
+ core,
+ root,
+ path,
+ inner: None,
+ }
+ }
+}
+
+impl oio::List for EtcdLazyLister {
+ async fn next(&mut self) -> Result<Option<Entry>> {
+ if self.inner.is_none() {
+ self.inner = Some(
+ EtcdLister::new(self.core.clone(), self.root.clone(), self.path.clone()).await?,
+ );
+ }
+
+ self.inner
+ .as_mut()
+ .expect("lister must be initialized")
+ .next()
+ .await
+ }
+}
diff --git a/core/services/foundationdb/src/backend.rs b/core/services/foundationdb/src/backend.rs
index 43f8c6748..fbc3a251d 100644
--- a/core/services/foundationdb/src/backend.rs
+++ b/core/services/foundationdb/src/backend.rs
@@ -202,73 +202,53 @@ impl Service for FoundationdbBackend {
}
}
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<FoundationdbReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(FoundationdbReader::new(self.clone(), path, args)),
- ))
+ fn read(&self, _ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<FoundationdbReader> = {
+ Ok(oio::StreamReader::new(FoundationdbReader::new(
+ self.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, FoundationdbWriter) = {
+ fn write(&self, _ctx: &OperationContext, path: &str, _: OpWrite) -> Result<Self::Writer> {
+ let output: FoundationdbWriter = {
let p = build_abs_path(&self.root, path);
- Ok((
- RpWrite::new(),
- FoundationdbWriter::new(self.core.clone(), p),
- ))
+ Ok(FoundationdbWriter::new(self.core.clone(), p))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<FoundationdbDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(FoundationdbDeleter::new(
- self.core.clone(),
- self.root.clone(),
- )),
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<FoundationdbDeleter> = {
+ Ok(oio::OneShotDeleter::new(FoundationdbDeleter::new(
+ self.core.clone(),
+ self.root.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- _path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _path: &str, _args: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/foyer/src/backend.rs b/core/services/foyer/src/backend.rs
index 082f42911..b156808dd 100644
--- a/core/services/foyer/src/backend.rs
+++ b/core/services/foyer/src/backend.rs
@@ -316,67 +316,53 @@ impl Service for FoyerBackend {
}
}
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<FoyerReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(FoyerReader::new(self.clone(), path, args)),
- ))
+ fn read(&self, _ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<FoyerReader> = {
+ Ok(oio::StreamReader::new(FoyerReader::new(
+ self.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, FoyerWriter) = {
+ fn write(&self, _ctx: &OperationContext, path: &str, _: OpWrite) -> Result<Self::Writer> {
+ let output: FoyerWriter = {
let p = build_abs_path(&self.root, path);
- Ok((RpWrite::new(), FoyerWriter::new(self.core.clone(), p)))
+ Ok(FoyerWriter::new(self.core.clone(), p))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<FoyerDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(FoyerDeleter::new(self.core.clone(), self.root.clone())),
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<FoyerDeleter> = {
+ Ok(oio::OneShotDeleter::new(FoyerDeleter::new(
+ self.core.clone(),
+ self.root.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- _path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _path: &str, _args: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/fs/src/backend.rs b/core/services/fs/src/backend.rs
index 934f8cb32..3c321f08c 100644
--- a/core/services/fs/src/backend.rs
+++ b/core/services/fs/src/backend.rs
@@ -216,12 +216,131 @@ impl oio::PositionRead for FsReader {
}
}
+pub struct FsLazyReader {
+ core: Arc<FsCore>,
+ path: String,
+}
+
+impl FsLazyReader {
+ fn new(core: Arc<FsCore>, path: &str) -> Self {
+ Self {
+ core,
+ path: path.to_string(),
+ }
+ }
+
+ async fn reader(&self) -> Result<oio::PositionReader<FsReader>> {
+ let file = self.core.fs_open(&self.path).await?;
+ Ok(oio::PositionReader::new(FsReader::new(
+ self.core.clone(),
+ file,
+ )))
+ }
+}
+
+impl oio::Read for FsLazyReader {
+ async fn open(&self, range: BytesRange) -> Result<(RpRead, Box<dyn oio::ReadStreamDyn>)> {
+ self.reader().await?.open(range).await
+ }
+
+ async fn read(&self, range: BytesRange) -> Result<(RpRead, Buffer)> {
+ self.reader().await?.read(range).await
+ }
+}
+
+pub struct FsLazyWriter {
+ core: Arc<FsCore>,
+ executor: Executor,
+ path: String,
+ op: OpWrite,
+ inner: Option<FsWriters>,
+}
+
+impl FsLazyWriter {
+ fn new(core: Arc<FsCore>, executor: Executor, path: &str, op: OpWrite) -> Self {
+ Self {
+ core,
+ executor,
+ path: path.to_string(),
+ op,
+ inner: None,
+ }
+ }
+
+ async fn inner(&mut self) -> Result<&mut FsWriters> {
+ if self.inner.is_none() {
+ let is_append = self.op.append();
+ let concurrent = self.op.concurrent();
+ let writer = FsWriter::create(self.core.clone(), &self.path, self.op.clone()).await?;
+ let writer = if is_append {
+ FsWriters::One(writer)
+ } else {
+ FsWriters::Two(oio::PositionWriter::new(
+ self.executor.clone(),
+ writer,
+ concurrent,
+ ))
+ };
+
+ self.inner = Some(writer);
+ }
+
+ Ok(self.inner.as_mut().expect("writer must be initialized"))
+ }
+}
+
+impl oio::Write for FsLazyWriter {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ self.inner().await?.write(bs).await
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ self.inner().await?.close().await
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ self.inner().await?.abort().await
+ }
+}
+
+pub struct FsLazyLister {
+ core: Arc<FsCore>,
+ path: String,
+ inner: Option<Option<FsLister<tokio::fs::ReadDir>>>,
+}
+
+impl FsLazyLister {
+ fn new(core: Arc<FsCore>, path: &str) -> Self {
+ Self {
+ core,
+ path: path.to_string(),
+ inner: None,
+ }
+ }
+}
+
+impl oio::List for FsLazyLister {
+ async fn next(&mut self) -> Result<Option<oio::Entry>> {
+ if self.inner.is_none() {
+ self.inner = Some(match self.core.fs_list(&self.path).await? {
+ Some(rd) => Some(FsLister::new(&self.core.root, &self.path, rd)),
+ None => None,
+ });
+ }
+
+ match self.inner.as_mut().expect("lister must be initialized") {
+ Some(lister) => lister.next().await,
+ None => Ok(None),
+ }
+ }
+}
+
impl Service for FsBackend {
- type Reader = oio::PositionReader<FsReader>;
- type Writer = FsWriters;
- type Lister = Option<FsLister<tokio::fs::ReadDir>>;
+ type Reader = FsLazyReader;
+ type Writer = FsLazyWriter;
+ type Lister = FsLazyLister;
type Deleter = oio::OneShotDeleter<FsDeleter>;
- type Copier = ();
+ type Copier = oio::OneShotCopier;
fn info(&self) -> ServiceInfo {
self.core.info.clone()
@@ -246,95 +365,42 @@ impl Service for FsBackend {
Ok(RpStat::new(m))
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::PositionReader<FsReader>) = {
- let file = self.core.fs_open(path).await?;
- Ok((
- RpRead::default(),
- oio::PositionReader::new(FsReader::new(self.core.clone(), file)),
- ))
- }?;
-
- Ok((rp, output))
+ fn read(&self, _ctx: &OperationContext, path: &str, _: OpRead) -> Result<Self::Reader> {
+ Ok(FsLazyReader::new(self.core.clone(), path))
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- op: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, FsWriters) = {
- let is_append = op.append();
- let concurrent = op.concurrent();
-
- let writer = FsWriter::create(self.core.clone(), path, op).await?;
-
- let writer = if is_append {
- FsWriters::One(writer)
- } else {
- FsWriters::Two(oio::PositionWriter::new(
- ctx.executor().clone(),
- writer,
- concurrent,
- ))
- };
-
- Ok((RpWrite::default(), writer))
- }?;
-
- Ok((rp, output))
+ fn write(&self, ctx: &OperationContext, path: &str, op: OpWrite) -> Result<Self::Writer> {
+ Ok(FsLazyWriter::new(
+ self.core.clone(),
+ ctx.executor().clone(),
+ path,
+ op,
+ ))
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<FsDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(FsDeleter::new(self.core.clone())),
- ))
- }?;
-
- Ok((rp, output))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ Ok(oio::OneShotDeleter::new(FsDeleter::new(self.core.clone())))
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, Option<FsLister<tokio::fs::ReadDir>>) = {
- match self.core.fs_list(path).await? {
- Some(f) => {
- let rd = FsLister::new(&self.core.root, path, f);
- Ok((RpList::default(), Some(rd)))
- }
- None => Ok((RpList::default(), None)),
- }
- }?;
-
- Ok((rp, output))
+ fn list(&self, _ctx: &OperationContext, path: &str, _: OpList) -> Result<Self::Lister> {
+ Ok(FsLazyLister::new(self.core.clone(), path))
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
from: &str,
to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let (rp, output): (_, ()) = {
- self.core.fs_copy(from, to).await?;
- Ok((RpCopy::default(), ()))
- }?;
-
- Ok((rp, output))
+ ) -> Result<Self::Copier> {
+ let core = self.core.clone();
+ let from = from.to_string();
+ let to = to.to_string();
+ Ok(oio::OneShotCopier::new(async move {
+ core.fs_copy(&from, &to).await?;
+ Ok(Metadata::default())
+ }))
}
async fn rename(
diff --git a/core/services/ftp/src/backend.rs b/core/services/ftp/src/backend.rs
index 08e63d423..5d0c68061 100644
--- a/core/services/ftp/src/backend.rs
+++ b/core/services/ftp/src/backend.rs
@@ -201,6 +201,125 @@ impl FtpReader {
}
}
+pub struct FtpLazyWriter {
+ core: Arc<FtpCore>,
+ path: String,
+ append: bool,
+ inner: Option<FtpWriter>,
+}
+
+impl FtpLazyWriter {
+ fn new(core: Arc<FtpCore>, path: &str, op: OpWrite) -> Self {
+ Self {
+ core,
+ path: path.to_string(),
+ append: op.append(),
+ inner: None,
+ }
+ }
+
+ async fn inner(&mut self) -> Result<&mut FtpWriter> {
+ if self.inner.is_none() {
+ let parent = get_parent(&self.path);
+ let paths: Vec<&str> = parent.split('/').collect();
+
+ // TODO: we can optimize this by checking dir existence first.
+ let mut ftp_stream = self.core.ftp_connect(Operation::Write).await?;
+ let mut curr_path = String::new();
+
+ for path in paths {
+ if path.is_empty() {
+ continue;
+ }
+ curr_path.push_str(path);
+ curr_path.push('/');
+ match ftp_stream.mkdir(&curr_path).await {
+ // Do nothing if status is FileUnavailable or OK(()) is return.
+ Err(FtpError::UnexpectedResponse(Response {
+ status: Status::FileUnavailable,
+ ..
+ }))
+ | Ok(()) => (),
+ Err(e) => {
+ return Err(format_ftp_error(e));
+ }
+ }
+ }
+
+ let tmp_path = (!self.append).then_some(build_tmp_path_of(&self.path));
+ let w = FtpWriter::new(ftp_stream, self.path.clone(), tmp_path);
+ self.inner = Some(w);
+ }
+
+ Ok(self.inner.as_mut().expect("ftp writer must be initialized"))
+ }
+}
+
+impl oio::Write for FtpLazyWriter {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ self.inner().await?.write(bs).await
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ match &mut self.inner {
+ Some(w) => w.close().await,
+ None => Ok(Metadata::default()),
+ }
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ match &mut self.inner {
+ Some(w) => w.abort().await,
+ None => Err(Error::new(
+ ErrorKind::Unsupported,
+ "FtpWriter doesn't support abort",
+ )),
+ }
+ }
+}
+
+pub struct FtpLazyLister {
+ core: Arc<FtpCore>,
+ path: String,
+ inner: Option<FtpLister>,
+}
+
+impl FtpLazyLister {
+ fn new(core: Arc<FtpCore>, path: &str) -> Self {
+ Self {
+ core,
+ path: path.to_string(),
+ inner: None,
+ }
+ }
+
+ async fn inner(&mut self) -> Result<&mut FtpLister> {
+ if self.inner.is_none() {
+ let mut ftp_stream = self.core.ftp_connect(Operation::List).await?;
+
+ let pathname = if self.path == "/" {
+ None
+ } else {
+ Some(self.path.as_str())
+ };
+ let files = ftp_stream.list(pathname).await.map_err(format_ftp_error)?;
+
+ self.inner = Some(FtpLister::new(
+ if self.path == "/" { "" } else { &self.path },
+ files,
+ ));
+ }
+
+ Ok(self.inner.as_mut().expect("ftp lister must be initialized"))
+ }
+}
+
+impl oio::List for FtpLazyLister {
+ async fn next(&mut self) -> Result<Option<oio::Entry>> {
+ self.inner().await?.next().await
+ }
+}
+
impl oio::StreamRead for FtpReader {
async fn open(&self, range: BytesRange) -> Result<(RpRead, Box<dyn oio::ReadStreamDyn>)> {
let backend = &self.backend;
@@ -216,8 +335,8 @@ impl oio::StreamRead for FtpReader {
impl Service for FtpBackend {
type Reader = oio::StreamReader<FtpReader>;
- type Writer = FtpWriter;
- type Lister = FtpLister;
+ type Writer = FtpLazyWriter;
+ type Lister = FtpLazyLister;
type Deleter = oio::OneShotDeleter<FtpDeleter>;
type Copier = ();
@@ -276,105 +395,41 @@ impl Service for FtpBackend {
Ok(RpStat::new(meta))
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<FtpReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(FtpReader::new(self.clone(), path, args)),
- ))
+ fn read(&self, _ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<FtpReader> = {
+ Ok(oio::StreamReader::new(FtpReader::new(
+ self.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- op: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, FtpWriter) = {
- // Ensure the parent dir exists.
- let parent = get_parent(path);
- let paths: Vec<&str> = parent.split('/').collect();
-
- // TODO: we can optimize this by checking dir existence first.
- let mut ftp_stream = self.core.ftp_connect(Operation::Write).await?;
- let mut curr_path = String::new();
-
- for path in paths {
- if path.is_empty() {
- continue;
- }
- curr_path.push_str(path);
- curr_path.push('/');
- match ftp_stream.mkdir(&curr_path).await {
- // Do nothing if status is FileUnavailable or OK(()) is return.
- Err(FtpError::UnexpectedResponse(Response {
- status: Status::FileUnavailable,
- ..
- }))
- | Ok(()) => (),
- Err(e) => {
- return Err(format_ftp_error(e));
- }
- }
- }
-
- let tmp_path = (!op.append()).then_some(build_tmp_path_of(path));
- let w = FtpWriter::new(ftp_stream, path.to_string(), tmp_path);
-
- Ok((RpWrite::new(), w))
- }?;
-
- Ok((rp, output))
+ fn write(&self, _ctx: &OperationContext, path: &str, op: OpWrite) -> Result<Self::Writer> {
+ Ok(FtpLazyWriter::new(self.core.clone(), path, op))
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<FtpDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(FtpDeleter::new(self.core.clone())),
- ))
- }?;
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<FtpDeleter> =
+ { Ok(oio::OneShotDeleter::new(FtpDeleter::new(self.core.clone()))) }?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, FtpLister) = {
- let mut ftp_stream = self.core.ftp_connect(Operation::List).await?;
-
- let pathname = if path == "/" { None } else { Some(path) };
- let files = ftp_stream.list(pathname).await.map_err(format_ftp_error)?;
-
- Ok((
- RpList::default(),
- FtpLister::new(if path == "/" { "" } else { path }, files),
- ))
- }?;
-
- Ok((rp, output))
+ fn list(&self, _ctx: &OperationContext, path: &str, _: OpList) -> Result<Self::Lister> {
+ Ok(FtpLazyLister::new(self.core.clone(), path))
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/gcs/src/backend.rs b/core/services/gcs/src/backend.rs
index 47212f413..f145b282e 100644
--- a/core/services/gcs/src/backend.rs
+++ b/core/services/gcs/src/backend.rs
@@ -513,62 +513,46 @@ impl Service for GcsBackend {
Ok(RpStat::new(m))
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<GcsReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(GcsReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<GcsReader> = {
+ Ok(oio::StreamReader::new(GcsReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, GcsWriters) = {
+ fn write(&self, ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: GcsWriters = {
let concurrent = args.concurrent();
let w = GcsWriter::new(self.core.clone(), ctx.clone(), path, args);
// Multipart uploads schedule work through the operation executor
// supplied by the caller.
let w = oio::MultipartWriter::new(ctx.executor().clone(), w, concurrent);
- Ok((RpWrite::default(), w))
+ Ok(w)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::BatchDeleter<GcsDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::BatchDeleter::new(
- GcsDeleter::new(self.core.clone(), ctx.clone()),
- self.core.capability.delete_max_size,
- ),
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::BatchDeleter<GcsDeleter> = {
+ Ok(oio::BatchDeleter::new(
+ GcsDeleter::new(self.core.clone(), ctx.clone()),
+ self.core.capability.delete_max_size,
))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<GcsLister>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<GcsLister> = {
let l = GcsLister::new(
self.core.clone(),
ctx.clone(),
@@ -578,26 +562,26 @@ impl Service for GcsBackend {
args.start_after(),
);
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
args: OpCopy,
opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let (rp, output): (_, GcsCopier) = {
+ ) -> Result<Self::Copier> {
+ let output: GcsCopier = {
let copier = GcsCopier::new(self.core.clone(), ctx.clone(), from, to, args, opts);
- Ok((RpCopy::default(), copier))
+ Ok(copier)
}?;
- Ok((rp, output))
+ Ok(output)
}
async fn rename(
diff --git a/core/services/gdrive/src/backend.rs b/core/services/gdrive/src/backend.rs
index 1727e226b..44f200889 100644
--- a/core/services/gdrive/src/backend.rs
+++ b/core/services/gdrive/src/backend.rs
@@ -111,7 +111,7 @@ impl Service for GdriveBackend {
type Writer = oio::OneShotWriter<GdriveWriter>;
type Lister = GdriveListers;
type Deleter = oio::OneShotDeleter<GdriveDeleter>;
- type Copier = ();
+ type Copier = oio::OneShotCopier;
fn info(&self) -> ServiceInfo {
self.core.info.clone()
@@ -207,98 +207,80 @@ impl Service for GdriveBackend {
}
Ok(RpStat::new(meta))
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<GdriveReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(GdriveReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<GdriveReader> = {
+ Ok(oio::StreamReader::new(GdriveReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, oio::OneShotWriter<GdriveWriter>) = {
+ fn write(&self, ctx: &OperationContext, path: &str, _: OpWrite) -> Result<Self::Writer> {
+ let output: oio::OneShotWriter<GdriveWriter> = {
let path = build_abs_path(&self.core.root, path);
- // As Google Drive allows files have the same name, we need to check if the file exists.
- // If the file exists, we will keep its ID and update it.
- let file_id = match self.core.resolve_path(ctx, &path).await? {
- Some(id) => Some(id),
- None => self.core.resolve_path_after_refresh(ctx, &path).await?,
- };
-
- Ok((
- RpWrite::default(),
- oio::OneShotWriter::new(GdriveWriter::new(
- self.core.clone(),
- ctx.clone(),
- path,
- file_id,
- )),
- ))
+ Ok(oio::OneShotWriter::new(GdriveWriter::new(
+ self.core.clone(),
+ ctx.clone(),
+ path,
+ None,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<GdriveDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(GdriveDeleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<GdriveDeleter> = {
+ Ok(oio::OneShotDeleter::new(GdriveDeleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, GdriveListers) = {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: GdriveListers = {
let path = build_abs_path(&self.core.root, path);
if args.recursive() {
// Use optimized batch-query recursive lister
let l = GdriveFlatLister::new(path, self.core.clone(), ctx.clone());
- Ok((RpList::default(), TwoWays::Two(l)))
+ Ok(TwoWays::Two(l))
} else {
// Use standard page-based lister for non-recursive
let l = GdriveLister::new(path, self.core.clone(), ctx.clone());
- Ok((RpList::default(), TwoWays::One(oio::PageLister::new(l))))
+ Ok(TwoWays::One(oio::PageLister::new(l)))
}
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
ctx: &OperationContext,
from: &str,
to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
- let (rp, output): (_, ()) = {
- let source = build_abs_path(&self.core.root, from);
- let target = build_abs_path(&self.core.root, to);
- let resp = self.core.gdrive_copy(ctx, from, to).await?;
+ ) -> Result<Self::Copier> {
+ let core = self.core.clone();
+ let ctx = ctx.clone();
+ let from = from.to_string();
+ let to = to.to_string();
+
+ Ok(oio::OneShotCopier::new(async move {
+ let source = build_abs_path(&core.root, &from);
+ let target = build_abs_path(&core.root, &to);
+ let resp = core.gdrive_copy(&ctx, &from, &to).await?;
match resp.status() {
StatusCode::OK => {
@@ -306,7 +288,7 @@ impl Service for GdriveBackend {
let meta: GdriveFile = serde_json::from_reader(body.reader())
.map_err(new_json_deserialize_error)?;
- let to_path = build_abs_path(&self.core.root, to);
+ let to_path = build_abs_path(&core.root, &to);
let mut metadata = if meta.mime_type == "application/vnd.google-apps.folder" {
Metadata::new(EntryMode::DIR)
} else {
@@ -321,25 +303,25 @@ impl Service for GdriveBackend {
}
if metadata.mode().is_dir() {
- self.core.cache_dir_id(&to_path, &meta.id).await;
+ core.cache_dir_id(&to_path, &meta.id).await;
} else {
- self.core.cache_file_id(&to_path, &meta.id).await;
+ core.cache_file_id(&to_path, &meta.id).await;
}
- self.core.record_recent_upsert(&to_path, metadata).await;
+ core.record_recent_upsert(&to_path, metadata).await;
- Ok((RpCopy::default(), ()))
+ Ok(Metadata::default())
}
StatusCode::NOT_FOUND => {
- self.core.refresh_path(&source).await;
- self.core.refresh_path(&target).await;
- let resp = self.core.gdrive_copy(ctx, from, to).await?;
+ core.refresh_path(&source).await;
+ core.refresh_path(&target).await;
+ let resp = core.gdrive_copy(&ctx, &from, &to).await?;
match resp.status() {
StatusCode::OK => {
let body = resp.into_body();
let meta: GdriveFile = serde_json::from_reader(body.reader())
.map_err(new_json_deserialize_error)?;
- let to_path = build_abs_path(&self.core.root, to);
+ let to_path = build_abs_path(&core.root, &to);
let mut metadata =
if meta.mime_type == "application/vnd.google-apps.folder" {
Metadata::new(EntryMode::DIR)
@@ -356,22 +338,20 @@ impl Service for GdriveBackend {
}
if metadata.mode().is_dir() {
- self.core.cache_dir_id(&to_path, &meta.id).await;
+ core.cache_dir_id(&to_path, &meta.id).await;
} else {
- self.core.cache_file_id(&to_path, &meta.id).await;
+ core.cache_file_id(&to_path, &meta.id).await;
}
- self.core.record_recent_upsert(&to_path, metadata).await;
+ core.record_recent_upsert(&to_path, metadata).await;
- Ok((RpCopy::default(), ()))
+ Ok(Metadata::default())
}
_ => Err(parse_error(resp)),
}
}
_ => Err(parse_error(resp)),
}
- }?;
-
- Ok((rp, output))
+ }))
}
async fn rename(
diff --git a/core/services/gdrive/src/writer.rs b/core/services/gdrive/src/writer.rs
index fb6a0dc10..24f52b8eb 100644
--- a/core/services/gdrive/src/writer.rs
+++ b/core/services/gdrive/src/writer.rs
@@ -56,7 +56,17 @@ impl oio::OneShotWrite for GdriveWriter {
async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
let size = bs.len();
- let mut current_file_id = self.file_id.clone();
+ let mut current_file_id = match &self.file_id {
+ Some(file_id) => Some(file_id.clone()),
+ None => match self.core.resolve_path(&self.ctx, &self.path).await? {
+ Some(id) => Some(id),
+ None => {
+ self.core
+ .resolve_path_after_refresh(&self.ctx, &self.path)
+ .await?
+ }
+ },
+ };
let mut retried = false;
loop {
diff --git a/core/services/ghac/src/backend.rs b/core/services/ghac/src/backend.rs
index 803ffbb8f..b8ade333e 100644
--- a/core/services/ghac/src/backend.rs
+++ b/core/services/ghac/src/backend.rs
@@ -29,7 +29,7 @@ use super::config::GhacConfig;
use super::core::GhacCore;
use super::core::*;
use super::error::parse_error;
-use super::writer::GhacWriter;
+use super::writer::GhacLazyWriter;
use opendal_core::raw::*;
use opendal_core::*;
@@ -244,7 +244,7 @@ impl oio::StreamRead for GhacReader {
impl Service for GhacBackend {
type Reader = oio::StreamReader<GhacReader>;
- type Writer = GhacWriter;
+ type Writer = GhacLazyWriter;
type Lister = ();
type Deleter = ();
type Copier = ();
@@ -286,73 +286,50 @@ impl Service for GhacBackend {
_ => Err(parse_error(resp)),
}
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<GhacReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(GhacReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<GhacReader> = {
+ Ok(oio::StreamReader::new(GhacReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, GhacWriter) = {
- let url = self.core.ghac_get_upload_url(ctx, path).await?;
-
- Ok((
- RpWrite::default(),
- GhacWriter::new(
- self.core.clone(),
- ctx.clone(),
- ctx.executor().clone(),
- path.to_string(),
- url,
- )?,
- ))
- }?;
-
- Ok((rp, output))
+ fn write(&self, ctx: &OperationContext, path: &str, _: OpWrite) -> Result<Self::Writer> {
+ Ok(GhacLazyWriter::new(
+ self.core.clone(),
+ ctx.clone(),
+ ctx.executor().clone(),
+ path.to_string(),
+ ))
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- _path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _path: &str, _args: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/ghac/src/writer.rs b/core/services/ghac/src/writer.rs
index 7b9fc566f..f649ffd20 100644
--- a/core/services/ghac/src/writer.rs
+++ b/core/services/ghac/src/writer.rs
@@ -144,6 +144,60 @@ impl GhacWriter {
}
}
+pub struct GhacLazyWriter {
+ core: Arc<GhacCore>,
+ ctx: OperationContext,
+ executor: Executor,
+ path: String,
+ inner: Option<GhacWriter>,
+}
+
+impl GhacLazyWriter {
+ pub fn new(
+ core: Arc<GhacCore>,
+ ctx: OperationContext,
+ executor: Executor,
+ path: String,
+ ) -> Self {
+ Self {
+ core,
+ ctx,
+ executor,
+ path,
+ inner: None,
+ }
+ }
+
+ async fn inner(&mut self) -> Result<&mut GhacWriter> {
+ if self.inner.is_none() {
+ let url = self.core.ghac_get_upload_url(&self.ctx, &self.path).await?;
+ self.inner = Some(GhacWriter::new(
+ self.core.clone(),
+ self.ctx.clone(),
+ self.executor.clone(),
+ self.path.clone(),
+ url,
+ )?);
+ }
+
+ Ok(self.inner.as_mut().expect("writer must be initialized"))
+ }
+}
+
+impl oio::Write for GhacLazyWriter {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ self.inner().await?.write(bs).await
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ self.inner().await?.abort().await
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ self.inner().await?.close().await
+ }
+}
+
impl oio::Write for GhacWriter {
async fn write(&mut self, bs: Buffer) -> Result<()> {
self.0.write(bs).await
diff --git a/core/services/github/src/backend.rs b/core/services/github/src/backend.rs
index 3024543fb..3fbd935c6 100644
--- a/core/services/github/src/backend.rs
+++ b/core/services/github/src/backend.rs
@@ -256,72 +256,59 @@ impl Service for GithubBackend {
_ => Err(parse_error(resp)),
}
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<GithubReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(GithubReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<GithubReader> = {
+ Ok(oio::StreamReader::new(GithubReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- _args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, GithubWriters) = {
+ fn write(&self, ctx: &OperationContext, path: &str, _args: OpWrite) -> Result<Self::Writer> {
+ let output: GithubWriters = {
let writer = GithubWriter::new(self.core.clone(), ctx.clone(), path.to_string());
let w = oio::OneShotWriter::new(writer);
- Ok((RpWrite::default(), w))
+ Ok(w)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<GithubDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(GithubDeleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<GithubDeleter> = {
+ Ok(oio::OneShotDeleter::new(GithubDeleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<GithubLister>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<GithubLister> = {
let l = GithubLister::new(self.core.clone(), ctx.clone(), path, args.recursive());
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/goosefs/src/backend.rs b/core/services/goosefs/src/backend.rs
index 822162c63..d590e40ff 100644
--- a/core/services/goosefs/src/backend.rs
+++ b/core/services/goosefs/src/backend.rs
@@ -369,69 +369,54 @@ impl Service for GoosefsBackend {
let file_info = self.core.get_status(path).await?;
Ok(RpStat::new(self.core.file_info_to_metadata(&file_info)))
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<GoosefsReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(GoosefsReader::new(self.clone(), path, args)),
- ))
+ fn read(&self, _ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<GoosefsReader> = {
+ Ok(oio::StreamReader::new(GoosefsReader::new(
+ self.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, GoosefsWriters) = {
+ fn write(&self, _ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ let output: GoosefsWriters = {
let w = GoosefsWriter::new(self.core.clone(), args.clone(), path.to_string());
- Ok((RpWrite::default(), w))
+ Ok(w)
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<GoosefsDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(GoosefsDeleter::new(self.core.clone())),
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<GoosefsDeleter> = {
+ Ok(oio::OneShotDeleter::new(GoosefsDeleter::new(
+ self.core.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<GoosefsLister>) = {
+ fn list(&self, _ctx: &OperationContext, path: &str, _args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<GoosefsLister> = {
let l = GoosefsLister::new(self.core.clone(), path);
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/gridfs/src/backend.rs b/core/services/gridfs/src/backend.rs
index 52bfefa8d..8de3e371b 100644
--- a/core/services/gridfs/src/backend.rs
+++ b/core/services/gridfs/src/backend.rs
@@ -266,67 +266,53 @@ impl Service for GridfsBackend {
}
}
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<GridfsReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(GridfsReader::new(self.clone(), path, args)),
- ))
+ fn read(&self, _ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<GridfsReader> = {
+ Ok(oio::StreamReader::new(GridfsReader::new(
+ self.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, GridfsWriter) = {
+ fn write(&self, _ctx: &OperationContext, path: &str, _: OpWrite) -> Result<Self::Writer> {
+ let output: GridfsWriter = {
let p = build_abs_path(&self.root, path);
- Ok((RpWrite::new(), GridfsWriter::new(self.core.clone(), p)))
+ Ok(GridfsWriter::new(self.core.clone(), p))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<GridfsDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(GridfsDeleter::new(self.core.clone(), self.root.clone())),
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<GridfsDeleter> = {
+ Ok(oio::OneShotDeleter::new(GridfsDeleter::new(
+ self.core.clone(),
+ self.root.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- _path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _path: &str, _args: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/hdfs-native/src/backend.rs b/core/services/hdfs-native/src/backend.rs
index d56bdafd2..3c5249035 100644
--- a/core/services/hdfs-native/src/backend.rs
+++ b/core/services/hdfs-native/src/backend.rs
@@ -198,10 +198,117 @@ impl oio::PositionRead for HdfsNativeReader {
}
}
+pub struct HdfsNativeLazyReader {
+ core: Arc<HdfsNativeCore>,
+ path: String,
+}
+
+impl HdfsNativeLazyReader {
+ fn new(core: Arc<HdfsNativeCore>, path: &str) -> Self {
+ Self {
+ core,
+ path: path.to_string(),
+ }
+ }
+
+ async fn reader(&self) -> Result<oio::PositionReader<HdfsNativeReader>> {
+ let file = self.core.hdfs_open(&self.path).await?;
+ Ok(oio::PositionReader::new(HdfsNativeReader::new(file)))
+ }
+}
+
+impl oio::Read for HdfsNativeLazyReader {
+ async fn open(&self, range: BytesRange) -> Result<(RpRead, Box<dyn oio::ReadStreamDyn>)> {
+ self.reader().await?.open(range).await
+ }
+
+ async fn read(&self, range: BytesRange) -> Result<(RpRead, Buffer)> {
+ self.reader().await?.read(range).await
+ }
+}
+
+pub struct HdfsNativeLazyWriter {
+ core: Arc<HdfsNativeCore>,
+ path: String,
+ args: OpWrite,
+ inner: Option<HdfsNativeWriter>,
+}
+
+impl HdfsNativeLazyWriter {
+ fn new(core: Arc<HdfsNativeCore>, path: &str, args: OpWrite) -> Self {
+ Self {
+ core,
+ path: path.to_string(),
+ args,
+ inner: None,
+ }
+ }
+
+ async fn inner(&mut self) -> Result<&mut HdfsNativeWriter> {
+ if self.inner.is_none() {
+ let (f, initial_size) = self.core.hdfs_write(&self.path, &self.args).await?;
+ self.inner = Some(HdfsNativeWriter::new(f, initial_size));
+ }
+
+ Ok(self.inner.as_mut().expect("writer must be initialized"))
+ }
+}
+
+impl oio::Write for HdfsNativeLazyWriter {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ self.inner().await?.write(bs).await
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ self.inner().await?.close().await
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ self.inner().await?.abort().await
+ }
+}
+
+pub struct HdfsNativeLazyLister {
+ core: Arc<HdfsNativeCore>,
+ path: String,
+ inner: Option<Option<HdfsNativeLister>>,
+}
+
+impl HdfsNativeLazyLister {
+ fn new(core: Arc<HdfsNativeCore>, path: &str) -> Self {
+ Self {
+ core,
+ path: path.to_string(),
+ inner: None,
+ }
+ }
+}
+
+impl oio::List for HdfsNativeLazyLister {
+ async fn next(&mut self) -> Result<Option<oio::Entry>> {
+ if self.inner.is_none() {
+ self.inner = Some(match self.core.hdfs_list(&self.path).await? {
+ Some((p, current_path)) => Some(HdfsNativeLister::new(
+ &self.core.root,
+ &self.core.client,
+ &p,
+ current_path,
+ )),
+ None => None,
+ });
+ }
+
+ match self.inner.as_mut().expect("lister must be initialized") {
+ Some(lister) => lister.next().await,
+ None => Ok(None),
+ }
+ }
+}
+
impl Service for HdfsNativeBackend {
- type Reader = oio::PositionReader<HdfsNativeReader>;
- type Writer = HdfsNativeWriter;
- type Lister = Option<HdfsNativeLister>;
+ type Reader = HdfsNativeLazyReader;
+ type Writer = HdfsNativeLazyWriter;
+ type Lister = HdfsNativeLazyLister;
type Deleter = oio::OneShotDeleter<HdfsNativeDeleter>;
type Copier = ();
@@ -227,81 +334,36 @@ impl Service for HdfsNativeBackend {
let m = self.core.hdfs_stat(path).await?;
Ok(RpStat::new(m))
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::PositionReader<HdfsNativeReader>) = {
- let file = self.core.hdfs_open(path).await?;
- Ok((
- RpRead::default(),
- oio::PositionReader::new(HdfsNativeReader::new(file)),
- ))
- }?;
-
- Ok((rp, output))
+ fn read(&self, _ctx: &OperationContext, path: &str, _: OpRead) -> Result<Self::Reader> {
+ Ok(HdfsNativeLazyReader::new(self.core.clone(), path))
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, HdfsNativeWriter) = {
- let (f, initial_size) = self.core.hdfs_write(path, &args).await?;
-
- Ok((RpWrite::new(), HdfsNativeWriter::new(f, initial_size)))
- }?;
-
- Ok((rp, output))
+ fn write(&self, _ctx: &OperationContext, path: &str, args: OpWrite) -> Result<Self::Writer> {
+ Ok(HdfsNativeLazyWriter::new(self.core.clone(), path, args))
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<HdfsNativeDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(HdfsNativeDeleter::new(Arc::clone(&self.core))),
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<HdfsNativeDeleter> = {
+ Ok(oio::OneShotDeleter::new(HdfsNativeDeleter::new(
+ Arc::clone(&self.core),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, Option<HdfsNativeLister>) = {
- match self.core.hdfs_list(path).await? {
- Some((p, current_path)) => Ok((
- RpList::default(),
- Some(HdfsNativeLister::new(
- &self.core.root,
- &self.core.client,
- &p,
- current_path,
- )),
- )),
- None => Ok((RpList::default(), None)),
- }
- }?;
-
- Ok((rp, output))
+ fn list(&self, _ctx: &OperationContext, path: &str, _args: OpList) -> Result<Self::Lister> {
+ Ok(HdfsNativeLazyLister::new(self.core.clone(), path))
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/hdfs/src/backend.rs b/core/services/hdfs/src/backend.rs
index a46a0051c..d5b9fbbb2 100644
--- a/core/services/hdfs/src/backend.rs
+++ b/core/services/hdfs/src/backend.rs
@@ -219,9 +219,59 @@ impl oio::StreamRead for HdfsReader {
}
}
+pub struct HdfsLazyWriter {
+ core: Arc<HdfsCore>,
+ path: String,
+ op: OpWrite,
+ inner: Option<HdfsWriter<hdrs::AsyncFile>>,
+}
+
+impl HdfsLazyWriter {
+ fn new(core: Arc<HdfsCore>, path: &str, op: OpWrite) -> Self {
+ Self {
+ core,
+ path: path.to_string(),
+ op,
+ inner: None,
+ }
+ }
+
+ async fn inner(&mut self) -> Result<&mut HdfsWriter<hdrs::AsyncFile>> {
+ if self.inner.is_none() {
+ let (target_path, tmp_path, f, target_exists, initial_size) =
+ self.core.hdfs_write(&self.path, &self.op).await?;
+
+ self.inner = Some(HdfsWriter::new(
+ target_path,
+ tmp_path,
+ f,
+ Arc::clone(&self.core.client),
+ target_exists,
+ initial_size,
+ ));
+ }
+
+ Ok(self.inner.as_mut().expect("writer must be initialized"))
+ }
+}
+
+impl oio::Write for HdfsLazyWriter {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ self.inner().await?.write(bs).await
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ self.inner().await?.close().await
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ self.inner().await?.abort().await
+ }
+}
+
impl Service for HdfsBackend {
type Reader = oio::StreamReader<HdfsReader>;
- type Writer = HdfsWriter<hdrs::AsyncFile>;
+ type Writer = HdfsLazyWriter;
type Lister = Option<HdfsLister>;
type Deleter = oio::OneShotDeleter<HdfsDeleter>;
type Copier = ();
@@ -248,86 +298,54 @@ impl Service for HdfsBackend {
let m = self.core.hdfs_stat(path)?;
Ok(RpStat::new(m))
}
- async fn read(
- &self,
- _ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<HdfsReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(HdfsReader::new(self.clone(), path, args)),
- ))
+ fn read(&self, _ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<HdfsReader> = {
+ Ok(oio::StreamReader::new(HdfsReader::new(
+ self.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- path: &str,
- op: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, HdfsWriter<hdrs::AsyncFile>) = {
- let (target_path, tmp_path, f, target_exists, initial_size) =
- self.core.hdfs_write(path, &op).await?;
-
- Ok((
- RpWrite::new(),
- HdfsWriter::new(
- target_path,
- tmp_path,
- f,
- Arc::clone(&self.core.client),
- target_exists,
- initial_size,
- ),
- ))
- }?;
-
- Ok((rp, output))
+ fn write(&self, _ctx: &OperationContext, path: &str, op: OpWrite) -> Result<Self::Writer> {
+ Ok(HdfsLazyWriter::new(self.core.clone(), path, op))
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<HdfsDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(HdfsDeleter::new(Arc::clone(&self.core))),
- ))
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<HdfsDeleter> = {
+ Ok(oio::OneShotDeleter::new(HdfsDeleter::new(Arc::clone(
+ &self.core,
+ ))))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- path: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, Option<HdfsLister>) = {
+ fn list(&self, _ctx: &OperationContext, path: &str, _: OpList) -> Result<Self::Lister> {
+ let output: Option<HdfsLister> = {
match self.core.hdfs_list(path)? {
Some(f) => {
let rd = HdfsLister::new(&self.core.root, f, path);
- Ok((RpList::default(), Some(rd)))
+ Ok(Some(rd))
}
- None => Ok((RpList::default(), None)),
+ None => Ok(None),
}
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/hf/src/backend.rs b/core/services/hf/src/backend.rs
index 2ed3206c0..16ab0c799 100644
--- a/core/services/hf/src/backend.rs
+++ b/core/services/hf/src/backend.rs
@@ -28,7 +28,7 @@ use super::deleter::HfDeleter;
use super::lister::HfLister;
use super::reader::HfReadStream;
use super::uri::{HfRepo, HfRepoType};
-use super::writer::HfWriter;
+use super::writer::HfLazyWriter;
use opendal_core::raw::*;
use opendal_core::*;
@@ -283,7 +283,7 @@ impl oio::StreamRead for HfReader {
impl Service for HfBackend {
type Reader = oio::StreamReader<HfReader>;
- type Writer = HfWriter;
+ type Writer = HfLazyWriter;
type Lister = oio::PageLister<HfLister>;
type Deleter = oio::BatchDeleter<HfDeleter>;
type Copier = ();
@@ -322,77 +322,59 @@ impl Service for HfBackend {
let info = self.core.path_info(ctx, path).await?;
Ok(RpStat::new(info.metadata()?))
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<HfReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(HfReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<HfReader> = {
+ Ok(oio::StreamReader::new(HfReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<HfLister>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, args: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<HfLister> = {
let lister = HfLister::new(
self.core.clone(),
ctx.clone(),
path.to_string(),
args.recursive(),
);
- Ok((RpList::default(), oio::PageLister::new(lister)))
+ Ok(oio::PageLister::new(lister))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- _args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, HfWriter) = {
- let writer =
- HfWriter::try_new(self.core.clone(), ctx.clone(), path.to_string()).await?;
- Ok((RpWrite::default(), writer))
- }?;
-
- Ok((rp, output))
+ fn write(&self, ctx: &OperationContext, path: &str, _args: OpWrite) -> Result<Self::Writer> {
+ Ok(HfLazyWriter::new(
+ self.core.clone(),
+ ctx.clone(),
+ path.to_string(),
+ ))
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::BatchDeleter<HfDeleter>) = {
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::BatchDeleter<HfDeleter> = {
let deleter = HfDeleter::new(self.core.clone(), ctx.clone());
let max_batch_size = self.core.capability.delete_max_size;
- Ok((
- RpDelete::default(),
- oio::BatchDeleter::new(deleter, max_batch_size),
- ))
+ Ok(oio::BatchDeleter::new(deleter, max_batch_size))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/hf/src/writer.rs b/core/services/hf/src/writer.rs
index f79e58a76..fce8b2bc3 100644
--- a/core/services/hf/src/writer.rs
+++ b/core/services/hf/src/writer.rs
@@ -35,6 +35,52 @@ pub struct HfWriter {
finished_info: Option<XetFileInfo>,
}
+/// Lazily creates [`HfWriter`] so `Service::write` can stay synchronous.
+pub struct HfLazyWriter {
+ core: Arc<HfCore>,
+ ctx: OperationContext,
+ path: String,
+ inner: Option<HfWriter>,
+}
+
+impl HfLazyWriter {
+ pub fn new(core: Arc<HfCore>, ctx: OperationContext, path: String) -> Self {
+ Self {
+ core,
+ ctx,
+ path,
+ inner: None,
+ }
+ }
+
+ async fn inner(&mut self) -> Result<&mut HfWriter> {
+ if self.inner.is_none() {
+ let writer =
+ HfWriter::try_new(self.core.clone(), self.ctx.clone(), self.path.clone()).await?;
+ self.inner = Some(writer);
+ }
+
+ Ok(self.inner.as_mut().expect("hf writer must be initialized"))
+ }
+}
+
+impl oio::Write for HfLazyWriter {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ self.inner().await?.write(bs).await
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ self.inner().await?.close().await
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ match &mut self.inner {
+ Some(w) => w.abort().await,
+ None => Ok(()),
+ }
+ }
+}
+
impl HfWriter {
/// Create a new writer for the given path.
///
diff --git a/core/services/http/src/backend.rs b/core/services/http/src/backend.rs
index 69d322626..c6cfe746f 100644
--- a/core/services/http/src/backend.rs
+++ b/core/services/http/src/backend.rs
@@ -260,61 +260,48 @@ impl Service for HttpBackend {
_ => Err(parse_error(resp)),
}
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<HttpReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(HttpReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<HttpReader> = {
+ Ok(oio::StreamReader::new(HttpReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- _path: &str,
- _args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _ctx: &OperationContext, _path: &str, _args: OpWrite) -> Result<Self::Writer> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- _ctx: &OperationContext,
- _path: &str,
- _args: OpList,
- ) -> Result<(RpList, Self::Lister)> {
+ fn list(&self, _ctx: &OperationContext, _path: &str, _args: OpList) -> Result<Self::Lister> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/ipfs/src/backend.rs b/core/services/ipfs/src/backend.rs
index 2c31b7b22..cfc8b4d71 100644
--- a/core/services/ipfs/src/backend.rs
+++ b/core/services/ipfs/src/backend.rs
@@ -215,63 +215,50 @@ impl Service for IpfsBackend {
Ok(RpStat::new(metadata))
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<IpfsReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(IpfsReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<IpfsReader> = {
+ Ok(oio::StreamReader::new(IpfsReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- _ctx: &OperationContext,
- _path: &str,
- _args: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
+ fn write(&self, _ctx: &OperationContext, _path: &str, _args: OpWrite) -> Result<Self::Writer> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn delete(&self, _ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
+ fn delete(&self, _ctx: &OperationContext) -> Result<Self::Deleter> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
))
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<DirStream>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, _: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<DirStream> = {
let l = DirStream::new(self.core.clone(), ctx.clone(), path);
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
&self,
_ctx: &OperationContext,
_from: &str,
_to: &str,
_args: OpCopy,
_opts: OpCopier,
- ) -> Result<(RpCopy, Self::Copier)> {
+ ) -> Result<Self::Copier> {
Err(Error::new(
ErrorKind::Unsupported,
"operation is not supported",
diff --git a/core/services/ipmfs/src/backend.rs b/core/services/ipmfs/src/backend.rs
index 74853e3e0..9e9b6fd49 100644
--- a/core/services/ipmfs/src/backend.rs
+++ b/core/services/ipmfs/src/backend.rs
@@ -137,75 +137,59 @@ impl Service for IpmfsBackend {
_ => Err(parse_error(resp)),
}
}
- async fn read(
- &self,
- ctx: &OperationContext,
- path: &str,
- args: OpRead,
- ) -> Result<(RpRead, Self::Reader)> {
- let (rp, output): (_, oio::StreamReader<IpmfsReader>) = {
- Ok((
- RpRead::default(),
- oio::StreamReader::new(IpmfsReader::new(self.clone(), ctx.clone(), path, args)),
- ))
+ fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
+ let output: oio::StreamReader<IpmfsReader> = {
+ Ok(oio::StreamReader::new(IpmfsReader::new(
+ self.clone(),
+ ctx.clone(),
+ path,
+ args,
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn write(
- &self,
- ctx: &OperationContext,
- path: &str,
- _: OpWrite,
- ) -> Result<(RpWrite, Self::Writer)> {
- let (rp, output): (_, oio::OneShotWriter<IpmfsWriter>) = {
- Ok((
- RpWrite::default(),
- oio::OneShotWriter::new(IpmfsWriter::new(
- self.core.clone(),
- ctx.clone(),
- path.to_string(),
- )),
- ))
+ fn write(&self, ctx: &OperationContext, path: &str, _: OpWrite) -> Result<Self::Writer> {
+ let output: oio::OneShotWriter<IpmfsWriter> = {
+ Ok(oio::OneShotWriter::new(IpmfsWriter::new(
+ self.core.clone(),
+ ctx.clone(),
+ path.to_string(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn delete(&self, ctx: &OperationContext) -> Result<(RpDelete, Self::Deleter)> {
- let (rp, output): (_, oio::OneShotDeleter<IpmfsDeleter>) = {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(IpmfsDeleter::new(self.core.clone(), ctx.clone())),
- ))
+ fn delete(&self, ctx: &OperationContext) -> Result<Self::Deleter> {
+ let output: oio::OneShotDeleter<IpmfsDeleter> = {
+ Ok(oio::OneShotDeleter::new(IpmfsDeleter::new(
+ self.core.clone(),
+ ctx.clone(),
+ )))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn list(
- &self,
- ctx: &OperationContext,
- path: &str,
- _: OpList,
- ) -> Result<(RpList, Self::Lister)> {
- let (rp, output): (_, oio::PageLister<IpmfsLister>) = {
+ fn list(&self, ctx: &OperationContext, path: &str, _: OpList) -> Result<Self::Lister> {
+ let output: oio::PageLister<IpmfsLister> = {
let l = IpmfsLister::new(self.core.clone(), ctx.clone(), &self.core.root, path);
- Ok((RpList::default(), oio::PageLister::new(l)))
+ Ok(oio::PageLister::new(l))
}?;
- Ok((rp, output))
+ Ok(output)
}
- async fn copy(
+ fn copy(
... 4824 lines suppressed ...