You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/08/25 09:45:17 UTC
(opendal) branch xuanwo/writer-copy-from-s3 created (now 1be31585c)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a change to branch xuanwo/writer-copy-from-s3
in repository https://gitbox.apache.org/repos/asf/opendal.git
at 1be31585c fix(services/s3): sign writer part copy with IAM
This branch includes the following new commits:
new f2ae14777 feat(services/s3): support Writer::copy_from
new 4eefc54f3 Merge remote-tracking branch 'origin/main' into xuanwo/writer-copy-from-s3
new 98a860057 test: make multipart copy test deterministic
new 0f5d5593a refactor: use Unsupported for writer copy fallback
new 03e22cecb refactor: gate writer copy with capability
new 25a753e8b refactor: align writer copy state naming
new f1c17ec11 Merge remote-tracking branch 'origin/main' into xuanwo/writer-copy-from-s3
new 1be31585c fix(services/s3): sign writer part copy with IAM
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opendal) 06/08: refactor: align writer copy state naming
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/writer-copy-from-s3
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 25a753e8b67d7d4ae4a284836b0a11f6f3df21fa
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Aug 25 17:38:27 2026 +0800
refactor: align writer copy state naming
---
core/core/src/types/context/write.rs | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/core/core/src/types/context/write.rs b/core/core/src/types/context/write.rs
index bb05e6b55..d7de75fd2 100644
--- a/core/core/src/types/context/write.rs
+++ b/core/core/src/types/context/write.rs
@@ -123,8 +123,8 @@ pub struct WriteGenerator<W> {
/// If `exact` is true, the size of the data written to the underlying storage is
/// exactly `chunk_size` bytes.
exact: bool,
- /// Whether the composed writer supports native source range copy.
- native_copy: bool,
+ /// Whether the composed writer can copy source ranges directly.
+ can_copy_from: bool,
buffer: oio::QueueBuf,
}
@@ -132,7 +132,7 @@ impl WriteGenerator<oio::Writer> {
/// Create a new exact buf writer.
pub fn create(ctx: Arc<WriteContext>) -> Result<Self> {
let (chunk_size, exact) = ctx.calculate_chunk_size();
- let native_copy = ctx.service().capability().write_can_copy_from && !ctx.args().append();
+ let can_copy_from = ctx.service().capability().write_can_copy_from && !ctx.args().append();
let w = ctx.srv.write(&ctx.ctx, ctx.path(), ctx.args().clone())?;
Ok(Self {
@@ -140,7 +140,7 @@ impl WriteGenerator<oio::Writer> {
ctx: Some(ctx),
chunk_size,
exact,
- native_copy,
+ can_copy_from,
buffer: oio::QueueBuf::new(),
})
}
@@ -153,7 +153,7 @@ impl WriteGenerator<oio::Writer> {
ctx: None,
chunk_size,
exact,
- native_copy: false,
+ can_copy_from: false,
buffer: oio::QueueBuf::new(),
}
}
@@ -278,7 +278,7 @@ impl WriteGenerator<oio::Writer> {
return Ok(());
}
- let Some(chunk_size) = self.chunk_size.filter(|_| self.native_copy) else {
+ let Some(chunk_size) = self.chunk_size.filter(|_| self.can_copy_from) else {
return self
.stream_range(
path,
(opendal) 04/08: refactor: use Unsupported for writer copy fallback
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/writer-copy-from-s3
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 0f5d5593af7216f66a7151bd2b5e7de5d02a4ff5
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Aug 25 16:35:27 2026 +0800
refactor: use Unsupported for writer copy fallback
---
core/core/src/docs/rfcs/8109_writer_copy_from.md | 26 ++++-----
core/core/src/layers/complete.rs | 65 +++++++++++++++++-----
core/core/src/layers/error_context.rs | 13 ++---
core/core/src/raw/enum_utils.rs | 7 +--
core/core/src/raw/oio/write/api.rs | 35 +++++-------
core/core/src/raw/oio/write/mod.rs | 1 -
core/core/src/raw/oio/write/multipart_write.rs | 68 +++++++++++++++---------
core/core/src/types/context/write.rs | 7 +--
core/services/s3/src/writer.rs | 2 +-
9 files changed, 125 insertions(+), 99 deletions(-)
diff --git a/core/core/src/docs/rfcs/8109_writer_copy_from.md b/core/core/src/docs/rfcs/8109_writer_copy_from.md
index a9c32eb7b..1194e9702 100644
--- a/core/core/src/docs/rfcs/8109_writer_copy_from.md
+++ b/core/core/src/docs/rfcs/8109_writer_copy_from.md
@@ -220,17 +220,11 @@ the normal reader and writer paths.
Extend `oio::Write` with an optional internal operation that accepts a source
path, `OpRead`, and exactly one bounded physical `BytesRange`. Following
RFC-7660, `OpRead` contains the source conditions but not the range. The raw
-operation rejects ranges that are not absolute and bounded. Its result
-distinguishes:
-
-- `Accepted`;
-- `Unsupported` with no writer mutation;
-- an execution error.
-
-Only the no-mutation result permits streaming fallback. An execution failure
-must not trigger fallback because the native operation may already have
-accepted a part. The writer enters its normal error state and remains
-abortable.
+operation rejects ranges that are not absolute and bounded. `Ok(())` means the
+writer accepted the range. `ErrorKind::Unsupported` means the call did not
+mutate the writer and permits streaming fallback. Every execution failure uses
+a different error kind because the native operation may already have accepted
+a part. The writer enters its normal error state and remains abortable.
Multipart and block helpers schedule local uploads and remote copies in one
ordered part queue under the same upload ID, part or block numbers, completion,
@@ -260,11 +254,11 @@ This proposal adds methods to the asynchronous `Writer` and an optional raw
writer method. It does not change the blocking API, `write`,
`write_from(Buf)`, `Operator::copy`, or `Copier`.
-The raw method defaults to the no-mutation unsupported result, so existing
-services remain correct through read-to-write fallback. Layers that affect
-paths, routing, or bytes must intercept or reject the native operation before
-any service enables its fast path. Services can then add native support
-incrementally.
+The raw method defaults to an `ErrorKind::Unsupported` error without mutation,
+so existing services remain correct through read-to-write fallback. Layers
+that affect paths, routing, or bytes must intercept or reject the native
+operation before any service enables its fast path. Services can then add
+native support incrementally.
Append writers accept `copy_from`. A service writer without a native append
transaction returns `Unsupported` without mutation, and the public writer uses
diff --git a/core/core/src/layers/complete.rs b/core/core/src/layers/complete.rs
index 0d653bcd9..b5f921997 100644
--- a/core/core/src/layers/complete.rs
+++ b/core/core/src/layers/complete.rs
@@ -381,12 +381,7 @@ where
Ok(())
}
- async fn copy_from(
- &mut self,
- path: &str,
- args: OpRead,
- range: BytesRange,
- ) -> Result<oio::CopyFromOutcome> {
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
let w = self.inner.as_mut().ok_or_else(|| {
Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
})?;
@@ -397,15 +392,18 @@ where
)
})?;
- let outcome = w
- .copy_from(path, args, range)
- .await
- .inspect_err(|_| self.state.transition(CompleteState::Error))?;
- if outcome == oio::CopyFromOutcome::Accepted {
- self.size += size;
- self.state.transition(CompleteState::Written);
+ match w.copy_from(path, args, range).await {
+ Ok(()) => {
+ self.size += size;
+ self.state.transition(CompleteState::Written);
+ Ok(())
+ }
+ Err(err) if err.kind() == ErrorKind::Unsupported => Err(err),
+ Err(err) => {
+ self.state.transition(CompleteState::Error);
+ Err(err)
+ }
}
- Ok(outcome)
}
async fn close(&mut self) -> Result<Metadata> {
@@ -453,6 +451,27 @@ where
#[cfg(test)]
mod tests {
use super::*;
+ use crate::raw::oio::Write as _;
+
+ #[derive(Default)]
+ struct MockWriteWriter {
+ size: u64,
+ }
+
+ impl oio::Write for MockWriteWriter {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ self.size += bs.len() as u64;
+ Ok(())
+ }
+
+ async fn close(&mut self) -> Result<Metadata> {
+ Ok(Metadata::new(EntryMode::FILE).with_content_length(self.size))
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ Ok(())
+ }
+ }
struct MockReadReader {
buffer: Buffer,
@@ -500,4 +519,22 @@ mod tests {
assert_eq!(err.kind(), ErrorKind::Unexpected);
}
+
+ #[tokio::test]
+ async fn test_writer_copy_from_unsupported_keeps_writer_usable() {
+ let mut writer = CompleteWriter::new(MockWriteWriter::default(), false);
+
+ let err = writer
+ .copy_from("source", OpRead::new(), BytesRange::new(0, Some(8)))
+ .await
+ .expect_err("default copy_from should be unsupported");
+ assert_eq!(err.kind(), ErrorKind::Unsupported);
+ assert_eq!(writer.state, CompleteState::Open);
+ assert_eq!(writer.size, 0);
+
+ writer.write(Buffer::from("fallback")).await.unwrap();
+ let metadata = writer.close().await.unwrap();
+ assert_eq!(metadata.content_length(), 8);
+ assert_eq!(writer.state, CompleteState::Closed);
+ }
}
diff --git a/core/core/src/layers/error_context.rs b/core/core/src/layers/error_context.rs
index 4c1cb2988..93f025ee5 100644
--- a/core/core/src/layers/error_context.rs
+++ b/core/core/src/layers/error_context.rs
@@ -292,12 +292,7 @@ impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
})
}
- async fn copy_from(
- &mut self,
- path: &str,
- args: OpRead,
- range: BytesRange,
- ) -> Result<oio::CopyFromOutcome> {
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
let size = range.size().ok_or_else(|| {
Error::new(
ErrorKind::Unexpected,
@@ -307,10 +302,8 @@ impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
self.inner
.copy_from(path, args, range)
.await
- .inspect(|outcome| {
- if *outcome == oio::CopyFromOutcome::Accepted {
- self.processed += size;
- }
+ .inspect(|_| {
+ self.processed += size;
})
.map_err(|err| {
err.with_operation(Operation::Write)
diff --git a/core/core/src/raw/enum_utils.rs b/core/core/src/raw/enum_utils.rs
index 05bb5ea14..4bc8ed49c 100644
--- a/core/core/src/raw/enum_utils.rs
+++ b/core/core/src/raw/enum_utils.rs
@@ -68,12 +68,7 @@ impl<ONE: oio::Write, TWO: oio::Write> oio::Write for TwoWays<ONE, TWO> {
}
}
- async fn copy_from(
- &mut self,
- path: &str,
- args: OpRead,
- range: BytesRange,
- ) -> Result<oio::CopyFromOutcome> {
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
match self {
Self::One(v) => v.copy_from(path, args, range).await,
Self::Two(v) => v.copy_from(path, args, range).await,
diff --git a/core/core/src/raw/oio/write/api.rs b/core/core/src/raw/oio/write/api.rs
index 9b9d76b5b..ca73845f7 100644
--- a/core/core/src/raw/oio/write/api.rs
+++ b/core/core/src/raw/oio/write/api.rs
@@ -24,15 +24,6 @@ use crate::*;
/// Writer is a type-erased [`Write`].
pub type Writer = Box<dyn WriteDyn>;
-/// The result of attempting a native writer-side copy.
-#[derive(Debug, Clone, Copy, Eq, PartialEq)]
-pub enum CopyFromOutcome {
- /// The source range has been accepted by the destination writer.
- Accepted,
- /// Native copy is unavailable and the writer has not been mutated.
- Unsupported,
-}
-
/// Write is the async sink used by services and layers.
pub trait Write: Unpin + Send + Sync {
/// Write the entire buffer into the writer.
@@ -43,16 +34,21 @@ pub trait Write: Unpin + Send + Sync {
/// Copy one absolute bounded source range into this writer.
///
- /// Implementations must return [`CopyFromOutcome::Unsupported`] before
- /// mutating the writer when native copy is unavailable. Any execution
- /// failure after accepting the operation must be returned as an error.
+ /// [`ErrorKind::Unsupported`] means native copy is unavailable and this
+ /// call has not mutated the writer. Any error after mutation must use a
+ /// different error kind so the caller will not start a streaming fallback.
fn copy_from(
&mut self,
_path: &str,
_args: OpRead,
_range: BytesRange,
- ) -> impl Future<Output = Result<CopyFromOutcome>> + MaybeSend {
- async { Ok(CopyFromOutcome::Unsupported) }
+ ) -> impl Future<Output = Result<()>> + MaybeSend {
+ async {
+ Err(Error::new(
+ ErrorKind::Unsupported,
+ "writer doesn't support native copy",
+ ))
+ }
}
/// Close the writer and make sure all data has been flushed.
@@ -93,7 +89,7 @@ pub trait WriteDyn: Unpin + Send + Sync {
path: &'a str,
args: OpRead,
range: BytesRange,
- ) -> BoxedFuture<'a, Result<CopyFromOutcome>>;
+ ) -> BoxedFuture<'a, Result<()>>;
/// The dyn version of [`Write::close`].
fn close_dyn(&mut self) -> BoxedFuture<'_, Result<Metadata>>;
@@ -112,7 +108,7 @@ impl<T: Write + ?Sized> WriteDyn for T {
path: &'a str,
args: OpRead,
range: BytesRange,
- ) -> BoxedFuture<'a, Result<CopyFromOutcome>> {
+ ) -> BoxedFuture<'a, Result<()>> {
Box::pin(self.copy_from(path, args, range))
}
@@ -130,12 +126,7 @@ impl<T: WriteDyn + ?Sized> Write for Box<T> {
self.deref_mut().write_dyn(bs).await
}
- async fn copy_from(
- &mut self,
- path: &str,
- args: OpRead,
- range: BytesRange,
- ) -> Result<CopyFromOutcome> {
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
self.deref_mut().copy_from_dyn(path, args, range).await
}
diff --git a/core/core/src/raw/oio/write/mod.rs b/core/core/src/raw/oio/write/mod.rs
index 2cb378f54..7240ece2c 100644
--- a/core/core/src/raw/oio/write/mod.rs
+++ b/core/core/src/raw/oio/write/mod.rs
@@ -16,7 +16,6 @@
// under the License.
mod api;
-pub use api::CopyFromOutcome;
pub use api::Write;
pub use api::WriteDyn;
pub use api::Writer;
diff --git a/core/core/src/raw/oio/write/multipart_write.rs b/core/core/src/raw/oio/write/multipart_write.rs
index f1a3a9803..26eb1518c 100644
--- a/core/core/src/raw/oio/write/multipart_write.rs
+++ b/core/core/src/raw/oio/write/multipart_write.rs
@@ -92,7 +92,7 @@ pub trait MultipartWrite: Send + Sync + Unpin + 'static {
) -> impl Future<Output = Result<MultipartPart>> + MaybeSend;
/// Return whether this multipart upload can accept native source ranges.
- fn supports_copy_from(&self) -> bool {
+ fn supports_copy_part(&self) -> bool {
false
}
@@ -279,6 +279,18 @@ impl<W: MultipartWrite> MultipartWriter<W> {
self.next_part_number += 1;
Ok(())
}
+
+ fn copy_execution_error(err: Error) -> Error {
+ if err.kind() != ErrorKind::Unsupported {
+ return err;
+ }
+
+ Error::new(
+ ErrorKind::Unexpected,
+ "native copy failed after the writer selected it for execution",
+ )
+ .set_source(err)
+ }
}
impl<W> oio::Write for MultipartWriter<W>
@@ -300,14 +312,12 @@ where
Ok(())
}
- async fn copy_from(
- &mut self,
- path: &str,
- args: OpRead,
- range: BytesRange,
- ) -> Result<oio::CopyFromOutcome> {
- if !self.w.supports_copy_from() {
- return Ok(oio::CopyFromOutcome::Unsupported);
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ if !self.w.supports_copy_part() {
+ return Err(Error::new(
+ ErrorKind::Unsupported,
+ "multipart writer doesn't support native copy",
+ ));
}
if range.is_suffix() || range.size().is_none() {
return Err(Error::new(
@@ -316,10 +326,11 @@ where
));
}
- let upload_id = self.upload_id().await?;
+ let upload_id = self.upload_id().await.map_err(Self::copy_execution_error)?;
if let Some(bytes) = self.cache.clone() {
self.schedule(upload_id.clone(), MultipartInput::Write(bytes))
- .await?;
+ .await
+ .map_err(Self::copy_execution_error)?;
self.cache = None;
}
self.schedule(
@@ -330,8 +341,9 @@ where
range,
},
)
- .await?;
- Ok(oio::CopyFromOutcome::Accepted)
+ .await
+ .map_err(Self::copy_execution_error)?;
+ Ok(())
}
async fn close(&mut self) -> Result<Metadata> {
@@ -486,7 +498,7 @@ mod tests {
})
}
- fn supports_copy_from(&self) -> bool {
+ fn supports_copy_part(&self) -> bool {
true
}
@@ -636,19 +648,13 @@ mod tests {
let mut writer = MultipartWriter::new(Executor::default(), inner.clone(), 1);
writer.write(Buffer::from("local-a")).await?;
- assert_eq!(
- writer
- .copy_from("source", OpRead::new(), BytesRange::new(10, Some(11)),)
- .await?,
- oio::CopyFromOutcome::Accepted
- );
+ writer
+ .copy_from("source", OpRead::new(), BytesRange::new(10, Some(11)))
+ .await?;
writer.write(Buffer::from("local-b")).await?;
- assert_eq!(
- writer
- .copy_from("source", OpRead::new(), BytesRange::new(30, Some(13)),)
- .await?,
- oio::CopyFromOutcome::Accepted
- );
+ writer
+ .copy_from("source", OpRead::new(), BytesRange::new(30, Some(13)))
+ .await?;
let metadata = loop {
match writer.close().await {
Ok(metadata) => break metadata,
@@ -661,4 +667,14 @@ mod tests {
assert_eq!(inner.lock().await.part_numbers, vec![0, 1, 2, 3]);
Ok(())
}
+
+ #[test]
+ fn test_copy_execution_error_does_not_return_unsupported() {
+ let err = MultipartWriter::<Arc<Mutex<TestWrite>>>::copy_execution_error(Error::new(
+ ErrorKind::Unsupported,
+ "copy part is unsupported",
+ ));
+
+ assert_eq!(err.kind(), ErrorKind::Unexpected);
+ }
}
diff --git a/core/core/src/types/context/write.rs b/core/core/src/types/context/write.rs
index 9bc470df3..77d6e6625 100644
--- a/core/core/src/types/context/write.rs
+++ b/core/core/src/types/context/write.rs
@@ -312,12 +312,13 @@ impl WriteGenerator<oio::Writer> {
remaining
};
let part_range = BytesRange::new(offset, Some(part_size));
- match self.w.copy_from(path, args.clone(), part_range).await? {
- oio::CopyFromOutcome::Accepted => {}
- oio::CopyFromOutcome::Unsupported => {
+ match self.w.copy_from(path, args.clone(), part_range).await {
+ Ok(()) => {}
+ Err(err) if err.kind() == ErrorKind::Unsupported => {
self.stream_range(path, args.clone(), options.clone(), part_range)
.await?;
}
+ Err(err) => return Err(err),
}
offset += part_size;
remaining -= part_size;
diff --git a/core/services/s3/src/writer.rs b/core/services/s3/src/writer.rs
index 58a21956d..c087dcaf1 100644
--- a/core/services/s3/src/writer.rs
+++ b/core/services/s3/src/writer.rs
@@ -152,7 +152,7 @@ impl oio::MultipartWrite for S3Writer {
}
}
- fn supports_copy_from(&self) -> bool {
+ fn supports_copy_part(&self) -> bool {
self.core.checksum_algorithm.is_none()
}
(opendal) 08/08: fix(services/s3): sign writer part copy with IAM
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/writer-copy-from-s3
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 1be31585cbe3595024f820eae63d31000c440605
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Aug 25 17:44:50 2026 +0800
fix(services/s3): sign writer part copy with IAM
---
core/services/s3/src/backend.rs | 39 +++++++++++++++++++++++++++++++++++++++
core/services/s3/src/writer.rs | 5 ++++-
2 files changed, 43 insertions(+), 1 deletion(-)
diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs
index e9afaefa3..58ed359c9 100644
--- a/core/services/s3/src/backend.rs
+++ b/core/services/s3/src/backend.rs
@@ -1690,6 +1690,45 @@ mod tests {
);
}
+ #[tokio::test]
+ async fn test_s3_express_writer_copy_from_uses_iam_credentials() {
+ let transport = S3ExpressMockTransport::new(CreateSessionOutcome::Success);
+ let op = s3_express_operator(transport.clone());
+
+ let mut writer = op.writer("target").await.expect("writer must open");
+ writer
+ .copy_from("source", 0..5 * 1024 * 1024_u64)
+ .await
+ .expect("writer copy must succeed");
+ writer.close().await.expect("writer must close");
+
+ let requests = transport.requests();
+ let part_copy = requests
+ .iter()
+ .find(|request| {
+ request.headers.contains_key(constants::X_AMZ_COPY_SOURCE)
+ && request
+ .uri
+ .query()
+ .is_some_and(|query| query.contains("partNumber="))
+ })
+ .expect("UploadPartCopy request must be captured");
+ assert_eq!(
+ header(part_copy, "x-amz-security-token"),
+ "source-session-token"
+ );
+ assert!(!part_copy.headers.contains_key("x-amz-s3session-token"));
+ assert!(header(part_copy, "authorization").contains("source-access-key/"));
+ assert_eq!(
+ requests
+ .iter()
+ .filter(|request| request.uri.query() == Some("session"))
+ .count(),
+ 1,
+ "writer multipart operations must share one session"
+ );
+ }
+
#[tokio::test]
async fn test_s3_express_create_session_error_mapping() {
for (outcome, kind, temporary) in [
diff --git a/core/services/s3/src/writer.rs b/core/services/s3/src/writer.rs
index e9539f9b1..74d4af150 100644
--- a/core/services/s3/src/writer.rs
+++ b/core/services/s3/src/writer.rs
@@ -186,7 +186,10 @@ impl oio::MultipartWrite for S3Writer {
operation: Operation::Write,
})?;
- let resp = self.core.send(&self.ctx, req).await?;
+ let resp = self
+ .core
+ .send(&self.ctx, req, self.core.signers.iam())
+ .await?;
match resp.status() {
StatusCode::OK => {
let (parts, body) = resp.into_parts();
(opendal) 07/08: Merge remote-tracking branch 'origin/main' into xuanwo/writer-copy-from-s3
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/writer-copy-from-s3
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit f1c17ec11f39ac059b57b80df8b4b6ab69c751fa
Merge: 25a753e8b 55530e90e
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Aug 25 17:40:57 2026 +0800
Merge remote-tracking branch 'origin/main' into xuanwo/writer-copy-from-s3
.github/services/s3/aws_s3_express/action.yml | 33 ++
.../OpenDAL.Tests/Behavior/DeleteBehaviorTest.cs | 42 ++
bindings/dotnet/OpenDAL/Capability.cs | 6 +
.../Interop/NativeObject/OpenDALCapability.cs | 1 +
bindings/dotnet/OpenDAL/Options/DeleteOptions.cs | 12 +
bindings/dotnet/src/capability.rs | 2 +
bindings/dotnet/src/options.rs | 2 +-
bindings/ruby/Cargo.lock | 37 +-
core/Cargo.lock | 18 +-
core/core/src/layers/simulate.rs | 51 +-
core/services/s3/Cargo.toml | 7 +-
core/services/s3/src/backend.rs | 627 ++++++++++++++++++++-
core/services/s3/src/copier.rs | 5 +-
core/services/s3/src/core.rs | 130 ++++-
core/services/s3/src/docs.md | 32 ++
core/services/s3/src/writer.rs | 15 +-
16 files changed, 940 insertions(+), 80 deletions(-)
diff --cc core/services/s3/src/copier.rs
index 38ef94dc3,f9b369ca7..833077cc0
--- a/core/services/s3/src/copier.rs
+++ b/core/services/s3/src/copier.rs
@@@ -192,10 -188,12 +192,13 @@@ impl oio::MultipartCopy for S3Copier
upload_id,
part_number,
range,
+ operation: Operation::Copy,
})?;
- let resp = self.core.send(&self.ctx, req).await?;
+ let resp = self
+ .core
+ .send(&self.ctx, req, self.core.signers.iam())
+ .await?;
match resp.status() {
StatusCode::OK => {
(opendal) 01/08: feat(services/s3): support Writer::copy_from
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/writer-copy-from-s3
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit f2ae147779c1557a4d028a5e7ffa4e233521198e
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Aug 24 16:24:23 2026 +0800
feat(services/s3): support Writer::copy_from
---
core/core/src/docs/rfcs/8109_writer_copy_from.md | 2 +-
core/core/src/layers/complete.rs | 27 +++
core/core/src/layers/error_context.rs | 30 ++++
core/core/src/raw/enum_utils.rs | 12 ++
core/core/src/raw/oio/write/api.rs | 49 +++++
core/core/src/raw/oio/write/mod.rs | 1 +
core/core/src/raw/oio/write/multipart_write.rs | 218 ++++++++++++++++++-----
core/core/src/types/context/read.rs | 21 +++
core/core/src/types/context/write.rs | 149 ++++++++++++++++
core/core/src/types/read/reader.rs | 4 +
core/core/src/types/write/writer.rs | 54 +++++-
core/services/s3/src/copier.rs | 5 +
core/services/s3/src/core.rs | 30 +++-
core/services/s3/src/writer.rs | 62 +++++++
core/tests/behavior/async_write.rs | 70 ++++++++
15 files changed, 687 insertions(+), 47 deletions(-)
diff --git a/core/core/src/docs/rfcs/8109_writer_copy_from.md b/core/core/src/docs/rfcs/8109_writer_copy_from.md
index fc6e52127..a9c32eb7b 100644
--- a/core/core/src/docs/rfcs/8109_writer_copy_from.md
+++ b/core/core/src/docs/rfcs/8109_writer_copy_from.md
@@ -1,7 +1,7 @@
- Proposal Name: `writer_copy_from`
- Start Date: 2026-08-19
- RFC PR: [apache/opendal#8109](https://github.com/apache/opendal/pull/8109)
-- Tracking Issue: [apache/opendal#0000](https://github.com/apache/opendal/issues/0000)
+- Tracking Issue: [apache/opendal#8134](https://github.com/apache/opendal/issues/8134)
# Summary
diff --git a/core/core/src/layers/complete.rs b/core/core/src/layers/complete.rs
index a2c84e160..0d653bcd9 100644
--- a/core/core/src/layers/complete.rs
+++ b/core/core/src/layers/complete.rs
@@ -381,6 +381,33 @@ where
Ok(())
}
+ async fn copy_from(
+ &mut self,
+ path: &str,
+ args: OpRead,
+ range: BytesRange,
+ ) -> Result<oio::CopyFromOutcome> {
+ let w = self.inner.as_mut().ok_or_else(|| {
+ Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
+ })?;
+ let size = range.size().ok_or_else(|| {
+ Error::new(
+ ErrorKind::Unexpected,
+ "native writer copy requires a bounded range",
+ )
+ })?;
+
+ let outcome = w
+ .copy_from(path, args, range)
+ .await
+ .inspect_err(|_| self.state.transition(CompleteState::Error))?;
+ if outcome == oio::CopyFromOutcome::Accepted {
+ self.size += size;
+ self.state.transition(CompleteState::Written);
+ }
+ Ok(outcome)
+ }
+
async fn close(&mut self) -> Result<Metadata> {
let w = self.inner.as_mut().ok_or_else(|| {
debug_assert_ne!(
diff --git a/core/core/src/layers/error_context.rs b/core/core/src/layers/error_context.rs
index 761112816..4c1cb2988 100644
--- a/core/core/src/layers/error_context.rs
+++ b/core/core/src/layers/error_context.rs
@@ -292,6 +292,36 @@ impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
})
}
+ async fn copy_from(
+ &mut self,
+ path: &str,
+ args: OpRead,
+ range: BytesRange,
+ ) -> Result<oio::CopyFromOutcome> {
+ let size = range.size().ok_or_else(|| {
+ Error::new(
+ ErrorKind::Unexpected,
+ "native writer copy requires a bounded range",
+ )
+ })?;
+ self.inner
+ .copy_from(path, args, range)
+ .await
+ .inspect(|outcome| {
+ if *outcome == oio::CopyFromOutcome::Accepted {
+ self.processed += size;
+ }
+ })
+ .map_err(|err| {
+ err.with_operation(Operation::Write)
+ .with_context("service", self.scheme)
+ .with_context("path", &self.path)
+ .with_context("source", path)
+ .with_context("range", range.to_string())
+ .with_context("written", self.processed.to_string())
+ })
+ }
+
async fn close(&mut self) -> Result<Metadata> {
self.inner.close().await.map_err(|err| {
err.with_operation(Operation::Write)
diff --git a/core/core/src/raw/enum_utils.rs b/core/core/src/raw/enum_utils.rs
index ce3f8b8b2..05bb5ea14 100644
--- a/core/core/src/raw/enum_utils.rs
+++ b/core/core/src/raw/enum_utils.rs
@@ -68,6 +68,18 @@ impl<ONE: oio::Write, TWO: oio::Write> oio::Write for TwoWays<ONE, TWO> {
}
}
+ async fn copy_from(
+ &mut self,
+ path: &str,
+ args: OpRead,
+ range: BytesRange,
+ ) -> Result<oio::CopyFromOutcome> {
+ match self {
+ Self::One(v) => v.copy_from(path, args, range).await,
+ Self::Two(v) => v.copy_from(path, args, range).await,
+ }
+ }
+
async fn close(&mut self) -> Result<Metadata> {
match self {
Self::One(v) => v.close().await,
diff --git a/core/core/src/raw/oio/write/api.rs b/core/core/src/raw/oio/write/api.rs
index 0d360c238..9b9d76b5b 100644
--- a/core/core/src/raw/oio/write/api.rs
+++ b/core/core/src/raw/oio/write/api.rs
@@ -24,6 +24,15 @@ use crate::*;
/// Writer is a type-erased [`Write`].
pub type Writer = Box<dyn WriteDyn>;
+/// The result of attempting a native writer-side copy.
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+pub enum CopyFromOutcome {
+ /// The source range has been accepted by the destination writer.
+ Accepted,
+ /// Native copy is unavailable and the writer has not been mutated.
+ Unsupported,
+}
+
/// Write is the async sink used by services and layers.
pub trait Write: Unpin + Send + Sync {
/// Write the entire buffer into the writer.
@@ -32,6 +41,20 @@ pub trait Write: Unpin + Send + Sync {
/// must return an error instead of treating a partial write as success.
fn write(&mut self, bs: Buffer) -> impl Future<Output = Result<()>> + MaybeSend;
+ /// Copy one absolute bounded source range into this writer.
+ ///
+ /// Implementations must return [`CopyFromOutcome::Unsupported`] before
+ /// mutating the writer when native copy is unavailable. Any execution
+ /// failure after accepting the operation must be returned as an error.
+ fn copy_from(
+ &mut self,
+ _path: &str,
+ _args: OpRead,
+ _range: BytesRange,
+ ) -> impl Future<Output = Result<CopyFromOutcome>> + MaybeSend {
+ async { Ok(CopyFromOutcome::Unsupported) }
+ }
+
/// Close the writer and make sure all data has been flushed.
fn close(&mut self) -> impl Future<Output = Result<Metadata>> + MaybeSend;
@@ -64,6 +87,14 @@ pub trait WriteDyn: Unpin + Send + Sync {
/// The dyn version of [`Write::write`].
fn write_dyn(&mut self, bs: Buffer) -> BoxedFuture<'_, Result<()>>;
+ /// The dyn version of [`Write::copy_from`].
+ fn copy_from_dyn<'a>(
+ &'a mut self,
+ path: &'a str,
+ args: OpRead,
+ range: BytesRange,
+ ) -> BoxedFuture<'a, Result<CopyFromOutcome>>;
+
/// The dyn version of [`Write::close`].
fn close_dyn(&mut self) -> BoxedFuture<'_, Result<Metadata>>;
@@ -76,6 +107,15 @@ impl<T: Write + ?Sized> WriteDyn for T {
Box::pin(self.write(bs))
}
+ fn copy_from_dyn<'a>(
+ &'a mut self,
+ path: &'a str,
+ args: OpRead,
+ range: BytesRange,
+ ) -> BoxedFuture<'a, Result<CopyFromOutcome>> {
+ Box::pin(self.copy_from(path, args, range))
+ }
+
fn close_dyn(&mut self) -> BoxedFuture<'_, Result<Metadata>> {
Box::pin(self.close())
}
@@ -90,6 +130,15 @@ impl<T: WriteDyn + ?Sized> Write for Box<T> {
self.deref_mut().write_dyn(bs).await
}
+ async fn copy_from(
+ &mut self,
+ path: &str,
+ args: OpRead,
+ range: BytesRange,
+ ) -> Result<CopyFromOutcome> {
+ self.deref_mut().copy_from_dyn(path, args, range).await
+ }
+
async fn close(&mut self) -> Result<Metadata> {
self.deref_mut().close_dyn().await
}
diff --git a/core/core/src/raw/oio/write/mod.rs b/core/core/src/raw/oio/write/mod.rs
index 7240ece2c..2cb378f54 100644
--- a/core/core/src/raw/oio/write/mod.rs
+++ b/core/core/src/raw/oio/write/mod.rs
@@ -16,6 +16,7 @@
// under the License.
mod api;
+pub use api::CopyFromOutcome;
pub use api::Write;
pub use api::WriteDyn;
pub use api::Writer;
diff --git a/core/core/src/raw/oio/write/multipart_write.rs b/core/core/src/raw/oio/write/multipart_write.rs
index 00e1be8e2..4b4e3ed6d 100644
--- a/core/core/src/raw/oio/write/multipart_write.rs
+++ b/core/core/src/raw/oio/write/multipart_write.rs
@@ -91,6 +91,28 @@ pub trait MultipartWrite: Send + Sync + Unpin + 'static {
body: Buffer,
) -> impl Future<Output = Result<MultipartPart>> + MaybeSend;
+ /// Return whether this multipart upload can accept native source ranges.
+ fn supports_copy_from(&self) -> bool {
+ false
+ }
+
+ /// Copy one absolute bounded source range into a multipart part.
+ fn copy_part(
+ &self,
+ _upload_id: &str,
+ _part_number: usize,
+ _path: &str,
+ _args: OpRead,
+ _range: BytesRange,
+ ) -> impl Future<Output = Result<MultipartPart>> + MaybeSend {
+ async {
+ Err(Error::new(
+ ErrorKind::Unsupported,
+ "multipart writer doesn't support native copy",
+ ))
+ }
+ }
+
/// complete_part will complete the multipart upload to build the final
/// file.
fn complete_part(
@@ -122,12 +144,21 @@ pub struct MultipartPart {
pub size: Option<u64>,
}
+enum MultipartInput {
+ Write(Buffer),
+ Copy {
+ path: String,
+ args: Box<OpRead>,
+ range: BytesRange,
+ },
+}
+
struct WriteInput<W: MultipartWrite> {
w: Arc<W>,
executor: Executor,
upload_id: Arc<String>,
part_number: usize,
- bytes: Buffer,
+ input: MultipartInput,
}
/// MultipartWriter will implement [`oio::Write`] based on multipart
@@ -163,12 +194,33 @@ impl<W: MultipartWrite> MultipartWriter<W> {
tasks: ConcurrentTasks::new(executor, concurrent, 8192, |input| {
Box::pin({
async move {
- let fut = input.w.write_part(
- &input.upload_id,
- input.part_number,
- input.bytes.len() as u64,
- input.bytes.clone(),
- );
+ let fut = async {
+ match &input.input {
+ MultipartInput::Write(bytes) => {
+ input
+ .w
+ .write_part(
+ &input.upload_id,
+ input.part_number,
+ bytes.len() as u64,
+ bytes.clone(),
+ )
+ .await
+ }
+ MultipartInput::Copy { path, args, range } => {
+ input
+ .w
+ .copy_part(
+ &input.upload_id,
+ input.part_number,
+ path,
+ args.as_ref().clone(),
+ *range,
+ )
+ .await
+ }
+ }
+ };
match input.executor.timeout() {
None => {
let result = fut.await;
@@ -202,47 +254,86 @@ impl<W: MultipartWrite> MultipartWriter<W> {
self.cache = Some(bs);
size
}
-}
-impl<W> oio::Write for MultipartWriter<W>
-where
- W: MultipartWrite,
-{
- async fn write(&mut self, bs: Buffer) -> Result<()> {
- let upload_id = match self.upload_id.clone() {
- Some(v) => v,
- None => {
- // Fill cache with the first write.
- if self.cache.is_none() {
- self.fill_cache(bs);
- return Ok(());
- }
+ async fn upload_id(&mut self) -> Result<Arc<String>> {
+ if let Some(upload_id) = &self.upload_id {
+ return Ok(upload_id.clone());
+ }
- let upload_id = self.w.initiate_part().await?;
- let upload_id = Arc::new(upload_id);
- self.upload_id = Some(upload_id.clone());
- upload_id
- }
- };
+ let upload_id = Arc::new(self.w.initiate_part().await?);
+ self.upload_id = Some(upload_id.clone());
+ Ok(upload_id)
+ }
- let bytes = self.cache.clone().expect("pending write must exist");
+ async fn schedule(&mut self, upload_id: Arc<String>, input: MultipartInput) -> Result<()> {
let part_number = self.next_part_number;
-
self.tasks
.execute(WriteInput {
w: self.w.clone(),
executor: self.executor.clone(),
- upload_id: upload_id.clone(),
+ upload_id,
part_number,
- bytes,
+ input,
})
.await?;
- self.cache = None;
self.next_part_number += 1;
+ Ok(())
+ }
+}
+
+impl<W> oio::Write for MultipartWriter<W>
+where
+ W: MultipartWrite,
+{
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ if self.cache.is_none() {
+ self.fill_cache(bs);
+ return Ok(());
+ }
+
+ let upload_id = self.upload_id().await?;
+ let bytes = self.cache.clone().expect("pending write must exist");
+ self.schedule(upload_id, MultipartInput::Write(bytes))
+ .await?;
+ self.cache = None;
self.fill_cache(bs);
Ok(())
}
+ async fn copy_from(
+ &mut self,
+ path: &str,
+ args: OpRead,
+ range: BytesRange,
+ ) -> Result<oio::CopyFromOutcome> {
+ if !self.w.supports_copy_from() {
+ return Ok(oio::CopyFromOutcome::Unsupported);
+ }
+ if range.is_suffix() || range.size().is_none() {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "native writer copy requires an absolute bounded range",
+ ));
+ }
+
+ let upload_id = self.upload_id().await?;
+ if let Some(bytes) = self.cache.clone() {
+ self.schedule(upload_id.clone(), MultipartInput::Write(bytes))
+ .await?;
+ self.cache = None;
+ }
+ self.schedule(
+ upload_id,
+ MultipartInput::Copy {
+ path: path.to_string(),
+ args: Box::new(args),
+ range,
+ },
+ )
+ .await?;
+ Ok(oio::CopyFromOutcome::Accepted)
+ }
+
async fn close(&mut self) -> Result<Metadata> {
let upload_id = match self.upload_id.clone() {
Some(v) => v,
@@ -261,19 +352,9 @@ where
};
if let Some(cache) = self.cache.clone() {
- let part_number = self.next_part_number;
-
- self.tasks
- .execute(WriteInput {
- w: self.w.clone(),
- executor: self.executor.clone(),
- upload_id: upload_id.clone(),
- part_number,
- bytes: cache,
- })
+ self.schedule(upload_id.clone(), MultipartInput::Write(cache))
.await?;
self.cache = None;
- self.next_part_number += 1;
}
loop {
@@ -395,6 +476,31 @@ mod tests {
})
}
+ fn supports_copy_from(&self) -> bool {
+ true
+ }
+
+ async fn copy_part(
+ &self,
+ upload_id: &str,
+ part_number: usize,
+ _: &str,
+ _: OpRead,
+ range: BytesRange,
+ ) -> Result<MultipartPart> {
+ let size = range.size().expect("test range must be bounded");
+ let mut test = self.lock().await;
+ assert_eq!(upload_id, test.upload_id);
+ test.part_numbers.push(part_number);
+ test.length += size;
+ Ok(MultipartPart {
+ part_number,
+ etag: "copied-etag".to_string(),
+ checksum: None,
+ size: Some(size),
+ })
+ }
+
async fn complete_part(
&self,
upload_id: &str,
@@ -513,4 +619,30 @@ mod tests {
assert_eq!(inner.content.clone().unwrap().to_bytes(), bs);
}
}
+
+ #[tokio::test]
+ async fn test_multipart_writer_interleaves_writes_and_copies() -> Result<()> {
+ let inner = TestWrite::new();
+ let mut writer = MultipartWriter::new(Executor::default(), inner.clone(), 1);
+
+ writer.write(Buffer::from("local-a")).await?;
+ assert_eq!(
+ writer
+ .copy_from("source", OpRead::new(), BytesRange::new(10, Some(11)),)
+ .await?,
+ oio::CopyFromOutcome::Accepted
+ );
+ writer.write(Buffer::from("local-b")).await?;
+ assert_eq!(
+ writer
+ .copy_from("source", OpRead::new(), BytesRange::new(30, Some(13)),)
+ .await?,
+ oio::CopyFromOutcome::Accepted
+ );
+ let metadata = writer.close().await?;
+
+ assert_eq!(metadata.content_length(), 7 + 11 + 7 + 13);
+ assert_eq!(inner.lock().await.part_numbers, vec![0, 1, 2, 3]);
+ Ok(())
+ }
}
diff --git a/core/core/src/types/context/read.rs b/core/core/src/types/context/read.rs
index 0f8e64fb5..3f0a01788 100644
--- a/core/core/src/types/context/read.rs
+++ b/core/core/src/types/context/read.rs
@@ -127,6 +127,18 @@ impl ReadContext {
if let Some(v) = self.args().version() {
op_stat = op_stat.with_version(v);
}
+ if let Some(v) = self.args().if_match() {
+ op_stat = op_stat.with_if_match(v);
+ }
+ if let Some(v) = self.args().if_none_match() {
+ op_stat = op_stat.with_if_none_match(v);
+ }
+ if let Some(v) = self.args().if_modified_since() {
+ op_stat = op_stat.with_if_modified_since(v);
+ }
+ if let Some(v) = self.args().if_unmodified_since() {
+ op_stat = op_stat.with_if_unmodified_since(v);
+ }
Ok(self
.srv
@@ -161,6 +173,15 @@ impl ReadContext {
}
};
+ if start > end {
+ return Err(Error::new(
+ ErrorKind::RangeNotSatisfied,
+ "range starts after the end of the object",
+ )
+ .with_context("offset", start)
+ .with_context("content_length", end));
+ }
+
Ok(start..end)
}
}
diff --git a/core/core/src/types/context/write.rs b/core/core/src/types/context/write.rs
index e006874d9..9bc470df3 100644
--- a/core/core/src/types/context/write.rs
+++ b/core/core/src/types/context/write.rs
@@ -17,6 +17,9 @@
use std::sync::Arc;
+use bytes::Buf;
+use futures::TryStreamExt;
+
use crate::raw::oio::Write;
use crate::raw::*;
use crate::*;
@@ -60,6 +63,18 @@ impl WriteContext {
&self.path
}
+ /// Get the composed operation context.
+ #[inline]
+ pub(crate) fn context(&self) -> &OperationContext {
+ &self.ctx
+ }
+
+ /// Get the composed service.
+ #[inline]
+ pub(crate) fn service(&self) -> &Servicer {
+ &self.srv
+ }
+
/// Get the arguments.
#[inline]
pub fn args(&self) -> &OpWrite {
@@ -101,6 +116,8 @@ impl WriteContext {
pub struct WriteGenerator<W> {
w: W,
+ ctx: Option<Arc<WriteContext>>,
+
/// The size for buffer, we will flush the underlying storage at the size of this buffer.
chunk_size: Option<usize>,
/// If `exact` is true, the size of the data written to the underlying storage is
@@ -117,6 +134,7 @@ impl WriteGenerator<oio::Writer> {
Ok(Self {
w,
+ ctx: Some(ctx),
chunk_size,
exact,
buffer: oio::QueueBuf::new(),
@@ -128,6 +146,7 @@ impl WriteGenerator<oio::Writer> {
fn new(w: oio::Writer, chunk_size: Option<usize>, exact: bool) -> Self {
Self {
w,
+ ctx: None,
chunk_size,
exact,
buffer: oio::QueueBuf::new(),
@@ -186,6 +205,136 @@ impl WriteGenerator<oio::Writer> {
Ok(n)
}
+ fn reader(&self, path: &str, args: OpRead, options: OpReader) -> Result<Reader> {
+ let ctx = self
+ .ctx
+ .as_ref()
+ .expect("write context must exist outside tests");
+ let reader = ctx.service().read(ctx.context(), path, args.clone())?;
+ Ok(Reader::new(ReadContext::new(
+ ctx.context().clone(),
+ ctx.service().clone(),
+ path.to_string(),
+ args,
+ options,
+ reader,
+ )))
+ }
+
+ async fn stream_range(
+ &mut self,
+ path: &str,
+ args: OpRead,
+ options: OpReader,
+ range: BytesRange,
+ ) -> Result<()> {
+ let reader = self.reader(path, args, options)?;
+ let mut stream = reader.into_stream(range).await?;
+ while let Some(mut bs) = stream.try_next().await? {
+ while !bs.is_empty() {
+ let n = self.write(bs.clone()).await?;
+ bs.advance(n);
+ }
+ }
+ Ok(())
+ }
+
+ /// Copy a source range into this writer, using native execution when possible.
+ pub async fn copy_from(
+ &mut self,
+ path: &str,
+ args: OpRead,
+ options: OpReader,
+ range: BytesRange,
+ ) -> Result<()> {
+ const MAX_COPY_FROM_SIZE: u64 = 5 * 1024 * 1024 * 1024;
+
+ if let Some(size) = range.size()
+ && size > MAX_COPY_FROM_SIZE
+ {
+ return Err(Error::new(
+ ErrorKind::RangeNotSatisfied,
+ "copy_from range exceeds the 5 GiB per-call limit",
+ )
+ .with_context("size", size));
+ }
+
+ let reader = self.reader(path, args.clone(), options.clone())?;
+ let absolute = reader.parse_into_range(range).await?;
+ let size = absolute.end - absolute.start;
+ if size > MAX_COPY_FROM_SIZE {
+ return Err(Error::new(
+ ErrorKind::RangeNotSatisfied,
+ "copy_from range exceeds the 5 GiB per-call limit",
+ )
+ .with_context("size", size));
+ }
+ if size == 0 {
+ return Ok(());
+ }
+
+ let Some(chunk_size) = self.chunk_size else {
+ return self
+ .stream_range(
+ path,
+ args,
+ options,
+ BytesRange::new(absolute.start, Some(size)),
+ )
+ .await;
+ };
+
+ let mut offset = absolute.start;
+ let mut remaining = size;
+
+ if self.buffer.len() >= chunk_size {
+ let buf = self.buffer.take().collect();
+ self.w.write(buf).await?;
+ }
+
+ if !self.buffer.is_empty() {
+ let boundary_size = (chunk_size as u64 - self.buffer.len() as u64).min(remaining);
+ self.stream_range(
+ path,
+ args.clone(),
+ options.clone(),
+ BytesRange::new(offset, Some(boundary_size)),
+ )
+ .await?;
+ offset += boundary_size;
+ remaining -= boundary_size;
+ }
+
+ while remaining >= chunk_size as u64 {
+ let part_size = if self.exact {
+ chunk_size as u64
+ } else {
+ remaining
+ };
+ let part_range = BytesRange::new(offset, Some(part_size));
+ match self.w.copy_from(path, args.clone(), part_range).await? {
+ oio::CopyFromOutcome::Accepted => {}
+ oio::CopyFromOutcome::Unsupported => {
+ self.stream_range(path, args.clone(), options.clone(), part_range)
+ .await?;
+ }
+ }
+ offset += part_size;
+ remaining -= part_size;
+ }
+
+ if remaining > 0 {
+ self.stream_range(
+ path,
+ args,
+ options,
+ BytesRange::new(offset, Some(remaining)),
+ )
+ .await?;
+ }
+ Ok(())
+ }
+
/// Finish the write process.
pub async fn close(&mut self) -> Result<Metadata> {
loop {
diff --git a/core/core/src/types/read/reader.rs b/core/core/src/types/read/reader.rs
index d69e9b64f..a8a388477 100644
--- a/core/core/src/types/read/reader.rs
+++ b/core/core/src/types/read/reader.rs
@@ -120,6 +120,10 @@ impl Reader {
Reader { ctx: Arc::new(ctx) }
}
+ pub(crate) async fn parse_into_range(&self, range: BytesRange) -> Result<Range<u64>> {
+ self.ctx.parse_into_range(range).await
+ }
+
/// Get complete object metadata observed by this reader.
///
/// This method doesn't perform I/O. It returns `None` if no read has
diff --git a/core/core/src/types/write/writer.rs b/core/core/src/types/write/writer.rs
index 36da8aa50..e1359ee41 100644
--- a/core/core/src/types/write/writer.rs
+++ b/core/core/src/types/write/writer.rs
@@ -99,7 +99,7 @@ use crate::*;
/// creating writer with `append` enabled.
pub struct Writer {
/// Keep a reference to write context in writer.
- _ctx: Arc<WriteContext>,
+ ctx: Arc<WriteContext>,
inner: WriteGenerator<oio::Writer>,
}
@@ -109,7 +109,7 @@ impl Writer {
let ctx = Arc::new(ctx);
let inner = std::future::ready(WriteGenerator::create(ctx.clone())).await?;
- Ok(Self { _ctx: ctx, inner })
+ Ok(Self { ctx, inner })
}
/// Write [`Buffer`] into writer.
@@ -182,6 +182,56 @@ impl Writer {
self.write(Buffer::from(chunks)).await
}
+ /// Copy a range of an existing object into this writer.
+ ///
+ /// The source path is resolved by the same [`Operator`] that created this
+ /// writer. Calls to `write` and `copy_from` may be interleaved, and their
+ /// call order defines the destination bytes. OpenDAL uses a native
+ /// writer-side copy when the composed service supports it and otherwise
+ /// streams the range through the client.
+ ///
+ /// Each call accepts at most 5 GiB. Applications must split larger source
+ /// ranges into multiple ordered calls.
+ pub async fn copy_from(&mut self, path: &str, range: impl Into<BytesRange>) -> Result<()> {
+ self.copy_from_options(
+ path,
+ options::ReadOptions {
+ range: range.into(),
+ ..Default::default()
+ },
+ )
+ .await
+ }
+
+ /// Copy an existing object range with read conditions and execution options.
+ pub async fn copy_from_options(
+ &mut self,
+ path: &str,
+ options: options::ReadOptions,
+ ) -> Result<()> {
+ let path = normalize_path(path);
+ if !validate_path(&path, EntryMode::FILE) {
+ return Err(
+ Error::new(ErrorKind::IsADirectory, "copy source path is a directory")
+ .with_operation("Writer::copy_from")
+ .with_context("service", self.ctx.service().info().scheme())
+ .with_context("path", &path),
+ );
+ }
+ if path == self.ctx.path() {
+ return Err(Error::new(
+ ErrorKind::IsSameFile,
+ "copy source and destination are the same",
+ )
+ .with_operation("Writer::copy_from")
+ .with_context("service", self.ctx.service().info().scheme())
+ .with_context("path", &path));
+ }
+
+ let (range, args, options) = options.into();
+ self.inner.copy_from(&path, args, options, range).await
+ }
+
/// Abort the writer and clean up all written data.
///
/// ## Notes
diff --git a/core/services/s3/src/copier.rs b/core/services/s3/src/copier.rs
index 1af4d81a8..38ef94dc3 100644
--- a/core/services/s3/src/copier.rs
+++ b/core/services/s3/src/copier.rs
@@ -185,9 +185,14 @@ impl oio::MultipartCopy for S3Copier {
from: &self.from,
to: &self.to,
source_version: self.args.source_version(),
+ if_match: None,
+ if_none_match: None,
+ if_modified_since: None,
+ if_unmodified_since: None,
upload_id,
part_number,
range,
+ operation: Operation::Copy,
})?;
let resp = self.core.send(&self.ctx, req).await?;
diff --git a/core/services/s3/src/core.rs b/core/services/s3/src/core.rs
index a2278a19b..b51331fa7 100644
--- a/core/services/s3/src/core.rs
+++ b/core/services/s3/src/core.rs
@@ -50,6 +50,10 @@ use opendal_core::*;
pub mod constants {
pub const X_AMZ_COPY_SOURCE: &str = "x-amz-copy-source";
pub const X_AMZ_COPY_SOURCE_RANGE: &str = "x-amz-copy-source-range";
+ pub const X_AMZ_COPY_SOURCE_IF_MATCH: &str = "x-amz-copy-source-if-match";
+ pub const X_AMZ_COPY_SOURCE_IF_NONE_MATCH: &str = "x-amz-copy-source-if-none-match";
+ pub const X_AMZ_COPY_SOURCE_IF_MODIFIED_SINCE: &str = "x-amz-copy-source-if-modified-since";
+ pub const X_AMZ_COPY_SOURCE_IF_UNMODIFIED_SINCE: &str = "x-amz-copy-source-if-unmodified-since";
pub const X_AMZ_SERVER_SIDE_ENCRYPTION: &str = "x-amz-server-side-encryption";
pub const X_AMZ_SERVER_REQUEST_PAYER: (&str, &str) = ("x-amz-request-payer", "requester");
@@ -112,9 +116,14 @@ pub(crate) struct S3UploadPartCopyRequest<'a> {
pub(crate) from: &'a str,
pub(crate) to: &'a str,
pub(crate) source_version: Option<&'a str>,
+ pub(crate) if_match: Option<&'a str>,
+ pub(crate) if_none_match: Option<&'a str>,
+ pub(crate) if_modified_since: Option<Timestamp>,
+ pub(crate) if_unmodified_since: Option<Timestamp>,
pub(crate) upload_id: &'a str,
pub(crate) part_number: usize,
pub(crate) range: BytesRange,
+ pub(crate) operation: Operation,
}
fn format_crc32c_iter(body: Buffer) -> String {
@@ -1009,6 +1018,25 @@ impl S3Core {
let mut req = Request::put(&url);
+ if let Some(v) = input.if_match {
+ req = req.header(constants::X_AMZ_COPY_SOURCE_IF_MATCH, v);
+ }
+ if let Some(v) = input.if_none_match {
+ req = req.header(constants::X_AMZ_COPY_SOURCE_IF_NONE_MATCH, v);
+ }
+ if let Some(v) = input.if_modified_since {
+ req = req.header(
+ constants::X_AMZ_COPY_SOURCE_IF_MODIFIED_SINCE,
+ v.format_http_date(),
+ );
+ }
+ if let Some(v) = input.if_unmodified_since {
+ req = req.header(
+ constants::X_AMZ_COPY_SOURCE_IF_UNMODIFIED_SINCE,
+ v.format_http_date(),
+ );
+ }
+
// Set SSE headers.
req = self.insert_sse_headers(req, true);
@@ -1052,7 +1080,7 @@ impl S3Core {
req = self.insert_request_payer_header(req);
let req = req
- .extension(Operation::Copy)
+ .extension(input.operation)
.extension(ServiceOperation("UploadPartCopy"))
.header(constants::X_AMZ_COPY_SOURCE, source)
.header(constants::X_AMZ_COPY_SOURCE_RANGE, input.range.to_header())
diff --git a/core/services/s3/src/writer.rs b/core/services/s3/src/writer.rs
index 8d0b9130a..58a21956d 100644
--- a/core/services/s3/src/writer.rs
+++ b/core/services/s3/src/writer.rs
@@ -152,6 +152,68 @@ impl oio::MultipartWrite for S3Writer {
}
}
+ fn supports_copy_from(&self) -> bool {
+ self.core.checksum_algorithm.is_none()
+ }
+
+ async fn copy_part(
+ &self,
+ upload_id: &str,
+ part_number: usize,
+ path: &str,
+ args: OpRead,
+ range: BytesRange,
+ ) -> Result<oio::MultipartPart> {
+ let size = range
+ .size()
+ .expect("multipart writer copy range must be bounded");
+ let part_number = part_number + 1;
+ let req = self
+ .core
+ .s3_upload_part_copy_request(S3UploadPartCopyRequest {
+ from: path,
+ to: &self.path,
+ source_version: args.version(),
+ if_match: args.if_match(),
+ if_none_match: args.if_none_match(),
+ if_modified_since: args.if_modified_since(),
+ if_unmodified_since: args.if_unmodified_since(),
+ upload_id,
+ part_number,
+ range,
+ operation: Operation::Write,
+ })?;
+
+ let resp = self.core.send(&self.ctx, req).await?;
+ match resp.status() {
+ StatusCode::OK => {
+ let (parts, body) = resp.into_parts();
+ let result: CopyObjectResult =
+ quick_xml::de::from_reader(body.reader()).map_err(new_xml_deserialize_error)?;
+
+ if result.etag.is_empty() {
+ return Err(from_s3_error(
+ S3Error {
+ code: result.code,
+ message: result.message,
+ resource: String::new(),
+ request_id: result.request_id,
+ },
+ parts,
+ ));
+ }
+
+ Ok(oio::MultipartPart {
+ part_number,
+ etag: result.etag,
+ checksum: None,
+ size: Some(size),
+ })
+ }
+ _ => Err(parse_error(resp)),
+ }
+ }
+
async fn complete_part(
&self,
upload_id: &str,
diff --git a/core/tests/behavior/async_write.rs b/core/tests/behavior/async_write.rs
index 5a17dc69c..2a3f3f065 100644
--- a/core/tests/behavior/async_write.rs
+++ b/core/tests/behavior/async_write.rs
@@ -57,6 +57,8 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
test_writer_futures_copy,
test_writer_futures_copy_with_concurrent,
test_writer_return_metadata,
+ test_writer_copy_from_interleaved,
+ test_writer_copy_from_rejects_self_copy,
test_writer_write_non_contiguous_data,
test_writer_write_with_if_not_exists,
test_writer_write_with_if_none_match,
@@ -352,6 +354,74 @@ pub async fn test_writer_write(op: Operator) -> Result<()> {
Ok(())
}
+/// Assemble a destination from local bytes and source ranges in call order.
+pub async fn test_writer_copy_from_interleaved(op: Operator) -> Result<()> {
+ if !op.info().capability().write_can_multi {
+ return Ok(());
+ }
+
+ let source_path = TEST_FIXTURE.new_file_path();
+ let target_path = TEST_FIXTURE.new_file_path();
+ let source = gen_fixed_bytes(18 * 1024 * 1024);
+ op.write(&source_path, source.clone()).await?;
+ let source_meta = op.stat(&source_path).await?;
+ let source_if_match =
+ if op.info().capability().read_with_if_match && op.info().capability().stat_with_if_match {
+ source_meta.etag().map(str::to_string)
+ } else {
+ None
+ };
+
+ let mut writer = op.writer(&target_path).await?;
+ writer.write("header").await?;
+ writer.copy_from(&source_path, 1024_u64..2048).await?;
+ writer
+ .copy_from_options(
+ &source_path,
+ options::ReadOptions {
+ range: (2048_u64..14 * 1024 * 1024).into(),
+ if_match: source_if_match.clone(),
+ ..Default::default()
+ },
+ )
+ .await?;
+ writer.write("footer").await?;
+ writer
+ .copy_from_options(
+ &source_path,
+ options::ReadOptions {
+ range: (15 * 1024 * 1024_u64..).into(),
+ if_match: source_if_match,
+ ..Default::default()
+ },
+ )
+ .await?;
+ writer.close().await?;
+
+ let mut expected = Vec::new();
+ expected.extend_from_slice(b"header");
+ expected.extend_from_slice(&source[1024..14 * 1024 * 1024]);
+ expected.extend_from_slice(b"footer");
+ expected.extend_from_slice(&source[15 * 1024 * 1024..]);
+ assert_eq!(op.read(&target_path).await?.to_bytes(), expected);
+ Ok(())
+}
+
+/// Reject self-copy before the destination writer is mutated.
+pub async fn test_writer_copy_from_rejects_self_copy(op: Operator) -> Result<()> {
+ let path = TEST_FIXTURE.new_file_path();
+ op.write(&path, "source").await?;
+
+ let mut writer = op.writer(&path).await?;
+ let err = writer
+ .copy_from(&path, ..)
+ .await
+ .expect_err("self-copy must be rejected");
+ assert_eq!(err.kind(), ErrorKind::IsSameFile);
+ writer.abort().await?;
+ Ok(())
+}
+
/// Append data into writer
pub async fn test_writer_write_with_concurrent(op: Operator) -> Result<()> {
if !(op.info().capability().write_can_multi) {
(opendal) 03/08: test: make multipart copy test deterministic
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/writer-copy-from-s3
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 98a860057baa39a0f16e09088bb82975012d78e2
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Aug 24 16:26:20 2026 +0800
test: make multipart copy test deterministic
---
core/core/src/raw/oio/write/multipart_write.rs | 24 ++++++++++++++++++++----
1 file changed, 20 insertions(+), 4 deletions(-)
diff --git a/core/core/src/raw/oio/write/multipart_write.rs b/core/core/src/raw/oio/write/multipart_write.rs
index 4b4e3ed6d..f1a3a9803 100644
--- a/core/core/src/raw/oio/write/multipart_write.rs
+++ b/core/core/src/raw/oio/write/multipart_write.rs
@@ -404,15 +404,25 @@ mod tests {
part_numbers: Vec<usize>,
length: u64,
content: Option<Buffer>,
+ fail_randomly: bool,
}
impl TestWrite {
pub fn new() -> Arc<Mutex<Self>> {
+ Self::new_with_failures(true)
+ }
+
+ pub fn new_stable() -> Arc<Mutex<Self>> {
+ Self::new_with_failures(false)
+ }
+
+ fn new_with_failures(fail_randomly: bool) -> Arc<Mutex<Self>> {
let v = Self {
upload_id: uuid::Uuid::new_v4().to_string(),
part_numbers: Vec::new(),
length: 0,
content: None,
+ fail_randomly,
};
Arc::new(Mutex::new(v))
@@ -423,7 +433,7 @@ mod tests {
async fn write_once(&self, size: u64, body: Buffer) -> Result<Metadata> {
sleep(Duration::from_nanos(50)).await;
- if rng().random_bool(1.0 / 10.0) {
+ if self.lock().await.fail_randomly && rng().random_bool(1.0 / 10.0) {
return Err(
Error::new(ErrorKind::Unexpected, "I'm a crazy monkey!").set_temporary()
);
@@ -456,7 +466,7 @@ mod tests {
sleep(Duration::from_nanos(50)).await;
// We will have 10% percent rate for write part to fail.
- if rng().random_bool(1.0 / 10.0) {
+ if self.lock().await.fail_randomly && rng().random_bool(1.0 / 10.0) {
return Err(
Error::new(ErrorKind::Unexpected, "I'm a crazy monkey!").set_temporary()
);
@@ -622,7 +632,7 @@ mod tests {
#[tokio::test]
async fn test_multipart_writer_interleaves_writes_and_copies() -> Result<()> {
- let inner = TestWrite::new();
+ let inner = TestWrite::new_stable();
let mut writer = MultipartWriter::new(Executor::default(), inner.clone(), 1);
writer.write(Buffer::from("local-a")).await?;
@@ -639,7 +649,13 @@ mod tests {
.await?,
oio::CopyFromOutcome::Accepted
);
- let metadata = writer.close().await?;
+ let metadata = loop {
+ match writer.close().await {
+ Ok(metadata) => break metadata,
+ Err(err) if err.is_temporary() => continue,
+ Err(err) => return Err(err),
+ }
+ };
assert_eq!(metadata.content_length(), 7 + 11 + 7 + 13);
assert_eq!(inner.lock().await.part_numbers, vec![0, 1, 2, 3]);
(opendal) 05/08: refactor: gate writer copy with capability
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/writer-copy-from-s3
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 03e22cecb4f993be088c8a0119de1f1923ff4d2a
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Aug 25 17:35:34 2026 +0800
refactor: gate writer copy with capability
---
core/core/src/docs/rfcs/8109_writer_copy_from.md | 44 ++++++++++++---------
core/core/src/layers/complete.rs | 49 ++++++++++--------------
core/core/src/raw/oio/write/api.rs | 6 +--
core/core/src/raw/oio/write/multipart_write.rs | 45 ++--------------------
core/core/src/types/capability.rs | 3 ++
core/core/src/types/context/write.rs | 16 ++++----
core/core/src/types/write/writer.rs | 4 +-
core/layers/async-backtrace/src/lib.rs | 5 +++
core/layers/await-tree/src/lib.rs | 7 ++++
core/layers/chaos/src/lib.rs | 4 +-
core/layers/concurrent-limit/src/lib.rs | 5 +++
core/layers/dtrace/src/lib.rs | 14 +++++++
core/layers/fastrace/src/lib.rs | 12 ++++++
core/layers/foyer/src/lib.rs | 4 +-
core/layers/hotpath/src/lib.rs | 4 ++
core/layers/logging/src/lib.rs | 28 ++++++++++++++
core/layers/observe-metrics-common/src/lib.rs | 14 +++++++
core/layers/oteltrace/src/lib.rs | 7 ++++
core/layers/retry/src/lib.rs | 32 ++++++++++++++++
core/layers/route/src/lib.rs | 4 +-
core/layers/tail-cut/src/lib.rs | 13 +++++++
core/layers/throttle/src/lib.rs | 4 ++
core/layers/timeout/src/lib.rs | 5 +++
core/layers/tracing/src/lib.rs | 7 ++++
core/services/s3/src/backend.rs | 21 ++++++++++
core/services/s3/src/writer.rs | 4 --
26 files changed, 251 insertions(+), 110 deletions(-)
diff --git a/core/core/src/docs/rfcs/8109_writer_copy_from.md b/core/core/src/docs/rfcs/8109_writer_copy_from.md
index 1194e9702..398a1d11b 100644
--- a/core/core/src/docs/rfcs/8109_writer_copy_from.md
+++ b/core/core/src/docs/rfcs/8109_writer_copy_from.md
@@ -149,8 +149,9 @@ constructed operators address compatible storage.
The public operation requires the existing read and write capabilities. An
unbounded range also requires OpenDAL to discover its length before writer
-mutation. Native range copy is an optional optimization, not a new public
-capability.
+mutation. `Capability::write_can_copy_from` reports whether the composed
+service can execute eligible ranges natively. The public operation remains
+available through streaming fallback when this capability is false.
Version 1 limits each `copy_from` call to 5 GiB. A bounded or suffix range whose
declared length exceeds that limit returns an input error before writer
@@ -221,10 +222,16 @@ Extend `oio::Write` with an optional internal operation that accepts a source
path, `OpRead`, and exactly one bounded physical `BytesRange`. Following
RFC-7660, `OpRead` contains the source conditions but not the range. The raw
operation rejects ranges that are not absolute and bounded. `Ok(())` means the
-writer accepted the range. `ErrorKind::Unsupported` means the call did not
-mutate the writer and permits streaming fallback. Every execution failure uses
-a different error kind because the native operation may already have accepted
-a part. The writer enters its normal error state and remains abortable.
+writer accepted the range. Every error is an execution failure because the
+native operation may already have accepted a part. The writer enters its
+normal error state and remains abortable. Execution errors never trigger
+streaming fallback.
+
+`WriteGenerator` reads `Capability::write_can_copy_from` when it creates the
+writer. When the capability is false or the writer appends, it uses streaming
+for the complete public call. Otherwise, it uses native execution for every
+eligible physical range. A service sets the capability only when every
+supported `OpRead` argument can be represented by its native operation.
Multipart and block helpers schedule local uploads and remote copies in one
ordered part queue under the same upload ID, part or block numbers, completion,
@@ -238,9 +245,9 @@ service provides a semantically equivalent specialized path.
The native operation passes through the complete `oio::Write` wrapper stack. A
writer wrapper may forward it only when the layer is an identity transform for
the source path and byte content and does not change the selected route target.
-Every other wrapper returns `Unsupported` without mutation. The public
-`Writer` then opens the source through the complete composed reader stack and
-uses streaming fallback, preserving the layer's read-to-write behavior.
+Every other layer clears `Capability::write_can_copy_from`. The public `Writer`
+then opens the source through the complete composed reader stack and uses
+streaming fallback, preserving the layer's read-to-write behavior.
Assembly and fallback belong to the public `Writer` and `WriteGenerator`.
Service writers only execute bounded native ranges inside their existing upload
@@ -250,19 +257,18 @@ boundary-materialized, and streamed bytes.
# Compatibility and migration
-This proposal adds methods to the asynchronous `Writer` and an optional raw
-writer method. It does not change the blocking API, `write`,
-`write_from(Buf)`, `Operator::copy`, or `Copier`.
+This proposal adds methods to the asynchronous `Writer`, an optional raw
+writer method, and `Capability::write_can_copy_from`. It does not change the
+blocking API, `write`, `write_from(Buf)`, `Operator::copy`, or `Copier`.
-The raw method defaults to an `ErrorKind::Unsupported` error without mutation,
-so existing services remain correct through read-to-write fallback. Layers
-that affect paths, routing, or bytes must intercept or reject the native
-operation before any service enables its fast path. Services can then add
-native support incrementally.
+The capability defaults to false, so existing services remain correct through
+read-to-write fallback. Layers that affect paths, routing, or bytes clear the
+capability. Services can then add native support incrementally. The raw method
+defaults to a terminal `ErrorKind::Unsupported` error so a false capability
+claim fails visibly instead of changing execution paths.
Append writers accept `copy_from`. A service writer without a native append
-transaction returns `Unsupported` without mutation, and the public writer uses
-streaming fallback.
+transaction uses streaming fallback for the complete public call.
# Drawbacks
diff --git a/core/core/src/layers/complete.rs b/core/core/src/layers/complete.rs
index b5f921997..1212d1d40 100644
--- a/core/core/src/layers/complete.rs
+++ b/core/core/src/layers/complete.rs
@@ -392,18 +392,12 @@ where
)
})?;
- match w.copy_from(path, args, range).await {
- Ok(()) => {
- self.size += size;
- self.state.transition(CompleteState::Written);
- Ok(())
- }
- Err(err) if err.kind() == ErrorKind::Unsupported => Err(err),
- Err(err) => {
- self.state.transition(CompleteState::Error);
- Err(err)
- }
- }
+ w.copy_from(path, args, range)
+ .await
+ .inspect_err(|_| self.state.transition(CompleteState::Error))?;
+ self.size += size;
+ self.state.transition(CompleteState::Written);
+ Ok(())
}
async fn close(&mut self) -> Result<Metadata> {
@@ -453,19 +447,19 @@ mod tests {
use super::*;
use crate::raw::oio::Write as _;
- #[derive(Default)]
- struct MockWriteWriter {
- size: u64,
- }
+ struct UnsupportedCopyWriter;
- impl oio::Write for MockWriteWriter {
- async fn write(&mut self, bs: Buffer) -> Result<()> {
- self.size += bs.len() as u64;
+ impl oio::Write for UnsupportedCopyWriter {
+ async fn write(&mut self, _: Buffer) -> Result<()> {
Ok(())
}
+ async fn copy_from(&mut self, _: &str, _: OpRead, _: BytesRange) -> Result<()> {
+ Err(Error::new(ErrorKind::Unsupported, "copy is unsupported"))
+ }
+
async fn close(&mut self) -> Result<Metadata> {
- Ok(Metadata::new(EntryMode::FILE).with_content_length(self.size))
+ Ok(Metadata::default())
}
async fn abort(&mut self) -> Result<()> {
@@ -521,20 +515,17 @@ mod tests {
}
#[tokio::test]
- async fn test_writer_copy_from_unsupported_keeps_writer_usable() {
- let mut writer = CompleteWriter::new(MockWriteWriter::default(), false);
+ async fn test_writer_copy_from_error_is_terminal() {
+ let mut writer = CompleteWriter::new(UnsupportedCopyWriter, false);
let err = writer
.copy_from("source", OpRead::new(), BytesRange::new(0, Some(8)))
.await
- .expect_err("default copy_from should be unsupported");
+ .expect_err("copy_from should fail");
assert_eq!(err.kind(), ErrorKind::Unsupported);
- assert_eq!(writer.state, CompleteState::Open);
- assert_eq!(writer.size, 0);
+ assert_eq!(writer.state, CompleteState::Error);
- writer.write(Buffer::from("fallback")).await.unwrap();
- let metadata = writer.close().await.unwrap();
- assert_eq!(metadata.content_length(), 8);
- assert_eq!(writer.state, CompleteState::Closed);
+ writer.abort().await.unwrap();
+ assert!(writer.inner.is_none());
}
}
diff --git a/core/core/src/raw/oio/write/api.rs b/core/core/src/raw/oio/write/api.rs
index ca73845f7..b81303f4c 100644
--- a/core/core/src/raw/oio/write/api.rs
+++ b/core/core/src/raw/oio/write/api.rs
@@ -34,9 +34,9 @@ pub trait Write: Unpin + Send + Sync {
/// Copy one absolute bounded source range into this writer.
///
- /// [`ErrorKind::Unsupported`] means native copy is unavailable and this
- /// call has not mutated the writer. Any error after mutation must use a
- /// different error kind so the caller will not start a streaming fallback.
+ /// Callers invoke this operation only when the composed service declares
+ /// [`Capability::write_can_copy_from`]. Every error is an execution failure
+ /// and must not trigger streaming fallback.
fn copy_from(
&mut self,
_path: &str,
diff --git a/core/core/src/raw/oio/write/multipart_write.rs b/core/core/src/raw/oio/write/multipart_write.rs
index 26eb1518c..ab9a08edd 100644
--- a/core/core/src/raw/oio/write/multipart_write.rs
+++ b/core/core/src/raw/oio/write/multipart_write.rs
@@ -91,11 +91,6 @@ pub trait MultipartWrite: Send + Sync + Unpin + 'static {
body: Buffer,
) -> impl Future<Output = Result<MultipartPart>> + MaybeSend;
- /// Return whether this multipart upload can accept native source ranges.
- fn supports_copy_part(&self) -> bool {
- false
- }
-
/// Copy one absolute bounded source range into a multipart part.
fn copy_part(
&self,
@@ -279,18 +274,6 @@ impl<W: MultipartWrite> MultipartWriter<W> {
self.next_part_number += 1;
Ok(())
}
-
- fn copy_execution_error(err: Error) -> Error {
- if err.kind() != ErrorKind::Unsupported {
- return err;
- }
-
- Error::new(
- ErrorKind::Unexpected,
- "native copy failed after the writer selected it for execution",
- )
- .set_source(err)
- }
}
impl<W> oio::Write for MultipartWriter<W>
@@ -313,12 +296,6 @@ where
}
async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
- if !self.w.supports_copy_part() {
- return Err(Error::new(
- ErrorKind::Unsupported,
- "multipart writer doesn't support native copy",
- ));
- }
if range.is_suffix() || range.size().is_none() {
return Err(Error::new(
ErrorKind::Unexpected,
@@ -326,11 +303,10 @@ where
));
}
- let upload_id = self.upload_id().await.map_err(Self::copy_execution_error)?;
+ let upload_id = self.upload_id().await?;
if let Some(bytes) = self.cache.clone() {
self.schedule(upload_id.clone(), MultipartInput::Write(bytes))
- .await
- .map_err(Self::copy_execution_error)?;
+ .await?;
self.cache = None;
}
self.schedule(
@@ -341,8 +317,7 @@ where
range,
},
)
- .await
- .map_err(Self::copy_execution_error)?;
+ .await?;
Ok(())
}
@@ -498,10 +473,6 @@ mod tests {
})
}
- fn supports_copy_part(&self) -> bool {
- true
- }
-
async fn copy_part(
&self,
upload_id: &str,
@@ -667,14 +638,4 @@ mod tests {
assert_eq!(inner.lock().await.part_numbers, vec![0, 1, 2, 3]);
Ok(())
}
-
- #[test]
- fn test_copy_execution_error_does_not_return_unsupported() {
- let err = MultipartWriter::<Arc<Mutex<TestWrite>>>::copy_execution_error(Error::new(
- ErrorKind::Unsupported,
- "copy part is unsupported",
- ));
-
- assert_eq!(err.kind(), ErrorKind::Unexpected);
- }
}
diff --git a/core/core/src/types/capability.rs b/core/core/src/types/capability.rs
index cca00a6e1..1b4702798 100644
--- a/core/core/src/types/capability.rs
+++ b/core/core/src/types/capability.rs
@@ -109,6 +109,9 @@ pub struct Capability {
pub write_can_empty: bool,
/// Indicates if append operations are supported.
pub write_can_append: bool,
+ /// Indicates if a non-append writer can natively copy source ranges into an in-progress write.
+ /// `Writer::copy_from` remains available through streaming fallback when false or when appending.
+ pub write_can_copy_from: bool,
/// Indicates if Content-Type can be specified during write operations.
pub write_with_content_type: bool,
/// Indicates if Content-Disposition can be specified during write operations.
diff --git a/core/core/src/types/context/write.rs b/core/core/src/types/context/write.rs
index 77d6e6625..bb05e6b55 100644
--- a/core/core/src/types/context/write.rs
+++ b/core/core/src/types/context/write.rs
@@ -123,6 +123,8 @@ pub struct WriteGenerator<W> {
/// If `exact` is true, the size of the data written to the underlying storage is
/// exactly `chunk_size` bytes.
exact: bool,
+ /// Whether the composed writer supports native source range copy.
+ native_copy: bool,
buffer: oio::QueueBuf,
}
@@ -130,6 +132,7 @@ impl WriteGenerator<oio::Writer> {
/// Create a new exact buf writer.
pub fn create(ctx: Arc<WriteContext>) -> Result<Self> {
let (chunk_size, exact) = ctx.calculate_chunk_size();
+ let native_copy = ctx.service().capability().write_can_copy_from && !ctx.args().append();
let w = ctx.srv.write(&ctx.ctx, ctx.path(), ctx.args().clone())?;
Ok(Self {
@@ -137,6 +140,7 @@ impl WriteGenerator<oio::Writer> {
ctx: Some(ctx),
chunk_size,
exact,
+ native_copy,
buffer: oio::QueueBuf::new(),
})
}
@@ -149,6 +153,7 @@ impl WriteGenerator<oio::Writer> {
ctx: None,
chunk_size,
exact,
+ native_copy: false,
buffer: oio::QueueBuf::new(),
}
}
@@ -273,7 +278,7 @@ impl WriteGenerator<oio::Writer> {
return Ok(());
}
- let Some(chunk_size) = self.chunk_size else {
+ let Some(chunk_size) = self.chunk_size.filter(|_| self.native_copy) else {
return self
.stream_range(
path,
@@ -312,14 +317,7 @@ impl WriteGenerator<oio::Writer> {
remaining
};
let part_range = BytesRange::new(offset, Some(part_size));
- match self.w.copy_from(path, args.clone(), part_range).await {
- Ok(()) => {}
- Err(err) if err.kind() == ErrorKind::Unsupported => {
- self.stream_range(path, args.clone(), options.clone(), part_range)
- .await?;
- }
- Err(err) => return Err(err),
- }
+ self.w.copy_from(path, args.clone(), part_range).await?;
offset += part_size;
remaining -= part_size;
}
diff --git a/core/core/src/types/write/writer.rs b/core/core/src/types/write/writer.rs
index e1359ee41..d4a4b1324 100644
--- a/core/core/src/types/write/writer.rs
+++ b/core/core/src/types/write/writer.rs
@@ -188,7 +188,9 @@ impl Writer {
/// writer. Calls to `write` and `copy_from` may be interleaved, and their
/// call order defines the destination bytes. OpenDAL uses a native
/// writer-side copy when the composed service supports it and otherwise
- /// streams the range through the client.
+ /// streams the range through the client. Check
+ /// [`Capability::write_can_copy_from`] to determine whether eligible
+ /// non-append writes use the native path.
///
/// Each call accepts at most 5 GiB. Applications must split larger source
/// ranges into multiple ordered calls.
diff --git a/core/layers/async-backtrace/src/lib.rs b/core/layers/async-backtrace/src/lib.rs
index 7e6e6c7f3..526ed519a 100644
--- a/core/layers/async-backtrace/src/lib.rs
+++ b/core/layers/async-backtrace/src/lib.rs
@@ -200,6 +200,11 @@ impl<R: oio::Write> oio::Write for AsyncBacktraceWrapper<R> {
self.inner.write(bs).await
}
+ #[async_backtrace::framed]
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ self.inner.copy_from(path, args, range).await
+ }
+
#[async_backtrace::framed]
async fn close(&mut self) -> Result<Metadata> {
self.inner.close().await
diff --git a/core/layers/await-tree/src/lib.rs b/core/layers/await-tree/src/lib.rs
index 8f2fc6e5d..c2105da3c 100644
--- a/core/layers/await-tree/src/lib.rs
+++ b/core/layers/await-tree/src/lib.rs
@@ -214,6 +214,13 @@ impl<R: oio::Write> oio::Write for AwaitTreeWrapper<R> {
.await
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ self.inner
+ .copy_from(path, args, range)
+ .instrument_await(format!("opendal::{}", Operation::Write.into_static()))
+ .await
+ }
+
async fn abort(&mut self) -> Result<()> {
self.inner
.abort()
diff --git a/core/layers/chaos/src/lib.rs b/core/layers/chaos/src/lib.rs
index 0e9ca5201..114ffd51e 100644
--- a/core/layers/chaos/src/lib.rs
+++ b/core/layers/chaos/src/lib.rs
@@ -118,7 +118,9 @@ impl Service for ChaosService {
}
fn capability(&self) -> Capability {
- self.inner.capability()
+ let mut capability = self.inner.capability();
+ capability.write_can_copy_from = false;
+ capability
}
async fn create_dir(
diff --git a/core/layers/concurrent-limit/src/lib.rs b/core/layers/concurrent-limit/src/lib.rs
index c539317e9..3bb5112d1 100644
--- a/core/layers/concurrent-limit/src/lib.rs
+++ b/core/layers/concurrent-limit/src/lib.rs
@@ -438,6 +438,11 @@ where
self.inner.write(bs).await
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ self.acquire().await;
+ self.inner.copy_from(path, args, range).await
+ }
+
async fn close(&mut self) -> Result<Metadata> {
self.acquire().await;
self.inner.close().await
diff --git a/core/layers/dtrace/src/lib.rs b/core/layers/dtrace/src/lib.rs
index 97def1880..c8b98b7af 100644
--- a/core/layers/dtrace/src/lib.rs
+++ b/core/layers/dtrace/src/lib.rs
@@ -412,6 +412,20 @@ impl<R: oio::Write> oio::Write for DtraceLayerWrapper<R> {
})
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ let c_path = CString::new(self.path.clone()).unwrap();
+ probe_lazy!(opendal, writer_write_start, c_path.as_ptr());
+ self.inner
+ .copy_from(path, args, range)
+ .await
+ .map(|_| {
+ probe_lazy!(opendal, writer_write_ok, c_path.as_ptr());
+ })
+ .inspect_err(|_| {
+ probe_lazy!(opendal, writer_write_error, c_path.as_ptr());
+ })
+ }
+
async fn abort(&mut self) -> Result<()> {
let c_path = CString::new(self.path.clone()).unwrap();
probe_lazy!(opendal, writer_poll_abort_start, c_path.as_ptr());
diff --git a/core/layers/fastrace/src/lib.rs b/core/layers/fastrace/src/lib.rs
index eea5a8dea..ea4784234 100644
--- a/core/layers/fastrace/src/lib.rs
+++ b/core/layers/fastrace/src/lib.rs
@@ -307,6 +307,18 @@ impl<R: oio::Write> oio::Write for FastraceWrapper<R> {
self.inner.write(bs)
}
+ fn copy_from(
+ &mut self,
+ path: &str,
+ args: OpRead,
+ range: BytesRange,
+ ) -> impl Future<Output = Result<()>> + MaybeSend {
+ let _guard = self.span.set_local_parent();
+ let _span = LocalSpan::enter_with_local_parent(Operation::Write.into_static());
+ let path = path.to_string();
+ async move { self.inner.copy_from(&path, args, range).await }
+ }
+
fn abort(&mut self) -> impl Future<Output = Result<()>> + MaybeSend {
let _guard = self.span.set_local_parent();
let _span = LocalSpan::enter_with_local_parent(Operation::Write.into_static());
diff --git a/core/layers/foyer/src/lib.rs b/core/layers/foyer/src/lib.rs
index c78af3623..e4307de51 100644
--- a/core/layers/foyer/src/lib.rs
+++ b/core/layers/foyer/src/lib.rs
@@ -279,7 +279,9 @@ impl Service for FoyerService {
}
fn capability(&self) -> Capability {
- self.inner.capability()
+ let mut capability = self.inner.capability();
+ capability.write_can_copy_from = false;
+ capability
}
fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
diff --git a/core/layers/hotpath/src/lib.rs b/core/layers/hotpath/src/lib.rs
index a7cfb6c80..14c3b9140 100644
--- a/core/layers/hotpath/src/lib.rs
+++ b/core/layers/hotpath/src/lib.rs
@@ -223,6 +223,10 @@ impl<R: oio::Write> oio::Write for HotpathWrapper<R> {
hotpath::measure_async(LABEL_WRITER_WRITE, self.inner.write(bs)).await
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ hotpath::measure_async(LABEL_WRITER_WRITE, self.inner.copy_from(path, args, range)).await
+ }
+
async fn close(&mut self) -> Result<Metadata> {
hotpath::measure_async(LABEL_WRITER_CLOSE, self.inner.close()).await
}
diff --git a/core/layers/logging/src/lib.rs b/core/layers/logging/src/lib.rs
index baa43d3b2..785f71194 100644
--- a/core/layers/logging/src/lib.rs
+++ b/core/layers/logging/src/lib.rs
@@ -639,6 +639,34 @@ impl<W: oio::Write, I: LoggingInterceptor> oio::Write for LoggingWriter<W, I> {
}
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ let size = range
+ .size()
+ .expect("writer copy range must be absolute and bounded");
+
+ match self.inner.copy_from(path, args, range).await {
+ Ok(()) => {
+ self.written += size;
+ Ok(())
+ }
+ Err(err) => {
+ self.logger.log(
+ &self.info,
+ Operation::Write,
+ &[
+ ("path", &self.path),
+ ("source", path),
+ ("written", &self.written.to_string()),
+ ("size", &size.to_string()),
+ ],
+ "failed",
+ Some(&err),
+ );
+ Err(err)
+ }
+ }
+ }
+
async fn abort(&mut self) -> Result<()> {
match self.inner.abort().await {
Ok(_) => {
diff --git a/core/layers/observe-metrics-common/src/lib.rs b/core/layers/observe-metrics-common/src/lib.rs
index 3ee72e400..24990c7d0 100644
--- a/core/layers/observe-metrics-common/src/lib.rs
+++ b/core/layers/observe-metrics-common/src/lib.rs
@@ -1217,6 +1217,20 @@ impl<R: oio::Write, I: MetricsIntercept> oio::Write for MetricsWrapper<R, I> {
.inspect_err(|err| self.record_error(err))
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ let size = range
+ .size()
+ .expect("writer copy range must be absolute and bounded");
+
+ self.inner
+ .copy_from(path, args, range)
+ .await
+ .inspect(|_| {
+ self.size += size;
+ })
+ .inspect_err(|err| self.record_error(err))
+ }
+
async fn close(&mut self) -> Result<Metadata> {
let result = self
.inner
diff --git a/core/layers/oteltrace/src/lib.rs b/core/layers/oteltrace/src/lib.rs
index d49f01a5a..837dadb58 100644
--- a/core/layers/oteltrace/src/lib.rs
+++ b/core/layers/oteltrace/src/lib.rs
@@ -265,6 +265,13 @@ impl<R: oio::Write> oio::Write for OtelTraceWrapper<R> {
self.inner.write(bs).with_context(self.cx.clone()).await
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ self.inner
+ .copy_from(path, args, range)
+ .with_context(self.cx.clone())
+ .await
+ }
+
async fn abort(&mut self) -> Result<()> {
self.inner.abort().with_context(self.cx.clone()).await
}
diff --git a/core/layers/retry/src/lib.rs b/core/layers/retry/src/lib.rs
index 7371c83bf..88602bac2 100644
--- a/core/layers/retry/src/lib.rs
+++ b/core/layers/retry/src/lib.rs
@@ -765,6 +765,38 @@ impl<R: oio::Write, I: RetryInterceptor> oio::Write for RetryWrapper<R, I> {
res.map_err(|err| err.set_persistent())
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ use backon::RetryableWithContext;
+
+ let inner = self.take_inner()?;
+ let path = path.to_string();
+ let mut attempt: u32 = 0;
+
+ let ((inner, _, _, _), res) = {
+ |(mut r, path, args, range): (R, String, OpRead, BytesRange)| async move {
+ let res = r.copy_from(&path, args.clone(), range).await;
+
+ ((r, path, args, range), res)
+ }
+ }
+ .retry(self.builder)
+ .when(|e| e.is_temporary())
+ .context((inner, path, args, range))
+ .notify(|err, dur| {
+ attempt += 1;
+ self.notify.intercept(RetryEvent {
+ op: Operation::Write,
+ err,
+ retry_after: dur,
+ attempt,
+ })
+ })
+ .await;
+
+ self.inner = Some(inner);
+ res.map_err(|err| err.set_persistent())
+ }
+
async fn abort(&mut self) -> Result<()> {
use backon::RetryableWithContext;
diff --git a/core/layers/route/src/lib.rs b/core/layers/route/src/lib.rs
index 64c762cd1..6dff6ad62 100644
--- a/core/layers/route/src/lib.rs
+++ b/core/layers/route/src/lib.rs
@@ -202,7 +202,9 @@ impl Service for RouteAccessor {
}
fn capability(&self) -> Capability {
- self.inner.capability()
+ let mut capability = self.inner.capability();
+ capability.write_can_copy_from = false;
+ capability
}
async fn create_dir(
diff --git a/core/layers/tail-cut/src/lib.rs b/core/layers/tail-cut/src/lib.rs
index e82490067..b800fb568 100644
--- a/core/layers/tail-cut/src/lib.rs
+++ b/core/layers/tail-cut/src/lib.rs
@@ -615,6 +615,19 @@ impl<R: oio::Write> oio::Write for TailCutWrapper<R> {
.await
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ let deadline = self.calculate_deadline(Operation::Write);
+ Self::with_io_deadline(
+ deadline,
+ self.config.percentile,
+ &self.stats,
+ self.size,
+ Operation::Write,
+ self.inner.copy_from(path, args, range),
+ )
+ .await
+ }
+
async fn close(&mut self) -> Result<Metadata> {
let deadline = self.calculate_deadline(Operation::Write);
Self::with_io_deadline(
diff --git a/core/layers/throttle/src/lib.rs b/core/layers/throttle/src/lib.rs
index 02e502ac5..4e488124b 100644
--- a/core/layers/throttle/src/lib.rs
+++ b/core/layers/throttle/src/lib.rs
@@ -258,6 +258,10 @@ impl<R: oio::Write> oio::Write for ThrottleWrapper<R> {
self.inner.write(bs).await
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ self.inner.copy_from(path, args, range).await
+ }
+
async fn close(&mut self) -> Result<Metadata> {
self.inner.close().await
}
diff --git a/core/layers/timeout/src/lib.rs b/core/layers/timeout/src/lib.rs
index 86a04da0d..6f5f81ece 100644
--- a/core/layers/timeout/src/lib.rs
+++ b/core/layers/timeout/src/lib.rs
@@ -374,6 +374,11 @@ impl<R: oio::Write> oio::Write for TimeoutWrapper<R> {
Self::io_timeout(self.timeout, Operation::Write.into_static(), fut).await
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ let fut = self.inner.copy_from(path, args, range);
+ Self::io_timeout(self.timeout, Operation::Write.into_static(), fut).await
+ }
+
async fn close(&mut self) -> Result<Metadata> {
let fut = self.inner.close();
Self::io_timeout(self.timeout, Operation::Write.into_static(), fut).await
diff --git a/core/layers/tracing/src/lib.rs b/core/layers/tracing/src/lib.rs
index b1f4dbe8f..f87948f42 100644
--- a/core/layers/tracing/src/lib.rs
+++ b/core/layers/tracing/src/lib.rs
@@ -354,6 +354,13 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
self.inner.write(bs).instrument(self.span.clone()).await
}
+ async fn copy_from(&mut self, path: &str, args: OpRead, range: BytesRange) -> Result<()> {
+ self.inner
+ .copy_from(path, args, range)
+ .instrument(self.span.clone())
+ .await
+ }
+
async fn abort(&mut self) -> Result<()> {
self.inner.abort().instrument(self.span.clone()).await
}
diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs
index 33081cc87..91d37c27b 100644
--- a/core/services/s3/src/backend.rs
+++ b/core/services/s3/src/backend.rs
@@ -963,6 +963,7 @@ impl Builder for S3Builder {
write_can_empty: true,
write_can_multi: true,
write_can_append: true,
+ write_can_copy_from: checksum_algorithm.is_none(),
write_with_cache_control: true,
write_with_content_type: true,
@@ -1247,6 +1248,26 @@ mod tests {
assert_eq!(builder.config.profile.as_deref(), Some("selected"));
}
+ #[test]
+ fn test_write_can_copy_from_capability() {
+ let backend = S3Builder::default()
+ .bucket("test")
+ .region("us-east-1")
+ .endpoint("http://127.0.0.1:9000")
+ .build()
+ .unwrap();
+ assert!(backend.capability().write_can_copy_from);
+
+ let backend = S3Builder::default()
+ .bucket("test")
+ .region("us-east-1")
+ .endpoint("http://127.0.0.1:9000")
+ .checksum_algorithm("crc32c")
+ .build()
+ .unwrap();
+ assert!(!backend.capability().write_can_copy_from);
+ }
+
#[test]
fn test_is_valid_bucket() {
let bucket_cases = vec![
diff --git a/core/services/s3/src/writer.rs b/core/services/s3/src/writer.rs
index c087dcaf1..0808b350a 100644
--- a/core/services/s3/src/writer.rs
+++ b/core/services/s3/src/writer.rs
@@ -152,10 +152,6 @@ impl oio::MultipartWrite for S3Writer {
}
}
- fn supports_copy_part(&self) -> bool {
- self.core.checksum_algorithm.is_none()
- }
-
async fn copy_part(
&self,
upload_id: &str,
(opendal) 02/08: Merge remote-tracking branch 'origin/main' into xuanwo/writer-copy-from-s3
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/writer-copy-from-s3
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 4eefc54f3bfeda9cc75c0d06f4f0f064bfc46e4e
Merge: f2ae14777 101362fc4
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Aug 24 16:24:30 2026 +0800
Merge remote-tracking branch 'origin/main' into xuanwo/writer-copy-from-s3
.github/services/s3/0_minio_s3/action.yml | 2 +-
.../services/s3/minio_s3_with_anonymous/action.yml | 2 +-
.../s3/minio_s3_with_list_objects_v1/action.yml | 2 +-
.../s3/minio_s3_with_versioning/action.yml | 2 +-
bindings/c/src/operator.rs | 1 +
bindings/c/src/presign.rs | 1 +
bindings/cpp/src/lib.rs | 1 +
bindings/dotnet/src/options.rs | 1 +
bindings/nodejs/generated.d.ts | 5 +++
bindings/nodejs/src/options.rs | 7 ++++
bindings/python/python/opendal/operator.pyi | 6 +++
bindings/python/python/opendal/options.py | 2 +
bindings/python/src/operator.rs | 16 ++++++--
bindings/python/src/options.rs | 4 ++
core/core/src/layers/correctness_check.rs | 8 ++++
core/core/src/raw/ops.rs | 17 ++++++++
core/core/src/types/capability.rs | 2 +
core/core/src/types/operator/operator_futures.rs | 6 +++
core/core/src/types/options.rs | 12 ++++++
core/services/s3/src/backend.rs | 1 +
core/services/s3/src/core.rs | 19 +++++++++
core/tests/behavior/async_delete.rs | 48 ++++++++++++++++++++++
dev/src/generate/options.rs | 8 +++-
23 files changed, 164 insertions(+), 9 deletions(-)