You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/07/24 08:21:06 UTC
[incubator-opendal] branch main updated: feat(oio): Add MultipartUploadWrite to easier the work for Writer (#2699)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new a16dbec21 feat(oio): Add MultipartUploadWrite to easier the work for Writer (#2699)
a16dbec21 is described below
commit a16dbec21f61e68cc59242525936a602e52c1658
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Jul 24 16:21:01 2023 +0800
feat(oio): Add MultipartUploadWrite to easier the work for Writer (#2699)
* feat(oio): Add MultipartUploadWrite to easier the work for Writer
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix typo
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Update core/src/raw/oio/write/multipart_upload_write.rs
Co-authored-by: Suyan <su...@gmail.com>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
Co-authored-by: Suyan <su...@gmail.com>
---
core/src/raw/oio/write/mod.rs | 5 +
core/src/raw/oio/write/multipart_upload_write.rs | 265 +++++++++++++++++++++++
core/src/services/s3/backend.rs | 2 +-
core/src/services/s3/core.rs | 14 +-
core/src/services/s3/writer.rs | 166 ++++----------
5 files changed, 319 insertions(+), 133 deletions(-)
diff --git a/core/src/raw/oio/write/mod.rs b/core/src/raw/oio/write/mod.rs
index eb809a871..490136e29 100644
--- a/core/src/raw/oio/write/mod.rs
+++ b/core/src/raw/oio/write/mod.rs
@@ -21,3 +21,8 @@ pub use api::BlockingWriter;
pub use api::Write;
pub use api::WriteOperation;
pub use api::Writer;
+
+mod multipart_upload_write;
+pub use multipart_upload_write::MultipartUploadPart;
+pub use multipart_upload_write::MultipartUploadWrite;
+pub use multipart_upload_write::MultipartUploadWriter;
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
new file mode 100644
index 000000000..0fe035439
--- /dev/null
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -0,0 +1,265 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use async_trait::async_trait;
+use bytes::Bytes;
+
+use crate::{raw::*, *};
+
+const DEFAULT_WRITE_MIN_SIZE: usize = 8 * 1024 * 1024;
+
+/// MultipartUploadWrite is used to implement [`Write`] based on multipart
+/// uploads. By implementing MultipartUploadWrite, services don't need to
+/// care about the details of buffering and uploading parts.
+///
+/// The layout after adopting [`MultipartUploadWrite`]:
+///
+/// - Services impl `MultipartUploadWrite`
+/// - `MultipartUploadWriter` impl `Write`
+/// - Expose `MultipartUploadWriter` as `Accessor::Writer`
+#[async_trait]
+pub trait MultipartUploadWrite: Send + Sync + Unpin {
+ /// write_once write all data at once.
+ ///
+ /// MultipartUploadWriter will call this API when the size of data is
+ /// already known.
+ async fn write_once(&self, size: u64, body: AsyncBody) -> Result<()>;
+
+ /// initiate_part will call start a multipart upload and return the upload id.
+ ///
+ /// MultipartUploadWriter will call this when:
+ ///
+ /// - the total size of data is unknown.
+ /// - the total size of data is known, but the size of current write
+ /// is less then the total size.
+ async fn initiate_part(&self) -> Result<String>;
+
+ /// write_part will write a part of the data and returns the result
+ /// [`MultipartUploadPart`].
+ ///
+ /// MultipartUploadWriter will call this API and stores the result in
+ /// order.
+ ///
+ /// - part_number is the index of the part, starting from 0.
+ async fn write_part(
+ &self,
+ upload_id: &str,
+ part_number: usize,
+ size: u64,
+ body: AsyncBody,
+ ) -> Result<MultipartUploadPart>;
+
+ /// complete_part will complete the multipart upload to build the final
+ /// file.
+ async fn complete_part(&self, upload_id: &str, parts: &[MultipartUploadPart]) -> Result<()>;
+
+ /// abort_part will cancel the multipart upload and purge all data.
+ async fn abort_part(&self, upload_id: &str) -> Result<()>;
+}
+
+/// The result of [`MultipartUploadWrite::write_part`].
+///
+/// services implement should convert MultipartUploadPart to their own represents.
+///
+/// - `part_number` is the index of the part, starting from 0.
+/// - `etag` is the `ETag` of the part.
+pub struct MultipartUploadPart {
+ /// The number of the part, starting from 0.
+ pub part_number: usize,
+ /// The etag of the part.
+ pub etag: String,
+}
+
+/// MultipartUploadWriter will implements [`Write`] based on multipart
+/// uploads.
+///
+/// ## TODO
+///
+/// - Add threshold for `write_once` to avoid unnecessary multipart uploads.
+/// - Allow users to switch to un-buffered mode if users write 16MiB every time.
+pub struct MultipartUploadWriter<W: MultipartUploadWrite> {
+ inner: W,
+ total_size: Option<u64>,
+
+ upload_id: Option<String>,
+ parts: Vec<MultipartUploadPart>,
+ buffer: oio::VectorCursor,
+ buffer_size: usize,
+}
+
+impl<W: MultipartUploadWrite> MultipartUploadWriter<W> {
+ /// Create a new MultipartUploadWriter.
+ pub fn new(inner: W, total_size: Option<u64>) -> Self {
+ Self {
+ inner,
+ total_size,
+
+ upload_id: None,
+ parts: Vec::new(),
+ buffer: oio::VectorCursor::new(),
+ buffer_size: DEFAULT_WRITE_MIN_SIZE,
+ }
+ }
+
+ /// Configure the write_min_size.
+ ///
+ /// write_min_size is used to control the size of internal buffer.
+ ///
+ /// MultipartUploadWriter will flush the buffer to upload a part when
+ /// the size of buffer is larger than write_min_size.
+ ///
+ /// This value is default to 8 MiB (as recommended by AWS).
+ pub fn with_write_min_size(mut self, v: usize) -> Self {
+ self.buffer_size = v;
+ self
+ }
+}
+
+#[async_trait]
+impl<W> oio::Write for MultipartUploadWriter<W>
+where
+ W: MultipartUploadWrite,
+{
+ async fn write(&mut self, bs: Bytes) -> Result<()> {
+ let upload_id = match &self.upload_id {
+ Some(upload_id) => upload_id,
+ None => {
+ if self.total_size.unwrap_or_default() == bs.len() as u64 {
+ return self
+ .inner
+ .write_once(bs.len() as u64, AsyncBody::Bytes(bs))
+ .await;
+ }
+
+ let upload_id = self.inner.initiate_part().await?;
+ self.upload_id = Some(upload_id);
+ self.upload_id.as_deref().unwrap()
+ }
+ };
+
+ // Ignore empty bytes
+ if bs.is_empty() {
+ return Ok(());
+ }
+
+ self.buffer.push(bs);
+ // Return directly if the buffer is not full
+ if self.buffer.len() <= self.buffer_size {
+ return Ok(());
+ }
+
+ let bs = self.buffer.peak_at_least(self.buffer_size);
+ let size = bs.len();
+
+ match self
+ .inner
+ .write_part(
+ upload_id,
+ self.parts.len(),
+ size as u64,
+ AsyncBody::Bytes(bs),
+ )
+ .await
+ {
+ Ok(part) => {
+ self.buffer.take(size);
+ self.parts.push(part);
+ Ok(())
+ }
+ Err(e) => {
+ // If the upload fails, we should pop the given bs to make sure
+ // write is re-enter safe.
+ self.buffer.pop();
+ Err(e)
+ }
+ }
+ }
+
+ async fn sink(&mut self, size: u64, s: oio::Streamer) -> Result<()> {
+ if !self.buffer.is_empty() {
+ return Err(Error::new(
+ ErrorKind::InvalidInput,
+ "Writer::sink should not be used mixed with existing buffer",
+ ));
+ }
+
+ let upload_id = match &self.upload_id {
+ Some(upload_id) => upload_id,
+ None => {
+ if self.total_size.unwrap_or_default() == size {
+ return self.inner.write_once(size, AsyncBody::Stream(s)).await;
+ }
+
+ let upload_id = self.inner.initiate_part().await?;
+ self.upload_id = Some(upload_id);
+ self.upload_id.as_deref().unwrap()
+ }
+ };
+
+ let part = self
+ .inner
+ .write_part(upload_id, self.parts.len(), size, AsyncBody::Stream(s))
+ .await?;
+ self.parts.push(part);
+
+ Ok(())
+ }
+
+ async fn close(&mut self) -> Result<()> {
+ let upload_id = if let Some(upload_id) = &self.upload_id {
+ upload_id
+ } else {
+ return Ok(());
+ };
+
+ // Make sure internal buffer has been flushed.
+ if !self.buffer.is_empty() {
+ let bs = self.buffer.peak_exact(self.buffer.len());
+
+ match self
+ .inner
+ .write_part(
+ upload_id,
+ self.parts.len(),
+ bs.len() as u64,
+ AsyncBody::Bytes(bs),
+ )
+ .await
+ {
+ Ok(part) => {
+ self.buffer.clear();
+ self.parts.push(part);
+ }
+ Err(e) => {
+ return Err(e);
+ }
+ }
+ }
+
+ self.inner.complete_part(upload_id, &self.parts).await
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ let upload_id = if let Some(upload_id) = &self.upload_id {
+ upload_id
+ } else {
+ return Ok(());
+ };
+
+ self.inner.abort_part(upload_id).await
+ }
+}
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 8a2d46fc9..fdf4e4f84 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -898,7 +898,7 @@ pub struct S3Backend {
impl Accessor for S3Backend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
- type Writer = S3Writer;
+ type Writer = oio::MultipartUploadWriter<S3Writer>;
type BlockingWriter = ();
type Appender = ();
type Pager = S3Pager;
diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs
index 027d6f36a..e06c1e6c6 100644
--- a/core/src/services/s3/core.rs
+++ b/core/src/services/s3/core.rs
@@ -547,7 +547,7 @@ impl S3Core {
path: &str,
upload_id: &str,
part_number: usize,
- size: Option<u64>,
+ size: u64,
body: AsyncBody,
) -> Result<Request<AsyncBody>> {
let p = build_abs_path(&self.root, path);
@@ -562,9 +562,7 @@ impl S3Core {
let mut req = Request::put(&url);
- if let Some(size) = size {
- req = req.header(CONTENT_LENGTH, size);
- }
+ req = req.header(CONTENT_LENGTH, size);
// Set SSE headers.
req = self.insert_sse_headers(req, true);
@@ -579,7 +577,7 @@ impl S3Core {
&self,
path: &str,
upload_id: &str,
- parts: &[CompleteMultipartUploadRequestPart],
+ parts: Vec<CompleteMultipartUploadRequestPart>,
) -> Result<Response<IncomingAsyncBody>> {
let p = build_abs_path(&self.root, path);
@@ -595,10 +593,8 @@ impl S3Core {
// Set SSE headers.
let req = self.insert_sse_headers(req, true);
- let content = quick_xml::se::to_string(&CompleteMultipartUploadRequest {
- part: parts.to_vec(),
- })
- .map_err(new_xml_deserialize_error)?;
+ let content = quick_xml::se::to_string(&CompleteMultipartUploadRequest { part: parts })
+ .map_err(new_xml_deserialize_error)?;
// Make sure content length has been set to avoid post with chunked encoding.
let req = req.header(CONTENT_LENGTH, content.len());
// Set content-type to `application/xml` to avoid mixed with form post.
diff --git a/core/src/services/s3/writer.rs b/core/src/services/s3/writer.rs
index 2ef197e43..508716d14 100644
--- a/core/src/services/s3/writer.rs
+++ b/core/src/services/s3/writer.rs
@@ -19,7 +19,6 @@ use std::sync::Arc;
use async_trait::async_trait;
use bytes::Buf;
-use bytes::Bytes;
use http::StatusCode;
use super::core::*;
@@ -32,29 +31,26 @@ pub struct S3Writer {
op: OpWrite,
path: String,
- upload_id: Option<String>,
-
- parts: Vec<CompleteMultipartUploadRequestPart>,
- buffer: oio::VectorCursor,
- buffer_size: usize,
}
impl S3Writer {
- pub fn new(core: Arc<S3Core>, path: &str, op: OpWrite) -> Self {
- let buffer_size = core.write_min_size;
- S3Writer {
+ pub fn new(core: Arc<S3Core>, path: &str, op: OpWrite) -> oio::MultipartUploadWriter<Self> {
+ let write_min_size = core.write_min_size;
+
+ let total_size = op.content_length();
+ let s3_writer = S3Writer {
core,
path: path.to_string(),
op,
+ };
- upload_id: None,
- parts: vec![],
- buffer: oio::VectorCursor::new(),
- buffer_size,
- }
+ oio::MultipartUploadWriter::new(s3_writer, total_size).with_write_min_size(write_min_size)
}
+}
- async fn write_oneshot(&self, size: u64, body: AsyncBody) -> Result<()> {
+#[async_trait]
+impl oio::MultipartUploadWrite for S3Writer {
+ async fn write_once(&self, size: u64, body: AsyncBody) -> Result<()> {
let mut req = self.core.s3_put_object_request(
&self.path,
Some(size),
@@ -79,7 +75,7 @@ impl S3Writer {
}
}
- async fn initiate_upload(&self) -> Result<String> {
+ async fn initiate_part(&self) -> Result<String> {
let resp = self
.core
.s3_initiate_multipart_upload(
@@ -108,18 +104,16 @@ impl S3Writer {
async fn write_part(
&self,
upload_id: &str,
- bs: Bytes,
- ) -> Result<CompleteMultipartUploadRequestPart> {
+ part_number: usize,
+ size: u64,
+ body: AsyncBody,
+ ) -> Result<oio::MultipartUploadPart> {
// AWS S3 requires part number must between [1..=10000]
- let part_number = self.parts.len() + 1;
+ let part_number = part_number + 1;
- let mut req = self.core.s3_upload_part_request(
- &self.path,
- upload_id,
- part_number,
- Some(bs.len() as u64),
- AsyncBody::Bytes(bs),
- )?;
+ let mut req =
+ self.core
+ .s3_upload_part_request(&self.path, upload_id, part_number, size, body)?;
self.core.sign(&mut req).await?;
@@ -140,78 +134,43 @@ impl S3Writer {
resp.into_body().consume().await?;
- Ok(CompleteMultipartUploadRequestPart { part_number, etag })
+ Ok(oio::MultipartUploadPart { part_number, etag })
}
_ => Err(parse_error(resp).await?),
}
}
-}
-#[async_trait]
-impl oio::Write for S3Writer {
- async fn write(&mut self, bs: Bytes) -> Result<()> {
- let upload_id = match &self.upload_id {
- Some(upload_id) => upload_id,
- None => {
- if self.op.content_length().unwrap_or_default() == bs.len() as u64 {
- return self
- .write_oneshot(bs.len() as u64, AsyncBody::Bytes(bs))
- .await;
- } else {
- let upload_id = self.initiate_upload().await?;
- self.upload_id = Some(upload_id);
- self.upload_id.as_deref().unwrap()
- }
- }
- };
+ async fn complete_part(
+ &self,
+ upload_id: &str,
+ parts: &[oio::MultipartUploadPart],
+ ) -> Result<()> {
+ let parts = parts
+ .iter()
+ .map(|p| CompleteMultipartUploadRequestPart {
+ part_number: p.part_number,
+ etag: p.etag.clone(),
+ })
+ .collect();
- // Ignore empty bytes
- if bs.is_empty() {
- return Ok(());
- }
+ let resp = self
+ .core
+ .s3_complete_multipart_upload(&self.path, upload_id, parts)
+ .await?;
- self.buffer.push(bs);
- // Return directly if the buffer is not full
- if self.buffer.len() <= self.buffer_size {
- return Ok(());
- }
+ let status = resp.status();
- let bs = self.buffer.peak_at_least(self.buffer_size);
- let size = bs.len();
+ match status {
+ StatusCode::OK => {
+ resp.into_body().consume().await?;
- match self.write_part(upload_id, bs).await {
- Ok(part) => {
- self.buffer.take(size);
- self.parts.push(part);
Ok(())
}
- Err(e) => {
- // If the upload fails, we should pop the given bs to make sure
- // write is re-enter safe.
- self.buffer.pop();
- Err(e)
- }
- }
- }
-
- async fn sink(&mut self, size: u64, s: oio::Streamer) -> Result<()> {
- if self.op.content_length().unwrap_or_default() == size {
- return self.write_oneshot(size, AsyncBody::Stream(s)).await;
- } else {
- return Err(Error::new(
- ErrorKind::Unsupported,
- "S3 does not support streaming multipart upload",
- ));
+ _ => Err(parse_error(resp).await?),
}
}
- async fn abort(&mut self) -> Result<()> {
- let upload_id = if let Some(upload_id) = &self.upload_id {
- upload_id
- } else {
- return Ok(());
- };
-
+ async fn abort_part(&self, upload_id: &str) -> Result<()> {
let resp = self
.core
.s3_abort_multipart_upload(&self.path, upload_id)
@@ -225,43 +184,4 @@ impl oio::Write for S3Writer {
_ => Err(parse_error(resp).await?),
}
}
-
- async fn close(&mut self) -> Result<()> {
- let upload_id = if let Some(upload_id) = &self.upload_id {
- upload_id
- } else {
- return Ok(());
- };
-
- // Make sure internal buffer has been flushed.
- if !self.buffer.is_empty() {
- let bs = self.buffer.peak_exact(self.buffer.len());
-
- match self.write_part(upload_id, bs).await {
- Ok(part) => {
- self.buffer.clear();
- self.parts.push(part);
- }
- Err(e) => {
- return Err(e);
- }
- }
- }
-
- let resp = self
- .core
- .s3_complete_multipart_upload(&self.path, upload_id, &self.parts)
- .await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK => {
- resp.into_body().consume().await?;
-
- Ok(())
- }
- _ => Err(parse_error(resp).await?),
- }
- }
}