You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/07/24 08:21:06 UTC

[incubator-opendal] branch main updated: feat(oio): Add MultipartUploadWrite to easier the work for Writer (#2699)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new a16dbec21 feat(oio): Add MultipartUploadWrite to easier the work for Writer (#2699)
a16dbec21 is described below

commit a16dbec21f61e68cc59242525936a602e52c1658
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Jul 24 16:21:01 2023 +0800

    feat(oio): Add MultipartUploadWrite to easier the work for Writer (#2699)
    
    * feat(oio): Add MultipartUploadWrite to easier the work for Writer
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    * Fix typo
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    * Update core/src/raw/oio/write/multipart_upload_write.rs
    
    Co-authored-by: Suyan <su...@gmail.com>
    
    ---------
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    Co-authored-by: Suyan <su...@gmail.com>
---
 core/src/raw/oio/write/mod.rs                    |   5 +
 core/src/raw/oio/write/multipart_upload_write.rs | 265 +++++++++++++++++++++++
 core/src/services/s3/backend.rs                  |   2 +-
 core/src/services/s3/core.rs                     |  14 +-
 core/src/services/s3/writer.rs                   | 166 ++++----------
 5 files changed, 319 insertions(+), 133 deletions(-)

diff --git a/core/src/raw/oio/write/mod.rs b/core/src/raw/oio/write/mod.rs
index eb809a871..490136e29 100644
--- a/core/src/raw/oio/write/mod.rs
+++ b/core/src/raw/oio/write/mod.rs
@@ -21,3 +21,8 @@ pub use api::BlockingWriter;
 pub use api::Write;
 pub use api::WriteOperation;
 pub use api::Writer;
+
+mod multipart_upload_write;
+pub use multipart_upload_write::MultipartUploadPart;
+pub use multipart_upload_write::MultipartUploadWrite;
+pub use multipart_upload_write::MultipartUploadWriter;
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
new file mode 100644
index 000000000..0fe035439
--- /dev/null
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -0,0 +1,265 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use async_trait::async_trait;
+use bytes::Bytes;
+
+use crate::{raw::*, *};
+
+const DEFAULT_WRITE_MIN_SIZE: usize = 8 * 1024 * 1024;
+
+/// MultipartUploadWrite is used to implement [`Write`] based on multipart
+/// uploads. By implementing MultipartUploadWrite, services don't need to
+/// care about the details of buffering and uploading parts.
+///
+/// The layout after adopting [`MultipartUploadWrite`]:
+///
+/// - Services impl `MultipartUploadWrite`
+/// - `MultipartUploadWriter` impl `Write`
+/// - Expose `MultipartUploadWriter` as `Accessor::Writer`
+#[async_trait]
+pub trait MultipartUploadWrite: Send + Sync + Unpin {
+    /// write_once write all data at once.
+    ///
+    /// MultipartUploadWriter will call this API when the size of data is
+    /// already known.
+    async fn write_once(&self, size: u64, body: AsyncBody) -> Result<()>;
+
+    /// initiate_part will call start a multipart upload and return the upload id.
+    ///
+    /// MultipartUploadWriter will call this when:
+    ///
+    /// - the total size of data is unknown.
+    /// - the total size of data is known, but the size of current write
+    /// is less then the total size.
+    async fn initiate_part(&self) -> Result<String>;
+
+    /// write_part will write a part of the data and returns the result
+    /// [`MultipartUploadPart`].
+    ///
+    /// MultipartUploadWriter will call this API and stores the result in
+    /// order.
+    ///
+    /// - part_number is the index of the part, starting from 0.
+    async fn write_part(
+        &self,
+        upload_id: &str,
+        part_number: usize,
+        size: u64,
+        body: AsyncBody,
+    ) -> Result<MultipartUploadPart>;
+
+    /// complete_part will complete the multipart upload to build the final
+    /// file.
+    async fn complete_part(&self, upload_id: &str, parts: &[MultipartUploadPart]) -> Result<()>;
+
+    /// abort_part will cancel the multipart upload and purge all data.
+    async fn abort_part(&self, upload_id: &str) -> Result<()>;
+}
+
+/// The result of [`MultipartUploadWrite::write_part`].
+///
+/// services implement should convert MultipartUploadPart to their own represents.
+///
+/// - `part_number` is the index of the part, starting from 0.
+/// - `etag` is the `ETag` of the part.
+pub struct MultipartUploadPart {
+    /// The number of the part, starting from 0.
+    pub part_number: usize,
+    /// The etag of the part.
+    pub etag: String,
+}
+
+/// MultipartUploadWriter will implements [`Write`] based on multipart
+/// uploads.
+///
+/// ## TODO
+///
+/// - Add threshold for `write_once` to avoid unnecessary multipart uploads.
+/// - Allow users to switch to un-buffered mode if users write 16MiB every time.
+pub struct MultipartUploadWriter<W: MultipartUploadWrite> {
+    inner: W,
+    total_size: Option<u64>,
+
+    upload_id: Option<String>,
+    parts: Vec<MultipartUploadPart>,
+    buffer: oio::VectorCursor,
+    buffer_size: usize,
+}
+
+impl<W: MultipartUploadWrite> MultipartUploadWriter<W> {
+    /// Create a new MultipartUploadWriter.
+    pub fn new(inner: W, total_size: Option<u64>) -> Self {
+        Self {
+            inner,
+            total_size,
+
+            upload_id: None,
+            parts: Vec::new(),
+            buffer: oio::VectorCursor::new(),
+            buffer_size: DEFAULT_WRITE_MIN_SIZE,
+        }
+    }
+
+    /// Configure the write_min_size.
+    ///
+    /// write_min_size is used to control the size of internal buffer.
+    ///
+    /// MultipartUploadWriter will flush the buffer to upload a part when
+    /// the size of buffer is larger than write_min_size.
+    ///
+    /// This value is default to 8 MiB (as recommended by AWS).
+    pub fn with_write_min_size(mut self, v: usize) -> Self {
+        self.buffer_size = v;
+        self
+    }
+}
+
+#[async_trait]
+impl<W> oio::Write for MultipartUploadWriter<W>
+where
+    W: MultipartUploadWrite,
+{
+    async fn write(&mut self, bs: Bytes) -> Result<()> {
+        let upload_id = match &self.upload_id {
+            Some(upload_id) => upload_id,
+            None => {
+                if self.total_size.unwrap_or_default() == bs.len() as u64 {
+                    return self
+                        .inner
+                        .write_once(bs.len() as u64, AsyncBody::Bytes(bs))
+                        .await;
+                }
+
+                let upload_id = self.inner.initiate_part().await?;
+                self.upload_id = Some(upload_id);
+                self.upload_id.as_deref().unwrap()
+            }
+        };
+
+        // Ignore empty bytes
+        if bs.is_empty() {
+            return Ok(());
+        }
+
+        self.buffer.push(bs);
+        // Return directly if the buffer is not full
+        if self.buffer.len() <= self.buffer_size {
+            return Ok(());
+        }
+
+        let bs = self.buffer.peak_at_least(self.buffer_size);
+        let size = bs.len();
+
+        match self
+            .inner
+            .write_part(
+                upload_id,
+                self.parts.len(),
+                size as u64,
+                AsyncBody::Bytes(bs),
+            )
+            .await
+        {
+            Ok(part) => {
+                self.buffer.take(size);
+                self.parts.push(part);
+                Ok(())
+            }
+            Err(e) => {
+                // If the upload fails, we should pop the given bs to make sure
+                // write is re-enter safe.
+                self.buffer.pop();
+                Err(e)
+            }
+        }
+    }
+
+    async fn sink(&mut self, size: u64, s: oio::Streamer) -> Result<()> {
+        if !self.buffer.is_empty() {
+            return Err(Error::new(
+                ErrorKind::InvalidInput,
+                "Writer::sink should not be used mixed with existing buffer",
+            ));
+        }
+
+        let upload_id = match &self.upload_id {
+            Some(upload_id) => upload_id,
+            None => {
+                if self.total_size.unwrap_or_default() == size {
+                    return self.inner.write_once(size, AsyncBody::Stream(s)).await;
+                }
+
+                let upload_id = self.inner.initiate_part().await?;
+                self.upload_id = Some(upload_id);
+                self.upload_id.as_deref().unwrap()
+            }
+        };
+
+        let part = self
+            .inner
+            .write_part(upload_id, self.parts.len(), size, AsyncBody::Stream(s))
+            .await?;
+        self.parts.push(part);
+
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<()> {
+        let upload_id = if let Some(upload_id) = &self.upload_id {
+            upload_id
+        } else {
+            return Ok(());
+        };
+
+        // Make sure internal buffer has been flushed.
+        if !self.buffer.is_empty() {
+            let bs = self.buffer.peak_exact(self.buffer.len());
+
+            match self
+                .inner
+                .write_part(
+                    upload_id,
+                    self.parts.len(),
+                    bs.len() as u64,
+                    AsyncBody::Bytes(bs),
+                )
+                .await
+            {
+                Ok(part) => {
+                    self.buffer.clear();
+                    self.parts.push(part);
+                }
+                Err(e) => {
+                    return Err(e);
+                }
+            }
+        }
+
+        self.inner.complete_part(upload_id, &self.parts).await
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        let upload_id = if let Some(upload_id) = &self.upload_id {
+            upload_id
+        } else {
+            return Ok(());
+        };
+
+        self.inner.abort_part(upload_id).await
+    }
+}
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 8a2d46fc9..fdf4e4f84 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -898,7 +898,7 @@ pub struct S3Backend {
 impl Accessor for S3Backend {
     type Reader = IncomingAsyncBody;
     type BlockingReader = ();
-    type Writer = S3Writer;
+    type Writer = oio::MultipartUploadWriter<S3Writer>;
     type BlockingWriter = ();
     type Appender = ();
     type Pager = S3Pager;
diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs
index 027d6f36a..e06c1e6c6 100644
--- a/core/src/services/s3/core.rs
+++ b/core/src/services/s3/core.rs
@@ -547,7 +547,7 @@ impl S3Core {
         path: &str,
         upload_id: &str,
         part_number: usize,
-        size: Option<u64>,
+        size: u64,
         body: AsyncBody,
     ) -> Result<Request<AsyncBody>> {
         let p = build_abs_path(&self.root, path);
@@ -562,9 +562,7 @@ impl S3Core {
 
         let mut req = Request::put(&url);
 
-        if let Some(size) = size {
-            req = req.header(CONTENT_LENGTH, size);
-        }
+        req = req.header(CONTENT_LENGTH, size);
 
         // Set SSE headers.
         req = self.insert_sse_headers(req, true);
@@ -579,7 +577,7 @@ impl S3Core {
         &self,
         path: &str,
         upload_id: &str,
-        parts: &[CompleteMultipartUploadRequestPart],
+        parts: Vec<CompleteMultipartUploadRequestPart>,
     ) -> Result<Response<IncomingAsyncBody>> {
         let p = build_abs_path(&self.root, path);
 
@@ -595,10 +593,8 @@ impl S3Core {
         // Set SSE headers.
         let req = self.insert_sse_headers(req, true);
 
-        let content = quick_xml::se::to_string(&CompleteMultipartUploadRequest {
-            part: parts.to_vec(),
-        })
-        .map_err(new_xml_deserialize_error)?;
+        let content = quick_xml::se::to_string(&CompleteMultipartUploadRequest { part: parts })
+            .map_err(new_xml_deserialize_error)?;
         // Make sure content length has been set to avoid post with chunked encoding.
         let req = req.header(CONTENT_LENGTH, content.len());
         // Set content-type to `application/xml` to avoid mixed with form post.
diff --git a/core/src/services/s3/writer.rs b/core/src/services/s3/writer.rs
index 2ef197e43..508716d14 100644
--- a/core/src/services/s3/writer.rs
+++ b/core/src/services/s3/writer.rs
@@ -19,7 +19,6 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 use bytes::Buf;
-use bytes::Bytes;
 use http::StatusCode;
 
 use super::core::*;
@@ -32,29 +31,26 @@ pub struct S3Writer {
 
     op: OpWrite,
     path: String,
-    upload_id: Option<String>,
-
-    parts: Vec<CompleteMultipartUploadRequestPart>,
-    buffer: oio::VectorCursor,
-    buffer_size: usize,
 }
 
 impl S3Writer {
-    pub fn new(core: Arc<S3Core>, path: &str, op: OpWrite) -> Self {
-        let buffer_size = core.write_min_size;
-        S3Writer {
+    pub fn new(core: Arc<S3Core>, path: &str, op: OpWrite) -> oio::MultipartUploadWriter<Self> {
+        let write_min_size = core.write_min_size;
+
+        let total_size = op.content_length();
+        let s3_writer = S3Writer {
             core,
             path: path.to_string(),
             op,
+        };
 
-            upload_id: None,
-            parts: vec![],
-            buffer: oio::VectorCursor::new(),
-            buffer_size,
-        }
+        oio::MultipartUploadWriter::new(s3_writer, total_size).with_write_min_size(write_min_size)
     }
+}
 
-    async fn write_oneshot(&self, size: u64, body: AsyncBody) -> Result<()> {
+#[async_trait]
+impl oio::MultipartUploadWrite for S3Writer {
+    async fn write_once(&self, size: u64, body: AsyncBody) -> Result<()> {
         let mut req = self.core.s3_put_object_request(
             &self.path,
             Some(size),
@@ -79,7 +75,7 @@ impl S3Writer {
         }
     }
 
-    async fn initiate_upload(&self) -> Result<String> {
+    async fn initiate_part(&self) -> Result<String> {
         let resp = self
             .core
             .s3_initiate_multipart_upload(
@@ -108,18 +104,16 @@ impl S3Writer {
     async fn write_part(
         &self,
         upload_id: &str,
-        bs: Bytes,
-    ) -> Result<CompleteMultipartUploadRequestPart> {
+        part_number: usize,
+        size: u64,
+        body: AsyncBody,
+    ) -> Result<oio::MultipartUploadPart> {
         // AWS S3 requires part number must between [1..=10000]
-        let part_number = self.parts.len() + 1;
+        let part_number = part_number + 1;
 
-        let mut req = self.core.s3_upload_part_request(
-            &self.path,
-            upload_id,
-            part_number,
-            Some(bs.len() as u64),
-            AsyncBody::Bytes(bs),
-        )?;
+        let mut req =
+            self.core
+                .s3_upload_part_request(&self.path, upload_id, part_number, size, body)?;
 
         self.core.sign(&mut req).await?;
 
@@ -140,78 +134,43 @@ impl S3Writer {
 
                 resp.into_body().consume().await?;
 
-                Ok(CompleteMultipartUploadRequestPart { part_number, etag })
+                Ok(oio::MultipartUploadPart { part_number, etag })
             }
             _ => Err(parse_error(resp).await?),
         }
     }
-}
 
-#[async_trait]
-impl oio::Write for S3Writer {
-    async fn write(&mut self, bs: Bytes) -> Result<()> {
-        let upload_id = match &self.upload_id {
-            Some(upload_id) => upload_id,
-            None => {
-                if self.op.content_length().unwrap_or_default() == bs.len() as u64 {
-                    return self
-                        .write_oneshot(bs.len() as u64, AsyncBody::Bytes(bs))
-                        .await;
-                } else {
-                    let upload_id = self.initiate_upload().await?;
-                    self.upload_id = Some(upload_id);
-                    self.upload_id.as_deref().unwrap()
-                }
-            }
-        };
+    async fn complete_part(
+        &self,
+        upload_id: &str,
+        parts: &[oio::MultipartUploadPart],
+    ) -> Result<()> {
+        let parts = parts
+            .iter()
+            .map(|p| CompleteMultipartUploadRequestPart {
+                part_number: p.part_number,
+                etag: p.etag.clone(),
+            })
+            .collect();
 
-        // Ignore empty bytes
-        if bs.is_empty() {
-            return Ok(());
-        }
+        let resp = self
+            .core
+            .s3_complete_multipart_upload(&self.path, upload_id, parts)
+            .await?;
 
-        self.buffer.push(bs);
-        // Return directly if the buffer is not full
-        if self.buffer.len() <= self.buffer_size {
-            return Ok(());
-        }
+        let status = resp.status();
 
-        let bs = self.buffer.peak_at_least(self.buffer_size);
-        let size = bs.len();
+        match status {
+            StatusCode::OK => {
+                resp.into_body().consume().await?;
 
-        match self.write_part(upload_id, bs).await {
-            Ok(part) => {
-                self.buffer.take(size);
-                self.parts.push(part);
                 Ok(())
             }
-            Err(e) => {
-                // If the upload fails, we should pop the given bs to make sure
-                // write is re-enter safe.
-                self.buffer.pop();
-                Err(e)
-            }
-        }
-    }
-
-    async fn sink(&mut self, size: u64, s: oio::Streamer) -> Result<()> {
-        if self.op.content_length().unwrap_or_default() == size {
-            return self.write_oneshot(size, AsyncBody::Stream(s)).await;
-        } else {
-            return Err(Error::new(
-                ErrorKind::Unsupported,
-                "S3 does not support streaming multipart upload",
-            ));
+            _ => Err(parse_error(resp).await?),
         }
     }
 
-    async fn abort(&mut self) -> Result<()> {
-        let upload_id = if let Some(upload_id) = &self.upload_id {
-            upload_id
-        } else {
-            return Ok(());
-        };
-
+    async fn abort_part(&self, upload_id: &str) -> Result<()> {
         let resp = self
             .core
             .s3_abort_multipart_upload(&self.path, upload_id)
@@ -225,43 +184,4 @@ impl oio::Write for S3Writer {
             _ => Err(parse_error(resp).await?),
         }
     }
-
-    async fn close(&mut self) -> Result<()> {
-        let upload_id = if let Some(upload_id) = &self.upload_id {
-            upload_id
-        } else {
-            return Ok(());
-        };
-
-        // Make sure internal buffer has been flushed.
-        if !self.buffer.is_empty() {
-            let bs = self.buffer.peak_exact(self.buffer.len());
-
-            match self.write_part(upload_id, bs).await {
-                Ok(part) => {
-                    self.buffer.clear();
-                    self.parts.push(part);
-                }
-                Err(e) => {
-                    return Err(e);
-                }
-            }
-        }
-
-        let resp = self
-            .core
-            .s3_complete_multipart_upload(&self.path, upload_id, &self.parts)
-            .await?;
-
-        let status = resp.status();
-
-        match status {
-            StatusCode::OK => {
-                resp.into_body().consume().await?;
-
-                Ok(())
-            }
-            _ => Err(parse_error(resp).await?),
-        }
-    }
 }