You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/01/05 12:57:40 UTC

(incubator-opendal) branch main updated: feat: implement concurrent `MultipartUploadWriter` (#3915)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 23db343b7e feat: implement concurrent `MultipartUploadWriter` (#3915)
23db343b7e is described below

commit 23db343b7e260e3ef768c6bd81d68dd7915be9d4
Author: Weny Xu <we...@gmail.com>
AuthorDate: Fri Jan 5 21:57:35 2024 +0900

    feat: implement concurrent `MultipartUploadWriter` (#3915)
    
    * feat: expose the concurrent field
    
    * feat: implement concurrent `MultipartUploadWriter`
    
    * chore: apply suggestions from CR
    
    * chore: apply suggestions from CR
    
    * chore: correct comments
    
    * fix: clear future queue while aborting
---
 core/src/raw/futures_util.rs                     |   9 +
 core/src/raw/oio/write/multipart_upload_write.rs | 213 ++++++++++++-----------
 core/src/raw/ops.rs                              |  12 ++
 core/src/services/b2/backend.rs                  |   3 +-
 core/src/services/cos/backend.rs                 |   2 +-
 core/src/services/obs/backend.rs                 |   2 +-
 core/src/services/oss/backend.rs                 |   2 +-
 core/src/services/s3/backend.rs                  |   3 +-
 core/src/services/upyun/backend.rs               |   3 +-
 core/src/types/operator/operator_futures.rs      |  12 ++
 10 files changed, 158 insertions(+), 103 deletions(-)

diff --git a/core/src/raw/futures_util.rs b/core/src/raw/futures_util.rs
index 08583fe6d8..d7a1168b63 100644
--- a/core/src/raw/futures_util.rs
+++ b/core/src/raw/futures_util.rs
@@ -101,6 +101,15 @@ where
         }
     }
 
+    /// Drop all tasks.
+    pub fn clear(&mut self) {
+        match &mut self.tasks {
+            Tasks::Once(fut) => *fut = None,
+            Tasks::Small(tasks) => tasks.clear(),
+            Tasks::Large(tasks) => *tasks = FuturesOrdered::new(),
+        }
+    }
+
     /// Return the length of current concurrent futures (both ongoing and ready).
     pub fn len(&self) -> usize {
         match &self.tasks {
diff --git a/core/src/raw/oio/write/multipart_upload_write.rs b/core/src/raw/oio/write/multipart_upload_write.rs
index a0f92aa98e..66105b4d4a 100644
--- a/core/src/raw/oio/write/multipart_upload_write.rs
+++ b/core/src/raw/oio/write/multipart_upload_write.rs
@@ -15,12 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::pin::Pin;
 use std::sync::Arc;
 use std::task::ready;
 use std::task::Context;
 use std::task::Poll;
 
 use async_trait::async_trait;
+use futures::Future;
+use futures::FutureExt;
+use futures::StreamExt;
 
 use crate::raw::*;
 use crate::*;
@@ -103,44 +107,76 @@ pub struct MultipartUploadPart {
     pub etag: String,
 }
 
+struct WritePartFuture(BoxedFuture<Result<MultipartUploadPart>>);
+
+/// # Safety
+///
+/// wasm32 is a special target that we only have one event-loop for this WritePartFuture.
+unsafe impl Send for WritePartFuture {}
+
+/// # Safety
+///
+/// We will only take `&mut Self` reference for WritePartFuture.
+unsafe impl Sync for WritePartFuture {}
+
+impl Future for WritePartFuture {
+    type Output = Result<MultipartUploadPart>;
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        self.get_mut().0.poll_unpin(cx)
+    }
+}
+
 /// MultipartUploadWriter will implements [`Write`] based on multipart
 /// uploads.
 pub struct MultipartUploadWriter<W: MultipartUploadWrite> {
-    state: State<W>,
+    state: State,
+    w: Arc<W>,
 
-    cache: Option<oio::ChunkedBytes>,
     upload_id: Option<Arc<String>>,
     parts: Vec<MultipartUploadPart>,
+    cache: Option<oio::ChunkedBytes>,
+    futures: ConcurrentFutures<WritePartFuture>,
+    next_part_number: usize,
 }
 
-enum State<W> {
-    Idle(Option<W>),
-    Init(BoxedFuture<(W, Result<String>)>),
-    Write(BoxedFuture<(W, Result<MultipartUploadPart>)>),
-    Close(BoxedFuture<(W, Result<()>)>),
-    Abort(BoxedFuture<(W, Result<()>)>),
+enum State {
+    Idle,
+    Init(BoxedFuture<Result<String>>),
+    Close(BoxedFuture<Result<()>>),
+    Abort(BoxedFuture<Result<()>>),
 }
 
 /// # Safety
 ///
 /// wasm32 is a special target that we only have one event-loop for this state.
-unsafe impl<S: MultipartUploadWrite> Send for State<S> {}
+unsafe impl Send for State {}
 /// # Safety
 ///
 /// We will only take `&mut Self` reference for State.
-unsafe impl<S: MultipartUploadWrite> Sync for State<S> {}
+unsafe impl Sync for State {}
 
 impl<W: MultipartUploadWrite> MultipartUploadWriter<W> {
     /// Create a new MultipartUploadWriter.
-    pub fn new(inner: W) -> Self {
+    pub fn new(inner: W, concurrent: usize) -> Self {
         Self {
-            state: State::Idle(Some(inner)),
+            state: State::Idle,
 
-            cache: None,
+            w: Arc::new(inner),
             upload_id: None,
             parts: Vec::new(),
+            cache: None,
+            futures: ConcurrentFutures::new(1.max(concurrent)),
+            next_part_number: 0,
         }
     }
+
+    fn fill_cache(&mut self, bs: &dyn oio::WriteBuf) -> usize {
+        let size = bs.remaining();
+        let bs = oio::ChunkedBytes::from_vec(bs.vectored_bytes(size));
+        assert!(self.cache.is_none());
+        self.cache = Some(bs);
+        size
+    }
 }
 
 impl<W> oio::Write for MultipartUploadWriter<W>
@@ -150,61 +186,49 @@ where
     fn poll_write(&mut self, cx: &mut Context<'_>, bs: &dyn oio::WriteBuf) -> Poll<Result<usize>> {
         loop {
             match &mut self.state {
-                State::Idle(w) => {
+                State::Idle => {
                     match self.upload_id.as_ref() {
                         Some(upload_id) => {
                             let upload_id = upload_id.clone();
-                            let part_number = self.parts.len();
-
-                            let bs = self.cache.clone().expect("cache must be valid").clone();
-                            let w = w.take().expect("writer must be valid");
-                            self.state = State::Write(Box::pin(async move {
-                                let size = bs.len();
-                                let part = w
-                                    .write_part(
+                            if self.futures.has_remaining() {
+                                let cache = self.cache.take().expect("pending write must exist");
+                                let part_number = self.next_part_number;
+                                self.next_part_number += 1;
+                                let w = self.w.clone();
+                                let size = cache.len();
+                                self.futures.push(WritePartFuture(Box::pin(async move {
+                                    w.write_part(
                                         &upload_id,
                                         part_number,
                                         size as u64,
-                                        AsyncBody::ChunkedBytes(bs),
+                                        AsyncBody::ChunkedBytes(cache),
                                     )
-                                    .await;
-
-                                (w, part)
-                            }));
+                                    .await
+                                })));
+                                let size = self.fill_cache(bs);
+                                return Poll::Ready(Ok(size));
+                            } else if let Some(part) = ready!(self.futures.poll_next_unpin(cx)) {
+                                self.parts.push(part?);
+                            }
                         }
                         None => {
                             // Fill cache with the first write.
                             if self.cache.is_none() {
-                                let size = bs.remaining();
-                                let cb = oio::ChunkedBytes::from_vec(bs.vectored_bytes(size));
-                                self.cache = Some(cb);
+                                let size = self.fill_cache(bs);
                                 return Poll::Ready(Ok(size));
                             }
 
-                            let w = w.take().expect("writer must be valid");
-                            self.state = State::Init(Box::pin(async move {
-                                let upload_id = w.initiate_part().await;
-                                (w, upload_id)
-                            }));
+                            let w = self.w.clone();
+                            self.state =
+                                State::Init(Box::pin(async move { w.initiate_part().await }));
                         }
                     }
                 }
                 State::Init(fut) => {
-                    let (w, upload_id) = ready!(fut.as_mut().poll(cx));
-                    self.state = State::Idle(Some(w));
+                    let upload_id = ready!(fut.as_mut().poll(cx));
+                    self.state = State::Idle;
                     self.upload_id = Some(Arc::new(upload_id?));
                 }
-                State::Write(fut) => {
-                    let (w, part) = ready!(fut.as_mut().poll(cx));
-                    self.state = State::Idle(Some(w));
-                    self.parts.push(part?);
-
-                    // Replace the cache when last write succeeded
-                    let size = bs.remaining();
-                    let cb = oio::ChunkedBytes::from_vec(bs.vectored_bytes(size));
-                    self.cache = Some(cb);
-                    return Poll::Ready(Ok(size));
-                }
                 State::Close(_) => {
                     unreachable!(
                         "MultipartUploadWriter must not go into State::Close during poll_write"
@@ -222,73 +246,71 @@ where
     fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
         loop {
             match &mut self.state {
-                State::Idle(w) => {
-                    let w = w.take().expect("writer must be valid");
+                State::Idle => {
                     match self.upload_id.clone() {
                         Some(upload_id) => {
-                            let parts = self.parts.clone();
-                            match self.cache.clone() {
-                                Some(bs) => {
-                                    let upload_id = upload_id.clone();
-                                    self.state = State::Write(Box::pin(async move {
-                                        let size = bs.len();
-                                        let part = w
-                                            .write_part(
+                            let w = self.w.clone();
+                            if self.futures.is_empty() && self.cache.is_none() {
+                                let upload_id = upload_id.clone();
+                                let parts = self.parts.clone();
+                                self.state = State::Close(Box::pin(async move {
+                                    w.complete_part(&upload_id, &parts).await
+                                }));
+                            } else {
+                                if self.futures.has_remaining() {
+                                    if let Some(cache) = self.cache.take() {
+                                        let upload_id = upload_id.clone();
+                                        let part_number = self.next_part_number;
+                                        self.next_part_number += 1;
+                                        let size = cache.len();
+                                        let w = self.w.clone();
+                                        self.futures.push(WritePartFuture(Box::pin(async move {
+                                            w.write_part(
                                                 &upload_id,
-                                                parts.len(),
+                                                part_number,
                                                 size as u64,
-                                                AsyncBody::ChunkedBytes(bs),
+                                                AsyncBody::ChunkedBytes(cache),
                                             )
-                                            .await;
-                                        (w, part)
-                                    }));
+                                            .await
+                                        })));
+                                    }
                                 }
-                                None => {
-                                    self.state = State::Close(Box::pin(async move {
-                                        let res = w.complete_part(&upload_id, &parts).await;
-                                        (w, res)
-                                    }));
+                                while let Some(part) = ready!(self.futures.poll_next_unpin(cx)) {
+                                    self.parts.push(part?);
                                 }
                             }
                         }
-                        None => match self.cache.clone() {
-                            Some(bs) => {
+                        None => match &self.cache {
+                            Some(cache) => {
+                                let w = self.w.clone();
+                                let bs = cache.clone();
                                 self.state = State::Close(Box::pin(async move {
                                     let size = bs.len();
-                                    let res = w
-                                        .write_once(size as u64, AsyncBody::ChunkedBytes(bs))
-                                        .await;
-                                    (w, res)
+                                    w.write_once(size as u64, AsyncBody::ChunkedBytes(bs)).await
                                 }));
                             }
                             None => {
+                                let w = self.w.clone();
                                 // Call write_once if there is no data in cache and no upload_id.
                                 self.state = State::Close(Box::pin(async move {
-                                    let res = w.write_once(0, AsyncBody::Empty).await;
-                                    (w, res)
+                                    w.write_once(0, AsyncBody::Empty).await
                                 }));
                             }
                         },
                     }
                 }
                 State::Close(fut) => {
-                    let (w, res) = futures::ready!(fut.as_mut().poll(cx));
-                    self.state = State::Idle(Some(w));
+                    let res = futures::ready!(fut.as_mut().poll(cx));
+                    self.state = State::Idle;
                     // We should check res first before clean up cache.
                     res?;
-
                     self.cache = None;
+
                     return Poll::Ready(Ok(()));
                 }
                 State::Init(_) => unreachable!(
                     "MultipartUploadWriter must not go into State::Init during poll_close"
                 ),
-                State::Write(fut) => {
-                    let (w, part) = ready!(fut.as_mut().poll(cx));
-                    self.state = State::Idle(Some(w));
-                    self.parts.push(part?);
-                    self.cache = None;
-                }
                 State::Abort(_) => unreachable!(
                     "MultipartUploadWriter must not go into State::Abort during poll_close"
                 ),
@@ -299,32 +321,29 @@ where
     fn poll_abort(&mut self, cx: &mut Context<'_>) -> Poll<Result<()>> {
         loop {
             match &mut self.state {
-                State::Idle(w) => {
-                    let w = w.take().expect("writer must be valid");
+                State::Idle => {
+                    let w = self.w.clone();
                     match self.upload_id.clone() {
                         Some(upload_id) => {
-                            self.state = State::Abort(Box::pin(async move {
-                                let res = w.abort_part(&upload_id).await;
-                                (w, res)
-                            }));
+                            self.futures.clear();
+                            self.state =
+                                State::Abort(Box::pin(
+                                    async move { w.abort_part(&upload_id).await },
+                                ));
                         }
                         None => {
-                            self.cache = None;
                             return Poll::Ready(Ok(()));
                         }
                     }
                 }
                 State::Abort(fut) => {
-                    let (w, res) = futures::ready!(fut.as_mut().poll(cx));
-                    self.state = State::Idle(Some(w));
+                    let res = futures::ready!(fut.as_mut().poll(cx));
+                    self.state = State::Idle;
                     return Poll::Ready(res);
                 }
                 State::Init(_) => unreachable!(
                     "MultipartUploadWriter must not go into State::Init during poll_abort"
                 ),
-                State::Write(_) => unreachable!(
-                    "MultipartUploadWriter must not go into State::Write during poll_abort"
-                ),
                 State::Close(_) => unreachable!(
                     "MultipartUploadWriter must not go into State::Close during poll_abort"
                 ),
diff --git a/core/src/raw/ops.rs b/core/src/raw/ops.rs
index dbf43bfaa7..955a588421 100644
--- a/core/src/raw/ops.rs
+++ b/core/src/raw/ops.rs
@@ -516,6 +516,7 @@ impl OpStat {
 pub struct OpWrite {
     append: bool,
     buffer: Option<usize>,
+    concurrent: usize,
 
     content_type: Option<String>,
     content_disposition: Option<String>,
@@ -601,6 +602,17 @@ impl OpWrite {
         self.cache_control = Some(cache_control.to_string());
         self
     }
+
+    /// Get the concurrent.
+    pub fn concurrent(&self) -> usize {
+        self.concurrent
+    }
+
+    /// Set the maximum concurrent write task amount.
+    pub fn with_concurrent(mut self, concurrent: usize) -> Self {
+        self.concurrent = concurrent;
+        self
+    }
 }
 
 /// Args for `copy` operation.
diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs
index 2c5128bad6..a6f29055fd 100644
--- a/core/src/services/b2/backend.rs
+++ b/core/src/services/b2/backend.rs
@@ -376,9 +376,10 @@ impl Accessor for B2Backend {
     }
 
     async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let concurrent = args.concurrent();
         let writer = B2Writer::new(self.core.clone(), path, args);
 
-        let w = oio::MultipartUploadWriter::new(writer);
+        let w = oio::MultipartUploadWriter::new(writer, concurrent);
 
         Ok((RpWrite::default(), w))
     }
diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs
index 5c6d458858..e0e58701fd 100644
--- a/core/src/services/cos/backend.rs
+++ b/core/src/services/cos/backend.rs
@@ -340,7 +340,7 @@ impl Accessor for CosBackend {
         let w = if args.append() {
             CosWriters::Two(oio::AppendObjectWriter::new(writer))
         } else {
-            CosWriters::One(oio::MultipartUploadWriter::new(writer))
+            CosWriters::One(oio::MultipartUploadWriter::new(writer, args.concurrent()))
         };
 
         Ok((RpWrite::default(), w))
diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs
index d00fbe423a..ad3b9c98fc 100644
--- a/core/src/services/obs/backend.rs
+++ b/core/src/services/obs/backend.rs
@@ -350,7 +350,7 @@ impl Accessor for ObsBackend {
         let w = if args.append() {
             ObsWriters::Two(oio::AppendObjectWriter::new(writer))
         } else {
-            ObsWriters::One(oio::MultipartUploadWriter::new(writer))
+            ObsWriters::One(oio::MultipartUploadWriter::new(writer, args.concurrent()))
         };
 
         Ok((RpWrite::default(), w))
diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs
index 6e109e1321..79624a7dde 100644
--- a/core/src/services/oss/backend.rs
+++ b/core/src/services/oss/backend.rs
@@ -492,7 +492,7 @@ impl Accessor for OssBackend {
         let w = if args.append() {
             OssWriters::Two(oio::AppendObjectWriter::new(writer))
         } else {
-            OssWriters::One(oio::MultipartUploadWriter::new(writer))
+            OssWriters::One(oio::MultipartUploadWriter::new(writer, args.concurrent()))
         };
 
         Ok((RpWrite::default(), w))
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 94acf49c37..2094fde0d9 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -1085,9 +1085,10 @@ impl Accessor for S3Backend {
     }
 
     async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let concurrent = args.concurrent();
         let writer = S3Writer::new(self.core.clone(), path, args);
 
-        let w = oio::MultipartUploadWriter::new(writer);
+        let w = oio::MultipartUploadWriter::new(writer, concurrent);
 
         Ok((RpWrite::default(), w))
     }
diff --git a/core/src/services/upyun/backend.rs b/core/src/services/upyun/backend.rs
index 091977f6f5..a13764b945 100644
--- a/core/src/services/upyun/backend.rs
+++ b/core/src/services/upyun/backend.rs
@@ -316,9 +316,10 @@ impl Accessor for UpyunBackend {
     }
 
     async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let concurrent = args.concurrent();
         let writer = UpyunWriter::new(self.core.clone(), args, path.to_string());
 
-        let w = oio::MultipartUploadWriter::new(writer);
+        let w = oio::MultipartUploadWriter::new(writer, concurrent);
 
         Ok((RpWrite::default(), w))
     }
diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs
index 8c75f56b3c..4faf5ea523 100644
--- a/core/src/types/operator/operator_futures.rs
+++ b/core/src/types/operator/operator_futures.rs
@@ -478,6 +478,12 @@ impl FutureWrite {
             .map_args(|(args, bs)| (args.with_cache_control(v), bs));
         self
     }
+
+    /// Set the maximum concurrent write task amount.
+    pub fn concurrent(mut self, v: usize) -> Self {
+        self.0 = self.0.map_args(|(args, bs)| (args.with_buffer(v), bs));
+        self
+    }
 }
 
 impl Future for FutureWrite {
@@ -543,6 +549,12 @@ impl FutureWriter {
         self.0 = self.0.map_args(|args| args.with_cache_control(v));
         self
     }
+
+    /// Set the maximum concurrent write task amount.
+    pub fn concurrent(mut self, v: usize) -> Self {
+        self.0 = self.0.map_args(|args| args.with_concurrent(v));
+        self
+    }
 }
 
 impl Future for FutureWriter {