You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by ho...@apache.org on 2024/04/11 05:27:09 UTC
(opendal) branch block-write-buffer updated: feat(raw/oio): block write change to buffer
This is an automated email from the ASF dual-hosted git repository.
hoslo pushed a commit to branch block-write-buffer
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/block-write-buffer by this push:
new b3154887fe feat(raw/oio): block write change to buffer
new 33c128316e Merge branch 'block-write-buffer' of https://github.com/apache/opendal into block-write-buffer
b3154887fe is described below
commit b3154887fee85429f9f5eabe40315110f54140be
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Apr 11 11:19:03 2024 +0800
feat(raw/oio): block write change to buffer
feat(raw/oio): block write change to buffer
---
core/src/raw/oio/write/block_write.rs | 31 ++++++++++++++++++++++---------
1 file changed, 22 insertions(+), 9 deletions(-)
diff --git a/core/src/raw/oio/write/block_write.rs b/core/src/raw/oio/write/block_write.rs
index a88b00cb46..f8be9c07be 100644
--- a/core/src/raw/oio/write/block_write.rs
+++ b/core/src/raw/oio/write/block_write.rs
@@ -20,7 +20,6 @@ use std::sync::Arc;
use std::task::Context;
use std::task::Poll;
-use bytes::Bytes;
use futures::Future;
use futures::FutureExt;
use futures::StreamExt;
@@ -110,7 +109,8 @@ pub trait BlockWrite: Send + Sync + Unpin + 'static {
/// WriteBlockResult is the result returned by [`WriteBlockFuture`].
///
/// The error part will carries input `(block_id, bytes, err)` so caller can retry them.
-type WriteBlockResult = std::result::Result<Uuid, (Uuid, Bytes, Error)>;
+type WriteBlockResult = std::result::Result<Uuid, (Uuid, Buffer, Error)>;
+type WriteBlockResult = std::result::Result<Uuid, (Uuid, Buffer, Error)>;
struct WriteBlockFuture(BoxedStaticFuture<WriteBlockResult>);
@@ -132,9 +132,16 @@ impl Future for WriteBlockFuture {
}
impl WriteBlockFuture {
- pub fn new<W: BlockWrite>(w: Arc<W>, block_id: Uuid, bytes: Bytes) -> Self {
+ pub fn new<W: BlockWrite>(w: Arc<W>, block_id: Uuid, bytes: Buffer) -> Self {
+ pub fn new<W: BlockWrite>(w: Arc<W>, block_id: Uuid, bytes: Buffer) -> Self {
let fut = async move {
- w.write_block(block_id, bytes.len() as u64, Buffer::from(bytes.clone()))
+ w.write_block(block_id, bytes.len() as u64, bytes.clone())
+ .await
+ // Return bytes while we got an error to allow retry.
+ .map_err(|err| (block_id, bytes, err))
+ // Return the successful block id.
+ .map(|_| block_id)
+ w.write_block(block_id, bytes.len() as u64, bytes.clone())
.await
// Return bytes while we got an error to allow retry.
.map_err(|err| (block_id, bytes, err))
@@ -152,7 +159,8 @@ pub struct BlockWriter<W: BlockWrite> {
w: Arc<W>,
block_ids: Vec<Uuid>,
- cache: Option<Bytes>,
+ cache: Option<Buffer>,
+ cache: Option<Buffer>,
futures: ConcurrentFutures<WriteBlockFuture>,
}
@@ -167,7 +175,8 @@ impl<W: BlockWrite> BlockWriter<W> {
}
}
- fn fill_cache(&mut self, bs: Bytes) -> usize {
+ fn fill_cache(&mut self, bs: Buffer) -> usize {
+ fn fill_cache(&mut self, bs: Buffer) -> usize {
let size = bs.len();
assert!(self.cache.is_none());
self.cache = Some(bs);
@@ -184,7 +193,8 @@ where
if self.futures.has_remaining() {
// Fill cache with the first write.
if self.cache.is_none() {
- let size = self.fill_cache(bs.to_bytes());
+ let size = self.fill_cache(bs);
+ let size = self.fill_cache(bs);
return Ok(size);
}
@@ -195,7 +205,8 @@ where
cache,
));
- let size = self.fill_cache(bs.to_bytes());
+ let size = self.fill_cache(bs);
+ let size = self.fill_cache(bs);
return Ok(size);
} else if let Some(res) = self.futures.next().await {
match res {
@@ -220,7 +231,9 @@ where
// No write block has been sent.
if self.futures.is_empty() && self.block_ids.is_empty() {
let (size, body) = match self.cache.clone() {
- Some(cache) => (cache.len(), Buffer::from(cache)),
+ Some(cache) => (cache.len(), cache),
+ None => (0, Buffer::new()),
+ Some(cache) => (cache.len(), cache),
None => (0, Buffer::new()),
};
self.w.write_once(size as u64, body).await?;