You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/03/06 11:02:40 UTC
(opendal) branch main updated: feat(core): Expose Reader::into_stream and Writer::into_sink (#5698)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 8586a9e01 feat(core): Expose Reader::into_stream and Writer::into_sink (#5698)
8586a9e01 is described below
commit 8586a9e0134b4b52a2de88347287e64bed6b68eb
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Mar 6 19:02:35 2025 +0800
feat(core): Expose Reader::into_stream and Writer::into_sink (#5698)
* feat(core): Expose Reader::into_stream and Writer::into_sink
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Polish comments
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/types/read/buffer_stream.rs | 26 +++++++-----
core/src/types/read/reader.rs | 82 ++++++++++++++++++++++++++++++++++--
core/src/types/write/buffer_sink.rs | 2 +-
core/src/types/write/writer.rs | 57 +++++++++++++++++++++++++
4 files changed, 151 insertions(+), 16 deletions(-)
diff --git a/core/src/types/read/buffer_stream.rs b/core/src/types/read/buffer_stream.rs
index 304a5c6b4..b9bde6f9c 100644
--- a/core/src/types/read/buffer_stream.rs
+++ b/core/src/types/read/buffer_stream.rs
@@ -24,7 +24,7 @@ use std::task::Poll;
use futures::ready;
use futures::Stream;
-use crate::raw::oio::Read;
+use crate::raw::oio::Read as _;
use crate::raw::*;
use crate::*;
@@ -125,16 +125,17 @@ impl oio::Read for ChunkedReader {
}
}
-/// BufferStream is a stream of buffers.
+/// BufferStream is a stream of buffers, created by [`Reader::into_stream`]
///
-/// # Notes
-///
-/// The underlying reader is either a StreamingReader or a ChunkedReader.
-///
-/// - If chunk is None, BufferStream will use StreamingReader to iterate
-/// data in streaming way.
-/// - Otherwise, BufferStream will use ChunkedReader to read data in chunks.
+/// `BufferStream` implements `Stream` trait.
pub struct BufferStream {
+ /// # Notes to maintainers
+ ///
+ /// The underlying reader is either a StreamingReader or a ChunkedReader.
+ ///
+ /// - If chunk is None, BufferStream will use StreamingReader to iterate
+ /// data in streaming way.
+ /// - Otherwise, BufferStream will use ChunkedReader to read data in chunks.
state: State,
}
@@ -145,7 +146,7 @@ enum State {
impl BufferStream {
/// Create a new buffer stream with already calculated offset and size.
- pub fn new(ctx: Arc<ReadContext>, offset: u64, size: Option<u64>) -> Self {
+ pub(crate) fn new(ctx: Arc<ReadContext>, offset: u64, size: Option<u64>) -> Self {
debug_assert!(
size.is_some() || ctx.options().chunk().is_none(),
"size must be known if chunk is set"
@@ -166,7 +167,10 @@ impl BufferStream {
///
/// If users is going to perform chunked read but the read size is unknown, we will parse
/// into range first.
- pub async fn create(ctx: Arc<ReadContext>, range: impl RangeBounds<u64>) -> Result<Self> {
+ pub(crate) async fn create(
+ ctx: Arc<ReadContext>,
+ range: impl RangeBounds<u64>,
+ ) -> Result<Self> {
let reader = if ctx.options().chunk().is_some() {
let range = ctx.parse_into_range(range).await?;
TwoWays::Two(ChunkedReader::new(ctx, range.into()))
diff --git a/core/src/types/read/reader.rs b/core/src/types/read/reader.rs
index ba7b6ea60..04d53ce84 100644
--- a/core/src/types/read/reader.rs
+++ b/core/src/types/read/reader.rs
@@ -195,10 +195,76 @@ impl Reader {
///
/// # Notes
///
- /// This API can be public but we are not sure if it's useful for users.
- /// And the name `BufferStream` is not good enough to expose to users.
- /// Let's keep it inside for now.
- async fn into_stream(self, range: impl RangeBounds<u64>) -> Result<BufferStream> {
+ /// BufferStream is a zero-cost abstraction. It doesn't involve extra copy of data.
+ /// It will return underlying [`Buffer`] directly.
+ ///
+ /// The [`Buffer`] this stream yields can be seen as an iterator of [`Bytes`].
+ ///
+ /// # Inputs
+ ///
+ /// - `range`: The range of data to read. range like `..` it will read all data from reader.
+ ///
+ /// # Examples
+ ///
+ /// ## Basic Usage
+ ///
+ /// ```
+ /// use std::io;
+ ///
+ /// use futures::TryStreamExt;
+ /// use opendal::{Buffer, Operator};
+ /// use opendal::Result;
+ /// use bytes::Bytes;
+ ///
+ /// async fn test(op: Operator) -> io::Result<()> {
+ /// let mut s = op
+ /// .reader("hello.txt")
+ /// .await?
+ /// .into_stream(1024..2048)
+ /// .await?;
+ ///
+ /// let bs: Vec<Buffer> = s.try_collect().await?;
+ /// // We can use those buffer as bytes if we want.
+ /// let bytes_vec: Vec<Bytes> = bs.clone().into_iter().flatten().collect();
+ /// // Or we can merge them into a single [`Buffer`] and later use it as [`bytes::Buf`].
+ /// let new_buffer: Buffer = bs.into_iter().flatten().collect::<Buffer>();
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ ///
+ /// ## Concurrent Read
+ ///
+ /// The following example reads data in 256B chunks with 8 concurrent.
+ ///
+ /// ```
+ /// use std::io;
+ /// use bytes::Bytes;
+ ///
+ /// use futures::TryStreamExt;
+ /// use opendal::{Buffer, Operator};
+ /// use opendal::Result;
+ ///
+ /// async fn test(op: Operator) -> io::Result<()> {
+ /// let s = op
+ /// .reader_with("hello.txt")
+ /// .concurrent(8)
+ /// .chunk(256)
+ /// .await?
+ /// .into_stream(1024..2048)
+ /// .await?;
+ ///
+ /// // Every buffer except the last one in the stream will be 256B.
+ /// let bs: Vec<Buffer> = s.try_collect().await?;
+ /// // We can use those buffer as bytes if we want.
+ /// let bytes_vec: Vec<Bytes> = bs.clone().into_iter().flatten().collect();
+ /// // Or we can merge them into a single [`Buffer`] and later use it as [`bytes::Buf`].
+ /// let new_buffer: Buffer = bs.into_iter().flatten().collect::<Buffer>();
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn into_stream(self, range: impl RangeBounds<u64>) -> Result<BufferStream> {
BufferStream::create(self.ctx, range).await
}
@@ -210,6 +276,10 @@ impl Reader {
/// FuturesAsyncReader is not a zero-cost abstraction. The underlying reader
/// returns an owned [`Buffer`], which involves an extra copy operation.
///
+ /// # Inputs
+ ///
+ /// - `range`: The range of data to read. range like `..` it will read all data from reader.
+ ///
/// # Examples
///
/// ## Basic Usage
@@ -270,6 +340,10 @@ impl Reader {
/// Convert reader into [`FuturesBytesStream`] which implements [`futures::Stream`].
///
+ /// # Inputs
+ ///
+ /// - `range`: The range of data to read. range like `..` it will read all data from reader.
+ ///
/// # Examples
///
/// ## Basic Usage
diff --git a/core/src/types/write/buffer_sink.rs b/core/src/types/write/buffer_sink.rs
index f993455d0..94f1ebc37 100644
--- a/core/src/types/write/buffer_sink.rs
+++ b/core/src/types/write/buffer_sink.rs
@@ -25,7 +25,7 @@ use bytes::Buf;
use crate::raw::*;
use crate::*;
-/// BufferSink is the adapter of [`futures::Sink`] generated by [`Writer`].
+/// BufferSink is the adapter of [`futures::Sink`] generated by [`Writer::into_sink`]
///
/// Users can use this adapter in cases where they need to use [`futures::Sink`]
pub struct BufferSink {
diff --git a/core/src/types/write/writer.rs b/core/src/types/write/writer.rs
index 66f7c948e..2aef6fdc2 100644
--- a/core/src/types/write/writer.rs
+++ b/core/src/types/write/writer.rs
@@ -179,6 +179,63 @@ impl Writer {
self.inner.close().await
}
+ /// Convert writer into [`BufferSink`] which implements [`Sink<Buffer>`].
+ ///
+ /// # Notes
+ ///
+ /// BufferSink is a zero-cost abstraction. The underlying writer
+ /// will reuse the Bytes and won't perform any copy operation over data.
+ ///
+ /// # Examples
+ ///
+ /// ## Basic Usage
+ ///
+ /// ```
+ /// use std::io;
+ ///
+ /// use bytes::Bytes;
+ /// use futures::SinkExt;
+ /// use opendal::{Buffer, Operator};
+ /// use opendal::Result;
+ ///
+ /// async fn test(op: Operator) -> io::Result<()> {
+ /// let mut s = op.writer("hello.txt").await?.into_sink();
+ /// let bs = "Hello, World!".as_bytes();
+ /// s.send(Buffer::from(bs)).await?;
+ /// s.close().await?;
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ ///
+ /// ## Concurrent Write
+ ///
+ /// ```
+ /// use std::io;
+ ///
+ /// use bytes::Bytes;
+ /// use futures::SinkExt;
+ /// use opendal::{Buffer, Operator};
+ /// use opendal::Result;
+ ///
+ /// async fn test(op: Operator) -> io::Result<()> {
+ /// let mut w = op
+ /// .writer_with("hello.txt")
+ /// .concurrent(8)
+ /// .chunk(256)
+ /// .await?
+ /// .into_sink();
+ /// let bs = "Hello, World!".as_bytes();
+ /// w.send(Buffer::from(bs)).await?;
+ /// w.close().await?;
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub fn into_sink(self) -> BufferSink {
+ BufferSink::new(self.inner)
+ }
+
/// Convert writer into [`FuturesAsyncWriter`] which implements [`futures::AsyncWrite`],
///
/// # Notes