You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/03/06 11:02:40 UTC

(opendal) branch main updated: feat(core): Expose Reader::into_stream and Writer::into_sink (#5698)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 8586a9e01 feat(core): Expose Reader::into_stream and Writer::into_sink (#5698)
8586a9e01 is described below

commit 8586a9e0134b4b52a2de88347287e64bed6b68eb
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Mar 6 19:02:35 2025 +0800

    feat(core): Expose Reader::into_stream and Writer::into_sink (#5698)
    
    * feat(core): Expose Reader::into_stream and Writer::into_sink
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    * Polish comments
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
    
    ---------
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 core/src/types/read/buffer_stream.rs | 26 +++++++-----
 core/src/types/read/reader.rs        | 82 ++++++++++++++++++++++++++++++++++--
 core/src/types/write/buffer_sink.rs  |  2 +-
 core/src/types/write/writer.rs       | 57 +++++++++++++++++++++++++
 4 files changed, 151 insertions(+), 16 deletions(-)

diff --git a/core/src/types/read/buffer_stream.rs b/core/src/types/read/buffer_stream.rs
index 304a5c6b4..b9bde6f9c 100644
--- a/core/src/types/read/buffer_stream.rs
+++ b/core/src/types/read/buffer_stream.rs
@@ -24,7 +24,7 @@ use std::task::Poll;
 use futures::ready;
 use futures::Stream;
 
-use crate::raw::oio::Read;
+use crate::raw::oio::Read as _;
 use crate::raw::*;
 use crate::*;
 
@@ -125,16 +125,17 @@ impl oio::Read for ChunkedReader {
     }
 }
 
-/// BufferStream is a stream of buffers.
+/// BufferStream is a stream of buffers, created by [`Reader::into_stream`]
 ///
-/// # Notes
-///
-/// The underlying reader is either a StreamingReader or a ChunkedReader.
-///
-/// - If chunk is None, BufferStream will use StreamingReader to iterate
-///   data in streaming way.
-/// - Otherwise, BufferStream will use ChunkedReader to read data in chunks.
+/// `BufferStream` implements `Stream` trait.
 pub struct BufferStream {
+    /// # Notes to maintainers
+    ///
+    /// The underlying reader is either a StreamingReader or a ChunkedReader.
+    ///
+    /// - If chunk is None, BufferStream will use StreamingReader to iterate
+    ///   data in streaming way.
+    /// - Otherwise, BufferStream will use ChunkedReader to read data in chunks.
     state: State,
 }
 
@@ -145,7 +146,7 @@ enum State {
 
 impl BufferStream {
     /// Create a new buffer stream with already calculated offset and size.
-    pub fn new(ctx: Arc<ReadContext>, offset: u64, size: Option<u64>) -> Self {
+    pub(crate) fn new(ctx: Arc<ReadContext>, offset: u64, size: Option<u64>) -> Self {
         debug_assert!(
             size.is_some() || ctx.options().chunk().is_none(),
             "size must be known if chunk is set"
@@ -166,7 +167,10 @@ impl BufferStream {
     ///
     /// If users is going to perform chunked read but the read size is unknown, we will parse
     /// into range first.
-    pub async fn create(ctx: Arc<ReadContext>, range: impl RangeBounds<u64>) -> Result<Self> {
+    pub(crate) async fn create(
+        ctx: Arc<ReadContext>,
+        range: impl RangeBounds<u64>,
+    ) -> Result<Self> {
         let reader = if ctx.options().chunk().is_some() {
             let range = ctx.parse_into_range(range).await?;
             TwoWays::Two(ChunkedReader::new(ctx, range.into()))
diff --git a/core/src/types/read/reader.rs b/core/src/types/read/reader.rs
index ba7b6ea60..04d53ce84 100644
--- a/core/src/types/read/reader.rs
+++ b/core/src/types/read/reader.rs
@@ -195,10 +195,76 @@ impl Reader {
     ///
     /// # Notes
     ///
-    /// This API can be public but we are not sure if it's useful for users.
-    /// And the name `BufferStream` is not good enough to expose to users.
-    /// Let's keep it inside for now.
-    async fn into_stream(self, range: impl RangeBounds<u64>) -> Result<BufferStream> {
+    /// BufferStream is a zero-cost abstraction. It doesn't involve extra copy of data.
+    /// It will return underlying [`Buffer`] directly.
+    ///
+    /// The [`Buffer`] this stream yields can be seen as an iterator of [`Bytes`].
+    ///
+    /// # Inputs
+    ///
+    /// - `range`: The range of data to read. range like `..` it will read all data from reader.
+    ///
+    /// # Examples
+    ///
+    /// ## Basic Usage
+    ///
+    /// ```
+    /// use std::io;
+    ///
+    /// use futures::TryStreamExt;
+    /// use opendal::{Buffer, Operator};
+    /// use opendal::Result;
+    /// use bytes::Bytes;
+    ///
+    /// async fn test(op: Operator) -> io::Result<()> {
+    ///     let mut s = op
+    ///         .reader("hello.txt")
+    ///         .await?
+    ///         .into_stream(1024..2048)
+    ///         .await?;
+    ///
+    ///     let bs: Vec<Buffer> = s.try_collect().await?;
+    ///     // We can use those buffer as bytes if we want.
+    ///     let bytes_vec: Vec<Bytes> = bs.clone().into_iter().flatten().collect();
+    ///     // Or we can merge them into a single [`Buffer`] and later use it as [`bytes::Buf`].
+    ///     let new_buffer: Buffer = bs.into_iter().flatten().collect::<Buffer>();
+    ///
+    ///     Ok(())
+    /// }
+    /// ```
+    ///
+    /// ## Concurrent Read
+    ///
+    /// The following example reads data in 256B chunks with 8 concurrent.
+    ///
+    /// ```
+    /// use std::io;
+    /// use bytes::Bytes;
+    ///
+    /// use futures::TryStreamExt;
+    /// use opendal::{Buffer, Operator};
+    /// use opendal::Result;
+    ///
+    /// async fn test(op: Operator) -> io::Result<()> {
+    ///     let s = op
+    ///         .reader_with("hello.txt")
+    ///         .concurrent(8)
+    ///         .chunk(256)
+    ///         .await?
+    ///         .into_stream(1024..2048)
+    ///         .await?;
+    ///
+    ///     // Every buffer except the last one in the stream will be 256B.
+    ///     let bs: Vec<Buffer> = s.try_collect().await?;
+    ///     // We can use those buffer as bytes if we want.
+    ///     let bytes_vec: Vec<Bytes> = bs.clone().into_iter().flatten().collect();
+    ///     // Or we can merge them into a single [`Buffer`] and later use it as [`bytes::Buf`].
+    ///     let new_buffer: Buffer = bs.into_iter().flatten().collect::<Buffer>();
+    ///
+    ///     Ok(())
+    /// }
+    /// ```
+    pub async fn into_stream(self, range: impl RangeBounds<u64>) -> Result<BufferStream> {
         BufferStream::create(self.ctx, range).await
     }
 
@@ -210,6 +276,10 @@ impl Reader {
     /// FuturesAsyncReader is not a zero-cost abstraction. The underlying reader
     /// returns an owned [`Buffer`], which involves an extra copy operation.
     ///
+    /// # Inputs
+    ///
+    /// - `range`: The range of data to read. range like `..` it will read all data from reader.
+    ///
     /// # Examples
     ///
     /// ## Basic Usage
@@ -270,6 +340,10 @@ impl Reader {
 
     /// Convert reader into [`FuturesBytesStream`] which implements [`futures::Stream`].
     ///
+    /// # Inputs
+    ///
+    /// - `range`: The range of data to read. range like `..` it will read all data from reader.
+    ///
     /// # Examples
     ///
     /// ## Basic Usage
diff --git a/core/src/types/write/buffer_sink.rs b/core/src/types/write/buffer_sink.rs
index f993455d0..94f1ebc37 100644
--- a/core/src/types/write/buffer_sink.rs
+++ b/core/src/types/write/buffer_sink.rs
@@ -25,7 +25,7 @@ use bytes::Buf;
 use crate::raw::*;
 use crate::*;
 
-/// BufferSink is the adapter of [`futures::Sink`] generated by [`Writer`].
+/// BufferSink is the adapter of [`futures::Sink`] generated by [`Writer::into_sink`]
 ///
 /// Users can use this adapter in cases where they need to use [`futures::Sink`]
 pub struct BufferSink {
diff --git a/core/src/types/write/writer.rs b/core/src/types/write/writer.rs
index 66f7c948e..2aef6fdc2 100644
--- a/core/src/types/write/writer.rs
+++ b/core/src/types/write/writer.rs
@@ -179,6 +179,63 @@ impl Writer {
         self.inner.close().await
     }
 
+    /// Convert writer into [`BufferSink`] which implements [`Sink<Buffer>`].
+    ///
+    /// # Notes
+    ///
+    /// BufferSink is a zero-cost abstraction. The underlying writer
+    /// will reuse the Bytes and won't perform any copy operation over data.
+    ///
+    /// # Examples
+    ///
+    /// ## Basic Usage
+    ///
+    /// ```
+    /// use std::io;
+    ///
+    /// use bytes::Bytes;
+    /// use futures::SinkExt;
+    /// use opendal::{Buffer, Operator};
+    /// use opendal::Result;
+    ///
+    /// async fn test(op: Operator) -> io::Result<()> {
+    ///     let mut s = op.writer("hello.txt").await?.into_sink();
+    ///     let bs = "Hello, World!".as_bytes();
+    ///     s.send(Buffer::from(bs)).await?;
+    ///     s.close().await?;
+    ///
+    ///     Ok(())
+    /// }
+    /// ```
+    ///
+    /// ## Concurrent Write
+    ///
+    /// ```
+    /// use std::io;
+    ///
+    /// use bytes::Bytes;
+    /// use futures::SinkExt;
+    /// use opendal::{Buffer, Operator};
+    /// use opendal::Result;
+    ///
+    /// async fn test(op: Operator) -> io::Result<()> {
+    ///     let mut w = op
+    ///         .writer_with("hello.txt")
+    ///         .concurrent(8)
+    ///         .chunk(256)
+    ///         .await?
+    ///         .into_sink();
+    ///     let bs = "Hello, World!".as_bytes();
+    ///     w.send(Buffer::from(bs)).await?;
+    ///     w.close().await?;
+    ///
+    ///     Ok(())
+    /// }
+    /// ```
+    pub fn into_sink(self) -> BufferSink {
+        BufferSink::new(self.inner)
+    }
+
     /// Convert writer into [`FuturesAsyncWriter`] which implements [`futures::AsyncWrite`],
     ///
     /// # Notes