You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/04/15 04:10:55 UTC

(opendal) branch main updated: feat(core): Add auto ranged read and concurrent read support (#4486)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 09b22b418e feat(core): Add auto ranged read and concurrent read support (#4486)
09b22b418e is described below

commit 09b22b418ee6bc804d499880e70f4565416d01b8
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Apr 15 12:10:42 2024 +0800

    feat(core): Add auto ranged read and concurrent read support (#4486)
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 core/src/raw/ops.rs                         | 65 +++++++++++++++++++++++++++-
 core/src/types/operator/operator.rs         | 13 +++---
 core/src/types/operator/operator_futures.rs | 42 +++++++++++++-----
 core/src/types/reader.rs                    | 67 +++++++++++++++++++++++++----
 4 files changed, 160 insertions(+), 27 deletions(-)

diff --git a/core/src/raw/ops.rs b/core/src/raw/ops.rs
index 3bbef0eb26..99185b3f04 100644
--- a/core/src/raw/ops.rs
+++ b/core/src/raw/ops.rs
@@ -24,7 +24,7 @@ use std::time::Duration;
 use flagset::FlagSet;
 
 use crate::raw::*;
-use crate::Metakey;
+use crate::*;
 
 /// Args for `create` operation.
 ///
@@ -382,6 +382,69 @@ impl OpRead {
     }
 }
 
+/// Args for reader operation.
+#[derive(Debug, Clone)]
+pub struct OpReader {
+    /// The range of the read request.
+    ///
+    /// Not available for `reader``.
+    range: BytesRange,
+    /// The concurrent requests that reader can send.
+    concurrent: usize,
+    /// The chunk size of each request.
+    chunk: usize,
+}
+
+impl Default for OpReader {
+    fn default() -> Self {
+        Self {
+            range: BytesRange::default(),
+            concurrent: 1,
+            chunk: 0,
+        }
+    }
+}
+
+impl OpReader {
+    /// Create a new `OpReader`.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Set the range of the option
+    pub fn with_range(mut self, range: BytesRange) -> Self {
+        self.range = range;
+        self
+    }
+
+    /// Get range from option
+    pub fn range(&self) -> BytesRange {
+        self.range
+    }
+
+    /// Set the concurrent of the option
+    pub fn with_concurrent(mut self, concurrent: usize) -> Self {
+        self.concurrent = concurrent.max(1);
+        self
+    }
+
+    /// Get concurrent from option
+    pub fn concurrent(&self) -> usize {
+        self.concurrent
+    }
+
+    /// Set the chunk of the option
+    pub fn with_chunk(mut self, chunk: usize) -> Self {
+        self.chunk = chunk;
+        self
+    }
+
+    /// Get chunk from option
+    pub fn chunk(&self) -> usize {
+        self.chunk
+    }
+}
+
 /// Args for `stat` operation.
 #[derive(Debug, Clone, Default)]
 pub struct OpStat {
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index 715401742b..ecc8935459 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -499,8 +499,8 @@ impl Operator {
         OperatorFuture::new(
             self.inner().clone(),
             path,
-            (OpRead::default(), BytesRange::default()),
-            |inner, path, (args, range)| async move {
+            (OpRead::default(), OpReader::default()),
+            |inner, path, (args, options)| async move {
                 if !validate_path(&path, EntryMode::FILE) {
                     return Err(
                         Error::new(ErrorKind::IsADirectory, "read path is a directory")
@@ -510,7 +510,8 @@ impl Operator {
                     );
                 }
 
-                let r = Reader::create(inner, &path, args).await?;
+                let range = options.range();
+                let r = Reader::create(inner, &path, args, options).await?;
                 let buf = r.read(range.to_range()).await?;
                 Ok(buf)
             },
@@ -570,8 +571,8 @@ impl Operator {
         OperatorFuture::new(
             self.inner().clone(),
             path,
-            OpRead::default(),
-            |inner, path, args| async move {
+            (OpRead::default(), OpReader::default()),
+            |inner, path, (args, options)| async move {
                 if !validate_path(&path, EntryMode::FILE) {
                     return Err(
                         Error::new(ErrorKind::IsADirectory, "read path is a directory")
@@ -581,7 +582,7 @@ impl Operator {
                     );
                 }
 
-                Reader::create(inner.clone(), &path, args).await
+                Reader::create(inner.clone(), &path, args, options).await
             },
         )
     }
diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs
index f327711957..1156d2b766 100644
--- a/core/src/types/operator/operator_futures.rs
+++ b/core/src/types/operator/operator_futures.rs
@@ -199,27 +199,37 @@ impl<F: Future<Output = Result<PresignedRequest>>> FuturePresignWrite<F> {
 /// Future that generated by [`Operator::read_with`] or [`Operator::reader_with`].
 ///
 /// Users can add more options by public functions provided by this struct.
-pub type FutureRead<F> = OperatorFuture<(OpRead, BytesRange), Buffer, F>;
+pub type FutureRead<F> = OperatorFuture<(OpRead, OpReader), Buffer, F>;
 
 impl<F: Future<Output = Result<Buffer>>> FutureRead<F> {
-    /// Set the range header for this operation.
-    pub fn range(self, range: impl RangeBounds<u64>) -> Self {
-        self.map(|(args, _)| (args, range.into()))
-    }
-
     /// Set the If-Match for this operation.
     pub fn if_match(self, v: &str) -> Self {
-        self.map(|(args, range)| (args.with_if_match(v), range))
+        self.map(|(args, op_reader)| (args.with_if_match(v), op_reader))
     }
 
     /// Set the If-None-Match for this operation.
     pub fn if_none_match(self, v: &str) -> Self {
-        self.map(|(args, range)| (args.with_if_none_match(v), range))
+        self.map(|(args, op_reader)| (args.with_if_none_match(v), op_reader))
     }
 
     /// Set the version for this operation.
     pub fn version(self, v: &str) -> Self {
-        self.map(|(args, range)| (args.with_version(v), range))
+        self.map(|(args, op_reader)| (args.with_version(v), op_reader))
+    }
+
+    /// Set the range header for this operation.
+    pub fn range(self, range: impl RangeBounds<u64>) -> Self {
+        self.map(|(args, op_reader)| (args, op_reader.with_range(range.into())))
+    }
+
+    /// Set the concurrent read task amount.
+    pub fn concurrent(self, concurrent: usize) -> Self {
+        self.map(|(args, op_reader)| (args, op_reader.with_concurrent(concurrent)))
+    }
+
+    /// Set the chunk size for this operation.
+    pub fn chunk(self, chunk_size: usize) -> Self {
+        self.map(|(args, op_reader)| (args, op_reader.with_chunk(chunk_size)))
     }
 }
 
@@ -230,12 +240,22 @@ impl<F: Future<Output = Result<Buffer>>> FutureRead<F> {
 /// # Notes
 ///
 /// `(OpRead, ())` is a trick to make sure `FutureReader` is different from `FutureRead`
-pub type FutureReader<F> = OperatorFuture<OpRead, Reader, F>;
+pub type FutureReader<F> = OperatorFuture<(OpRead, OpReader), Reader, F>;
 
 impl<F: Future<Output = Result<Reader>>> FutureReader<F> {
     /// Set the version for this operation.
     pub fn version(self, v: &str) -> Self {
-        self.map(|args| args.with_version(v))
+        self.map(|(op_read, op_reader)| (op_read.with_version(v), op_reader))
+    }
+
+    /// Set the concurrent read task amount.
+    pub fn concurrent(self, concurrent: usize) -> Self {
+        self.map(|(op_read, op_reader)| (op_read, op_reader.with_concurrent(concurrent)))
+    }
+
+    /// Set the chunk size for this operation.
+    pub fn chunk(self, chunk_size: usize) -> Self {
+        self.map(|(op_read, op_reader)| (op_read, op_reader.with_chunk(chunk_size)))
     }
 }
 
diff --git a/core/src/types/reader.rs b/core/src/types/reader.rs
index cc82a645e0..4ecad6e3f5 100644
--- a/core/src/types/reader.rs
+++ b/core/src/types/reader.rs
@@ -41,6 +41,7 @@ use crate::*;
 #[derive(Clone)]
 pub struct Reader {
     inner: oio::Reader,
+    options: OpReader,
 }
 
 impl Reader {
@@ -51,10 +52,15 @@ impl Reader {
     ///
     /// We don't want to expose those details to users so keep this function
     /// in crate only.
-    pub(crate) async fn create(acc: FusedAccessor, path: &str, op: OpRead) -> Result<Self> {
-        let (_, r) = acc.read(path, op).await?;
-
-        Ok(Reader { inner: r })
+    pub(crate) async fn create(
+        acc: FusedAccessor,
+        path: &str,
+        args: OpRead,
+        options: OpReader,
+    ) -> Result<Self> {
+        let (_, r) = acc.read(path, args).await?;
+
+        Ok(Reader { inner: r, options })
     }
 
     /// Read give range from reader into [`Buffer`].
@@ -100,9 +106,10 @@ impl Reader {
         &self,
         range: impl RangeBounds<u64>,
     ) -> impl Stream<Item = Result<Buffer>> + Unpin + MaybeSend + 'static {
-        let futs = into_stream::ReadFutureIterator::new(self.inner.clone(), range);
+        let futs =
+            into_stream::ReadFutureIterator::new(self.inner.clone(), self.options.chunk(), range);
 
-        stream::iter(futs).then(|f| f)
+        stream::iter(futs).buffered(self.options.concurrent())
     }
 
     /// Convert reader into [`FuturesIoAsyncReader`] which implements [`futures::AsyncRead`],
@@ -132,6 +139,7 @@ pub mod into_stream {
 
     pub struct ReadFutureIterator {
         r: oio::Reader,
+        chunk: usize,
 
         offset: u64,
         end: Option<u64>,
@@ -139,7 +147,7 @@ pub mod into_stream {
     }
 
     impl ReadFutureIterator {
-        pub fn new(r: oio::Reader, range: impl RangeBounds<u64>) -> Self {
+        pub fn new(r: oio::Reader, chunk: usize, range: impl RangeBounds<u64>) -> Self {
             let start = match range.start_bound().cloned() {
                 Bound::Included(start) => start,
                 Bound::Excluded(start) => start + 1,
@@ -153,6 +161,7 @@ pub mod into_stream {
 
             ReadFutureIterator {
                 r,
+                chunk,
                 offset: start,
                 end,
                 finished: Arc::default(),
@@ -174,8 +183,11 @@ pub mod into_stream {
             let offset = self.offset;
             let limit = self
                 .end
-                .map(|end| end - self.offset)
-                .unwrap_or(4 * 1024 * 1024) as usize;
+                .map(|end| (end - self.offset) as usize)
+                // TODO: replace with services preferred chunk size.
+                .unwrap_or(4 * 1024 * 1024)
+                // Align with reader's chunk size.
+                .max(self.chunk);
             let finished = self.finished.clone();
             let r = self.r.clone();
 
@@ -490,6 +502,43 @@ mod tests {
         assert_eq!(buf.to_bytes(), content);
     }
 
+    #[tokio::test]
+    async fn test_reader_read_with_chunk() {
+        let op = Operator::new(services::Memory::default()).unwrap().finish();
+        let path = "test_file";
+
+        let content = gen_random_bytes();
+        op.write(path, content.clone())
+            .await
+            .expect("write must succeed");
+
+        let reader = op.reader_with(path).chunk(16).await.unwrap();
+        let buf = reader.read(..).await.expect("read to end must succeed");
+
+        assert_eq!(buf.to_bytes(), content);
+    }
+
+    #[tokio::test]
+    async fn test_reader_read_with_concurrent() {
+        let op = Operator::new(services::Memory::default()).unwrap().finish();
+        let path = "test_file";
+
+        let content = gen_random_bytes();
+        op.write(path, content.clone())
+            .await
+            .expect("write must succeed");
+
+        let reader = op
+            .reader_with(path)
+            .chunk(128)
+            .concurrent(16)
+            .await
+            .unwrap();
+        let buf = reader.read(..).await.expect("read to end must succeed");
+
+        assert_eq!(buf.to_bytes(), content);
+    }
+
     #[tokio::test]
     async fn test_reader_read_into() {
         let op = Operator::new(services::Memory::default()).unwrap().finish();