You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/04/15 04:10:55 UTC
(opendal) branch main updated: feat(core): Add auto ranged read and concurrent read support (#4486)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 09b22b418e feat(core): Add auto ranged read and concurrent read support (#4486)
09b22b418e is described below
commit 09b22b418ee6bc804d499880e70f4565416d01b8
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Apr 15 12:10:42 2024 +0800
feat(core): Add auto ranged read and concurrent read support (#4486)
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/raw/ops.rs | 65 +++++++++++++++++++++++++++-
core/src/types/operator/operator.rs | 13 +++---
core/src/types/operator/operator_futures.rs | 42 +++++++++++++-----
core/src/types/reader.rs | 67 +++++++++++++++++++++++++----
4 files changed, 160 insertions(+), 27 deletions(-)
diff --git a/core/src/raw/ops.rs b/core/src/raw/ops.rs
index 3bbef0eb26..99185b3f04 100644
--- a/core/src/raw/ops.rs
+++ b/core/src/raw/ops.rs
@@ -24,7 +24,7 @@ use std::time::Duration;
use flagset::FlagSet;
use crate::raw::*;
-use crate::Metakey;
+use crate::*;
/// Args for `create` operation.
///
@@ -382,6 +382,69 @@ impl OpRead {
}
}
+/// Args for reader operation.
+#[derive(Debug, Clone)]
+pub struct OpReader {
+ /// The range of the read request.
+ ///
+ /// Not available for `reader``.
+ range: BytesRange,
+ /// The concurrent requests that reader can send.
+ concurrent: usize,
+ /// The chunk size of each request.
+ chunk: usize,
+}
+
+impl Default for OpReader {
+ fn default() -> Self {
+ Self {
+ range: BytesRange::default(),
+ concurrent: 1,
+ chunk: 0,
+ }
+ }
+}
+
+impl OpReader {
+ /// Create a new `OpReader`.
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Set the range of the option
+ pub fn with_range(mut self, range: BytesRange) -> Self {
+ self.range = range;
+ self
+ }
+
+ /// Get range from option
+ pub fn range(&self) -> BytesRange {
+ self.range
+ }
+
+ /// Set the concurrent of the option
+ pub fn with_concurrent(mut self, concurrent: usize) -> Self {
+ self.concurrent = concurrent.max(1);
+ self
+ }
+
+ /// Get concurrent from option
+ pub fn concurrent(&self) -> usize {
+ self.concurrent
+ }
+
+ /// Set the chunk of the option
+ pub fn with_chunk(mut self, chunk: usize) -> Self {
+ self.chunk = chunk;
+ self
+ }
+
+ /// Get chunk from option
+ pub fn chunk(&self) -> usize {
+ self.chunk
+ }
+}
+
/// Args for `stat` operation.
#[derive(Debug, Clone, Default)]
pub struct OpStat {
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index 715401742b..ecc8935459 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -499,8 +499,8 @@ impl Operator {
OperatorFuture::new(
self.inner().clone(),
path,
- (OpRead::default(), BytesRange::default()),
- |inner, path, (args, range)| async move {
+ (OpRead::default(), OpReader::default()),
+ |inner, path, (args, options)| async move {
if !validate_path(&path, EntryMode::FILE) {
return Err(
Error::new(ErrorKind::IsADirectory, "read path is a directory")
@@ -510,7 +510,8 @@ impl Operator {
);
}
- let r = Reader::create(inner, &path, args).await?;
+ let range = options.range();
+ let r = Reader::create(inner, &path, args, options).await?;
let buf = r.read(range.to_range()).await?;
Ok(buf)
},
@@ -570,8 +571,8 @@ impl Operator {
OperatorFuture::new(
self.inner().clone(),
path,
- OpRead::default(),
- |inner, path, args| async move {
+ (OpRead::default(), OpReader::default()),
+ |inner, path, (args, options)| async move {
if !validate_path(&path, EntryMode::FILE) {
return Err(
Error::new(ErrorKind::IsADirectory, "read path is a directory")
@@ -581,7 +582,7 @@ impl Operator {
);
}
- Reader::create(inner.clone(), &path, args).await
+ Reader::create(inner.clone(), &path, args, options).await
},
)
}
diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs
index f327711957..1156d2b766 100644
--- a/core/src/types/operator/operator_futures.rs
+++ b/core/src/types/operator/operator_futures.rs
@@ -199,27 +199,37 @@ impl<F: Future<Output = Result<PresignedRequest>>> FuturePresignWrite<F> {
/// Future that generated by [`Operator::read_with`] or [`Operator::reader_with`].
///
/// Users can add more options by public functions provided by this struct.
-pub type FutureRead<F> = OperatorFuture<(OpRead, BytesRange), Buffer, F>;
+pub type FutureRead<F> = OperatorFuture<(OpRead, OpReader), Buffer, F>;
impl<F: Future<Output = Result<Buffer>>> FutureRead<F> {
- /// Set the range header for this operation.
- pub fn range(self, range: impl RangeBounds<u64>) -> Self {
- self.map(|(args, _)| (args, range.into()))
- }
-
/// Set the If-Match for this operation.
pub fn if_match(self, v: &str) -> Self {
- self.map(|(args, range)| (args.with_if_match(v), range))
+ self.map(|(args, op_reader)| (args.with_if_match(v), op_reader))
}
/// Set the If-None-Match for this operation.
pub fn if_none_match(self, v: &str) -> Self {
- self.map(|(args, range)| (args.with_if_none_match(v), range))
+ self.map(|(args, op_reader)| (args.with_if_none_match(v), op_reader))
}
/// Set the version for this operation.
pub fn version(self, v: &str) -> Self {
- self.map(|(args, range)| (args.with_version(v), range))
+ self.map(|(args, op_reader)| (args.with_version(v), op_reader))
+ }
+
+ /// Set the range header for this operation.
+ pub fn range(self, range: impl RangeBounds<u64>) -> Self {
+ self.map(|(args, op_reader)| (args, op_reader.with_range(range.into())))
+ }
+
+ /// Set the concurrent read task amount.
+ pub fn concurrent(self, concurrent: usize) -> Self {
+ self.map(|(args, op_reader)| (args, op_reader.with_concurrent(concurrent)))
+ }
+
+ /// Set the chunk size for this operation.
+ pub fn chunk(self, chunk_size: usize) -> Self {
+ self.map(|(args, op_reader)| (args, op_reader.with_chunk(chunk_size)))
}
}
@@ -230,12 +240,22 @@ impl<F: Future<Output = Result<Buffer>>> FutureRead<F> {
/// # Notes
///
/// `(OpRead, ())` is a trick to make sure `FutureReader` is different from `FutureRead`
-pub type FutureReader<F> = OperatorFuture<OpRead, Reader, F>;
+pub type FutureReader<F> = OperatorFuture<(OpRead, OpReader), Reader, F>;
impl<F: Future<Output = Result<Reader>>> FutureReader<F> {
/// Set the version for this operation.
pub fn version(self, v: &str) -> Self {
- self.map(|args| args.with_version(v))
+ self.map(|(op_read, op_reader)| (op_read.with_version(v), op_reader))
+ }
+
+ /// Set the concurrent read task amount.
+ pub fn concurrent(self, concurrent: usize) -> Self {
+ self.map(|(op_read, op_reader)| (op_read, op_reader.with_concurrent(concurrent)))
+ }
+
+ /// Set the chunk size for this operation.
+ pub fn chunk(self, chunk_size: usize) -> Self {
+ self.map(|(op_read, op_reader)| (op_read, op_reader.with_chunk(chunk_size)))
}
}
diff --git a/core/src/types/reader.rs b/core/src/types/reader.rs
index cc82a645e0..4ecad6e3f5 100644
--- a/core/src/types/reader.rs
+++ b/core/src/types/reader.rs
@@ -41,6 +41,7 @@ use crate::*;
#[derive(Clone)]
pub struct Reader {
inner: oio::Reader,
+ options: OpReader,
}
impl Reader {
@@ -51,10 +52,15 @@ impl Reader {
///
/// We don't want to expose those details to users so keep this function
/// in crate only.
- pub(crate) async fn create(acc: FusedAccessor, path: &str, op: OpRead) -> Result<Self> {
- let (_, r) = acc.read(path, op).await?;
-
- Ok(Reader { inner: r })
+ pub(crate) async fn create(
+ acc: FusedAccessor,
+ path: &str,
+ args: OpRead,
+ options: OpReader,
+ ) -> Result<Self> {
+ let (_, r) = acc.read(path, args).await?;
+
+ Ok(Reader { inner: r, options })
}
/// Read give range from reader into [`Buffer`].
@@ -100,9 +106,10 @@ impl Reader {
&self,
range: impl RangeBounds<u64>,
) -> impl Stream<Item = Result<Buffer>> + Unpin + MaybeSend + 'static {
- let futs = into_stream::ReadFutureIterator::new(self.inner.clone(), range);
+ let futs =
+ into_stream::ReadFutureIterator::new(self.inner.clone(), self.options.chunk(), range);
- stream::iter(futs).then(|f| f)
+ stream::iter(futs).buffered(self.options.concurrent())
}
/// Convert reader into [`FuturesIoAsyncReader`] which implements [`futures::AsyncRead`],
@@ -132,6 +139,7 @@ pub mod into_stream {
pub struct ReadFutureIterator {
r: oio::Reader,
+ chunk: usize,
offset: u64,
end: Option<u64>,
@@ -139,7 +147,7 @@ pub mod into_stream {
}
impl ReadFutureIterator {
- pub fn new(r: oio::Reader, range: impl RangeBounds<u64>) -> Self {
+ pub fn new(r: oio::Reader, chunk: usize, range: impl RangeBounds<u64>) -> Self {
let start = match range.start_bound().cloned() {
Bound::Included(start) => start,
Bound::Excluded(start) => start + 1,
@@ -153,6 +161,7 @@ pub mod into_stream {
ReadFutureIterator {
r,
+ chunk,
offset: start,
end,
finished: Arc::default(),
@@ -174,8 +183,11 @@ pub mod into_stream {
let offset = self.offset;
let limit = self
.end
- .map(|end| end - self.offset)
- .unwrap_or(4 * 1024 * 1024) as usize;
+ .map(|end| (end - self.offset) as usize)
+ // TODO: replace with services preferred chunk size.
+ .unwrap_or(4 * 1024 * 1024)
+ // Align with reader's chunk size.
+ .max(self.chunk);
let finished = self.finished.clone();
let r = self.r.clone();
@@ -490,6 +502,43 @@ mod tests {
assert_eq!(buf.to_bytes(), content);
}
+ #[tokio::test]
+ async fn test_reader_read_with_chunk() {
+ let op = Operator::new(services::Memory::default()).unwrap().finish();
+ let path = "test_file";
+
+ let content = gen_random_bytes();
+ op.write(path, content.clone())
+ .await
+ .expect("write must succeed");
+
+ let reader = op.reader_with(path).chunk(16).await.unwrap();
+ let buf = reader.read(..).await.expect("read to end must succeed");
+
+ assert_eq!(buf.to_bytes(), content);
+ }
+
+ #[tokio::test]
+ async fn test_reader_read_with_concurrent() {
+ let op = Operator::new(services::Memory::default()).unwrap().finish();
+ let path = "test_file";
+
+ let content = gen_random_bytes();
+ op.write(path, content.clone())
+ .await
+ .expect("write must succeed");
+
+ let reader = op
+ .reader_with(path)
+ .chunk(128)
+ .concurrent(16)
+ .await
+ .unwrap();
+ let buf = reader.read(..).await.expect("read to end must succeed");
+
+ assert_eq!(buf.to_bytes(), content);
+ }
+
#[tokio::test]
async fn test_reader_read_into() {
let op = Operator::new(services::Memory::default()).unwrap().finish();