You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/04/11 09:03:25 UTC
(opendal) branch main updated: feat(core): Implement read and read_into for Reader (#4467)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 5f6669b0dc feat(core): Implement read and read_into for Reader (#4467)
5f6669b0dc is described below
commit 5f6669b0dc8bb799446ba60c5838bfd64d15fd33
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Thu Apr 11 17:03:19 2024 +0800
feat(core): Implement read and read_into for Reader (#4467)
* feat(core): Implement read and read_into for Reader
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix binding c
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix binding java
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix integration dav-server
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix object_store
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix ofs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix python and nodejs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
bin/ofs/src/fuse.rs | 4 +-
bindings/c/src/error.rs | 3 +-
bindings/c/src/types.rs | 4 +-
bindings/java/src/blocking_operator.rs | 4 +-
bindings/java/src/operator.rs | 4 +-
bindings/nodejs/src/lib.rs | 18 ++-
bindings/python/src/operator.rs | 4 +-
core/benches/ops/read.rs | 5 +-
core/benches/vs_s3/src/main.rs | 6 +-
core/edge/s3_read_on_wasm/src/lib.rs | 2 +-
core/fuzz/fuzz_writer.rs | 2 +-
core/src/layers/retry.rs | 2 +-
core/src/layers/timeout.rs | 2 +-
core/src/raw/tests/read.rs | 14 +--
core/src/types/blocking_reader.rs | 95 ++++++++++------
core/src/types/buffer.rs | 157 +++++++++++++++++++++++---
core/src/types/operator/blocking_operator.rs | 7 +-
core/src/types/operator/operator.rs | 9 +-
core/src/types/operator/operator_functions.rs | 4 +-
core/src/types/reader.rs | 103 +++++++++++------
core/tests/behavior/async_copy.rs | 24 +++-
core/tests/behavior/async_read.rs | 41 +++++--
core/tests/behavior/async_rename.rs | 18 ++-
core/tests/behavior/async_write.rs | 20 ++--
core/tests/behavior/blocking_copy.rs | 6 +-
core/tests/behavior/blocking_read.rs | 22 +++-
core/tests/behavior/blocking_rename.rs | 6 +-
core/tests/behavior/blocking_write.rs | 4 +-
integrations/dav-server/src/file.rs | 7 +-
integrations/object_store/src/lib.rs | 2 +-
30 files changed, 426 insertions(+), 173 deletions(-)
diff --git a/bin/ofs/src/fuse.rs b/bin/ofs/src/fuse.rs
index c8bb50dbc6..ff58033a2b 100644
--- a/bin/ofs/src/fuse.rs
+++ b/bin/ofs/src/fuse.rs
@@ -515,7 +515,9 @@ impl PathFilesystem for Fuse {
self.set_opened_file_offset(FileKey::try_from(fh)?, path, offset + data.len() as u64)?;
- Ok(ReplyData { data: data.into() })
+ Ok(ReplyData {
+ data: data.to_bytes(),
+ })
}
async fn write(
diff --git a/bindings/c/src/error.rs b/bindings/c/src/error.rs
index 3599846201..31696a29d1 100644
--- a/bindings/c/src/error.rs
+++ b/bindings/c/src/error.rs
@@ -16,6 +16,7 @@
// under the License.
use ::opendal as core;
+use opendal::Buffer;
use crate::types::opendal_bytes;
@@ -96,7 +97,7 @@ impl opendal_error {
/// free this error.
pub fn new(err: core::Error) -> *mut opendal_error {
let code = opendal_code::from(err.kind());
- let message = opendal_bytes::new(err.to_string().into_bytes());
+ let message = opendal_bytes::new(Buffer::from(err.to_string()));
Box::into_raw(Box::new(opendal_error { code, message }))
}
diff --git a/bindings/c/src/types.rs b/bindings/c/src/types.rs
index 3e8b88c48c..c0f19eb841 100644
--- a/bindings/c/src/types.rs
+++ b/bindings/c/src/types.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use opendal::Buffer;
use std::collections::HashMap;
use std::os::raw::c_char;
@@ -35,7 +36,8 @@ pub struct opendal_bytes {
impl opendal_bytes {
/// Construct a [`opendal_bytes`] from the Rust [`Vec`] of bytes
- pub(crate) fn new(vec: Vec<u8>) -> Self {
+ pub(crate) fn new(buf: Buffer) -> Self {
+ let vec = buf.to_vec();
let data = vec.as_ptr();
let len = vec.len();
std::mem::forget(vec);
diff --git a/bindings/java/src/blocking_operator.rs b/bindings/java/src/blocking_operator.rs
index 47b1abd53e..3234cf84fc 100644
--- a/bindings/java/src/blocking_operator.rs
+++ b/bindings/java/src/blocking_operator.rs
@@ -75,8 +75,8 @@ pub unsafe extern "system" fn Java_org_apache_opendal_BlockingOperator_read(
fn intern_read(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString) -> Result<jbyteArray> {
let path = jstring_to_string(env, &path)?;
- let content = op.read(&path)?;
- let result = env.byte_array_from_slice(content.as_slice())?;
+ let content = op.read(&path)?.to_bytes();
+ let result = env.byte_array_from_slice(&content)?;
Ok(result.into_raw())
}
diff --git a/bindings/java/src/operator.rs b/bindings/java/src/operator.rs
index 80e55b452a..ceb00e0491 100644
--- a/bindings/java/src/operator.rs
+++ b/bindings/java/src/operator.rs
@@ -262,10 +262,10 @@ fn intern_read(
}
async fn do_read<'local>(op: &mut Operator, path: String) -> Result<JObject<'local>> {
- let content = op.read(&path).await?;
+ let content = op.read(&path).await?.to_bytes();
let env = unsafe { get_current_env() };
- let result = env.byte_array_from_slice(content.as_slice())?;
+ let result = env.byte_array_from_slice(&content)?;
Ok(result.into())
}
diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs
index b0bab1b4cc..d47d5d5155 100644
--- a/bindings/nodejs/src/lib.rs
+++ b/bindings/nodejs/src/lib.rs
@@ -174,7 +174,12 @@ impl Operator {
/// ```
#[napi]
pub async fn read(&self, path: String) -> Result<Buffer> {
- let res = self.0.read(&path).await.map_err(format_napi_error)?;
+ let res = self
+ .0
+ .read(&path)
+ .await
+ .map_err(format_napi_error)?
+ .to_vec();
Ok(res.into())
}
@@ -198,7 +203,12 @@ impl Operator {
/// ```
#[napi]
pub fn read_sync(&self, path: String) -> Result<Buffer> {
- let res = self.0.blocking().read(&path).map_err(format_napi_error)?;
+ let res = self
+ .0
+ .blocking()
+ .read(&path)
+ .map_err(format_napi_error)?
+ .to_vec();
Ok(res.into())
}
@@ -679,8 +689,8 @@ impl Reader {
/// Read bytes from this reader into given buffer.
#[napi]
pub async unsafe fn read(&mut self, mut buf: Buffer) -> Result<usize> {
- let mut buf = buf.as_mut();
- let n = self.inner.read(&mut buf).await.map_err(format_napi_error)?;
+ let buf = buf.as_mut();
+ let n = self.inner.read(buf).await.map_err(format_napi_error)?;
Ok(n)
}
}
diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs
index 32e82cdd6c..86d03fce21 100644
--- a/bindings/python/src/operator.rs
+++ b/bindings/python/src/operator.rs
@@ -95,7 +95,7 @@ impl Operator {
/// Read the whole path into bytes.
pub fn read<'p>(&'p self, py: Python<'p>, path: &str) -> PyResult<&'p PyAny> {
- let buffer = self.0.read(path).map_err(format_pyerr)?;
+ let buffer = self.0.read(path).map_err(format_pyerr)?.to_vec();
Buffer::new(buffer).into_bytes_ref(py)
}
@@ -262,7 +262,7 @@ impl AsyncOperator {
pub fn read<'p>(&'p self, py: Python<'p>, path: String) -> PyResult<&'p PyAny> {
let this = self.0.clone();
future_into_py(py, async move {
- let res: Vec<u8> = this.read(&path).await.map_err(format_pyerr)?;
+ let res: Vec<u8> = this.read(&path).await.map_err(format_pyerr)?.to_vec();
Python::with_gil(|py| Buffer::new(res).into_bytes(py))
})
}
diff --git a/core/benches/ops/read.rs b/core/benches/ops/read.rs
index 84a1e9b48c..6df1f9f587 100644
--- a/core/benches/ops/read.rs
+++ b/core/benches/ops/read.rs
@@ -89,11 +89,8 @@ fn bench_read_parallel(c: &mut Criterion, name: &str, op: Operator) {
b.to_async(&*TEST_RUNTIME).iter(|| async {
let futures = (0..parallel)
.map(|_| async {
- let mut buf = Vec::with_capacity(*buf_size);
let r = op.reader_with(path).await.unwrap();
- r.read_range(&mut buf, offset..=offset + size.bytes() as u64)
- .await
- .unwrap();
+ let _ = r.read(offset..offset + *buf_size as u64).await.unwrap();
let mut d = 0;
// mock same little cpu work
diff --git a/core/benches/vs_s3/src/main.rs b/core/benches/vs_s3/src/main.rs
index dd28226257..e3205fc0f7 100644
--- a/core/benches/vs_s3/src/main.rs
+++ b/core/benches/vs_s3/src/main.rs
@@ -74,8 +74,7 @@ fn bench_read(c: &mut Criterion, op: Operator, s3_client: aws_sdk_s3::Client, bu
group.bench_function("opendal_s3_reader", |b| {
b.to_async(&*TEST_RUNTIME).iter(|| async {
let r = op.reader("file").await.unwrap();
- let mut bs = Vec::new();
- let _ = r.read_to_end(&mut bs).await.unwrap();
+ let _ = r.read(..).await.unwrap();
});
});
group.bench_function("aws_s3_sdk_into_async_read", |b| {
@@ -97,8 +96,7 @@ fn bench_read(c: &mut Criterion, op: Operator, s3_client: aws_sdk_s3::Client, bu
group.bench_function("opendal_s3_reader_with_capacity", |b| {
b.to_async(&*TEST_RUNTIME).iter(|| async {
let r = op.reader("file").await.unwrap();
- let mut bs = Vec::with_capacity(16 * 1024 * 1024);
- let _ = r.read_to_end(&mut bs).await.unwrap();
+ let _ = r.read(..16 * 1024 * 1024).await.unwrap();
});
});
group.bench_function("aws_s3_sdk_into_async_read_with_capacity", |b| {
diff --git a/core/edge/s3_read_on_wasm/src/lib.rs b/core/edge/s3_read_on_wasm/src/lib.rs
index d0fadff2a5..439342f736 100644
--- a/core/edge/s3_read_on_wasm/src/lib.rs
+++ b/core/edge/s3_read_on_wasm/src/lib.rs
@@ -37,7 +37,7 @@ pub async fn hello_world() -> String {
)
.await
.unwrap();
- let bs = op.read("test").await.unwrap();
+ let bs = op.read("test").await.unwrap().to_bytes();
String::from_utf8_lossy(&bs).to_string()
}
diff --git a/core/fuzz/fuzz_writer.rs b/core/fuzz/fuzz_writer.rs
index 02183311e2..6528240734 100644
--- a/core/fuzz/fuzz_writer.rs
+++ b/core/fuzz/fuzz_writer.rs
@@ -97,7 +97,7 @@ async fn fuzz_writer(op: Operator, input: FuzzInput) -> Result<()> {
writer.close().await?;
- let result = op.read(&path).await?;
+ let result = op.read(&path).await?.to_bytes();
checker.check(&result);
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index a59a7c32e1..5c8239150d 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -1107,7 +1107,7 @@ mod tests {
let r = op.reader("retryable_error").await.unwrap();
let mut content = Vec::new();
let size = r
- .read_to_end(&mut content)
+ .read_into(&mut content, ..)
.await
.expect("read must succeed");
assert_eq!(size, 13);
diff --git a/core/src/layers/timeout.rs b/core/src/layers/timeout.rs
index 31c15a39ad..6272f3aa6e 100644
--- a/core/src/layers/timeout.rs
+++ b/core/src/layers/timeout.rs
@@ -424,7 +424,7 @@ mod tests {
let reader = op.reader("test").await.unwrap();
- let res = reader.read(&mut Vec::default(), 0, 4).await;
+ let res = reader.read(0..4).await;
assert!(res.is_err());
let err = res.unwrap_err();
assert_eq!(err.kind(), ErrorKind::Unexpected);
diff --git a/core/src/raw/tests/read.rs b/core/src/raw/tests/read.rs
index 326001a94f..99744c632c 100644
--- a/core/src/raw/tests/read.rs
+++ b/core/src/raw/tests/read.rs
@@ -104,12 +104,11 @@ impl ReadChecker {
for action in actions {
match *action {
ReadAction::Read(offset, size) => {
- let mut bs = Vec::with_capacity(size);
- let n = r
- .read(&mut bs, offset as u64, size)
+ let bs = r
+ .read(offset as u64..(offset + size) as u64)
.await
.expect("read must success");
- self.check_read(offset, size, &bs[..n]);
+ self.check_read(offset, size, bs.to_bytes().as_ref());
}
}
}
@@ -122,11 +121,10 @@ impl ReadChecker {
for action in actions {
match *action {
ReadAction::Read(offset, size) => {
- let mut bs = Vec::with_capacity(size);
- let n = r
- .read(&mut bs, offset as u64, size)
+ let bs = r
+ .read(offset as u64..(offset + size) as u64)
.expect("read must success");
- self.check_read(offset, size, &bs[..n]);
+ self.check_read(offset, size, bs.to_bytes().as_ref());
}
}
}
diff --git a/core/src/types/blocking_reader.rs b/core/src/types/blocking_reader.rs
index 37d2d6c914..a82c2caa9c 100644
--- a/core/src/types/blocking_reader.rs
+++ b/core/src/types/blocking_reader.rs
@@ -46,21 +46,62 @@ impl BlockingReader {
Ok(BlockingReader { inner: r })
}
- /// Read from underlying storage and write data into the specified buffer, starting at
- /// the given offset and up to the limit.
+ /// Read give range from reader into [`Buffer`].
///
- /// A return value of `n` signifies that `n` bytes of data have been read into `buf`.
- /// If `n < limit`, it indicates that the reader has reached EOF (End of File).
- #[inline]
- pub fn read(&self, buf: &mut impl BufMut, offset: u64, limit: usize) -> Result<usize> {
- let bs = self.inner.read_at(offset, limit)?;
- let n = bs.remaining();
- buf.put(bs);
- Ok(n)
+ /// This operation is zero-copy, which means it keeps the [`Bytes`] returned by underlying
+ /// storage services without any extra copy or intensive memory allocations.
+ ///
+ /// # Notes
+ ///
+ /// - Buffer length smaller than range means we have reached the end of file.
+ pub fn read(&self, range: impl RangeBounds<u64>) -> Result<Buffer> {
+ let start = match range.start_bound().cloned() {
+ Bound::Included(start) => start,
+ Bound::Excluded(start) => start + 1,
+ Bound::Unbounded => 0,
+ };
+
+ let end = match range.end_bound().cloned() {
+ Bound::Included(end) => Some(end + 1),
+ Bound::Excluded(end) => Some(end),
+ Bound::Unbounded => None,
+ };
+
+ // If range is empty, return Ok(0) directly.
+ if let Some(end) = end {
+ if end <= start {
+ return Ok(Buffer::new());
+ }
+ }
+
+ let mut bufs = Vec::new();
+ let mut offset = start;
+
+ loop {
+ // TODO: use service preferred io size instead.
+ let limit = end.map(|end| end - offset).unwrap_or(4 * 1024 * 1024) as usize;
+ let bs = self.inner.read_at(offset, limit)?;
+ let n = bs.remaining();
+ bufs.push(bs);
+ if n < limit {
+ return Ok(bufs.into_iter().flatten().collect());
+ }
+
+ offset += n as u64;
+ if Some(offset) == end {
+ return Ok(bufs.into_iter().flatten().collect());
+ }
+ }
}
- /// Read given range bytes of data from reader.
- pub fn read_range(&self, buf: &mut impl BufMut, range: impl RangeBounds<u64>) -> Result<usize> {
+ ///
+ /// This operation will copy and write bytes into given [`BufMut`]. Allocation happens while
+ /// [`BufMut`] doesn't have enough space.
+ ///
+ /// # Notes
+ ///
+ /// - Returning length smaller than range means we have reached the end of file.
+ pub fn read_into(&self, buf: &mut impl BufMut, range: impl RangeBounds<u64>) -> Result<usize> {
let start = match range.start_bound().cloned() {
Bound::Included(start) => start,
Bound::Excluded(start) => start + 1,
@@ -81,38 +122,26 @@ impl BlockingReader {
}
let mut offset = start;
- let mut size = end.map(|end| end - start);
-
let mut read = 0;
+
loop {
- let bs = self
- .inner
- // TODO: use service preferred io size instead.
- .read_at(offset, size.unwrap_or(4 * 1024 * 1024) as usize)?;
+ // TODO: use service preferred io size instead.
+ let limit = end.map(|end| end - offset).unwrap_or(4 * 1024 * 1024) as usize;
+ let bs = self.inner.read_at(offset, limit)?;
let n = bs.remaining();
- read += n;
buf.put(bs);
- if n == 0 {
- return Ok(read);
+ read += n as u64;
+ if n < limit {
+ return Ok(read as _);
}
offset += n as u64;
-
- size = size.map(|v| v - n as u64);
- if size == Some(0) {
- return Ok(read);
+ if Some(offset) == end {
+ return Ok(read as _);
}
}
}
- /// Read all data from reader.
- ///
- /// This API is exactly the same with `BlockingReader::read_range(buf, ..)`.
- #[inline]
- pub fn read_to_end(&self, buf: &mut impl BufMut) -> Result<usize> {
- self.read_range(buf, ..)
- }
-
/// Convert reader into [`FuturesIoAsyncReader`] which implements [`futures::AsyncRead`],
/// [`futures::AsyncSeek`] and [`futures::AsyncBufRead`].
#[inline]
diff --git a/core/src/types/buffer.rs b/core/src/types/buffer.rs
index eb40610c21..dc9cc6ec2b 100644
--- a/core/src/types/buffer.rs
+++ b/core/src/types/buffer.rs
@@ -18,19 +18,90 @@
use std::collections::VecDeque;
use std::convert::Infallible;
use std::fmt::{Debug, Formatter};
+use std::io::IoSlice;
use std::mem;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
-use bytes::Buf;
-use bytes::Bytes;
+use bytes::{Buf, BytesMut};
+use bytes::{BufMut, Bytes};
use futures::Stream;
/// Buffer is a wrapper of contiguous `Bytes` and non contiguous `[Bytes]`.
///
/// We designed buffer to allow underlying storage to return non-contiguous bytes. For example,
/// http based storage like s3 could generate non-contiguous bytes by stream.
+///
+/// ## Features
+///
+/// - [`Buffer`] can be used as [`Buf`], [`Iterator`], [`Stream`] directly.
+/// - [`Buffer`] is cheap to clone like [`Bytes`], only update reference count, no allocation.
+/// - [`Buffer`] is vectorized write friendly, you can convert it to [`IoSlice`] for vectored write.
+///
+/// ## Examples
+///
+/// ### As `Buf`
+///
+/// `Buffer` implements `Buf` trait:
+///
+/// ```rust
+/// use bytes::Buf;
+/// use serde_json;
+/// use opendal::Buffer;
+///
+/// fn test(mut buf: Buffer) -> Vec<String> {
+/// serde_json::from_reader(buf.reader()).unwrap()
+/// }
+/// ```
+///
+/// ### As Bytes `Iterator`
+///
+/// `Buffer` implements `Iterator<Item=Bytes>` trait:
+///
+/// ```rust
+/// use bytes::Bytes;
+/// use opendal::Buffer;
+///
+/// fn test(mut buf: Buffer) -> Vec<Bytes> {
+/// buf.into_iter().collect()
+/// }
+/// ```
+///
+/// ### As Bytes `Stream`
+///
+/// `Buffer` implements `Stream<Item=Result<Bytes, Infallible>>` trait:
+///
+/// ```rust
+/// use bytes::Bytes;
+/// use opendal::Buffer;
+/// use futures::TryStreamExt;
+///
+/// async fn test(mut buf: Buffer) -> Vec<Bytes> {
+/// buf.into_iter().try_collect().await.unwrap()
+/// }
+/// ```
+///
+/// ### As one contiguous Bytes
+///
+/// `Buffer` can make contiguous by transform into `Bytes` or `Vec<u8>`.
+/// Please keep in mind that this operation involves new allocation and bytes copy, and we can't
+/// reuse the same memory region anymore.
+///
+/// ```rust
+/// use bytes::Bytes;
+/// use opendal::Buffer;
+///
+/// fn test_to_vec(buf: Buffer) -> Vec<u8> {
+/// buf.to_vec()
+/// }
+///
+/// fn test_to_bytes(buf: Buffer) -> Bytes {
+/// buf.to_bytes()
+/// }
+/// ```
+///
+///
#[derive(Clone)]
pub struct Buffer(Inner);
@@ -86,18 +157,6 @@ impl Buffer {
Self(Inner::Contiguous(Bytes::new()))
}
- /// Clone internal bytes to a new `Bytes`.
- ///
- /// # Notes
- ///
- /// This operation is not efficient and should be used with caution.
- /// Please never use this method in read/write related APIs.
- #[inline]
- pub fn to_bytes(&self) -> Bytes {
- let mut bs = self.clone();
- bs.copy_to_bytes(bs.remaining())
- }
-
/// Get the length of the buffer.
#[inline]
pub fn len(&self) -> usize {
@@ -125,6 +184,45 @@ impl Buffer {
}
}
}
+
+ /// Combine all bytes together into one single [`Bytes`].
+ ///
+ /// This operation is not zero copy, it will copy all bytes into one single [`Bytes`].
+ /// Please use API from [`Buf`], [`Iterator`] or [`Stream`] whenever possible.
+ #[inline]
+ pub fn to_bytes(&self) -> Bytes {
+ let mut ret = BytesMut::with_capacity(self.len());
+ ret.put(self.clone());
+ ret.freeze()
+ }
+
+ /// Combine all bytes together into one single [`Vec<u8>`].
+ ///
+ /// This operation is not zero copy, it will copy all bytes into one single [`Vec<u8>`].
+ /// Please use API from [`Buf`], [`Iterator`] or [`Stream`] whenever possible.
+ #[inline]
+ pub fn to_vec(&self) -> Vec<u8> {
+ let mut ret = Vec::with_capacity(self.len());
+ ret.put(self.clone());
+ ret
+ }
+
+ /// Convert buffer into a slice of [`IoSlice`] for vectored write.
+ #[inline]
+ pub fn to_io_slice(&self) -> Vec<IoSlice<'_>> {
+ match &self.0 {
+ Inner::Contiguous(bs) => vec![IoSlice::new(bs.chunk())],
+ Inner::NonContiguous {
+ parts, idx, offset, ..
+ } => {
+ let mut ret = Vec::with_capacity(parts.len() - *idx);
+ for part in parts.iter().skip(*idx) {
+ ret.push(IoSlice::new(&part[*offset..]));
+ }
+ ret
+ }
+ }
+ }
}
impl From<Vec<u8>> for Buffer {
@@ -242,6 +340,37 @@ impl Buf for Buffer {
}
}
+ fn chunks_vectored<'a>(&'a self, dst: &mut [IoSlice<'a>]) -> usize {
+ match &self.0 {
+ Inner::Contiguous(b) => {
+ if dst.is_empty() {
+ return 0;
+ }
+
+ dst[0] = IoSlice::new(b.chunk());
+ 1
+ }
+ Inner::NonContiguous {
+ parts, idx, offset, ..
+ } => {
+ if dst.is_empty() {
+ return 0;
+ }
+
+ let mut i = 0;
+ for part in parts.iter().skip(*idx) {
+ if i >= dst.len() {
+ break;
+ }
+
+ dst[i] = IoSlice::new(&part[*offset..]);
+ i += 1;
+ }
+ i
+ }
+ }
+ }
+
#[inline]
fn advance(&mut self, cnt: usize) {
match &mut self.0 {
diff --git a/core/src/types/operator/blocking_operator.rs b/core/src/types/operator/blocking_operator.rs
index 03cedacf15..458aa2342b 100644
--- a/core/src/types/operator/blocking_operator.rs
+++ b/core/src/types/operator/blocking_operator.rs
@@ -358,7 +358,7 @@ impl BlockingOperator {
/// # Ok(())
/// # }
/// ```
- pub fn read(&self, path: &str) -> Result<Vec<u8>> {
+ pub fn read(&self, path: &str) -> Result<Buffer> {
self.read_with(path).call()
}
@@ -396,11 +396,8 @@ impl BlockingOperator {
);
}
- let size_hint = range.size();
-
let r = BlockingReader::create(inner, &path, args)?;
- let mut buf = Vec::with_capacity(size_hint.unwrap_or_default() as _);
- r.read_range(&mut buf, range.to_range())?;
+ let buf = r.read(range.to_range())?;
Ok(buf)
},
))
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index 3f3e8fecc0..4e6bea5e64 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -405,7 +405,7 @@ impl Operator {
/// # Ok(())
/// # }
/// ```
- pub async fn read(&self, path: &str) -> Result<Vec<u8>> {
+ pub async fn read(&self, path: &str) -> Result<Buffer> {
self.read_with(path).await
}
@@ -494,7 +494,7 @@ impl Operator {
/// # Ok(())
/// # }
/// ```
- pub fn read_with(&self, path: &str) -> FutureRead<impl Future<Output = Result<Vec<u8>>>> {
+ pub fn read_with(&self, path: &str) -> FutureRead<impl Future<Output = Result<Buffer>>> {
let path = normalize_path(path);
OperatorFuture::new(
@@ -511,11 +511,8 @@ impl Operator {
);
}
- let size_hint = range.size();
-
let r = Reader::create(inner, &path, args).await?;
- let mut buf = Vec::with_capacity(size_hint.unwrap_or_default() as _);
- r.read_range(&mut buf, range.to_range()).await?;
+ let buf = r.read(range.to_range()).await?;
Ok(buf)
},
)
diff --git a/core/src/types/operator/operator_functions.rs b/core/src/types/operator/operator_functions.rs
index 1e40f78d41..0a7711544a 100644
--- a/core/src/types/operator/operator_functions.rs
+++ b/core/src/types/operator/operator_functions.rs
@@ -320,7 +320,7 @@ impl FunctionLister {
/// Function that generated by [`BlockingOperator::read_with`].
///
/// Users can add more options by public functions provided by this struct.
-pub struct FunctionRead(pub(crate) OperatorFunction<(OpRead, BytesRange), Vec<u8>>);
+pub struct FunctionRead(pub(crate) OperatorFunction<(OpRead, BytesRange), Buffer>);
impl FunctionRead {
/// Set the range for this operation.
@@ -331,7 +331,7 @@ impl FunctionRead {
/// Call the function to consume all the input and generate a
/// result.
- pub fn call(self) -> Result<Vec<u8>> {
+ pub fn call(self) -> Result<Buffer> {
self.0.call()
}
}
diff --git a/core/src/types/reader.rs b/core/src/types/reader.rs
index 7a6f17d919..3ff7cb7588 100644
--- a/core/src/types/reader.rs
+++ b/core/src/types/reader.rs
@@ -54,21 +54,63 @@ impl Reader {
Ok(Reader { inner: r })
}
- /// Read from underlying storage and write data into the specified buffer, starting at
- /// the given offset and up to the limit.
+ /// Read give range from reader into [`Buffer`].
///
- /// A return value of `n` signifies that `n` bytes of data have been read into `buf`.
- /// If `n < limit`, it indicates that the reader has reached EOF (End of File).
- #[inline]
- pub async fn read(&self, buf: &mut impl BufMut, offset: u64, limit: usize) -> Result<usize> {
- let bs = self.inner.read_at_dyn(offset, limit).await?;
- let n = bs.remaining();
- buf.put(bs);
- Ok(n)
+ /// This operation is zero-copy, which means it keeps the [`Bytes`] returned by underlying
+ /// storage services without any extra copy or intensive memory allocations.
+ ///
+ /// # Notes
+ ///
+ /// - Buffer length smaller than range means we have reached the end of file.
+ pub async fn read(&self, range: impl RangeBounds<u64>) -> Result<Buffer> {
+ let start = match range.start_bound().cloned() {
+ Bound::Included(start) => start,
+ Bound::Excluded(start) => start + 1,
+ Bound::Unbounded => 0,
+ };
+
+ let end = match range.end_bound().cloned() {
+ Bound::Included(end) => Some(end + 1),
+ Bound::Excluded(end) => Some(end),
+ Bound::Unbounded => None,
+ };
+
+ // If range is empty, return Ok(0) directly.
+ if let Some(end) = end {
+ if end <= start {
+ return Ok(Buffer::new());
+ }
+ }
+
+ let mut bufs = Vec::new();
+ let mut offset = start;
+
+ loop {
+ // TODO: use service preferred io size instead.
+ let limit = end.map(|end| end - offset).unwrap_or(4 * 1024 * 1024) as usize;
+ let bs = self.inner.read_at_dyn(offset, limit).await?;
+ let n = bs.remaining();
+ bufs.push(bs);
+ if n < limit {
+ return Ok(bufs.into_iter().flatten().collect());
+ }
+
+ offset += n as u64;
+ if Some(offset) == end {
+ return Ok(bufs.into_iter().flatten().collect());
+ }
+ }
}
- /// Read given range bytes of data from reader.
- pub async fn read_range(
+ /// Read all data from reader into given [`BufMut`].
+ ///
+ /// This operation will copy and write bytes into given [`BufMut`]. Allocation happens while
+ /// [`BufMut`] doesn't have enough space.
+ ///
+ /// # Notes
+ ///
+ /// - Returning length smaller than range means we have reached the end of file.
+ pub async fn read_into(
&self,
buf: &mut impl BufMut,
range: impl RangeBounds<u64>,
@@ -93,36 +135,26 @@ impl Reader {
}
let mut offset = start;
- let mut size = end.map(|end| end - start);
-
let mut read = 0;
+
loop {
// TODO: use service preferred io size instead.
- let limit = size.unwrap_or(4 * 1024 * 1024) as usize;
+ let limit = end.map(|end| end - offset).unwrap_or(4 * 1024 * 1024) as usize;
let bs = self.inner.read_at_dyn(offset, limit).await?;
let n = bs.remaining();
- read += n;
buf.put(bs);
+ read += n as u64;
if n < limit {
- return Ok(read);
+ return Ok(read as _);
}
offset += n as u64;
- size = size.map(|v| v - n as u64);
- if size == Some(0) {
- return Ok(read);
+ if Some(offset) == end {
+ return Ok(read as _);
}
}
}
- /// Read all data from reader.
- ///
- /// This API is exactly the same with `Reader::read_range(buf, ..)`.
- #[inline]
- pub async fn read_to_end(&self, buf: &mut impl BufMut) -> Result<usize> {
- self.read_range(buf, ..).await
- }
-
/// Convert reader into [`FuturesIoAsyncReader`] which implements [`futures::AsyncRead`],
/// [`futures::AsyncSeek`] and [`futures::AsyncBufRead`].
#[inline]
@@ -412,7 +444,7 @@ mod tests {
}
#[tokio::test]
- async fn test_reader_async_read() {
+ async fn test_reader_read() {
let op = Operator::new(services::Memory::default()).unwrap().finish();
let path = "test_file";
@@ -422,17 +454,13 @@ mod tests {
.expect("write must succeed");
let reader = op.reader(path).await.unwrap();
- let mut buf = Vec::new();
- reader
- .read_to_end(&mut buf)
- .await
- .expect("read to end must succeed");
+ let buf = reader.read(..).await.expect("read to end must succeed");
- assert_eq!(buf, content);
+ assert_eq!(buf.to_bytes(), content);
}
#[tokio::test]
- async fn test_reader_async_seek() {
+ async fn test_reader_read_into() {
let op = Operator::new(services::Memory::default()).unwrap().finish();
let path = "test_file";
@@ -444,9 +472,10 @@ mod tests {
let reader = op.reader(path).await.unwrap();
let mut buf = Vec::new();
reader
- .read_to_end(&mut buf)
+ .read_into(&mut buf, ..)
.await
.expect("read to end must succeed");
+
assert_eq!(buf, content);
}
}
diff --git a/core/tests/behavior/async_copy.rs b/core/tests/behavior/async_copy.rs
index 82dc4637ed..56a372fe73 100644
--- a/core/tests/behavior/async_copy.rs
+++ b/core/tests/behavior/async_copy.rs
@@ -50,7 +50,11 @@ pub async fn test_copy_file_with_ascii_name(op: Operator) -> Result<()> {
op.copy(&source_path, &target_path).await?;
- let target_content = op.read(&target_path).await.expect("read must succeed");
+ let target_content = op
+ .read(&target_path)
+ .await
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
@@ -75,7 +79,11 @@ pub async fn test_copy_file_with_non_ascii_name(op: Operator) -> Result<()> {
op.write(source_path, source_content.clone()).await?;
op.copy(source_path, target_path).await?;
- let target_content = op.read(target_path).await.expect("read must succeed");
+ let target_content = op
+ .read(target_path)
+ .await
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
@@ -177,7 +185,11 @@ pub async fn test_copy_nested(op: Operator) -> Result<()> {
op.copy(&source_path, &target_path).await?;
- let target_content = op.read(&target_path).await.expect("read must succeed");
+ let target_content = op
+ .read(&target_path)
+ .await
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
@@ -203,7 +215,11 @@ pub async fn test_copy_overwrite(op: Operator) -> Result<()> {
op.copy(&source_path, &target_path).await?;
- let target_content = op.read(&target_path).await.expect("read must succeed");
+ let target_content = op
+ .read(&target_path)
+ .await
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
diff --git a/core/tests/behavior/async_read.rs b/core/tests/behavior/async_read.rs
index 0ba7db507c..2e77e83457 100644
--- a/core/tests/behavior/async_read.rs
+++ b/core/tests/behavior/async_read.rs
@@ -68,7 +68,7 @@ pub async fn test_read_full(op: Operator) -> anyhow::Result<()> {
.await
.expect("write must succeed");
- let bs = op.read(&path).await?;
+ let bs = op.read(&path).await?.to_bytes();
assert_eq!(size, bs.len(), "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -88,7 +88,11 @@ pub async fn test_read_range(op: Operator) -> anyhow::Result<()> {
.await
.expect("write must succeed");
- let bs = op.read_with(&path).range(offset..offset + length).await?;
+ let bs = op
+ .read_with(&path)
+ .range(offset..offset + length)
+ .await?
+ .to_bytes();
assert_eq!(bs.len() as u64, length, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -111,7 +115,11 @@ pub async fn test_read_large_range(op: Operator) -> anyhow::Result<()> {
.await
.expect("write must succeed");
- let bs = op.read_with(&path).range(offset..u32::MAX as u64).await?;
+ let bs = op
+ .read_with(&path)
+ .range(offset..u32::MAX as u64)
+ .await?
+ .to_bytes();
assert_eq!(
bs.len() as u64,
size as u64 - offset,
@@ -159,7 +167,8 @@ pub async fn test_read_with_if_match(op: Operator) -> anyhow::Result<()> {
.read_with(&path)
.if_match(meta.etag().expect("etag must exist"))
.await
- .expect("read must succeed");
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(bs, content);
Ok(())
@@ -190,7 +199,8 @@ pub async fn test_read_with_if_none_match(op: Operator) -> anyhow::Result<()> {
.read_with(&path)
.if_none_match("\"invalid_etag\"")
.await
- .expect("read must succeed");
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(bs, content);
Ok(())
@@ -233,7 +243,7 @@ pub async fn test_read_with_special_chars(op: Operator) -> anyhow::Result<()> {
.await
.expect("write must succeed");
- let bs = op.read(&path).await?;
+ let bs = op.read(&path).await?.to_bytes();
assert_eq!(size, bs.len(), "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -388,7 +398,7 @@ pub async fn test_read_with_override_content_type(op: Operator) -> anyhow::Resul
/// Read full content should match.
pub async fn test_read_only_read_full(op: Operator) -> anyhow::Result<()> {
- let bs = op.read("normal_file.txt").await?;
+ let bs = op.read("normal_file.txt").await?.to_bytes();
assert_eq!(bs.len(), 30482, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -401,7 +411,10 @@ pub async fn test_read_only_read_full(op: Operator) -> anyhow::Result<()> {
/// Read full content should match.
pub async fn test_read_only_read_full_with_special_chars(op: Operator) -> anyhow::Result<()> {
- let bs = op.read("special_file !@#$%^&()_+-=;',.txt").await?;
+ let bs = op
+ .read("special_file !@#$%^&()_+-=;',.txt")
+ .await?
+ .to_bytes();
assert_eq!(bs.len(), 30482, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -414,7 +427,11 @@ pub async fn test_read_only_read_full_with_special_chars(op: Operator) -> anyhow
/// Read full content should match.
pub async fn test_read_only_read_with_range(op: Operator) -> anyhow::Result<()> {
- let bs = op.read_with("normal_file.txt").range(1024..2048).await?;
+ let bs = op
+ .read_with("normal_file.txt")
+ .range(1024..2048)
+ .await?
+ .to_bytes();
assert_eq!(bs.len(), 1024, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -465,7 +482,8 @@ pub async fn test_read_only_read_with_if_match(op: Operator) -> anyhow::Result<(
.read_with(path)
.if_match(meta.etag().expect("etag must exist"))
.await
- .expect("read must succeed");
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(bs.len(), 30482, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -497,7 +515,8 @@ pub async fn test_read_only_read_with_if_none_match(op: Operator) -> anyhow::Res
.read_with(path)
.if_none_match("invalid_etag")
.await
- .expect("read must succeed");
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(bs.len(), 30482, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
diff --git a/core/tests/behavior/async_rename.rs b/core/tests/behavior/async_rename.rs
index d3cd7f921f..e29bbb1f3d 100644
--- a/core/tests/behavior/async_rename.rs
+++ b/core/tests/behavior/async_rename.rs
@@ -52,7 +52,11 @@ pub async fn test_rename_file(op: Operator) -> Result<()> {
let err = op.stat(&source_path).await.expect_err("stat must fail");
assert_eq!(err.kind(), ErrorKind::NotFound);
- let target_content = op.read(&target_path).await.expect("read must succeed");
+ let target_content = op
+ .read(&target_path)
+ .await
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
@@ -157,7 +161,11 @@ pub async fn test_rename_nested(op: Operator) -> Result<()> {
let err = op.stat(&source_path).await.expect_err("stat must fail");
assert_eq!(err.kind(), ErrorKind::NotFound);
- let target_content = op.read(&target_path).await.expect("read must succeed");
+ let target_content = op
+ .read(&target_path)
+ .await
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
@@ -186,7 +194,11 @@ pub async fn test_rename_overwrite(op: Operator) -> Result<()> {
let err = op.stat(&source_path).await.expect_err("stat must fail");
assert_eq!(err.kind(), ErrorKind::NotFound);
- let target_content = op.read(&target_path).await.expect("read must succeed");
+ let target_content = op
+ .read(&target_path)
+ .await
+ .expect("read must succeed")
+ .to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
diff --git a/core/tests/behavior/async_write.rs b/core/tests/behavior/async_write.rs
index 8f724033a6..dc37ba541d 100644
--- a/core/tests/behavior/async_write.rs
+++ b/core/tests/behavior/async_write.rs
@@ -274,7 +274,7 @@ pub async fn test_writer_write(op: Operator) -> Result<()> {
let meta = op.stat(&path).await.expect("stat must succeed");
assert_eq!(meta.content_length(), (size * 2) as u64);
- let bs = op.read(&path).await?;
+ let bs = op.read(&path).await?.to_bytes();
assert_eq!(bs.len(), size * 2, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs[..size])),
@@ -309,7 +309,7 @@ pub async fn test_writer_write_with_concurrent(op: Operator) -> Result<()> {
let meta = op.stat(&path).await.expect("stat must succeed");
assert_eq!(meta.content_length(), (size * 2) as u64);
- let bs = op.read(&path).await?;
+ let bs = op.read(&path).await?.to_bytes();
assert_eq!(bs.len(), size * 2, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs[..size])),
@@ -345,7 +345,7 @@ pub async fn test_writer_sink(op: Operator) -> Result<()> {
let meta = op.stat(&path).await.expect("stat must succeed");
assert_eq!(meta.content_length(), (size * 2) as u64);
- let bs = op.read(&path).await?;
+ let bs = op.read(&path).await?.to_bytes();
assert_eq!(bs.len(), size * 2, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs[..size])),
@@ -385,7 +385,7 @@ pub async fn test_writer_sink_with_concurrent(op: Operator) -> Result<()> {
let meta = op.stat(&path).await.expect("stat must succeed");
assert_eq!(meta.content_length(), (size * 2) as u64);
- let bs = op.read(&path).await?;
+ let bs = op.read(&path).await?.to_bytes();
assert_eq!(bs.len(), size * 2, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs[..size])),
@@ -425,7 +425,7 @@ pub async fn test_writer_futures_copy(op: Operator) -> Result<()> {
let meta = op.stat(&path).await.expect("stat must succeed");
assert_eq!(meta.content_length(), size as u64);
- let bs = op.read(&path).await?;
+ let bs = op.read(&path).await?.to_bytes();
assert_eq!(bs.len(), size, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs[..size])),
@@ -461,7 +461,7 @@ pub async fn test_writer_futures_copy_with_concurrent(op: Operator) -> Result<()
let meta = op.stat(&path).await.expect("stat must succeed");
assert_eq!(meta.content_length(), size as u64);
- let bs = op.read(&path).await?;
+ let bs = op.read(&path).await?.to_bytes();
assert_eq!(bs.len(), size, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs[..size])),
@@ -491,7 +491,11 @@ pub async fn test_write_with_append(op: Operator) -> Result<()> {
.await
.expect("append to an existing file must success");
- let bs = op.read(&path).await.expect("read file must success");
+ let bs = op
+ .read(&path)
+ .await
+ .expect("read file must success")
+ .to_bytes();
assert_eq!(bs.len(), size_one + size_two);
assert_eq!(bs[..size_one], content_one);
@@ -520,7 +524,7 @@ pub async fn test_writer_with_append(op: Operator) -> Result<()> {
let meta = op.stat(&path).await.expect("stat must succeed");
assert_eq!(meta.content_length(), size as u64);
- let bs = op.read(&path).await?;
+ let bs = op.read(&path).await?.to_bytes();
assert_eq!(bs.len(), size, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs[..size])),
diff --git a/core/tests/behavior/blocking_copy.rs b/core/tests/behavior/blocking_copy.rs
index c8a8a811bd..7e091e2ac9 100644
--- a/core/tests/behavior/blocking_copy.rs
+++ b/core/tests/behavior/blocking_copy.rs
@@ -49,7 +49,7 @@ pub fn test_blocking_copy_file(op: BlockingOperator) -> Result<()> {
op.copy(&source_path, &target_path)?;
- let target_content = op.read(&target_path).expect("read must succeed");
+ let target_content = op.read(&target_path).expect("read must succeed").to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
@@ -147,7 +147,7 @@ pub fn test_blocking_copy_nested(op: BlockingOperator) -> Result<()> {
op.copy(&source_path, &target_path)?;
- let target_content = op.read(&target_path).expect("read must succeed");
+ let target_content = op.read(&target_path).expect("read must succeed").to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
@@ -173,7 +173,7 @@ pub fn test_blocking_copy_overwrite(op: BlockingOperator) -> Result<()> {
op.copy(&source_path, &target_path)?;
- let target_content = op.read(&target_path).expect("read must succeed");
+ let target_content = op.read(&target_path).expect("read must succeed").to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
diff --git a/core/tests/behavior/blocking_read.rs b/core/tests/behavior/blocking_read.rs
index 006d3349d9..b448642671 100644
--- a/core/tests/behavior/blocking_read.rs
+++ b/core/tests/behavior/blocking_read.rs
@@ -57,7 +57,7 @@ pub fn test_blocking_read_full(op: BlockingOperator) -> Result<()> {
op.write(&path, content.clone())
.expect("write must succeed");
- let bs = op.read(&path)?;
+ let bs = op.read(&path)?.to_bytes();
assert_eq!(size, bs.len(), "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -79,7 +79,11 @@ pub fn test_blocking_read_range(op: BlockingOperator) -> Result<()> {
op.write(&path, content.clone())
.expect("write must succeed");
- let bs = op.read_with(&path).range(offset..offset + length).call()?;
+ let bs = op
+ .read_with(&path)
+ .range(offset..offset + length)
+ .call()?
+ .to_bytes();
assert_eq!(bs.len() as u64, length, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -104,7 +108,11 @@ pub fn test_blocking_read_large_range(op: BlockingOperator) -> Result<()> {
op.write(&path, content.clone())
.expect("write must succeed");
- let bs = op.read_with(&path).range(offset..u32::MAX as u64).call()?;
+ let bs = op
+ .read_with(&path)
+ .range(offset..u32::MAX as u64)
+ .call()?
+ .to_bytes();
assert_eq!(
bs.len() as u64,
size as u64 - offset,
@@ -168,7 +176,7 @@ pub fn test_blocking_read_only_stat_not_exist(op: BlockingOperator) -> Result<()
/// Read full content should match.
pub fn test_blocking_read_only_read_full(op: BlockingOperator) -> Result<()> {
- let bs = op.read("normal_file.txt")?;
+ let bs = op.read("normal_file.txt")?.to_bytes();
assert_eq!(bs.len(), 30482, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -181,7 +189,11 @@ pub fn test_blocking_read_only_read_full(op: BlockingOperator) -> Result<()> {
/// Read full content should match.
pub fn test_blocking_read_only_read_with_range(op: BlockingOperator) -> Result<()> {
- let bs = op.read_with("normal_file.txt").range(1024..2048).call()?;
+ let bs = op
+ .read_with("normal_file.txt")
+ .range(1024..2048)
+ .call()?
+ .to_bytes();
assert_eq!(bs.len(), 1024, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
diff --git a/core/tests/behavior/blocking_rename.rs b/core/tests/behavior/blocking_rename.rs
index 90b737aeb5..4de1b9a724 100644
--- a/core/tests/behavior/blocking_rename.rs
+++ b/core/tests/behavior/blocking_rename.rs
@@ -52,7 +52,7 @@ pub fn test_blocking_rename_file(op: BlockingOperator) -> Result<()> {
let err = op.stat(&source_path).expect_err("stat must fail");
assert_eq!(err.kind(), ErrorKind::NotFound);
- let target_content = op.read(&target_path).expect("read must succeed");
+ let target_content = op.read(&target_path).expect("read must succeed").to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
@@ -153,7 +153,7 @@ pub fn test_blocking_rename_nested(op: BlockingOperator) -> Result<()> {
let err = op.stat(&source_path).expect_err("stat must fail");
assert_eq!(err.kind(), ErrorKind::NotFound);
- let target_content = op.read(&target_path).expect("read must succeed");
+ let target_content = op.read(&target_path).expect("read must succeed").to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
@@ -182,7 +182,7 @@ pub fn test_blocking_rename_overwrite(op: BlockingOperator) -> Result<()> {
let err = op.stat(&source_path).expect_err("stat must fail");
assert_eq!(err.kind(), ErrorKind::NotFound);
- let target_content = op.read(&target_path).expect("read must succeed");
+ let target_content = op.read(&target_path).expect("read must succeed").to_bytes();
assert_eq!(
format!("{:x}", Sha256::digest(target_content)),
format!("{:x}", Sha256::digest(&source_content)),
diff --git a/core/tests/behavior/blocking_write.rs b/core/tests/behavior/blocking_write.rs
index 9b3a9a2321..86ca0025f2 100644
--- a/core/tests/behavior/blocking_write.rs
+++ b/core/tests/behavior/blocking_write.rs
@@ -116,7 +116,7 @@ pub fn test_blocking_write_with_append(op: BlockingOperator) -> Result<()> {
.call()
.expect("append to an existing file must success");
- let bs = op.read(&path).expect("read file must success");
+ let bs = op.read(&path).expect("read file must success").to_bytes();
assert_eq!(bs.len(), size_one + size_two);
assert_eq!(bs[..size_one], content_one);
@@ -143,7 +143,7 @@ pub fn test_blocking_writer_with_append(op: BlockingOperator) -> Result<()> {
let meta = op.stat(&path).expect("stat must succeed");
assert_eq!(meta.content_length(), size as u64);
- let bs = op.read(&path)?;
+ let bs = op.read(&path)?.to_bytes();
assert_eq!(bs.len(), size, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs[..size])),
diff --git a/integrations/dav-server/src/file.rs b/integrations/dav-server/src/file.rs
index c8983d1fc2..2e19a6ec7e 100644
--- a/integrations/dav-server/src/file.rs
+++ b/integrations/dav-server/src/file.rs
@@ -43,12 +43,13 @@ impl DavFile for WebdavFile {
fn read_bytes(&mut self, count: usize) -> FsFuture<Bytes> {
async move {
let file_path = self.path.as_url_string();
- self.op
+ let buf = self
+ .op
.read_with(&file_path)
.range(0..count as u64)
.await
- .map(Bytes::from)
- .map_err(convert_error)
+ .map_err(convert_error)?;
+ Ok(buf.to_bytes())
}
.boxed()
}
diff --git a/integrations/object_store/src/lib.rs b/integrations/object_store/src/lib.rs
index dd523cc919..981482f19e 100644
--- a/integrations/object_store/src/lib.rs
+++ b/integrations/object_store/src/lib.rs
@@ -156,7 +156,7 @@ impl ObjectStore for OpendalStore {
.await
.map_err(|err| format_object_store_error(err, location.as_ref()))?;
- Ok(Bytes::from(bs))
+ Ok(bs.to_bytes())
}
async fn head(&self, location: &Path) -> Result<ObjectMeta> {