You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/06/02 17:42:16 UTC
(opendal) branch main updated: feat(core): Streaming reading while chunk is not set (#4658)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 20d9c36e1f feat(core): Streaming reading while chunk is not set (#4658)
20d9c36e1f is described below
commit 20d9c36e1f9d7a535991f217dbf6cf8f737050ce
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Jun 3 01:42:09 2024 +0800
feat(core): Streaming reading while chunk is not set (#4658)
---
bin/ofs/Cargo.lock | 4 +-
bindings/c/src/types.rs | 6 +-
core/Cargo.toml | 1 +
core/src/layers/blocking.rs | 4 +-
core/src/layers/chaos.rs | 8 +-
core/src/layers/complete.rs | 34 +--
core/src/layers/concurrent_limit.rs | 8 +-
core/src/layers/dtrace.rs | 8 +-
core/src/layers/error_context.rs | 12 +-
core/src/layers/logging.rs | 8 +-
core/src/layers/madsim.rs | 7 +-
core/src/layers/metrics.rs | 8 +-
core/src/layers/minitrace.rs | 8 +-
core/src/layers/oteltrace.rs | 8 +-
core/src/layers/prometheus.rs | 8 +-
core/src/layers/prometheus_client.rs | 8 +-
core/src/layers/retry.rs | 110 +++++-----
core/src/layers/throttle.rs | 8 +-
core/src/layers/timeout.rs | 6 +-
core/src/layers/tracing.rs | 8 +-
core/src/layers/type_eraser.rs | 5 +-
core/src/raw/adapters/kv/backend.rs | 8 +-
core/src/raw/adapters/typed_kv/backend.rs | 8 +-
core/src/raw/enum_utils.rs | 48 ++---
core/src/raw/futures_util.rs | 12 ++
core/src/raw/http_util/body.rs | 118 ++++++++++
core/src/raw/http_util/bytes_range.rs | 11 +
core/src/raw/http_util/client.rs | 65 +++---
core/src/raw/http_util/mod.rs | 3 +
core/src/raw/oio/read/api.rs | 143 +++++-------
core/src/raw/ops.rs | 28 ++-
core/src/services/aliyun_drive/backend.rs | 34 ++-
core/src/services/aliyun_drive/mod.rs | 1 -
core/src/services/aliyun_drive/reader.rs | 71 ------
core/src/services/alluxio/backend.rs | 14 +-
core/src/services/alluxio/core.rs | 4 +-
core/src/services/alluxio/mod.rs | 1 -
core/src/services/alluxio/reader.rs | 53 -----
core/src/services/azblob/backend.rs | 19 +-
core/src/services/azblob/core.rs | 4 +-
core/src/services/azblob/mod.rs | 1 -
core/src/services/azblob/reader.rs | 60 ------
core/src/services/azdls/backend.rs | 19 +-
core/src/services/azdls/core.rs | 4 +-
core/src/services/azdls/mod.rs | 1 -
core/src/services/azdls/reader.rs | 57 -----
core/src/services/azfile/backend.rs | 19 +-
core/src/services/azfile/core.rs | 4 +-
core/src/services/azfile/mod.rs | 1 -
core/src/services/azfile/reader.rs | 57 -----
core/src/services/b2/backend.rs | 24 ++-
core/src/services/b2/core.rs | 4 +-
core/src/services/b2/mod.rs | 1 -
core/src/services/b2/reader.rs | 60 ------
core/src/services/chainsafe/backend.rs | 19 +-
core/src/services/chainsafe/core.rs | 8 +-
core/src/services/chainsafe/mod.rs | 1 -
core/src/services/chainsafe/reader.rs | 57 -----
core/src/services/cos/backend.rs | 22 +-
core/src/services/cos/core.rs | 4 +-
core/src/services/cos/mod.rs | 1 -
core/src/services/cos/reader.rs | 60 ------
core/src/services/dbfs/backend.rs | 11 +-
core/src/services/dbfs/mod.rs | 1 -
core/src/services/dbfs/reader.rs | 90 --------
core/src/services/dropbox/backend.rs | 21 +-
core/src/services/dropbox/core.rs | 4 +-
core/src/services/dropbox/mod.rs | 1 -
core/src/services/dropbox/reader.rs | 57 -----
core/src/services/fs/backend.rs | 40 +++-
core/src/services/fs/reader.rs | 122 +++++------
core/src/services/ftp/backend.rs | 5 +-
core/src/services/ftp/reader.rs | 64 +++---
core/src/services/ftp/writer.rs | 6 +-
core/src/services/gcs/backend.rs | 22 +-
core/src/services/gcs/core.rs | 4 +-
core/src/services/gcs/mod.rs | 1 -
core/src/services/gcs/reader.rs | 60 ------
core/src/services/gdrive/backend.rs | 19 +-
core/src/services/gdrive/core.rs | 4 +-
core/src/services/gdrive/mod.rs | 1 -
core/src/services/gdrive/reader.rs | 57 -----
core/src/services/ghac/backend.rs | 21 +-
core/src/services/ghac/mod.rs | 1 -
core/src/services/ghac/reader.rs | 56 -----
core/src/services/github/backend.rs | 22 +-
core/src/services/github/core.rs | 4 +-
core/src/services/github/mod.rs | 1 -
core/src/services/github/reader.rs | 57 -----
core/src/services/hdfs/backend.rs | 60 +++---
core/src/services/hdfs/reader.rs | 113 ++++++----
core/src/services/hdfs_native/reader.rs | 2 +-
core/src/services/http/backend.rs | 22 +-
core/src/services/http/mod.rs | 1 -
core/src/services/http/reader.rs | 55 -----
core/src/services/huggingface/backend.rs | 22 +-
core/src/services/huggingface/core.rs | 4 +-
core/src/services/huggingface/mod.rs | 1 -
core/src/services/huggingface/reader.rs | 57 -----
core/src/services/icloud/backend.rs | 20 +-
core/src/services/icloud/core.rs | 6 +-
core/src/services/icloud/mod.rs | 1 -
core/src/services/icloud/reader.rs | 57 -----
core/src/services/ipfs/backend.rs | 22 +-
core/src/services/ipfs/mod.rs | 1 -
core/src/services/ipfs/reader.rs | 55 -----
core/src/services/ipmfs/backend.rs | 25 ++-
core/src/services/ipmfs/mod.rs | 1 -
core/src/services/ipmfs/reader.rs | 55 -----
core/src/services/koofr/backend.rs | 21 +-
core/src/services/koofr/core.rs | 4 +-
core/src/services/koofr/mod.rs | 1 -
core/src/services/koofr/reader.rs | 57 -----
core/src/services/obs/backend.rs | 22 +-
core/src/services/obs/core.rs | 4 +-
core/src/services/obs/mod.rs | 1 -
core/src/services/obs/reader.rs | 60 ------
core/src/services/onedrive/backend.rs | 25 ++-
core/src/services/onedrive/mod.rs | 1 -
core/src/services/onedrive/reader.rs | 55 -----
core/src/services/oss/backend.rs | 22 +-
core/src/services/oss/core.rs | 4 +-
core/src/services/oss/mod.rs | 1 -
core/src/services/oss/reader.rs | 60 ------
core/src/services/pcloud/backend.rs | 22 +-
core/src/services/pcloud/core.rs | 4 +-
core/src/services/pcloud/mod.rs | 1 -
core/src/services/pcloud/reader.rs | 57 -----
core/src/services/s3/backend.rs | 22 +-
core/src/services/s3/core.rs | 4 +-
core/src/services/s3/mod.rs | 1 -
core/src/services/s3/reader.rs | 56 -----
core/src/services/seafile/backend.rs | 24 ++-
core/src/services/seafile/core.rs | 4 +-
core/src/services/seafile/mod.rs | 1 -
core/src/services/seafile/reader.rs | 57 -----
core/src/services/sftp/backend.rs | 24 ++-
core/src/services/sftp/reader.rs | 102 +++++----
core/src/services/supabase/backend.rs | 21 +-
core/src/services/supabase/core.rs | 4 +-
core/src/services/supabase/mod.rs | 1 -
core/src/services/supabase/reader.rs | 57 -----
core/src/services/swift/backend.rs | 20 +-
core/src/services/swift/core.rs | 4 +-
core/src/services/swift/mod.rs | 1 -
core/src/services/swift/reader.rs | 57 -----
core/src/services/upyun/backend.rs | 22 +-
core/src/services/upyun/core.rs | 4 +-
core/src/services/upyun/mod.rs | 1 -
core/src/services/upyun/reader.rs | 57 -----
core/src/services/vercel_artifacts/backend.rs | 23 +-
core/src/services/vercel_artifacts/mod.rs | 1 -
core/src/services/vercel_artifacts/reader.rs | 58 -----
core/src/services/vercel_blob/backend.rs | 22 +-
core/src/services/vercel_blob/core.rs | 4 +-
core/src/services/vercel_blob/mod.rs | 1 -
core/src/services/vercel_blob/reader.rs | 57 -----
core/src/services/webdav/backend.rs | 22 +-
core/src/services/webdav/core.rs | 4 +-
core/src/services/webdav/mod.rs | 1 -
core/src/services/webdav/reader.rs | 57 -----
core/src/services/webhdfs/backend.rs | 25 ++-
core/src/services/webhdfs/mod.rs | 1 -
core/src/services/webhdfs/reader.rs | 55 -----
core/src/services/yandex_disk/backend.rs | 28 ++-
core/src/services/yandex_disk/mod.rs | 1 -
core/src/services/yandex_disk/reader.rs | 66 ------
core/src/types/blocking_read/blocking_reader.rs | 78 +++----
core/src/types/blocking_read/buffer_iterator.rs | 94 ++++++++
core/src/types/blocking_read/mod.rs | 3 +
core/src/types/blocking_read/std_bytes_iterator.rs | 52 ++---
core/src/types/blocking_read/std_reader.rs | 103 ++++-----
core/src/types/buffer.rs | 2 +
core/src/types/context.rs | 202 +++++++++++++++++
core/src/types/mod.rs | 4 +
core/src/types/operator/blocking_operator.rs | 16 +-
core/src/types/operator/operator.rs | 15 +-
core/src/types/operator/operator_functions.rs | 3 +-
core/src/types/operator/operator_futures.rs | 2 +-
core/src/types/read/buffer_stream.rs | 239 ++++++++++++++-------
core/src/types/read/futures_async_reader.rs | 122 +++++++----
core/src/types/read/futures_bytes_stream.rs | 75 +++++--
core/src/types/read/mod.rs | 2 +
core/src/types/read/reader.rs | 93 ++++----
184 files changed, 2042 insertions(+), 3193 deletions(-)
diff --git a/bin/ofs/Cargo.lock b/bin/ofs/Cargo.lock
index 2ba0e09aeb..3cbe072632 100644
--- a/bin/ofs/Cargo.lock
+++ b/bin/ofs/Cargo.lock
@@ -331,9 +331,9 @@ dependencies = [
[[package]]
name = "crc32c"
-version = "0.6.5"
+version = "0.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2"
+checksum = "0227b9f93e535d49bc7ce914c066243424ce85ed90864cebd0874b184e9b6947"
dependencies = [
"rustc_version",
]
diff --git a/bindings/c/src/types.rs b/bindings/c/src/types.rs
index c0f19eb841..049f92f8ce 100644
--- a/bindings/c/src/types.rs
+++ b/bindings/c/src/types.rs
@@ -58,10 +58,10 @@ impl opendal_bytes {
}
#[allow(clippy::from_over_into)]
-impl Into<bytes::Bytes> for opendal_bytes {
- fn into(self) -> bytes::Bytes {
+impl Into<Buffer> for opendal_bytes {
+ fn into(self) -> Buffer {
let slice = unsafe { std::slice::from_raw_parts(self.data, self.len) };
- bytes::Bytes::copy_from_slice(slice)
+ Buffer::from(bytes::Bytes::copy_from_slice(slice))
}
}
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 32fe17522f..85776ac8bc 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -56,6 +56,7 @@ tests = [
"services-fs",
"services-http",
"services-memory",
+ "internal-tokio-rt",
"services-s3",
]
diff --git a/core/src/layers/blocking.rs b/core/src/layers/blocking.rs
index fa9f97cd49..70830981c8 100644
--- a/core/src/layers/blocking.rs
+++ b/core/src/layers/blocking.rs
@@ -282,8 +282,8 @@ impl<I> BlockingWrapper<I> {
}
impl<I: oio::Read + 'static> oio::BlockingRead for BlockingWrapper<I> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.handle.block_on(self.inner.read_at(offset, size))
+ fn read(&mut self) -> Result<Buffer> {
+ self.handle.block_on(self.inner.read())
}
}
diff --git a/core/src/layers/chaos.rs b/core/src/layers/chaos.rs
index 98855dc157..083e28c825 100644
--- a/core/src/layers/chaos.rs
+++ b/core/src/layers/chaos.rs
@@ -171,9 +171,9 @@ impl<R> ChaosReader<R> {
}
impl<R: oio::Read> oio::Read for ChaosReader<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ async fn read(&mut self) -> Result<Buffer> {
if self.i_feel_lucky() {
- self.inner.read_at(offset, size).await
+ self.inner.read().await
} else {
Err(Self::unexpected_eof())
}
@@ -181,9 +181,9 @@ impl<R: oio::Read> oio::Read for ChaosReader<R> {
}
impl<R: oio::BlockingRead> oio::BlockingRead for ChaosReader<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ fn read(&mut self) -> Result<Buffer> {
if self.i_feel_lucky() {
- self.inner.read_at(offset, size)
+ self.inner.read()
} else {
Err(Self::unexpected_eof())
}
diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs
index 0d38a4a450..8e265b2171 100644
--- a/core/src/layers/complete.rs
+++ b/core/src/layers/complete.rs
@@ -587,39 +587,15 @@ pub type CompleteLister<A, P> =
pub struct CompleteReader<R>(R);
impl<R: oio::Read> oio::Read for CompleteReader<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- if size == 0 {
- return Ok(Buffer::new());
- }
-
- let buf = self.0.read_at(offset, size).await?;
- if buf.len() != size {
- return Err(Error::new(
- ErrorKind::RangeNotSatisfied,
- "service didn't return the expected size",
- )
- .with_context("expect", size.to_string())
- .with_context("actual", buf.len().to_string()));
- }
+ async fn read(&mut self) -> Result<Buffer> {
+ let buf = self.0.read().await?;
Ok(buf)
}
}
impl<R: oio::BlockingRead> oio::BlockingRead for CompleteReader<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- if size == 0 {
- return Ok(Buffer::new());
- }
-
- let buf = self.0.read_at(offset, size)?;
- if buf.len() != size {
- return Err(Error::new(
- ErrorKind::RangeNotSatisfied,
- "service didn't return the expected size",
- )
- .with_context("expect", size.to_string())
- .with_context("actual", buf.len().to_string()));
- }
+ fn read(&mut self) -> Result<Buffer> {
+ let buf = self.0.read()?;
Ok(buf)
}
}
@@ -742,7 +718,7 @@ mod tests {
}
async fn read(&self, _: &str, _: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((RpRead::new(), Arc::new(bytes::Bytes::new())))
+ Ok((RpRead::new(), Box::new(bytes::Bytes::new())))
}
async fn write(&self, _: &str, _: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/layers/concurrent_limit.rs b/core/src/layers/concurrent_limit.rs
index b3f334ff99..a1a61ad01d 100644
--- a/core/src/layers/concurrent_limit.rs
+++ b/core/src/layers/concurrent_limit.rs
@@ -250,14 +250,14 @@ impl<R> ConcurrentLimitWrapper<R> {
}
impl<R: oio::Read> oio::Read for ConcurrentLimitWrapper<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size).await
+ async fn read(&mut self) -> Result<Buffer> {
+ self.inner.read().await
}
}
impl<R: oio::BlockingRead> oio::BlockingRead for ConcurrentLimitWrapper<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size)
+ fn read(&mut self) -> Result<Buffer> {
+ self.inner.read()
}
}
diff --git a/core/src/layers/dtrace.rs b/core/src/layers/dtrace.rs
index 8fe54039ce..c71277ed9a 100644
--- a/core/src/layers/dtrace.rs
+++ b/core/src/layers/dtrace.rs
@@ -340,10 +340,10 @@ impl<R> DtraceLayerWrapper<R> {
}
impl<R: oio::Read> oio::Read for DtraceLayerWrapper<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ async fn read(&mut self) -> Result<Buffer> {
let c_path = CString::new(self.path.clone()).unwrap();
probe_lazy!(opendal, reader_read_start, c_path.as_ptr());
- match self.inner.read_at(offset, size).await {
+ match self.inner.read().await {
Ok(bs) => {
probe_lazy!(opendal, reader_read_ok, c_path.as_ptr(), bs.remaining());
Ok(bs)
@@ -357,11 +357,11 @@ impl<R: oio::Read> oio::Read for DtraceLayerWrapper<R> {
}
impl<R: oio::BlockingRead> oio::BlockingRead for DtraceLayerWrapper<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ fn read(&mut self) -> Result<Buffer> {
let c_path = CString::new(self.path.clone()).unwrap();
probe_lazy!(opendal, blocking_reader_read_start, c_path.as_ptr());
self.inner
- .read_at(offset, size)
+ .read()
.map(|bs| {
probe_lazy!(
opendal,
diff --git a/core/src/layers/error_context.rs b/core/src/layers/error_context.rs
index 27b577649e..eeb429a3e7 100644
--- a/core/src/layers/error_context.rs
+++ b/core/src/layers/error_context.rs
@@ -339,25 +339,21 @@ pub struct ErrorContextWrapper<T> {
}
impl<T: oio::Read> oio::Read for ErrorContextWrapper<T> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size).await.map_err(|err| {
+ async fn read(&mut self) -> Result<Buffer> {
+ self.inner.read().await.map_err(|err| {
err.with_operation(ReadOperation::Read)
.with_context("service", self.scheme)
.with_context("path", &self.path)
- .with_context("offset", offset.to_string())
- .with_context("size", size.to_string())
})
}
}
impl<T: oio::BlockingRead> oio::BlockingRead for ErrorContextWrapper<T> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size).map_err(|err| {
+ fn read(&mut self) -> Result<Buffer> {
+ self.inner.read().map_err(|err| {
err.with_operation(ReadOperation::BlockingRead)
.with_context("service", self.scheme)
.with_context("path", &self.path)
- .with_context("offset", offset.to_string())
- .with_context("size", size.to_string())
})
}
}
diff --git a/core/src/layers/logging.rs b/core/src/layers/logging.rs
index 844bd12ac3..1e0d80d264 100644
--- a/core/src/layers/logging.rs
+++ b/core/src/layers/logging.rs
@@ -978,8 +978,8 @@ impl<R> Drop for LoggingReader<R> {
}
impl<R: oio::Read> oio::Read for LoggingReader<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- match self.inner.read_at(offset, size).await {
+ async fn read(&mut self) -> Result<Buffer> {
+ match self.inner.read().await {
Ok(bs) => {
self.read
.fetch_add(bs.remaining() as u64, Ordering::Relaxed);
@@ -1014,8 +1014,8 @@ impl<R: oio::Read> oio::Read for LoggingReader<R> {
}
impl<R: oio::BlockingRead> oio::BlockingRead for LoggingReader<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- match self.inner.read_at(offset, size) {
+ fn read(&mut self) -> Result<Buffer> {
+ match self.inner.read() {
Ok(bs) => {
self.read
.fetch_add(bs.remaining() as u64, Ordering::Relaxed);
diff --git a/core/src/layers/madsim.rs b/core/src/layers/madsim.rs
index 7607d45f31..5244820eff 100644
--- a/core/src/layers/madsim.rs
+++ b/core/src/layers/madsim.rs
@@ -261,10 +261,9 @@ pub struct MadsimReader {
}
impl oio::Read for MadsimReader {
- async fn read_at(&self, offset: u64, size: usize) -> crate::Result<Buffer> {
- if let Some(ref data) = self.data {
- let size = min(size, data.len());
- Ok(data.clone().split_to(size).into())
+ async fn read(&mut self) -> crate::Result<Buffer> {
+ if let Some(data) = self.data.take() {
+ Ok(Buffer::from(data))
} else {
Ok(Buffer::new())
}
diff --git a/core/src/layers/metrics.rs b/core/src/layers/metrics.rs
index 9c9a343c5d..219aebdf61 100644
--- a/core/src/layers/metrics.rs
+++ b/core/src/layers/metrics.rs
@@ -747,10 +747,10 @@ impl<R> MetricWrapper<R> {
}
impl<R: oio::Read> oio::Read for MetricWrapper<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ async fn read(&mut self) -> Result<Buffer> {
let start = Instant::now();
- match self.inner.read_at(offset, size).await {
+ match self.inner.read().await {
Ok(bs) => {
self.bytes_counter.increment(bs.remaining() as u64);
self.requests_duration_seconds
@@ -766,11 +766,11 @@ impl<R: oio::Read> oio::Read for MetricWrapper<R> {
}
impl<R: oio::BlockingRead> oio::BlockingRead for MetricWrapper<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ fn read(&mut self) -> Result<Buffer> {
let start = Instant::now();
self.inner
- .read_at(offset, size)
+ .read()
.map(|bs| {
self.bytes_counter.increment(bs.remaining() as u64);
self.requests_duration_seconds
diff --git a/core/src/layers/minitrace.rs b/core/src/layers/minitrace.rs
index 8ec44d086f..983d18b6b8 100644
--- a/core/src/layers/minitrace.rs
+++ b/core/src/layers/minitrace.rs
@@ -292,16 +292,16 @@ impl<R> MinitraceWrapper<R> {
impl<R: oio::Read> oio::Read for MinitraceWrapper<R> {
#[trace(enter_on_poll = true)]
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size).await
+ async fn read(&mut self) -> Result<Buffer> {
+ self.inner.read().await
}
}
impl<R: oio::BlockingRead> oio::BlockingRead for MinitraceWrapper<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ fn read(&mut self) -> Result<Buffer> {
let _g = self.span.set_local_parent();
let _span = LocalSpan::enter_with_local_parent(ReadOperation::BlockingRead.into_static());
- self.inner.read_at(offset, size)
+ self.inner.read()
}
}
diff --git a/core/src/layers/oteltrace.rs b/core/src/layers/oteltrace.rs
index 8085d6d983..6fb5d582ba 100644
--- a/core/src/layers/oteltrace.rs
+++ b/core/src/layers/oteltrace.rs
@@ -272,14 +272,14 @@ impl<R> OtelTraceWrapper<R> {
}
impl<R: oio::Read> oio::Read for OtelTraceWrapper<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size).await
+ async fn read(&mut self) -> Result<Buffer> {
+ self.inner.read().await
}
}
impl<R: oio::BlockingRead> oio::BlockingRead for OtelTraceWrapper<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size)
+ fn read(&mut self) -> Result<Buffer> {
+ self.inner.read()
}
}
diff --git a/core/src/layers/prometheus.rs b/core/src/layers/prometheus.rs
index 741a864dd7..bd300619f3 100644
--- a/core/src/layers/prometheus.rs
+++ b/core/src/layers/prometheus.rs
@@ -680,13 +680,13 @@ impl<R> PrometheusMetricWrapper<R> {
}
impl<R: oio::Read> oio::Read for PrometheusMetricWrapper<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ async fn read(&mut self) -> Result<Buffer> {
let labels = self.stats.generate_metric_label(
self.scheme.into_static(),
Operation::Read.into_static(),
&self.path,
);
- match self.inner.read_at(offset, size).await {
+ match self.inner.read().await {
Ok(bytes) => {
self.stats
.bytes_total
@@ -703,14 +703,14 @@ impl<R: oio::Read> oio::Read for PrometheusMetricWrapper<R> {
}
impl<R: oio::BlockingRead> oio::BlockingRead for PrometheusMetricWrapper<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ fn read(&mut self) -> Result<Buffer> {
let labels = self.stats.generate_metric_label(
self.scheme.into_static(),
Operation::BlockingRead.into_static(),
&self.path,
);
self.inner
- .read_at(offset, size)
+ .read()
.map(|bs| {
self.stats
.bytes_total
diff --git a/core/src/layers/prometheus_client.rs b/core/src/layers/prometheus_client.rs
index ac436d8089..e72b765216 100644
--- a/core/src/layers/prometheus_client.rs
+++ b/core/src/layers/prometheus_client.rs
@@ -533,10 +533,10 @@ impl<R> PrometheusMetricWrapper<R> {
}
impl<R: oio::Read> oio::Read for PrometheusMetricWrapper<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ async fn read(&mut self) -> Result<Buffer> {
let start = Instant::now();
- match self.inner.read_at(offset, size).await {
+ match self.inner.read().await {
Ok(bs) => {
self.metrics
.observe_bytes_total(self.scheme, self.op, bs.remaining());
@@ -554,10 +554,10 @@ impl<R: oio::Read> oio::Read for PrometheusMetricWrapper<R> {
}
impl<R: oio::BlockingRead> oio::BlockingRead for PrometheusMetricWrapper<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ fn read(&mut self) -> Result<Buffer> {
let start = Instant::now();
self.inner
- .read_at(offset, size)
+ .read()
.map(|bs| {
self.metrics
.observe_bytes_total(self.scheme, self.op, bs.remaining());
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index 1bd4505bdb..55551fb16f 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -27,7 +27,6 @@ use futures::FutureExt;
use log::warn;
use crate::raw::oio::ListOperation;
-use crate::raw::oio::ReadOperation;
use crate::raw::oio::WriteOperation;
use crate::raw::*;
use crate::*;
@@ -656,49 +655,35 @@ impl<R, I> RetryWrapper<R, I> {
}
impl<R: oio::Read, I: RetryInterceptor> oio::Read for RetryWrapper<R, I> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- {
- || {
- self.inner
- .as_ref()
- .expect("inner must be valid")
- .read_at(offset, size)
- }
- }
- .retry(&self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| {
- self.notify.intercept(
- err,
- dur,
- &[
- ("operation", ReadOperation::Read.into_static()),
- ("path", &self.path),
- ],
- )
- })
- .await
- .map_err(|e| e.set_persistent())
+ /// FIXME: we should bring retry back.
+ async fn read(&mut self) -> Result<Buffer> {
+ self.inner
+ .as_mut()
+ .expect("inner must be valid")
+ .read()
+ .await
+ // { || self.inner.as_ref().expect("inner must be valid").read() }
+ // .retry(&self.builder)
+ // .when(|e| e.is_temporary())
+ // .notify(|err, dur| {
+ // self.notify.intercept(
+ // err,
+ // dur,
+ // &[
+ // ("operation", ReadOperation::Read.into_static()),
+ // ("path", &self.path),
+ // ],
+ // )
+ // })
+ // .await
+ // .map_err(|e| e.set_persistent())
}
}
impl<R: oio::BlockingRead, I: RetryInterceptor> oio::BlockingRead for RetryWrapper<R, I> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- { || self.inner.as_ref().unwrap().read_at(offset, size) }
- .retry(&self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| {
- self.notify.intercept(
- err,
- dur,
- &[
- ("operation", ReadOperation::BlockingRead.into_static()),
- ("path", &self.path),
- ],
- );
- })
- .call()
- .map_err(|e| e.set_persistent())
+ /// FIXME: we should bring retry back.
+ fn read(&mut self) -> Result<Buffer> {
+ self.inner.as_mut().expect("inner must be valid").read()
}
}
@@ -1034,7 +1019,7 @@ mod tests {
}
impl oio::Read for MockReader {
- async fn read_at(&self, _: u64, _: usize) -> Result<Buffer> {
+ async fn read(&mut self) -> Result<Buffer> {
let mut attempt = self.attempt.lock().unwrap();
*attempt += 1;
@@ -1095,27 +1080,28 @@ mod tests {
}
}
- #[tokio::test]
- async fn test_retry_read() {
- let _ = tracing_subscriber::fmt().with_test_writer().try_init();
-
- let builder = MockBuilder::default();
- let op = Operator::new(builder.clone())
- .unwrap()
- .layer(RetryLayer::new())
- .finish();
-
- let r = op.reader("retryable_error").await.unwrap();
- let mut content = Vec::new();
- let size = r
- .read_into(&mut content, ..)
- .await
- .expect("read must succeed");
- assert_eq!(size, 13);
- assert_eq!(content, "Hello, World!".as_bytes());
- // The error is retryable, we should request it 3 times.
- assert_eq!(*builder.attempt.lock().unwrap(), 3);
- }
+ // TODO: we need to bring this test back after retry on reader is supported.
+ // #[tokio::test]
+ // async fn test_retry_read() {
+ // let _ = tracing_subscriber::fmt().with_test_writer().try_init();
+ //
+ // let builder = MockBuilder::default();
+ // let op = Operator::new(builder.clone())
+ // .unwrap()
+ // .layer(RetryLayer::new())
+ // .finish();
+ //
+ // let r = op.reader("retryable_error").await.unwrap();
+ // let mut content = Vec::new();
+ // let size = r
+ // .read_into(&mut content, ..)
+ // .await
+ // .expect("read must succeed");
+ // assert_eq!(size, 13);
+ // assert_eq!(content, "Hello, World!".as_bytes());
+ // // The error is retryable, we should request it 3 times.
+ // assert_eq!(*builder.attempt.lock().unwrap(), 3);
+ // }
#[tokio::test]
async fn test_retry_list() {
diff --git a/core/src/layers/throttle.rs b/core/src/layers/throttle.rs
index fbeb3a6436..3b66e327f3 100644
--- a/core/src/layers/throttle.rs
+++ b/core/src/layers/throttle.rs
@@ -179,14 +179,14 @@ impl<R> ThrottleWrapper<R> {
}
impl<R: oio::Read> oio::Read for ThrottleWrapper<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size).await
+ async fn read(&mut self) -> Result<Buffer> {
+ self.inner.read().await
}
}
impl<R: oio::BlockingRead> oio::BlockingRead for ThrottleWrapper<R> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size)
+ fn read(&mut self) -> Result<Buffer> {
+ self.inner.read()
}
}
diff --git a/core/src/layers/timeout.rs b/core/src/layers/timeout.rs
index 9703f2edc6..ace1b644fb 100644
--- a/core/src/layers/timeout.rs
+++ b/core/src/layers/timeout.rs
@@ -287,8 +287,8 @@ impl<R> TimeoutWrapper<R> {
}
impl<R: oio::Read> oio::Read for TimeoutWrapper<R> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let fut = self.inner.read_at(offset, size);
+ async fn read(&mut self) -> Result<Buffer> {
+ let fut = self.inner.read();
Self::io_timeout(self.timeout, ReadOperation::Read.into_static(), fut).await
}
}
@@ -376,7 +376,7 @@ mod tests {
struct MockReader;
impl oio::Read for MockReader {
- fn read_at(&self, _: u64, _: usize) -> impl Future<Output = Result<Buffer>> {
+ fn read(&mut self) -> impl Future<Output = Result<Buffer>> {
pending()
}
}
diff --git a/core/src/layers/tracing.rs b/core/src/layers/tracing.rs
index be01d97e62..4ba829a6ec 100644
--- a/core/src/layers/tracing.rs
+++ b/core/src/layers/tracing.rs
@@ -266,8 +266,8 @@ impl<R: oio::Read> oio::Read for TracingWrapper<R> {
parent = &self.span,
level = "trace",
skip_all)]
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size).await
+ async fn read(&mut self) -> Result<Buffer> {
+ self.inner.read().await
}
}
@@ -276,8 +276,8 @@ impl<R: oio::BlockingRead> oio::BlockingRead for TracingWrapper<R> {
parent = &self.span,
level = "trace",
skip_all)]
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.inner.read_at(offset, size)
+ fn read(&mut self) -> Result<Buffer> {
+ self.inner.read()
}
}
diff --git a/core/src/layers/type_eraser.rs b/core/src/layers/type_eraser.rs
index 72eee9d0e5..09e0a7fb19 100644
--- a/core/src/layers/type_eraser.rs
+++ b/core/src/layers/type_eraser.rs
@@ -17,7 +17,6 @@
use std::fmt::Debug;
use std::fmt::Formatter;
-use std::sync::Arc;
use crate::raw::*;
use crate::*;
@@ -68,7 +67,7 @@ impl<A: Access> LayeredAccess for TypeEraseAccessor<A> {
self.inner
.read(path, args)
.await
- .map(|(rp, r)| (rp, Arc::new(r) as oio::Reader))
+ .map(|(rp, r)| (rp, Box::new(r) as oio::Reader))
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
@@ -88,7 +87,7 @@ impl<A: Access> LayeredAccess for TypeEraseAccessor<A> {
fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
self.inner
.blocking_read(path, args)
- .map(|(rp, r)| (rp, Arc::new(r) as oio::BlockingReader))
+ .map(|(rp, r)| (rp, Box::new(r) as oio::BlockingReader))
}
fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
diff --git a/core/src/raw/adapters/kv/backend.rs b/core/src/raw/adapters/kv/backend.rs
index 0cf3278ce4..bb08e4cb19 100644
--- a/core/src/raw/adapters/kv/backend.rs
+++ b/core/src/raw/adapters/kv/backend.rs
@@ -87,22 +87,22 @@ impl<S: Adapter> Access for Backend<S> {
am
}
- async fn read(&self, path: &str, _: OpRead) -> Result<(RpRead, Self::Reader)> {
+ async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
let p = build_abs_path(&self.root, path);
let bs = match self.kv.get(&p).await? {
Some(bs) => bs,
None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't have this path")),
};
- Ok((RpRead::new(), bs))
+ Ok((RpRead::new(), bs.slice(args.range().to_range_as_usize())))
}
- fn blocking_read(&self, path: &str, _: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
+ fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
let p = build_abs_path(&self.root, path);
let bs = match self.kv.blocking_get(&p)? {
Some(bs) => bs,
None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't have this path")),
};
- Ok((RpRead::new(), bs))
+ Ok((RpRead::new(), bs.slice(args.range().to_range_as_usize())))
}
async fn write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/raw/adapters/typed_kv/backend.rs b/core/src/raw/adapters/typed_kv/backend.rs
index f19b6dee3b..ecce2eb879 100644
--- a/core/src/raw/adapters/typed_kv/backend.rs
+++ b/core/src/raw/adapters/typed_kv/backend.rs
@@ -94,7 +94,7 @@ impl<S: Adapter> Access for Backend<S> {
am
}
- async fn read(&self, path: &str, _: OpRead) -> Result<(RpRead, Self::Reader)> {
+ async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
let p = build_abs_path(&self.root, path);
let bs = match self.kv.get(&p).await? {
@@ -103,10 +103,10 @@ impl<S: Adapter> Access for Backend<S> {
None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't have this path")),
};
- Ok((RpRead::new(), bs))
+ Ok((RpRead::new(), bs.slice(args.range().to_range_as_usize())))
}
- fn blocking_read(&self, path: &str, _: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
+ fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
let p = build_abs_path(&self.root, path);
let bs = match self.kv.blocking_get(&p)? {
@@ -115,7 +115,7 @@ impl<S: Adapter> Access for Backend<S> {
None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't have this path")),
};
- Ok((RpRead::new(), bs))
+ Ok((RpRead::new(), bs.slice(args.range().to_range_as_usize())))
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/raw/enum_utils.rs b/core/src/raw/enum_utils.rs
index 6c614a7524..c22411904d 100644
--- a/core/src/raw/enum_utils.rs
+++ b/core/src/raw/enum_utils.rs
@@ -52,19 +52,19 @@ pub enum TwoWays<ONE, TWO> {
}
impl<ONE: oio::Read, TWO: oio::Read> oio::Read for TwoWays<ONE, TWO> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ async fn read(&mut self) -> Result<Buffer> {
match self {
- TwoWays::One(v) => v.read_at(offset, size).await,
- TwoWays::Two(v) => v.read_at(offset, size).await,
+ TwoWays::One(v) => v.read().await,
+ TwoWays::Two(v) => v.read().await,
}
}
}
impl<ONE: oio::BlockingRead, TWO: oio::BlockingRead> oio::BlockingRead for TwoWays<ONE, TWO> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ fn read(&mut self) -> Result<Buffer> {
match self {
- Self::One(v) => v.read_at(offset, size),
- Self::Two(v) => v.read_at(offset, size),
+ Self::One(v) => v.read(),
+ Self::Two(v) => v.read(),
}
}
}
@@ -105,11 +105,11 @@ pub enum ThreeWays<ONE, TWO, THREE> {
}
impl<ONE: oio::Read, TWO: oio::Read, THREE: oio::Read> oio::Read for ThreeWays<ONE, TWO, THREE> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ async fn read(&mut self) -> Result<Buffer> {
match self {
- ThreeWays::One(v) => v.read_at(offset, size).await,
- ThreeWays::Two(v) => v.read_at(offset, size).await,
- ThreeWays::Three(v) => v.read_at(offset, size).await,
+ ThreeWays::One(v) => v.read().await,
+ ThreeWays::Two(v) => v.read().await,
+ ThreeWays::Three(v) => v.read().await,
}
}
}
@@ -117,11 +117,11 @@ impl<ONE: oio::Read, TWO: oio::Read, THREE: oio::Read> oio::Read for ThreeWays<O
impl<ONE: oio::BlockingRead, TWO: oio::BlockingRead, THREE: oio::BlockingRead> oio::BlockingRead
for ThreeWays<ONE, TWO, THREE>
{
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ fn read(&mut self) -> Result<Buffer> {
match self {
- Self::One(v) => v.read_at(offset, size),
- Self::Two(v) => v.read_at(offset, size),
- Self::Three(v) => v.read_at(offset, size),
+ Self::One(v) => v.read(),
+ Self::Two(v) => v.read(),
+ Self::Three(v) => v.read(),
}
}
}
@@ -175,12 +175,12 @@ where
THREE: oio::Read,
FOUR: oio::Read,
{
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ async fn read(&mut self) -> Result<Buffer> {
match self {
- FourWays::One(v) => v.read_at(offset, size).await,
- FourWays::Two(v) => v.read_at(offset, size).await,
- FourWays::Three(v) => v.read_at(offset, size).await,
- FourWays::Four(v) => v.read_at(offset, size).await,
+ FourWays::One(v) => v.read().await,
+ FourWays::Two(v) => v.read().await,
+ FourWays::Three(v) => v.read().await,
+ FourWays::Four(v) => v.read().await,
}
}
}
@@ -192,12 +192,12 @@ where
THREE: oio::BlockingRead,
FOUR: oio::BlockingRead,
{
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
+ fn read(&mut self) -> Result<Buffer> {
match self {
- Self::One(v) => v.read_at(offset, size),
- Self::Two(v) => v.read_at(offset, size),
- Self::Three(v) => v.read_at(offset, size),
- Self::Four(v) => v.read_at(offset, size),
+ Self::One(v) => v.read(),
+ Self::Two(v) => v.read(),
+ Self::Three(v) => v.read(),
+ Self::Four(v) => v.read(),
}
}
}
diff --git a/core/src/raw/futures_util.rs b/core/src/raw/futures_util.rs
index 0010c2946c..1ebca8ab76 100644
--- a/core/src/raw/futures_util.rs
+++ b/core/src/raw/futures_util.rs
@@ -122,6 +122,18 @@ impl<I: Send + 'static, O: Send + 'static> ConcurrentTasks<I, O> {
self.results.clear();
}
+ /// Check if there are remaining space to push new tasks.
+ #[inline]
+ pub fn has_remaining(&self) -> bool {
+ self.tasks.len() < self.tasks.capacity()
+ }
+
+ /// Chunk if there are remaining results to fetch.
+ #[inline]
+ pub fn has_result(&self) -> bool {
+ !self.results.is_empty()
+ }
+
/// Execute the task with given input.
///
/// - Execute the task in the current thread if is not concurrent.
diff --git a/core/src/raw/http_util/body.rs b/core/src/raw/http_util/body.rs
new file mode 100644
index 0000000000..2db622f6e2
--- /dev/null
+++ b/core/src/raw/http_util/body.rs
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::cmp::Ordering;
+
+use futures::{Stream, StreamExt};
+use oio::Read;
+
+use crate::raw::*;
+use crate::*;
+
+/// HttpBody is the streaming body that opendal's HttpClient returned.
+///
+/// It implements the `oio::Read` trait, service implementors can return it as
+/// `Access::Read`.
+pub struct HttpBody {
+ #[cfg(not(target_arch = "wasm32"))]
+ stream: Box<dyn Stream<Item = Result<Buffer>> + Send + Sync + Unpin + 'static>,
+ #[cfg(target_arch = "wasm32")]
+ stream: Box<dyn Stream<Item = Result<Buffer>> + Unpin + 'static>,
+ size: Option<u64>,
+ consumed: u64,
+}
+
+/// # Safety
+///
+/// HttpBody is send on non wasm32 targets.
+unsafe impl Send for HttpBody {}
+
+/// # Safety
+///
+/// HttpBody is sync on non wasm32 targets.
+unsafe impl Sync for HttpBody {}
+
+impl HttpBody {
+ /// Create a new `HttpBody` with given stream and optional size.
+ #[cfg(not(target_arch = "wasm32"))]
+ pub fn new<S>(stream: S, size: Option<u64>) -> Self
+ where
+ S: Stream<Item = Result<Buffer>> + Send + Sync + Unpin + 'static,
+ {
+ HttpBody {
+ stream: Box::new(stream),
+ size,
+ consumed: 0,
+ }
+ }
+
+ /// Create a new `HttpBody` with given stream and optional size.
+ #[cfg(target_arch = "wasm32")]
+ pub fn new<S>(stream: S, size: Option<u64>) -> Self
+ where
+ S: Stream<Item = Result<Buffer>> + Unpin + 'static,
+ {
+ HttpBody {
+ stream: Box::new(stream),
+ size,
+ consumed: 0,
+ }
+ }
+
+ /// Check if the consumed data is equal to the expected content length.
+ #[inline]
+ fn check(&self) -> Result<()> {
+ let Some(expect) = self.size else {
+ return Ok(());
+ };
+
+ let actual = self.consumed;
+ match actual.cmp(&expect) {
+ Ordering::Equal => Ok(()),
+ Ordering::Less => Err(Error::new(
+ ErrorKind::Unexpected,
+ &format!("http response got too little data, expect: {expect}, actual: {actual}"),
+ )
+ .set_temporary()),
+ Ordering::Greater => Err(Error::new(
+ ErrorKind::Unexpected,
+ &format!("http response got too much data, expect: {expect}, actual: {actual}"),
+ )
+ .set_temporary()),
+ }
+ }
+
+ /// Read all data from the stream.
+ pub async fn to_buffer(&mut self) -> Result<Buffer> {
+ self.read_all().await
+ }
+}
+
+impl oio::Read for HttpBody {
+ async fn read(&mut self) -> Result<Buffer> {
+ match self.stream.next().await.transpose()? {
+ Some(buf) => {
+ self.consumed += buf.len() as u64;
+ Ok(buf)
+ }
+ None => {
+ self.check()?;
+ Ok(Buffer::new())
+ }
+ }
+ }
+}
diff --git a/core/src/raw/http_util/bytes_range.rs b/core/src/raw/http_util/bytes_range.rs
index f968e54d52..50e74718c5 100644
--- a/core/src/raw/http_util/bytes_range.rs
+++ b/core/src/raw/http_util/bytes_range.rs
@@ -93,6 +93,17 @@ impl BytesRange {
},
)
}
+
+ /// Convert bytes range into rust range with usize.
+ pub(crate) fn to_range_as_usize(self) -> impl RangeBounds<usize> {
+ (
+ Bound::Included(self.0 as usize),
+ match self.1 {
+ Some(size) => Bound::Excluded((self.0 + size) as usize),
+ None => Bound::Unbounded,
+ },
+ )
+ }
}
impl Display for BytesRange {
diff --git a/core/src/raw/http_util/client.rs b/core/src/raw/http_util/client.rs
index 4ecd743ead..8fc0b81ef0 100644
--- a/core/src/raw/http_util/client.rs
+++ b/core/src/raw/http_util/client.rs
@@ -15,21 +15,20 @@
// specific language governing permissions and limitations
// under the License.
-use std::cmp::Ordering;
use std::fmt::Debug;
use std::fmt::Formatter;
use std::future;
use std::mem;
use std::str::FromStr;
-use bytes::Buf;
-use bytes::Bytes;
use futures::TryStreamExt;
use http::Request;
use http::Response;
+use raw::oio::Read;
use super::parse_content_encoding;
use super::parse_content_length;
+use super::HttpBody;
use crate::*;
/// HttpClient that used across opendal.
@@ -72,6 +71,13 @@ impl HttpClient {
/// Send a request in async way.
pub async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
+ let (parts, mut body) = self.fetch(req).await?.into_parts();
+ let buffer = body.read_all().await?;
+ Ok(Response::from_parts(parts, buffer))
+ }
+
+ /// Fetch a request in async way.
+ pub async fn fetch(&self, req: Request<Buffer>) -> Result<Response<HttpBody>> {
// Uri stores all string alike data in `Bytes` which means
// the clone here is cheap.
let uri = req.uri().clone();
@@ -138,48 +144,25 @@ impl HttpClient {
// Swap headers directly instead of copy the entire map.
mem::swap(hr.headers_mut().unwrap(), resp.headers_mut());
- let bs: Vec<Bytes> = resp
- .bytes_stream()
- .try_filter(|v| future::ready(!v.is_empty()))
- .try_collect()
- .await
- .map_err(|err| {
- Error::new(ErrorKind::Unexpected, "read data from http response")
- .with_operation("http_util::Client::send")
- .with_context("url", uri.to_string())
- .with_temporary(is_temporary_error(&err))
- .set_source(err)
- })?;
-
- let buffer = Buffer::from(bs);
-
- if let Some(expect) = content_length {
- check(expect, buffer.remaining() as u64)?;
- }
-
- let resp = hr.body(buffer).expect("response must build succeed");
-
+ let bs = HttpBody::new(
+ resp.bytes_stream()
+ .try_filter(|v| future::ready(!v.is_empty()))
+ .map_ok(Buffer::from)
+ .map_err(move |err| {
+ Error::new(ErrorKind::Unexpected, "read data from http response")
+ .with_operation("http_util::Client::send")
+ .with_context("url", uri.to_string())
+ .with_temporary(is_temporary_error(&err))
+ .set_source(err)
+ }),
+ content_length,
+ );
+
+ let resp = hr.body(bs).expect("response must build succeed");
Ok(resp)
}
}
-#[inline]
-fn check(expect: u64, actual: u64) -> Result<()> {
- match actual.cmp(&expect) {
- Ordering::Equal => Ok(()),
- Ordering::Less => Err(Error::new(
- ErrorKind::Unexpected,
- &format!("http response got too little data, expect: {expect}, actual: {actual}"),
- )
- .set_temporary()),
- Ordering::Greater => Err(Error::new(
- ErrorKind::Unexpected,
- &format!("http response got too much data, expect: {expect}, actual: {actual}"),
- )
- .set_temporary()),
- }
-}
-
#[inline]
fn is_temporary_error(err: &reqwest::Error) -> bool {
// error sending request
diff --git a/core/src/raw/http_util/mod.rs b/core/src/raw/http_util/mod.rs
index db7ff18b05..965ea0bd49 100644
--- a/core/src/raw/http_util/mod.rs
+++ b/core/src/raw/http_util/mod.rs
@@ -25,6 +25,9 @@
mod client;
pub use client::HttpClient;
+mod body;
+pub use body::HttpBody;
+
mod header;
pub use header::build_header_value;
pub use header::format_authorization_by_basic;
diff --git a/core/src/raw/oio/read/api.rs b/core/src/raw/oio/read/api.rs
index 49847032f6..88bfcdedc4 100644
--- a/core/src/raw/oio/read/api.rs
+++ b/core/src/raw/oio/read/api.rs
@@ -17,8 +17,8 @@
use std::fmt::Display;
use std::fmt::Formatter;
-use std::ops::Deref;
-use std::sync::Arc;
+use std::mem;
+use std::ops::DerefMut;
use bytes::Bytes;
use futures::Future;
@@ -61,7 +61,7 @@ impl From<ReadOperation> for &'static str {
}
/// Reader is a type erased [`Read`].
-pub type Reader = Arc<dyn ReadDyn>;
+pub type Reader = Box<dyn ReadDyn>;
/// Read is the internal trait used by OpenDAL to read data from storage.
///
@@ -79,60 +79,42 @@ pub type Reader = Arc<dyn ReadDyn>;
/// once, at the outermost level of our API.
pub trait Read: Unpin + Send + Sync {
/// Read at the given offset with the given size.
- fn read_at(&self, offset: u64, size: usize)
- -> impl Future<Output = Result<Buffer>> + MaybeSend;
+ fn read(&mut self) -> impl Future<Output = Result<Buffer>> + MaybeSend;
+
+ /// Read all data from the reader.
+ fn read_all(&mut self) -> impl Future<Output = Result<Buffer>> + MaybeSend {
+ async {
+ let mut bufs = vec![];
+ loop {
+ match self.read().await {
+ Ok(buf) if buf.is_empty() => break,
+ Ok(buf) => bufs.push(buf),
+ Err(err) => return Err(err),
+ }
+ }
+ Ok(bufs.into_iter().flatten().collect())
+ }
+ }
}
impl Read for () {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let _ = (offset, size);
-
+ async fn read(&mut self) -> Result<Buffer> {
Err(Error::new(
ErrorKind::Unsupported,
- "output reader doesn't support streaming",
+ "output reader doesn't support read",
))
}
}
impl Read for Bytes {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let offset = offset as usize;
-
- if offset >= self.len() {
- return Err(Error::new(
- ErrorKind::RangeNotSatisfied,
- "offset out of range",
- ));
- }
- if size > self.len() - offset {
- return Err(Error::new(
- ErrorKind::RangeNotSatisfied,
- "size out of range",
- ));
- }
-
- Ok(Buffer::from(self.slice(offset..offset + size)))
+ async fn read(&mut self) -> Result<Buffer> {
+ Ok(Buffer::from(self.split_off(0)))
}
}
impl Read for Buffer {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let offset = offset as usize;
-
- if offset >= self.len() {
- return Err(Error::new(
- ErrorKind::RangeNotSatisfied,
- "offset out of range",
- ));
- }
- if size > self.len() - offset {
- return Err(Error::new(
- ErrorKind::RangeNotSatisfied,
- "size out of range",
- ));
- }
-
- Ok(self.slice(offset..offset + size))
+ async fn read(&mut self) -> Result<Buffer> {
+ Ok(mem::take(self))
}
}
@@ -142,12 +124,19 @@ pub trait ReadDyn: Unpin + Send + Sync {
/// The dyn version of [`Read::read_at`].
///
/// This function returns a boxed future to make it object safe.
- fn read_at_dyn(&self, offset: u64, size: usize) -> BoxedFuture<Result<Buffer>>;
+ fn read_dyn(&mut self) -> BoxedFuture<Result<Buffer>>;
+
+ /// The dyn version of [`Read::read_all`]
+ fn read_all_dyn(&mut self) -> BoxedFuture<Result<Buffer>>;
}
impl<T: Read + ?Sized> ReadDyn for T {
- fn read_at_dyn(&self, offset: u64, size: usize) -> BoxedFuture<Result<Buffer>> {
- Box::pin(self.read_at(offset, size))
+ fn read_dyn(&mut self) -> BoxedFuture<Result<Buffer>> {
+ Box::pin(self.read())
+ }
+
+ fn read_all_dyn(&mut self) -> BoxedFuture<Result<Buffer>> {
+ Box::pin(self.read_all())
}
}
@@ -156,80 +145,46 @@ impl<T: Read + ?Sized> ReadDyn for T {
/// Take care about the `deref_mut()` here. This makes sure that we are calling functions
/// upon `&mut T` instead of `&mut Box<T>`. The later could result in infinite recursion.
impl<T: ReadDyn + ?Sized> Read for Box<T> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.deref().read_at_dyn(offset, size).await
+ async fn read(&mut self) -> Result<Buffer> {
+ self.deref_mut().read_dyn().await
}
-}
-impl<T: ReadDyn + ?Sized> Read for Arc<T> {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- self.deref().read_at_dyn(offset, size).await
+ async fn read_all(&mut self) -> Result<Buffer> {
+ self.deref_mut().read_all_dyn().await
}
}
/// BlockingReader is a arc dyn `BlockingRead`.
-pub type BlockingReader = Arc<dyn BlockingRead>;
+pub type BlockingReader = Box<dyn BlockingRead>;
/// Read is the trait that OpenDAL returns to callers.
pub trait BlockingRead: Send + Sync {
/// Read data from the reader at the given offset with the given size.
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer>;
+ fn read(&mut self) -> Result<Buffer>;
}
impl BlockingRead for () {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let _ = (offset, size);
-
+ fn read(&mut self) -> Result<Buffer> {
unimplemented!("read is required to be implemented for oio::BlockingRead")
}
}
impl BlockingRead for Bytes {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let offset = offset as usize;
-
- if offset >= self.len() {
- return Err(Error::new(
- ErrorKind::RangeNotSatisfied,
- "offset out of range",
- ));
- }
- if size > self.len() - offset {
- return Err(Error::new(
- ErrorKind::RangeNotSatisfied,
- "size out of range",
- ));
- }
-
- Ok(Buffer::from(self.slice(offset..offset + size)))
+ fn read(&mut self) -> Result<Buffer> {
+ Ok(Buffer::from(self.split_off(0)))
}
}
impl BlockingRead for Buffer {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let offset = offset as usize;
-
- if offset >= self.len() {
- return Err(Error::new(
- ErrorKind::RangeNotSatisfied,
- "offset out of range",
- ));
- }
- if size > self.len() - offset {
- return Err(Error::new(
- ErrorKind::RangeNotSatisfied,
- "size out of range",
- ));
- }
-
- Ok(self.slice(offset..offset + size))
+ fn read(&mut self) -> Result<Buffer> {
+ Ok(mem::take(self))
}
}
/// `Arc<dyn BlockingRead>` won't implement `BlockingRead` automatically.
/// To make BlockingReader work as expected, we must add this impl.
-impl<T: BlockingRead + ?Sized> BlockingRead for Arc<T> {
- fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- (**self).read_at(offset, size)
+impl<T: BlockingRead + ?Sized> BlockingRead for Box<T> {
+ fn read(&mut self) -> Result<Buffer> {
+ (**self).read()
}
}
diff --git a/core/src/raw/ops.rs b/core/src/raw/ops.rs
index e22b25f31e..00ed8fe845 100644
--- a/core/src/raw/ops.rs
+++ b/core/src/raw/ops.rs
@@ -300,6 +300,7 @@ impl BatchOperation {
/// Args for `read` operation.
#[derive(Debug, Clone, Default)]
pub struct OpRead {
+ range: BytesRange,
if_match: Option<String>,
if_none_match: Option<String>,
override_content_type: Option<String>,
@@ -315,6 +316,17 @@ impl OpRead {
Self::default()
}
+ /// Set the range of the option
+ pub fn with_range(mut self, range: BytesRange) -> Self {
+ self.range = range;
+ self
+ }
+
+ /// Get range from option
+ pub fn range(&self) -> BytesRange {
+ self.range
+ }
+
/// Sets the content-disposition header that should be send back by the remote read operation.
pub fn with_override_content_disposition(mut self, content_disposition: &str) -> Self {
self.override_content_disposition = Some(content_disposition.into());
@@ -411,10 +423,6 @@ impl OpRead {
/// Args for reader operation.
#[derive(Debug, Clone)]
pub struct OpReader {
- /// The range of the read request.
- ///
- /// Not available for `reader`.
- range: BytesRange,
/// The concurrent requests that reader can send.
concurrent: usize,
/// The chunk size of each request.
@@ -426,7 +434,6 @@ pub struct OpReader {
impl Default for OpReader {
fn default() -> Self {
Self {
- range: BytesRange::default(),
concurrent: 1,
chunk: None,
gap: None,
@@ -440,17 +447,6 @@ impl OpReader {
Self::default()
}
- /// Set the range of the option
- pub fn with_range(mut self, range: BytesRange) -> Self {
- self.range = range;
- self
- }
-
- /// Get range from option
- pub fn range(&self) -> BytesRange {
- self.range
- }
-
/// Set the concurrent of the option
pub fn with_concurrent(mut self, concurrent: usize) -> Self {
self.concurrent = concurrent.max(1);
diff --git a/core/src/services/aliyun_drive/backend.rs b/core/src/services/aliyun_drive/backend.rs
index 47598e7190..091d0bc47a 100644
--- a/core/src/services/aliyun_drive/backend.rs
+++ b/core/src/services/aliyun_drive/backend.rs
@@ -21,14 +21,18 @@ use std::sync::Arc;
use bytes::Buf;
use chrono::Utc;
+use http::header;
+use http::Request;
+use http::Response;
+use http::StatusCode;
use log::debug;
use serde::Deserialize;
use tokio::sync::Mutex;
use super::core::*;
+use super::error::parse_error;
use super::lister::AliyunDriveLister;
use super::lister::AliyunDriveParent;
-use super::reader::AliyunDriveReader;
use super::writer::{AliyunDriveWriter, AliyunDriveWriters};
use crate::raw::*;
use crate::*;
@@ -260,7 +264,7 @@ pub struct AliyunDriveBackend {
}
impl Access for AliyunDriveBackend {
- type Reader = AliyunDriveReader;
+ type Reader = HttpBody;
type Writer = AliyunDriveWriters;
type Lister = oio::PageLister<AliyunDriveLister>;
type BlockingReader = ();
@@ -412,16 +416,24 @@ impl Access for AliyunDriveBackend {
let file: AliyunDriveFile =
serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?;
- let Some(size) = file.size else {
- return Err(Error::new(ErrorKind::Unexpected, "cannot get file size"));
- };
-
let download_url = self.core.get_download_url(&file.file_id).await?;
-
- Ok((
- RpRead::default(),
- AliyunDriveReader::new(self.core.clone(), &download_url, size, args),
- ))
+ let req = Request::get(&download_url)
+ .header(header::RANGE, args.range().to_header())
+ .body(Buffer::new())
+ .map_err(new_request_build_error)?;
+
+ let resp = self.core.client.fetch(req).await?;
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn delete(&self, path: &str, _args: OpDelete) -> Result<RpDelete> {
diff --git a/core/src/services/aliyun_drive/mod.rs b/core/src/services/aliyun_drive/mod.rs
index b807e036af..efdc2b8eef 100644
--- a/core/src/services/aliyun_drive/mod.rs
+++ b/core/src/services/aliyun_drive/mod.rs
@@ -18,7 +18,6 @@
mod backend;
mod error;
mod lister;
-mod reader;
mod writer;
pub use backend::AliyunDriveBuilder as AliyunDrive;
pub use backend::AliyunDriveConfig;
diff --git a/core/src/services/aliyun_drive/reader.rs b/core/src/services/aliyun_drive/reader.rs
deleted file mode 100644
index f6268d3f3d..0000000000
--- a/core/src/services/aliyun_drive/reader.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::{header, Request, StatusCode};
-
-use crate::raw::*;
-use crate::*;
-
-use super::core::AliyunDriveCore;
-use super::error::parse_error;
-
-pub struct AliyunDriveReader {
- core: Arc<AliyunDriveCore>,
-
- download_url: String,
- size: u64,
- _op: OpRead,
-}
-
-impl AliyunDriveReader {
- pub fn new(core: Arc<AliyunDriveCore>, download_url: &str, size: u64, op: OpRead) -> Self {
- AliyunDriveReader {
- core,
- download_url: download_url.to_string(),
- size,
- _op: op,
- }
- }
-}
-
-impl oio::Read for AliyunDriveReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- // AliyunDrive responds with status OK even if the range is not statisfiable.
- // and then the whole file will be read.
- let limit = if offset >= self.size {
- return Ok(Buffer::new());
- } else if offset + (size as u64) - 1 > self.size {
- self.size - offset
- } else {
- size as u64
- };
- let range = BytesRange::new(offset, Some(limit));
- let req = Request::get(self.download_url.as_str())
- .header(header::RANGE, range.to_header())
- .body(Buffer::new())
- .map_err(new_request_build_error)?;
- let res = self.core.client.send(req).await?;
- let status = res.status();
- match status {
- StatusCode::OK => Ok(Buffer::new()),
- StatusCode::PARTIAL_CONTENT => Ok(res.into_body()),
- _ => Err(parse_error(res).await?),
- }
- }
-}
diff --git a/core/src/services/alluxio/backend.rs b/core/src/services/alluxio/backend.rs
index 6eba21e431..78cad123c4 100644
--- a/core/src/services/alluxio/backend.rs
+++ b/core/src/services/alluxio/backend.rs
@@ -20,15 +20,16 @@ use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
+use http::Response;
use log::debug;
use serde::Deserialize;
use super::core::AlluxioCore;
+use super::error::parse_error;
use super::lister::AlluxioLister;
use super::writer::AlluxioWriter;
use super::writer::AlluxioWriters;
use crate::raw::*;
-use crate::services::alluxio::reader::AlluxioReader;
use crate::*;
/// Config for alluxio services support.
@@ -180,7 +181,7 @@ pub struct AlluxioBackend {
}
impl Access for AlluxioBackend {
- type Reader = AlluxioReader;
+ type Reader = HttpBody;
type Writer = AlluxioWriters;
type Lister = oio::PageLister<AlluxioLister>;
type BlockingReader = ();
@@ -228,8 +229,13 @@ impl Access for AlluxioBackend {
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
let stream_id = self.core.open_file(path).await?;
- let r = AlluxioReader::new(self.core.clone(), stream_id, args.clone());
- Ok((RpRead::new(), r))
+ let resp = self.core.read(stream_id, args.range()).await?;
+ if !resp.status().is_success() {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ return Err(parse_error(Response::from_parts(part, buf)).await?);
+ }
+ Ok((RpRead::new(), resp.into_body()))
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/alluxio/core.rs b/core/src/services/alluxio/core.rs
index 58c1513db1..52aab52924 100644
--- a/core/src/services/alluxio/core.rs
+++ b/core/src/services/alluxio/core.rs
@@ -291,7 +291,7 @@ impl AlluxioCore {
/// TODO: we should implement range support correctly.
///
/// Please refer to [alluxio-py](https://github.com/Alluxio/alluxio-py/blob/main/alluxio/const.py#L18)
- pub async fn read(&self, stream_id: u64, _: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn read(&self, stream_id: u64, _: BytesRange) -> Result<Response<HttpBody>> {
let req = Request::post(format!(
"{}/api/v1/streams/{}/read",
self.endpoint, stream_id,
@@ -299,7 +299,7 @@ impl AlluxioCore {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
pub(super) async fn write(&self, stream_id: u64, body: Buffer) -> Result<usize> {
diff --git a/core/src/services/alluxio/mod.rs b/core/src/services/alluxio/mod.rs
index 226d013035..95e0216d34 100644
--- a/core/src/services/alluxio/mod.rs
+++ b/core/src/services/alluxio/mod.rs
@@ -22,5 +22,4 @@ pub use backend::AlluxioConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/alluxio/reader.rs b/core/src/services/alluxio/reader.rs
deleted file mode 100644
index 53dd23912e..0000000000
--- a/core/src/services/alluxio/reader.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use super::core::*;
-use crate::raw::*;
-use crate::services::alluxio::error::parse_error;
-use crate::*;
-
-pub struct AlluxioReader {
- core: Arc<AlluxioCore>,
-
- stream_id: u64,
- _op: OpRead,
-}
-
-impl AlluxioReader {
- pub fn new(core: Arc<AlluxioCore>, stream_id: u64, op: OpRead) -> Self {
- AlluxioReader {
- core,
- stream_id,
- _op: op,
- }
- }
-}
-
-impl oio::Read for AlluxioReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.read(self.stream_id, range).await?;
-
- if !resp.status().is_success() {
- return Err(parse_error(resp).await?);
- }
- Ok(resp.into_body())
- }
-}
diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs
index 2dfb9a44c2..3423439110 100644
--- a/core/src/services/azblob/backend.rs
+++ b/core/src/services/azblob/backend.rs
@@ -24,6 +24,7 @@ use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use bytes::Buf;
use http::header::CONTENT_TYPE;
+use http::Response;
use http::StatusCode;
use log::debug;
use reqsign::AzureStorageConfig;
@@ -38,7 +39,6 @@ use super::lister::AzblobLister;
use super::writer::AzblobWriter;
use crate::raw::*;
use crate::services::azblob::core::AzblobCore;
-use crate::services::azblob::reader::AzblobReader;
use crate::services::azblob::writer::AzblobWriters;
use crate::*;
@@ -542,7 +542,7 @@ pub struct AzblobBackend {
}
impl Access for AzblobBackend {
- type Reader = AzblobReader;
+ type Reader = HttpBody;
type Writer = AzblobWriters;
type Lister = oio::PageLister<AzblobLister>;
type BlockingReader = ();
@@ -605,10 +605,17 @@ impl Access for AzblobBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- AzblobReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.azblob_get_blob(path, args.range(), &args).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/azblob/core.rs b/core/src/services/azblob/core.rs
index 0293e60c87..155a0d5484 100644
--- a/core/src/services/azblob/core.rs
+++ b/core/src/services/azblob/core.rs
@@ -217,12 +217,12 @@ impl AzblobCore {
path: &str,
range: BytesRange,
args: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let mut req = self.azblob_get_blob_request(path, range, args)?;
self.sign(&mut req).await?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub fn azblob_put_blob_request(
diff --git a/core/src/services/azblob/mod.rs b/core/src/services/azblob/mod.rs
index bad56f999f..da314f5b3e 100644
--- a/core/src/services/azblob/mod.rs
+++ b/core/src/services/azblob/mod.rs
@@ -22,5 +22,4 @@ pub use backend::AzblobConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/azblob/reader.rs b/core/src/services/azblob/reader.rs
deleted file mode 100644
index 918779438e..0000000000
--- a/core/src/services/azblob/reader.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::AzblobCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct AzblobReader {
- core: Arc<AzblobCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl AzblobReader {
- pub fn new(core: Arc<AzblobCore>, path: &str, op: OpRead) -> Self {
- AzblobReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for AzblobReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self
- .core
- .azblob_get_blob(&self.path, range, &self.op)
- .await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/azdls/backend.rs b/core/src/services/azdls/backend.rs
index 4fcf9489b8..86975606a8 100644
--- a/core/src/services/azdls/backend.rs
+++ b/core/src/services/azdls/backend.rs
@@ -20,6 +20,7 @@ use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
+use http::Response;
use http::StatusCode;
use log::debug;
use reqsign::AzureStorageConfig;
@@ -33,7 +34,6 @@ use super::lister::AzdlsLister;
use super::writer::AzdlsWriter;
use super::writer::AzdlsWriters;
use crate::raw::*;
-use crate::services::azdls::reader::AzdlsReader;
use crate::*;
/// Known endpoint suffix Azure Data Lake Storage Gen2 URI syntax.
@@ -244,7 +244,7 @@ pub struct AzdlsBackend {
}
impl Access for AzdlsBackend {
- type Reader = AzdlsReader;
+ type Reader = HttpBody;
type Writer = AzdlsWriters;
type Lister = oio::PageLister<AzdlsLister>;
type BlockingReader = ();
@@ -342,10 +342,17 @@ impl Access for AzdlsBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- AzdlsReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.azdls_read(path, args.range()).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/azdls/core.rs b/core/src/services/azdls/core.rs
index 8ebe6202e6..714a5adc7b 100644
--- a/core/src/services/azdls/core.rs
+++ b/core/src/services/azdls/core.rs
@@ -97,7 +97,7 @@ impl AzdlsCore {
}
impl AzdlsCore {
- pub async fn azdls_read(&self, path: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn azdls_read(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
let p = build_abs_path(&self.root, path);
let url = format!(
@@ -116,7 +116,7 @@ impl AzdlsCore {
let mut req = req.body(Buffer::new()).map_err(new_request_build_error)?;
self.sign(&mut req).await?;
- self.send(req).await
+ self.client.fetch(req).await
}
/// resource should be one of `file` or `directory`
diff --git a/core/src/services/azdls/mod.rs b/core/src/services/azdls/mod.rs
index e407c395ee..4284e4595e 100644
--- a/core/src/services/azdls/mod.rs
+++ b/core/src/services/azdls/mod.rs
@@ -21,5 +21,4 @@ pub use backend::AzdlsBuilder as Azdls;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/azdls/reader.rs b/core/src/services/azdls/reader.rs
deleted file mode 100644
index e8f049295b..0000000000
--- a/core/src/services/azdls/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::AzdlsCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct AzdlsReader {
- core: Arc<AzdlsCore>,
-
- path: String,
- _op: OpRead,
-}
-
-impl AzdlsReader {
- pub fn new(core: Arc<AzdlsCore>, path: &str, op: OpRead) -> Self {
- AzdlsReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for AzdlsReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.azdls_read(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs
index d651b5c386..195483d1db 100644
--- a/core/src/services/azfile/backend.rs
+++ b/core/src/services/azfile/backend.rs
@@ -20,6 +20,7 @@ use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
+use http::Response;
use http::StatusCode;
use log::debug;
use reqsign::AzureStorageConfig;
@@ -33,7 +34,6 @@ use super::writer::AzfileWriter;
use super::writer::AzfileWriters;
use crate::raw::*;
use crate::services::azfile::lister::AzfileLister;
-use crate::services::azfile::reader::AzfileReader;
use crate::*;
/// Default endpoint of Azure File services.
@@ -265,7 +265,7 @@ pub struct AzfileBackend {
}
impl Access for AzfileBackend {
- type Reader = AzfileReader;
+ type Reader = HttpBody;
type Writer = AzfileWriters;
type Lister = oio::PageLister<AzfileLister>;
type BlockingReader = ();
@@ -340,10 +340,17 @@ impl Access for AzfileBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- AzfileReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.azfile_read(path, args.range()).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/azfile/core.rs b/core/src/services/azfile/core.rs
index 19454d6da5..77f434b4a4 100644
--- a/core/src/services/azfile/core.rs
+++ b/core/src/services/azfile/core.rs
@@ -97,7 +97,7 @@ impl AzfileCore {
self.client.send(req).await
}
- pub async fn azfile_read(&self, path: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn azfile_read(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
let p = build_abs_path(&self.root, path);
let url = format!(
@@ -115,7 +115,7 @@ impl AzfileCore {
let mut req = req.body(Buffer::new()).map_err(new_request_build_error)?;
self.sign(&mut req).await?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub async fn azfile_create_file(
diff --git a/core/src/services/azfile/mod.rs b/core/src/services/azfile/mod.rs
index a94ed0be84..0ae9da7291 100644
--- a/core/src/services/azfile/mod.rs
+++ b/core/src/services/azfile/mod.rs
@@ -21,5 +21,4 @@ mod backend;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/azfile/reader.rs b/core/src/services/azfile/reader.rs
deleted file mode 100644
index 58066d67a4..0000000000
--- a/core/src/services/azfile/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::AzfileCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct AzfileReader {
- core: Arc<AzfileCore>,
-
- path: String,
- _op: OpRead,
-}
-
-impl AzfileReader {
- pub fn new(core: Arc<AzfileCore>, path: &str, op: OpRead) -> Self {
- Self {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for AzfileReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.azfile_read(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs
index 2c99176b32..ddf441057a 100644
--- a/core/src/services/b2/backend.rs
+++ b/core/src/services/b2/backend.rs
@@ -22,6 +22,7 @@ use std::sync::Arc;
use bytes::Buf;
use http::Request;
+use http::Response;
use http::StatusCode;
use log::debug;
use serde::Deserialize;
@@ -37,7 +38,6 @@ use super::writer::B2Writers;
use crate::raw::*;
use crate::services::b2::core::B2Signer;
use crate::services::b2::core::ListFileNamesResponse;
-use crate::services::b2::reader::B2Reader;
use crate::*;
/// Config for backblaze b2 services support.
@@ -267,7 +267,7 @@ pub struct B2Backend {
}
impl Access for B2Backend {
- type Reader = B2Reader;
+ type Reader = HttpBody;
type Writer = B2Writers;
type Lister = oio::PageLister<B2Lister>;
type BlockingReader = ();
@@ -352,10 +352,22 @@ impl Access for B2Backend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- B2Reader::new(self.core.clone(), path, args),
- ))
+ let resp = self
+ .core
+ .download_file_by_name(path, args.range(), &args)
+ .await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/b2/core.rs b/core/src/services/b2/core.rs
index 6d30586c24..6aea7a6092 100644
--- a/core/src/services/b2/core.rs
+++ b/core/src/services/b2/core.rs
@@ -134,7 +134,7 @@ impl B2Core {
path: &str,
range: BytesRange,
_args: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let path = build_abs_path(&self.root, path);
let auth_info = self.get_auth_info().await?;
@@ -157,7 +157,7 @@ impl B2Core {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub(super) async fn get_upload_url(&self) -> Result<GetUploadUrlResponse> {
diff --git a/core/src/services/b2/mod.rs b/core/src/services/b2/mod.rs
index 025a80c3a8..80745f01e8 100644
--- a/core/src/services/b2/mod.rs
+++ b/core/src/services/b2/mod.rs
@@ -22,5 +22,4 @@ pub use backend::B2Config;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/b2/reader.rs b/core/src/services/b2/reader.rs
deleted file mode 100644
index 727ef647d5..0000000000
--- a/core/src/services/b2/reader.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::*;
-use super::error::*;
-use crate::raw::*;
-use crate::*;
-
-pub struct B2Reader {
- core: Arc<B2Core>,
-
- path: String,
- op: OpRead,
-}
-
-impl B2Reader {
- pub fn new(core: Arc<B2Core>, path: &str, op: OpRead) -> Self {
- B2Reader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for B2Reader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self
- .core
- .download_file_by_name(&self.path, range, &self.op)
- .await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/chainsafe/backend.rs b/core/src/services/chainsafe/backend.rs
index a14915e09d..03249ecf0a 100644
--- a/core/src/services/chainsafe/backend.rs
+++ b/core/src/services/chainsafe/backend.rs
@@ -21,6 +21,7 @@ use std::fmt::Formatter;
use std::sync::Arc;
use bytes::Buf;
+use http::Response;
use http::StatusCode;
use log::debug;
use serde::Deserialize;
@@ -30,7 +31,6 @@ use super::core::ChainsafeCore;
use super::core::ObjectInfoResponse;
use super::error::parse_error;
use super::lister::ChainsafeLister;
-use super::reader::ChainsafeReader;
use super::writer::ChainsafeWriter;
use super::writer::ChainsafeWriters;
use crate::raw::*;
@@ -203,7 +203,7 @@ pub struct ChainsafeBackend {
}
impl Access for ChainsafeBackend {
- type Reader = ChainsafeReader;
+ type Reader = HttpBody;
type Writer = ChainsafeWriters;
type Lister = oio::PageLister<ChainsafeLister>;
type BlockingReader = ();
@@ -264,10 +264,17 @@ impl Access for ChainsafeBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- ChainsafeReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.download_object(path, args.range()).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/chainsafe/core.rs b/core/src/services/chainsafe/core.rs
index e864c41cb2..d4e4db179c 100644
--- a/core/src/services/chainsafe/core.rs
+++ b/core/src/services/chainsafe/core.rs
@@ -58,7 +58,11 @@ impl ChainsafeCore {
}
impl ChainsafeCore {
- pub async fn download_object(&self, path: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn download_object(
+ &self,
+ path: &str,
+ range: BytesRange,
+ ) -> Result<Response<HttpBody>> {
let path = build_abs_path(&self.root, path);
let url = format!(
@@ -81,7 +85,7 @@ impl ChainsafeCore {
.body(body)
.map_err(new_request_build_error)?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub async fn object_info(&self, path: &str) -> Result<Response<Buffer>> {
diff --git a/core/src/services/chainsafe/mod.rs b/core/src/services/chainsafe/mod.rs
index cecf6a3002..ed5335a738 100644
--- a/core/src/services/chainsafe/mod.rs
+++ b/core/src/services/chainsafe/mod.rs
@@ -22,5 +22,4 @@ pub use backend::ChainsafeConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/chainsafe/reader.rs b/core/src/services/chainsafe/reader.rs
deleted file mode 100644
index 1d1e27118a..0000000000
--- a/core/src/services/chainsafe/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::ChainsafeCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct ChainsafeReader {
- core: Arc<ChainsafeCore>,
-
- path: String,
- _op: OpRead,
-}
-
-impl ChainsafeReader {
- pub fn new(core: Arc<ChainsafeCore>, path: &str, op: OpRead) -> Self {
- ChainsafeReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for ChainsafeReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.download_object(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs
index 4347a46407..17b872bdb1 100644
--- a/core/src/services/cos/backend.rs
+++ b/core/src/services/cos/backend.rs
@@ -19,6 +19,7 @@ use std::collections::HashMap;
use std::fmt::Debug;
use std::sync::Arc;
+use http::Response;
use http::StatusCode;
use http::Uri;
use log::debug;
@@ -32,7 +33,6 @@ use super::error::parse_error;
use super::lister::CosLister;
use super::writer::CosWriter;
use crate::raw::*;
-use crate::services::cos::reader::CosReader;
use crate::services::cos::writer::CosWriters;
use crate::*;
@@ -253,7 +253,7 @@ pub struct CosBackend {
}
impl Access for CosBackend {
- type Reader = CosReader;
+ type Reader = HttpBody;
type Writer = CosWriters;
type Lister = oio::PageLister<CosLister>;
type BlockingReader = ();
@@ -324,10 +324,20 @@ impl Access for CosBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- CosReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.cos_get_object(path, args.range(), &args).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/cos/core.rs b/core/src/services/cos/core.rs
index 7ba298265c..f7dbdef609 100644
--- a/core/src/services/cos/core.rs
+++ b/core/src/services/cos/core.rs
@@ -109,12 +109,12 @@ impl CosCore {
path: &str,
range: BytesRange,
args: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let mut req = self.cos_get_object_request(path, range, args)?;
self.sign(&mut req).await?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub fn cos_get_object_request(
diff --git a/core/src/services/cos/mod.rs b/core/src/services/cos/mod.rs
index 07f1e54ce9..780c2c4796 100644
--- a/core/src/services/cos/mod.rs
+++ b/core/src/services/cos/mod.rs
@@ -21,5 +21,4 @@ pub use backend::CosBuilder as Cos;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/cos/reader.rs b/core/src/services/cos/reader.rs
deleted file mode 100644
index 15dbfdf695..0000000000
--- a/core/src/services/cos/reader.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::CosCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct CosReader {
- core: Arc<CosCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl CosReader {
- pub fn new(core: Arc<CosCore>, path: &str, op: OpRead) -> Self {
- CosReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for CosReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self
- .core
- .cos_get_object(&self.path, range, &self.op)
- .await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/dbfs/backend.rs b/core/src/services/dbfs/backend.rs
index 2e4b3b8041..925a1b0b50 100644
--- a/core/src/services/dbfs/backend.rs
+++ b/core/src/services/dbfs/backend.rs
@@ -28,7 +28,6 @@ use serde::Deserialize;
use super::core::DbfsCore;
use super::error::parse_error;
use super::lister::DbfsLister;
-use super::reader::DbfsReader;
use super::writer::DbfsWriter;
use crate::raw::*;
use crate::*;
@@ -166,7 +165,7 @@ pub struct DbfsBackend {
}
impl Access for DbfsBackend {
- type Reader = DbfsReader;
+ type Reader = ();
type Writer = oio::OneShotWriter<DbfsWriter>;
type Lister = oio::PageLister<DbfsLister>;
type BlockingReader = ();
@@ -180,8 +179,6 @@ impl Access for DbfsBackend {
.set_native_capability(Capability {
stat: true,
- read: true,
-
write: true,
create_dir: true,
delete: true,
@@ -240,12 +237,6 @@ impl Access for DbfsBackend {
}
}
- async fn read(&self, path: &str, _: OpRead) -> Result<(RpRead, Self::Reader)> {
- let op = DbfsReader::new(self.core.clone(), path.to_string());
-
- Ok((RpRead::new(), op))
- }
-
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
Ok((
RpWrite::default(),
diff --git a/core/src/services/dbfs/mod.rs b/core/src/services/dbfs/mod.rs
index 5a310d0880..c3437eceae 100644
--- a/core/src/services/dbfs/mod.rs
+++ b/core/src/services/dbfs/mod.rs
@@ -21,5 +21,4 @@ pub use backend::DbfsBuilder as Dbfs;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/dbfs/reader.rs b/core/src/services/dbfs/reader.rs
deleted file mode 100644
index 1030c15e08..0000000000
--- a/core/src/services/dbfs/reader.rs
+++ /dev/null
@@ -1,90 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use base64::engine::general_purpose;
-use base64::Engine;
-use bytes::Bytes;
-use serde::Deserialize;
-
-use super::core::DbfsCore;
-use crate::raw::*;
-use crate::*;
-
-// The number of bytes to read starting from the offset. This has a limit of 1 MB
-// Reference: https://docs.databricks.com/api/azure/workspace/dbfs/read
-// const DBFS_READ_LIMIT: usize = 1024 * 1024;
-
-#[allow(dead_code)]
-pub struct DbfsReader {
- core: Arc<DbfsCore>,
- path: String,
- has_filled: u64,
-}
-
-impl DbfsReader {
- pub fn new(core: Arc<DbfsCore>, path: String) -> Self {
- DbfsReader {
- core,
- path,
- has_filled: 0,
- }
- }
-
- #[allow(dead_code)]
- fn serde_json_decode(&self, bs: &Bytes) -> Result<Bytes> {
- let response_body = match serde_json::from_slice::<ReadContentJsonResponse>(bs) {
- Ok(v) => v,
- Err(err) => {
- return Err(
- Error::new(ErrorKind::Unexpected, "parse response content failed")
- .with_operation("http_util::IncomingDbfsAsyncBody::poll_read")
- .set_source(err),
- );
- }
- };
-
- let decoded_data = general_purpose::STANDARD
- .decode(response_body.data)
- .map_err(|err| {
- Error::new(ErrorKind::Unexpected, "decode response content failed")
- .with_operation("http_util::IncomingDbfsAsyncBody::poll_read")
- .set_source(err)
- })?;
-
- Ok(decoded_data.into())
- }
-}
-
-/// # Safety
-///
-/// We will only take `&mut Self` reference for DbfsReader.
-unsafe impl Sync for DbfsReader {}
-
-impl oio::Read for DbfsReader {
- async fn read_at(&self, _offset: u64, _limit: usize) -> Result<Buffer> {
- todo!()
- }
-}
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(default)]
-struct ReadContentJsonResponse {
- bytes_read: u64,
- data: String,
-}
diff --git a/core/src/services/dropbox/backend.rs b/core/src/services/dropbox/backend.rs
index 914190ac09..324ecab25a 100644
--- a/core/src/services/dropbox/backend.rs
+++ b/core/src/services/dropbox/backend.rs
@@ -20,12 +20,12 @@ use std::sync::Arc;
use backon::Retryable;
use bytes::Buf;
+use http::Response;
use http::StatusCode;
use super::core::*;
use super::error::*;
use super::lister::DropboxLister;
-use super::reader::DropboxReader;
use super::writer::DropboxWriter;
use crate::raw::*;
use crate::*;
@@ -36,7 +36,7 @@ pub struct DropboxBackend {
}
impl Access for DropboxBackend {
- type Reader = DropboxReader;
+ type Reader = HttpBody;
type Writer = oio::OneShotWriter<DropboxWriter>;
type Lister = oio::PageLister<DropboxLister>;
type BlockingReader = ();
@@ -143,10 +143,19 @@ impl Access for DropboxBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- DropboxReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.dropbox_get(path, args.range(), &args).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/dropbox/core.rs b/core/src/services/dropbox/core.rs
index ec953c8a48..3c4927d1bd 100644
--- a/core/src/services/dropbox/core.rs
+++ b/core/src/services/dropbox/core.rs
@@ -129,7 +129,7 @@ impl DropboxCore {
path: &str,
range: BytesRange,
_: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let url: String = "https://content.dropboxapi.com/2/files/download".to_string();
let download_args = DropboxDownloadArgs {
path: build_rooted_abs_path(&self.root, path),
@@ -148,7 +148,7 @@ impl DropboxCore {
let mut request = req.body(Buffer::new()).map_err(new_request_build_error)?;
self.sign(&mut request).await?;
- self.client.send(request).await
+ self.client.fetch(request).await
}
pub async fn dropbox_update(
diff --git a/core/src/services/dropbox/mod.rs b/core/src/services/dropbox/mod.rs
index 0a551c3924..3aaa93db9a 100644
--- a/core/src/services/dropbox/mod.rs
+++ b/core/src/services/dropbox/mod.rs
@@ -20,7 +20,6 @@ mod builder;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
pub use builder::DropboxBuilder as Dropbox;
diff --git a/core/src/services/dropbox/reader.rs b/core/src/services/dropbox/reader.rs
deleted file mode 100644
index 2675bb35be..0000000000
--- a/core/src/services/dropbox/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::DropboxCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct DropboxReader {
- core: Arc<DropboxCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl DropboxReader {
- pub fn new(core: Arc<DropboxCore>, path: &str, op: OpRead) -> Self {
- DropboxReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for DropboxReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.dropbox_get(&self.path, range, &self.op).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs
index 8dc46c7535..89f2c1ca68 100644
--- a/core/src/services/fs/backend.rs
+++ b/core/src/services/fs/backend.rs
@@ -16,6 +16,7 @@
// under the License.
use std::collections::HashMap;
+use std::io::SeekFrom;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
@@ -167,10 +168,10 @@ pub struct FsBackend {
}
impl Access for FsBackend {
- type Reader = FsReader;
+ type Reader = FsReader<tokio::fs::File>;
type Writer = FsWriter<tokio::fs::File>;
type Lister = Option<FsLister<tokio::fs::ReadDir>>;
- type BlockingReader = FsReader;
+ type BlockingReader = FsReader<std::fs::File>;
type BlockingWriter = FsWriter<std::fs::File>;
type BlockingLister = Option<FsLister<std::fs::ReadDir>>;
@@ -244,16 +245,28 @@ impl Access for FsBackend {
/// - open file first, and than use `seek`. (100ns)
///
/// Benchmark could be found [here](https://gist.github.com/Xuanwo/48f9cfbc3022ea5f865388bb62e1a70f)
- async fn read(&self, path: &str, _: OpRead) -> Result<(RpRead, Self::Reader)> {
+ async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
let p = self.core.root.join(path.trim_end_matches('/'));
- let f = tokio::fs::OpenOptions::new()
+ let mut f = tokio::fs::OpenOptions::new()
.read(true)
.open(&p)
.await
.map_err(new_std_io_error)?;
- let r = FsReader::new(self.core.clone(), f.into_std().await);
+ if args.range().offset() != 0 {
+ use tokio::io::AsyncSeekExt;
+
+ f.seek(SeekFrom::Start(args.range().offset()))
+ .await
+ .map_err(new_std_io_error)?;
+ }
+
+ let r = FsReader::new(
+ self.core.clone(),
+ f,
+ args.range().size().unwrap_or(u64::MAX) as _,
+ );
Ok((RpRead::new(), r))
}
@@ -407,15 +420,26 @@ impl Access for FsBackend {
Ok(RpStat::new(m))
}
- fn blocking_read(&self, path: &str, _: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
+ fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
let p = self.core.root.join(path.trim_end_matches('/'));
- let f = std::fs::OpenOptions::new()
+ let mut f = std::fs::OpenOptions::new()
.read(true)
.open(p)
.map_err(new_std_io_error)?;
- let r = FsReader::new(self.core.clone(), f);
+ if args.range().offset() != 0 {
+ use std::io::Seek;
+
+ f.seek(SeekFrom::Start(args.range().offset()))
+ .map_err(new_std_io_error)?;
+ }
+
+ let r = FsReader::new(
+ self.core.clone(),
+ f,
+ args.range().size().unwrap_or(u64::MAX) as _,
+ );
Ok((RpRead::new(), r))
}
diff --git a/core/src/services/fs/reader.rs b/core/src/services/fs/reader.rs
index afb5e18a05..53059bfc02 100644
--- a/core/src/services/fs/reader.rs
+++ b/core/src/services/fs/reader.rs
@@ -15,75 +15,83 @@
// specific language governing permissions and limitations
// under the License.
+use std::io::Read;
use std::sync::Arc;
+use tokio::io::AsyncReadExt;
use tokio::io::ReadBuf;
use super::core::*;
use crate::raw::*;
use crate::*;
-pub struct FsReader {
+pub struct FsReader<F> {
core: Arc<FsCore>,
- f: std::fs::File,
+ f: F,
+ read: usize,
+ size: usize,
+ buf_size: usize,
}
-impl FsReader {
- pub fn new(core: Arc<FsCore>, f: std::fs::File) -> Self {
- Self { core, f }
- }
-
- fn try_clone(&self) -> Result<Self> {
- let f = self.f.try_clone().map_err(|err| {
- Error::new(
- ErrorKind::Unexpected,
- "tokio fs clone file description failed",
- )
- .set_source(err)
- })?;
-
- Ok(Self {
- core: self.core.clone(),
+impl<F> FsReader<F> {
+ pub fn new(core: Arc<FsCore>, f: F, size: usize) -> Self {
+ Self {
+ core,
f,
- })
+ read: 0,
+ size,
+ // Use 2 MiB as default value.
+ buf_size: 2 * 1024 * 1024,
+ }
}
+}
- #[cfg(target_family = "unix")]
- pub fn read_at_inner(&self, buf: &mut [u8], offset: u64) -> Result<usize> {
- use std::os::unix::fs::FileExt;
- self.f.read_at(buf, offset).map_err(new_std_io_error)
- }
+impl oio::Read for FsReader<tokio::fs::File> {
+ async fn read(&mut self) -> Result<Buffer> {
+ if self.read >= self.size {
+ return Ok(Buffer::new());
+ }
- #[cfg(target_family = "windows")]
- pub fn read_at_inner(&self, buf: &mut [u8], offset: u64) -> Result<usize> {
- use std::os::windows::fs::FileExt;
- self.f.seek_read(buf, offset).map_err(new_std_io_error)
- }
-}
+ let mut bs = self.core.buf_pool.get();
+ bs.reserve(self.buf_size);
+
+ let size = (self.size - self.read).min(self.buf_size);
+ let buf = &mut bs.spare_capacity_mut()[..size];
+ let mut read_buf: ReadBuf = ReadBuf::uninit(buf);
-impl oio::Read for FsReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let handle = self.try_clone()?;
-
- match tokio::runtime::Handle::try_current() {
- Ok(runtime) => runtime
- .spawn_blocking(move || oio::BlockingRead::read_at(&handle, offset, size))
- .await
- .map_err(|err| {
- Error::new(ErrorKind::Unexpected, "tokio spawn io task failed").set_source(err)
- })?,
- Err(_) => Err(Error::new(
- ErrorKind::Unexpected,
- "no tokio runtime found, failed to run io task",
- )),
+ // SAFETY: Read at most `limit` bytes into `read_buf`.
+ unsafe {
+ read_buf.assume_init(size);
}
+
+ let n = self
+ .f
+ .read_buf(&mut read_buf)
+ .await
+ .map_err(new_std_io_error)?;
+ self.read += n;
+
+ // Safety: We make sure that bs contains `n` more bytes.
+ let filled = read_buf.filled().len();
+ unsafe { bs.set_len(filled) }
+
+ let frozen = bs.split().freeze();
+ // Return the buffer to the pool.
+ self.core.buf_pool.put(bs);
+
+ Ok(Buffer::from(frozen))
}
}
-impl oio::BlockingRead for FsReader {
- fn read_at(&self, mut offset: u64, size: usize) -> Result<Buffer> {
+impl oio::BlockingRead for FsReader<std::fs::File> {
+ fn read(&mut self) -> Result<Buffer> {
+ if self.read >= self.size {
+ return Ok(Buffer::new());
+ }
+
let mut bs = self.core.buf_pool.get();
- bs.reserve(size);
+ bs.reserve(self.buf_size);
+ let size = (self.size - self.read).min(self.buf_size);
let buf = &mut bs.spare_capacity_mut()[..size];
let mut read_buf: ReadBuf = ReadBuf::uninit(buf);
@@ -92,18 +100,12 @@ impl oio::BlockingRead for FsReader {
read_buf.assume_init(size);
}
- loop {
- // If the buffer is full, we are done.
- if read_buf.initialize_unfilled().is_empty() {
- break;
- }
- let n = self.read_at_inner(read_buf.initialize_unfilled(), offset)?;
- if n == 0 {
- break;
- }
- read_buf.advance(n);
- offset += n as u64;
- }
+ let n = self
+ .f
+ .read(read_buf.initialize_unfilled())
+ .map_err(new_std_io_error)?;
+ read_buf.advance(n);
+ self.read += n;
// Safety: We make sure that bs contains `n` more bytes.
let filled = read_buf.filled().len();
diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs
index fddfaca091..4dfb45e290 100644
--- a/core/src/services/ftp/backend.rs
+++ b/core/src/services/ftp/backend.rs
@@ -358,7 +358,10 @@ impl Access for FtpBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((RpRead::new(), FtpReader::new(self.clone(), path, args)))
+ let ftp_stream = self.ftp_connect(Operation::Read).await?;
+
+ let reader = FtpReader::new(ftp_stream, path.to_string(), args).await?;
+ Ok((RpRead::new(), reader))
}
async fn write(&self, path: &str, op: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/ftp/reader.rs b/core/src/services/ftp/reader.rs
index cf37429ccf..238a5538fe 100644
--- a/core/src/services/ftp/reader.rs
+++ b/core/src/services/ftp/reader.rs
@@ -15,48 +15,64 @@
// specific language governing permissions and limitations
// under the License.
-use futures::AsyncReadExt;
+use bb8::PooledConnection;
+use bytes::BytesMut;
+use futures::{AsyncRead, AsyncReadExt};
-use super::backend::FtpBackend;
+use super::backend::Manager;
use super::err::parse_error;
use crate::raw::*;
use crate::*;
pub struct FtpReader {
- core: FtpBackend,
+ /// Keep the connection alive while data stream is alive.
+ _ftp_stream: PooledConnection<'static, Manager>,
- path: String,
- _op: OpRead,
+ data_stream: Box<dyn AsyncRead + Sync + Send + Unpin + 'static>,
+ chunk: usize,
+ buf: BytesMut,
}
impl FtpReader {
- pub fn new(core: FtpBackend, path: &str, op: OpRead) -> Self {
- FtpReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for FtpReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let mut ftp_stream = self.core.ftp_connect(Operation::Read).await?;
-
+ pub async fn new(
+ mut ftp_stream: PooledConnection<'static, Manager>,
+ path: String,
+ args: OpRead,
+ ) -> Result<Self> {
+ let (offset, size) = (
+ args.range().offset(),
+ args.range().size().unwrap_or(u64::MAX),
+ );
if offset != 0 {
ftp_stream
.resume_transfer(offset as usize)
.await
.map_err(parse_error)?;
}
-
- let mut ds = ftp_stream
- .retr_as_stream(&self.path)
+ let ds = ftp_stream
+ .retr_as_stream(path)
.await
.map_err(parse_error)?
.take(size as _);
- let mut bs = Vec::with_capacity(size);
- ds.read_to_end(&mut bs).await.map_err(new_std_io_error)?;
- Ok(Buffer::from(bs))
+
+ Ok(Self {
+ _ftp_stream: ftp_stream,
+
+ data_stream: Box::new(ds),
+ chunk: 1024 * 1024,
+ buf: BytesMut::new(),
+ })
+ }
+}
+
+impl oio::Read for FtpReader {
+ async fn read(&mut self) -> Result<Buffer> {
+ self.buf.resize(self.chunk, 0);
+ let n = self
+ .data_stream
+ .read(&mut self.buf)
+ .await
+ .map_err(new_std_io_error)?;
+ Ok(Buffer::from(self.buf.split_to(n).freeze()))
}
}
diff --git a/core/src/services/ftp/writer.rs b/core/src/services/ftp/writer.rs
index 0efa7667ac..ec21609c18 100644
--- a/core/src/services/ftp/writer.rs
+++ b/core/src/services/ftp/writer.rs
@@ -17,7 +17,6 @@
use bb8::PooledConnection;
use bytes::Buf;
-use futures::AsyncRead;
use futures::AsyncWrite;
use futures::AsyncWriteExt;
@@ -26,14 +25,11 @@ use crate::raw::*;
use crate::services::ftp::err::parse_error;
use crate::*;
-trait DataStream: AsyncRead + AsyncWrite {}
-impl<T> DataStream for T where T: AsyncRead + AsyncWrite {}
-
pub struct FtpWriter {
target_path: String,
tmp_path: Option<String>,
ftp_stream: PooledConnection<'static, Manager>,
- data_stream: Option<Box<dyn DataStream + Sync + Send + Unpin + 'static>>,
+ data_stream: Option<Box<dyn AsyncWrite + Sync + Send + Unpin + 'static>>,
}
/// # TODO
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index b3e51a5246..5e9e0a396d 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -21,6 +21,7 @@ use std::fmt::Formatter;
use std::sync::Arc;
use bytes::Buf;
+use http::Response;
use http::StatusCode;
use log::debug;
use reqsign::GoogleCredentialLoader;
@@ -33,7 +34,6 @@ use serde_json;
use super::core::*;
use super::error::parse_error;
use super::lister::GcsLister;
-use super::reader::GcsReader;
use super::writer::GcsWriter;
use super::writer::GcsWriters;
use crate::raw::*;
@@ -331,7 +331,7 @@ pub struct GcsBackend {
}
impl Access for GcsBackend {
- type Reader = GcsReader;
+ type Reader = HttpBody;
type Writer = GcsWriters;
type Lister = oio::PageLister<GcsLister>;
type BlockingReader = ();
@@ -416,10 +416,20 @@ impl Access for GcsBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- GcsReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.gcs_get_object(path, args.range(), &args).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs
index 8bc25ceb7a..4a7f0b4657 100644
--- a/core/src/services/gcs/core.rs
+++ b/core/src/services/gcs/core.rs
@@ -204,11 +204,11 @@ impl GcsCore {
path: &str,
range: BytesRange,
args: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let mut req = self.gcs_get_object_request(path, range, args)?;
self.sign(&mut req).await?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub fn gcs_insert_object_request(
diff --git a/core/src/services/gcs/mod.rs b/core/src/services/gcs/mod.rs
index 756b247472..1dcaa9ca06 100644
--- a/core/src/services/gcs/mod.rs
+++ b/core/src/services/gcs/mod.rs
@@ -22,6 +22,5 @@ pub use backend::GcsConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod uri;
mod writer;
diff --git a/core/src/services/gcs/reader.rs b/core/src/services/gcs/reader.rs
deleted file mode 100644
index 70ad4f4a5c..0000000000
--- a/core/src/services/gcs/reader.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::GcsCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct GcsReader {
- core: Arc<GcsCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl GcsReader {
- pub fn new(core: Arc<GcsCore>, path: &str, op: OpRead) -> Self {
- GcsReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for GcsReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self
- .core
- .gcs_get_object(&self.path, range, &self.op)
- .await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs
index 14b241d612..fbf34c8255 100644
--- a/core/src/services/gdrive/backend.rs
+++ b/core/src/services/gdrive/backend.rs
@@ -22,6 +22,7 @@ use bytes::Buf;
use bytes::Bytes;
use chrono::Utc;
use http::Request;
+use http::Response;
use http::StatusCode;
use serde_json::json;
@@ -29,7 +30,6 @@ use super::core::GdriveCore;
use super::core::GdriveFile;
use super::error::parse_error;
use super::lister::GdriveLister;
-use super::reader::GdriveReader;
use super::writer::GdriveWriter;
use crate::raw::*;
use crate::*;
@@ -40,7 +40,7 @@ pub struct GdriveBackend {
}
impl Access for GdriveBackend {
- type Reader = GdriveReader;
+ type Reader = HttpBody;
type Writer = oio::OneShotWriter<GdriveWriter>;
type Lister = oio::PageLister<GdriveLister>;
type BlockingReader = ();
@@ -107,10 +107,17 @@ impl Access for GdriveBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- GdriveReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.gdrive_get(path, args.range()).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs
index 0b72e1c9db..ddc3968b88 100644
--- a/core/src/services/gdrive/core.rs
+++ b/core/src/services/gdrive/core.rs
@@ -76,7 +76,7 @@ impl GdriveCore {
self.client.send(req).await
}
- pub async fn gdrive_get(&self, path: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn gdrive_get(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
let path = build_abs_path(&self.root, path);
let path_id = self.path_cache.get(&path).await?.ok_or(Error::new(
ErrorKind::NotFound,
@@ -94,7 +94,7 @@ impl GdriveCore {
.map_err(new_request_build_error)?;
self.sign(&mut req).await?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
pub async fn gdrive_list(
diff --git a/core/src/services/gdrive/mod.rs b/core/src/services/gdrive/mod.rs
index 2b9b906cbc..015f177b28 100644
--- a/core/src/services/gdrive/mod.rs
+++ b/core/src/services/gdrive/mod.rs
@@ -22,5 +22,4 @@ mod error;
pub use builder::GdriveBuilder as Gdrive;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/gdrive/reader.rs b/core/src/services/gdrive/reader.rs
deleted file mode 100644
index 1cce64a1af..0000000000
--- a/core/src/services/gdrive/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::GdriveCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct GdriveReader {
- core: Arc<GdriveCore>,
-
- path: String,
- _op: OpRead,
-}
-
-impl GdriveReader {
- pub fn new(core: Arc<GdriveCore>, path: &str, op: OpRead) -> Self {
- GdriveReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for GdriveReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.gdrive_get(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/ghac/backend.rs b/core/src/services/ghac/backend.rs
index 68397729a7..0ba8316cbc 100644
--- a/core/src/services/ghac/backend.rs
+++ b/core/src/services/ghac/backend.rs
@@ -37,7 +37,6 @@ use serde::Serialize;
use super::error::parse_error;
use super::writer::GhacWriter;
use crate::raw::*;
-use crate::services::ghac::reader::GhacReader;
use crate::*;
/// The base url for cache url.
@@ -227,7 +226,7 @@ pub struct GhacBackend {
}
impl Access for GhacBackend {
- type Reader = GhacReader;
+ type Reader = HttpBody;
type Writer = GhacWriter;
type Lister = ();
type BlockingReader = ();
@@ -310,10 +309,20 @@ impl Access for GhacBackend {
return Err(parse_error(resp).await?);
};
- Ok((
- RpRead::default(),
- GhacReader::new(self.clone(), &location, args),
- ))
+ let req = self.ghac_get_location(&location, args.range()).await?;
+ let resp = self.client.fetch(req).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/ghac/mod.rs b/core/src/services/ghac/mod.rs
index 8d5fe5623d..b819c409dd 100644
--- a/core/src/services/ghac/mod.rs
+++ b/core/src/services/ghac/mod.rs
@@ -19,5 +19,4 @@ mod backend;
pub use backend::GhacBuilder as Ghac;
mod error;
-mod reader;
mod writer;
diff --git a/core/src/services/ghac/reader.rs b/core/src/services/ghac/reader.rs
deleted file mode 100644
index 2674dfcbed..0000000000
--- a/core/src/services/ghac/reader.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use http::StatusCode;
-
-use super::error::parse_error;
-use crate::raw::*;
-use crate::services::ghac::backend::GhacBackend;
-use crate::*;
-
-pub struct GhacReader {
- core: GhacBackend,
-
- location: String,
- _op: OpRead,
-}
-
-impl GhacReader {
- pub fn new(core: GhacBackend, location: &str, op: OpRead) -> Self {
- GhacReader {
- core,
- location: location.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for GhacReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let req = self.core.ghac_get_location(&self.location, range).await?;
- let resp = self.core.client.send(req).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/github/backend.rs b/core/src/services/github/backend.rs
index 4ef136493b..e46125f56a 100644
--- a/core/src/services/github/backend.rs
+++ b/core/src/services/github/backend.rs
@@ -21,6 +21,7 @@ use std::fmt::Formatter;
use std::sync::Arc;
use bytes::Buf;
+use http::Response;
use http::StatusCode;
use log::debug;
use serde::Deserialize;
@@ -32,7 +33,6 @@ use super::lister::GithubLister;
use super::writer::GithubWriter;
use super::writer::GithubWriters;
use crate::raw::*;
-use crate::services::github::reader::GithubReader;
use crate::*;
/// Config for backblaze Github services support.
@@ -217,7 +217,7 @@ pub struct GithubBackend {
}
impl Access for GithubBackend {
- type Reader = GithubReader;
+ type Reader = HttpBody;
type Writer = GithubWriters;
@@ -296,10 +296,20 @@ impl Access for GithubBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- GithubReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.get(path, args.range()).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, _args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/github/core.rs b/core/src/services/github/core.rs
index 86e3f762e9..ed76d61305 100644
--- a/core/src/services/github/core.rs
+++ b/core/src/services/github/core.rs
@@ -128,7 +128,7 @@ impl GithubCore {
self.send(req).await
}
- pub async fn get(&self, path: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn get(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
let path = build_abs_path(&self.root, path);
let url = format!(
@@ -148,7 +148,7 @@ impl GithubCore {
.body(Buffer::new())
.map_err(new_request_build_error)?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub async fn upload(&self, path: &str, bs: Buffer) -> Result<Response<Buffer>> {
diff --git a/core/src/services/github/mod.rs b/core/src/services/github/mod.rs
index 9855cb0808..06418c1934 100644
--- a/core/src/services/github/mod.rs
+++ b/core/src/services/github/mod.rs
@@ -22,5 +22,4 @@ pub use backend::GithubConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/github/reader.rs b/core/src/services/github/reader.rs
deleted file mode 100644
index 9425aa60b0..0000000000
--- a/core/src/services/github/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::GithubCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct GithubReader {
- core: Arc<GithubCore>,
-
- path: String,
- _op: OpRead,
-}
-
-impl GithubReader {
- pub fn new(core: Arc<GithubCore>, path: &str, op: OpRead) -> Self {
- GithubReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for GithubReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.get(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/hdfs/backend.rs b/core/src/services/hdfs/backend.rs
index 78b944f001..f0caf134d1 100644
--- a/core/src/services/hdfs/backend.rs
+++ b/core/src/services/hdfs/backend.rs
@@ -19,6 +19,7 @@ use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Formatter;
use std::io;
+use std::io::SeekFrom;
use std::path::PathBuf;
use std::sync::Arc;
@@ -244,10 +245,10 @@ unsafe impl Send for HdfsBackend {}
unsafe impl Sync for HdfsBackend {}
impl Access for HdfsBackend {
- type Reader = HdfsReader;
+ type Reader = HdfsReader<hdrs::AsyncFile>;
type Writer = HdfsWriter<hdrs::AsyncFile>;
type Lister = Option<HdfsLister>;
- type BlockingReader = HdfsReader;
+ type BlockingReader = HdfsReader<hdrs::File>;
type BlockingWriter = HdfsWriter<hdrs::File>;
type BlockingLister = Option<HdfsLister>;
@@ -304,30 +305,29 @@ impl Access for HdfsBackend {
Ok(RpStat::new(m))
}
- async fn read(&self, path: &str, _: OpRead) -> Result<(RpRead, Self::Reader)> {
+ async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
let p = build_rooted_abs_path(&self.root, path);
let client = self.client.clone();
- let f = match tokio::runtime::Handle::try_current() {
- Ok(runtime) => runtime
- .spawn_blocking(move || {
- client
- .open_file()
- .read(true)
- .open(&p)
- .map_err(new_std_io_error)
- })
+ let mut f = client
+ .open_file()
+ .read(true)
+ .async_open(&p)
+ .await
+ .map_err(new_std_io_error)?;
+
+ if args.range().offset() != 0 {
+ use futures::AsyncSeekExt;
+
+ f.seek(SeekFrom::Start(args.range().offset()))
.await
- .map_err(|err| {
- Error::new(ErrorKind::Unexpected, "tokio spawn io task failed").set_source(err)
- })?,
- Err(_) => Err(Error::new(
- ErrorKind::Unexpected,
- "no tokio runtime found, failed to run io task",
- )),
- }?;
-
- Ok((RpRead::new(), HdfsReader::new(f)))
+ .map_err(new_std_io_error)?;
+ }
+
+ Ok((
+ RpRead::new(),
+ HdfsReader::new(f, args.range().size().unwrap_or(u64::MAX) as _),
+ ))
}
async fn write(&self, path: &str, op: OpWrite) -> Result<(RpWrite, Self::Writer)> {
@@ -506,17 +506,27 @@ impl Access for HdfsBackend {
Ok(RpStat::new(m))
}
- fn blocking_read(&self, path: &str, _: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
+ fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
let p = build_rooted_abs_path(&self.root, path);
- let f = self
+ let mut f = self
.client
.open_file()
.read(true)
.open(&p)
.map_err(new_std_io_error)?;
- Ok((RpRead::new(), HdfsReader::new(f)))
+ if args.range().offset() != 0 {
+ use std::io::Seek;
+
+ f.seek(SeekFrom::Start(args.range().offset()))
+ .map_err(new_std_io_error)?;
+ }
+
+ Ok((
+ RpRead::new(),
+ HdfsReader::new(f, args.range().size().unwrap_or(u64::MAX) as _),
+ ))
}
fn blocking_write(&self, path: &str, op: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
diff --git a/core/src/services/hdfs/reader.rs b/core/src/services/hdfs/reader.rs
index 6f5565b687..32bad0a8c0 100644
--- a/core/src/services/hdfs/reader.rs
+++ b/core/src/services/hdfs/reader.rs
@@ -15,73 +15,100 @@
// specific language governing permissions and limitations
// under the License.
-use std::sync::Arc;
+use bytes::BytesMut;
-use hdrs::File;
+use futures::AsyncReadExt;
+use hdrs::{AsyncFile, File};
+use std::io::Read;
use tokio::io::ReadBuf;
use crate::raw::*;
use crate::*;
-pub struct HdfsReader {
- f: Arc<File>,
+pub struct HdfsReader<F> {
+ f: F,
+ read: usize,
+ size: usize,
+ buf_size: usize,
+ buf: BytesMut,
}
-impl HdfsReader {
- pub fn new(f: File) -> Self {
- Self { f: Arc::new(f) }
+impl<F> HdfsReader<F> {
+ pub fn new(f: F, size: usize) -> Self {
+ Self {
+ f,
+ read: 0,
+ size,
+ // Use 2 MiB as default value.
+ buf_size: 2 * 1024 * 1024,
+ buf: BytesMut::new(),
+ }
}
}
-impl oio::Read for HdfsReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let r = Self { f: self.f.clone() };
- match tokio::runtime::Handle::try_current() {
- Ok(runtime) => runtime
- .spawn_blocking(move || oio::BlockingRead::read_at(&r, offset, size))
- .await
- .map_err(|err| {
- Error::new(ErrorKind::Unexpected, "tokio spawn io task failed").set_source(err)
- })?,
- Err(_) => Err(Error::new(
- ErrorKind::Unexpected,
- "no tokio runtime found, failed to run io task",
- )),
+impl oio::Read for HdfsReader<AsyncFile> {
+ async fn read(&mut self) -> Result<Buffer> {
+ if self.read >= self.size {
+ return Ok(Buffer::new());
}
+
+ let size = (self.size - self.read).min(self.buf_size);
+ self.buf.reserve(size);
+
+ let buf = &mut self.buf.spare_capacity_mut()[..size];
+ let mut read_buf: ReadBuf = ReadBuf::uninit(buf);
+
+ // SAFETY: Read at most `limit` bytes into `read_buf`.
+ unsafe {
+ read_buf.assume_init(size);
+ }
+
+ let n = self
+ .f
+ .read(read_buf.initialize_unfilled())
+ .await
+ .map_err(new_std_io_error)?;
+ read_buf.advance(n);
+ self.read += n;
+
+ // Safety: We make sure that bs contains `n` more bytes.
+ let filled = read_buf.filled().len();
+ unsafe { self.buf.set_len(filled) }
+
+ let frozen = self.buf.split().freeze();
+ Ok(Buffer::from(frozen))
}
}
-impl oio::BlockingRead for HdfsReader {
- fn read_at(&self, mut offset: u64, size: usize) -> Result<Buffer> {
- let mut bs = Vec::with_capacity(size);
+impl oio::BlockingRead for HdfsReader<File> {
+ fn read(&mut self) -> Result<Buffer> {
+ if self.read >= self.size {
+ return Ok(Buffer::new());
+ }
+
+ let size = (self.size - self.read).min(self.buf_size);
+ self.buf.reserve(size);
- let buf = bs.spare_capacity_mut();
+ let buf = &mut self.buf.spare_capacity_mut()[..size];
let mut read_buf: ReadBuf = ReadBuf::uninit(buf);
- // SAFETY: Read at most `size` bytes into `read_buf`.
+ // SAFETY: Read at most `limit` bytes into `read_buf`.
unsafe {
read_buf.assume_init(size);
}
- loop {
- // If the buffer is full, we are done.
- if read_buf.initialize_unfilled().is_empty() {
- break;
- }
- let n = self
- .f
- .read_at(read_buf.initialize_unfilled(), offset)
- .map_err(new_std_io_error)?;
- if n == 0 {
- break;
- }
- read_buf.advance(n);
- offset += n as u64;
- }
+ let n = self
+ .f
+ .read(read_buf.initialize_unfilled())
+ .map_err(new_std_io_error)?;
+ read_buf.advance(n);
+ self.read += n;
// Safety: We make sure that bs contains `n` more bytes.
let filled = read_buf.filled().len();
- unsafe { bs.set_len(filled) }
- Ok(Buffer::from(bs))
+ unsafe { self.buf.set_len(filled) }
+
+ let frozen = self.buf.split().freeze();
+ Ok(Buffer::from(frozen))
}
}
diff --git a/core/src/services/hdfs_native/reader.rs b/core/src/services/hdfs_native/reader.rs
index 1554af2bcd..0a51eaf8c8 100644
--- a/core/src/services/hdfs_native/reader.rs
+++ b/core/src/services/hdfs_native/reader.rs
@@ -31,7 +31,7 @@ impl HdfsNativeReader {
}
impl oio::Read for HdfsNativeReader {
- async fn read_at(&self, _offset: u64, _limit: usize) -> Result<Buffer> {
+ async fn read(&mut self) -> Result<Buffer> {
todo!()
}
}
diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs
index 7ff36b1e94..6b2fce60f6 100644
--- a/core/src/services/http/backend.rs
+++ b/core/src/services/http/backend.rs
@@ -30,7 +30,6 @@ use serde::Deserialize;
use super::error::parse_error;
use crate::raw::*;
-use crate::services::http::reader::HttpReader;
use crate::*;
/// Config for Http service support.
@@ -222,7 +221,7 @@ impl Debug for HttpBackend {
}
impl Access for HttpBackend {
- type Reader = HttpReader;
+ type Reader = HttpBody;
type Writer = ();
type Lister = ();
type BlockingReader = ();
@@ -271,7 +270,20 @@ impl Access for HttpBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((RpRead::default(), HttpReader::new(self.clone(), path, args)))
+ let resp = self.http_get(path, args.range(), &args).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
}
@@ -281,7 +293,7 @@ impl HttpBackend {
path: &str,
range: BytesRange,
args: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let p = build_rooted_abs_path(&self.root, path);
let url = format!("{}{}", self.endpoint, percent_encode_path(&p));
@@ -306,7 +318,7 @@ impl HttpBackend {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
async fn http_head(&self, path: &str, args: &OpStat) -> Result<Response<Buffer>> {
diff --git a/core/src/services/http/mod.rs b/core/src/services/http/mod.rs
index 4bab91f879..7424d97c2b 100644
--- a/core/src/services/http/mod.rs
+++ b/core/src/services/http/mod.rs
@@ -20,4 +20,3 @@ pub use backend::HttpBuilder as Http;
pub use backend::HttpConfig;
mod error;
-mod reader;
diff --git a/core/src/services/http/reader.rs b/core/src/services/http/reader.rs
deleted file mode 100644
index e4a57722fa..0000000000
--- a/core/src/services/http/reader.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use http::StatusCode;
-
-use super::error::parse_error;
-use crate::raw::*;
-use crate::services::http::backend::HttpBackend;
-use crate::*;
-
-pub struct HttpReader {
- core: HttpBackend,
-
- path: String,
- op: OpRead,
-}
-
-impl HttpReader {
- pub fn new(core: HttpBackend, path: &str, op: OpRead) -> Self {
- HttpReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for HttpReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.http_get(&self.path, range, &self.op).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/huggingface/backend.rs b/core/src/services/huggingface/backend.rs
index 90fa4cf38b..93c26d3017 100644
--- a/core/src/services/huggingface/backend.rs
+++ b/core/src/services/huggingface/backend.rs
@@ -21,6 +21,7 @@ use std::fmt::Formatter;
use std::sync::Arc;
use bytes::Buf;
+use http::Response;
use http::StatusCode;
use log::debug;
use serde::Deserialize;
@@ -30,7 +31,6 @@ use super::core::HuggingfaceStatus;
use super::error::parse_error;
use super::lister::HuggingfaceLister;
use crate::raw::*;
-use crate::services::huggingface::reader::HuggingfaceReader;
use crate::*;
/// Configuration for Huggingface service support.
@@ -243,7 +243,7 @@ pub struct HuggingfaceBackend {
}
impl Access for HuggingfaceBackend {
- type Reader = HuggingfaceReader;
+ type Reader = HttpBody;
type Writer = ();
type Lister = oio::PageLister<HuggingfaceLister>;
type BlockingReader = ();
@@ -310,10 +310,20 @@ impl Access for HuggingfaceBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- HuggingfaceReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.hf_resolve(path, args.range(), &args).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
diff --git a/core/src/services/huggingface/core.rs b/core/src/services/huggingface/core.rs
index ff4e16856a..488d18e277 100644
--- a/core/src/services/huggingface/core.rs
+++ b/core/src/services/huggingface/core.rs
@@ -124,7 +124,7 @@ impl HuggingfaceCore {
path: &str,
range: BytesRange,
_args: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let p = build_abs_path(&self.root, path)
.trim_end_matches('/')
.to_string();
@@ -157,7 +157,7 @@ impl HuggingfaceCore {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
}
diff --git a/core/src/services/huggingface/mod.rs b/core/src/services/huggingface/mod.rs
index 73149c15ad..3a692fa45e 100644
--- a/core/src/services/huggingface/mod.rs
+++ b/core/src/services/huggingface/mod.rs
@@ -22,4 +22,3 @@ pub use backend::HuggingfaceConfig;
mod core;
mod error;
mod lister;
-mod reader;
diff --git a/core/src/services/huggingface/reader.rs b/core/src/services/huggingface/reader.rs
deleted file mode 100644
index 40e0a8d7f6..0000000000
--- a/core/src/services/huggingface/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::HuggingfaceCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct HuggingfaceReader {
- core: Arc<HuggingfaceCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl HuggingfaceReader {
- pub fn new(core: Arc<HuggingfaceCore>, path: &str, op: OpRead) -> Self {
- HuggingfaceReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for HuggingfaceReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.hf_resolve(&self.path, range, &self.op).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/icloud/backend.rs b/core/src/services/icloud/backend.rs
index fdf494b770..6f56e8ae95 100644
--- a/core/src/services/icloud/backend.rs
+++ b/core/src/services/icloud/backend.rs
@@ -20,12 +20,13 @@ use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
+use http::Response;
+use http::StatusCode;
use serde::Deserialize;
use tokio::sync::Mutex;
use super::core::*;
use crate::raw::*;
-use crate::services::icloud::reader::IcloudReader;
use crate::*;
/// Config for icloud services support.
@@ -265,7 +266,7 @@ pub struct IcloudBackend {
}
impl Access for IcloudBackend {
- type Reader = IcloudReader;
+ type Reader = HttpBody;
type BlockingReader = ();
type Writer = ();
type BlockingWriter = ();
@@ -310,9 +311,16 @@ impl Access for IcloudBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- IcloudReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.read(path, args.range(), &args).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
}
diff --git a/core/src/services/icloud/core.rs b/core/src/services/icloud/core.rs
index 9a55d4583b..5c4e44848f 100644
--- a/core/src/services/icloud/core.rs
+++ b/core/src/services/icloud/core.rs
@@ -373,7 +373,7 @@ impl IcloudCore {
zone: &str,
range: BytesRange,
args: OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let mut signer = self.signer.lock().await;
let uri = format!(
@@ -413,7 +413,7 @@ impl IcloudCore {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- let resp = signer.client.send(req).await?;
+ let resp = signer.client.fetch(req).await?;
Ok(resp)
}
@@ -423,7 +423,7 @@ impl IcloudCore {
path: &str,
range: BytesRange,
args: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let path = build_rooted_abs_path(&self.root, path);
let base = get_basename(&path);
diff --git a/core/src/services/icloud/mod.rs b/core/src/services/icloud/mod.rs
index 9366484735..d76c934ddd 100644
--- a/core/src/services/icloud/mod.rs
+++ b/core/src/services/icloud/mod.rs
@@ -19,4 +19,3 @@ mod backend;
pub use backend::IcloudBuilder as Icloud;
mod core;
-mod reader;
diff --git a/core/src/services/icloud/reader.rs b/core/src/services/icloud/reader.rs
deleted file mode 100644
index 84fd95fdf9..0000000000
--- a/core/src/services/icloud/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::parse_error;
-use super::core::IcloudCore;
-use crate::raw::*;
-use crate::*;
-
-pub struct IcloudReader {
- core: Arc<IcloudCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl IcloudReader {
- pub fn new(core: Arc<IcloudCore>, path: &str, op: OpRead) -> Self {
- IcloudReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for IcloudReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.read(&self.path, range, &self.op).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/ipfs/backend.rs b/core/src/services/ipfs/backend.rs
index a81f144138..fec0b37bb3 100644
--- a/core/src/services/ipfs/backend.rs
+++ b/core/src/services/ipfs/backend.rs
@@ -29,7 +29,6 @@ use prost::Message;
use super::error::parse_error;
use super::ipld::PBNode;
use crate::raw::*;
-use crate::services::ipfs::reader::IpfsReader;
use crate::*;
/// IPFS file system support based on [IPFS HTTP Gateway](https://docs.ipfs.tech/concepts/ipfs-gateway/).
@@ -160,7 +159,7 @@ impl Debug for IpfsBackend {
}
impl Access for IpfsBackend {
- type Reader = IpfsReader;
+ type Reader = HttpBody;
type Writer = ();
type Lister = oio::PageLister<DirStream>;
type BlockingReader = ();
@@ -336,7 +335,20 @@ impl Access for IpfsBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((RpRead::default(), IpfsReader::new(self.clone(), path, args)))
+ let resp = self.ipfs_get(path, args.range()).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Lister)> {
@@ -346,7 +358,7 @@ impl Access for IpfsBackend {
}
impl IpfsBackend {
- pub async fn ipfs_get(&self, path: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn ipfs_get(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
let p = build_rooted_abs_path(&self.root, path);
let url = format!("{}{}", self.endpoint, percent_encode_path(&p));
@@ -359,7 +371,7 @@ impl IpfsBackend {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
async fn ipfs_head(&self, path: &str) -> Result<Response<Buffer>> {
diff --git a/core/src/services/ipfs/mod.rs b/core/src/services/ipfs/mod.rs
index 85aeb674f0..4bd46822fb 100644
--- a/core/src/services/ipfs/mod.rs
+++ b/core/src/services/ipfs/mod.rs
@@ -20,4 +20,3 @@ pub use backend::IpfsBuilder as Ipfs;
mod error;
mod ipld;
-mod reader;
diff --git a/core/src/services/ipfs/reader.rs b/core/src/services/ipfs/reader.rs
deleted file mode 100644
index 4b7e0157d5..0000000000
--- a/core/src/services/ipfs/reader.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use http::StatusCode;
-
-use super::error::parse_error;
-use crate::raw::*;
-use crate::services::ipfs::backend::IpfsBackend;
-use crate::*;
-
-pub struct IpfsReader {
- core: IpfsBackend,
-
- path: String,
- _op: OpRead,
-}
-
-impl IpfsReader {
- pub fn new(core: IpfsBackend, path: &str, op: OpRead) -> Self {
- IpfsReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for IpfsReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.ipfs_get(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/ipmfs/backend.rs b/core/src/services/ipmfs/backend.rs
index 25916b777b..ae855d7b79 100644
--- a/core/src/services/ipmfs/backend.rs
+++ b/core/src/services/ipmfs/backend.rs
@@ -30,7 +30,6 @@ use super::error::parse_error;
use super::lister::IpmfsLister;
use super::writer::IpmfsWriter;
use crate::raw::*;
-use crate::services::ipmfs::reader::IpmfsReader;
use crate::*;
/// IPFS Mutable File System (IPMFS) backend.
@@ -62,7 +61,7 @@ impl IpmfsBackend {
}
impl Access for IpmfsBackend {
- type Reader = IpmfsReader;
+ type Reader = HttpBody;
type Writer = oio::OneShotWriter<IpmfsWriter>;
type Lister = oio::PageLister<IpmfsLister>;
type BlockingReader = ();
@@ -133,10 +132,20 @@ impl Access for IpmfsBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- IpmfsReader::new(self.clone(), path, args),
- ))
+ let resp = self.ipmfs_read(path, args.range()).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::Writer)> {
@@ -179,7 +188,7 @@ impl IpmfsBackend {
self.client.send(req).await
}
- pub async fn ipmfs_read(&self, path: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn ipmfs_read(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
let p = build_rooted_abs_path(&self.root, path);
let mut url = format!(
@@ -196,7 +205,7 @@ impl IpmfsBackend {
let req = Request::post(url);
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
async fn ipmfs_rm(&self, path: &str) -> Result<Response<Buffer>> {
diff --git a/core/src/services/ipmfs/mod.rs b/core/src/services/ipmfs/mod.rs
index 2e511206ca..2a7ce5c0ad 100644
--- a/core/src/services/ipmfs/mod.rs
+++ b/core/src/services/ipmfs/mod.rs
@@ -21,5 +21,4 @@ pub use builder::IpmfsBuilder as Ipmfs;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/ipmfs/reader.rs b/core/src/services/ipmfs/reader.rs
deleted file mode 100644
index 48449edd6a..0000000000
--- a/core/src/services/ipmfs/reader.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use http::StatusCode;
-
-use super::error::parse_error;
-use crate::raw::*;
-use crate::services::ipmfs::backend::IpmfsBackend;
-use crate::*;
-
-pub struct IpmfsReader {
- core: IpmfsBackend,
-
- path: String,
- _op: OpRead,
-}
-
-impl IpmfsReader {
- pub fn new(core: IpmfsBackend, path: &str, op: OpRead) -> Self {
- IpmfsReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for IpmfsReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.ipmfs_read(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/koofr/backend.rs b/core/src/services/koofr/backend.rs
index c26b0c1d48..90a6863ecc 100644
--- a/core/src/services/koofr/backend.rs
+++ b/core/src/services/koofr/backend.rs
@@ -21,6 +21,7 @@ use std::fmt::Formatter;
use std::sync::Arc;
use bytes::Buf;
+use http::Response;
use http::StatusCode;
use log::debug;
use serde::Deserialize;
@@ -35,7 +36,6 @@ use super::lister::KoofrLister;
use super::writer::KoofrWriter;
use super::writer::KoofrWriters;
use crate::raw::*;
-use crate::services::koofr::reader::KoofrReader;
use crate::*;
/// Config for backblaze Koofr services support.
@@ -235,7 +235,7 @@ pub struct KoofrBackend {
}
impl Access for KoofrBackend {
- type Reader = KoofrReader;
+ type Reader = HttpBody;
type Writer = KoofrWriters;
type Lister = oio::PageLister<KoofrLister>;
type BlockingReader = ();
@@ -310,10 +310,19 @@ impl Access for KoofrBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- KoofrReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.get(path, args.range()).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, _args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/koofr/core.rs b/core/src/services/koofr/core.rs
index 892baf0d68..6b2b66bfa6 100644
--- a/core/src/services/koofr/core.rs
+++ b/core/src/services/koofr/core.rs
@@ -238,7 +238,7 @@ impl KoofrCore {
self.send(req).await
}
- pub async fn get(&self, path: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn get(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
let path = build_rooted_abs_path(&self.root, path);
let mount_id = self.get_mount_id().await?;
@@ -256,7 +256,7 @@ impl KoofrCore {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub async fn put(&self, path: &str, bs: Buffer) -> Result<Response<Buffer>> {
diff --git a/core/src/services/koofr/mod.rs b/core/src/services/koofr/mod.rs
index fce486b246..445d69fbb8 100644
--- a/core/src/services/koofr/mod.rs
+++ b/core/src/services/koofr/mod.rs
@@ -22,5 +22,4 @@ pub use backend::KoofrConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/koofr/reader.rs b/core/src/services/koofr/reader.rs
deleted file mode 100644
index 36c2bc0fcc..0000000000
--- a/core/src/services/koofr/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::KoofrCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct KoofrReader {
- core: Arc<KoofrCore>,
-
- path: String,
- _op: OpRead,
-}
-
-impl KoofrReader {
- pub fn new(core: Arc<KoofrCore>, path: &str, op: OpRead) -> Self {
- KoofrReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for KoofrReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.get(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs
index efeced3436..8f21522329 100644
--- a/core/src/services/obs/backend.rs
+++ b/core/src/services/obs/backend.rs
@@ -19,6 +19,7 @@ use std::collections::HashMap;
use std::fmt::Debug;
use std::sync::Arc;
+use http::Response;
use http::StatusCode;
use http::Uri;
use log::debug;
@@ -31,7 +32,6 @@ use super::error::parse_error;
use super::lister::ObsLister;
use super::writer::ObsWriter;
use crate::raw::*;
-use crate::services::obs::reader::ObsReader;
use crate::services::obs::writer::ObsWriters;
use crate::*;
@@ -247,7 +247,7 @@ pub struct ObsBackend {
}
impl Access for ObsBackend {
- type Reader = ObsReader;
+ type Reader = HttpBody;
type Writer = ObsWriters;
type Lister = oio::PageLister<ObsLister>;
type BlockingReader = ();
@@ -321,10 +321,20 @@ impl Access for ObsBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- ObsReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.obs_get_object(path, args.range(), &args).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/obs/core.rs b/core/src/services/obs/core.rs
index f735b2632d..e331f327c7 100644
--- a/core/src/services/obs/core.rs
+++ b/core/src/services/obs/core.rs
@@ -106,12 +106,12 @@ impl ObsCore {
path: &str,
range: BytesRange,
args: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let mut req = self.obs_get_object_request(path, range, args)?;
self.sign(&mut req).await?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub fn obs_get_object_request(
diff --git a/core/src/services/obs/mod.rs b/core/src/services/obs/mod.rs
index 93eb1328da..f8a78290f0 100644
--- a/core/src/services/obs/mod.rs
+++ b/core/src/services/obs/mod.rs
@@ -21,5 +21,4 @@ pub use backend::ObsBuilder as Obs;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/obs/reader.rs b/core/src/services/obs/reader.rs
deleted file mode 100644
index 55e9468482..0000000000
--- a/core/src/services/obs/reader.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::ObsCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct ObsReader {
- core: Arc<ObsCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl ObsReader {
- pub fn new(core: Arc<ObsCore>, path: &str, op: OpRead) -> Self {
- ObsReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for ObsReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self
- .core
- .obs_get_object(&self.path, range, &self.op)
- .await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs
index 1fe1c9766a..9ba74b5bbc 100644
--- a/core/src/services/onedrive/backend.rs
+++ b/core/src/services/onedrive/backend.rs
@@ -32,7 +32,6 @@ use super::graph_model::OnedriveGetItemBody;
use super::lister::OnedriveLister;
use super::writer::OneDriveWriter;
use crate::raw::*;
-use crate::services::onedrive::reader::OnedriveReader;
use crate::*;
#[derive(Clone)]
@@ -62,7 +61,7 @@ impl Debug for OnedriveBackend {
}
impl Access for OnedriveBackend {
- type Reader = OnedriveReader;
+ type Reader = HttpBody;
type Writer = oio::OneShotWriter<OneDriveWriter>;
type Lister = oio::PageLister<OnedriveLister>;
type BlockingReader = ();
@@ -150,10 +149,20 @@ impl Access for OnedriveBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- OnedriveReader::new(self.clone(), path, args),
- ))
+ let resp = self.onedrive_get_content(path, args.range()).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
@@ -221,7 +230,7 @@ impl OnedriveBackend {
&self,
path: &str,
range: BytesRange,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let path = build_rooted_abs_path(&self.root, path);
let url: String = format!(
"https://graph.microsoft.com/v1.0/me/drive/root:{}{}",
@@ -236,7 +245,7 @@ impl OnedriveBackend {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
pub async fn onedrive_upload_simple(
diff --git a/core/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs
index b5211c55e4..672fb6c7e1 100644
--- a/core/src/services/onedrive/mod.rs
+++ b/core/src/services/onedrive/mod.rs
@@ -24,5 +24,4 @@ pub use builder::OnedriveBuilder as Onedrive;
pub use builder::OnedriveConfig;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/onedrive/reader.rs b/core/src/services/onedrive/reader.rs
deleted file mode 100644
index 8a8bbe987e..0000000000
--- a/core/src/services/onedrive/reader.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use http::StatusCode;
-
-use super::error::parse_error;
-use crate::raw::*;
-use crate::services::onedrive::backend::OnedriveBackend;
-use crate::*;
-
-pub struct OnedriveReader {
- core: OnedriveBackend,
-
- path: String,
- _op: OpRead,
-}
-
-impl OnedriveReader {
- pub fn new(core: OnedriveBackend, path: &str, op: OpRead) -> Self {
- OnedriveReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for OnedriveReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.onedrive_get_content(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs
index fc57fabcf4..c76dc665b5 100644
--- a/core/src/services/oss/backend.rs
+++ b/core/src/services/oss/backend.rs
@@ -22,6 +22,7 @@ use std::fmt::Formatter;
use std::sync::Arc;
use bytes::Buf;
+use http::Response;
use http::StatusCode;
use http::Uri;
use log::debug;
@@ -34,7 +35,6 @@ use super::error::parse_error;
use super::lister::OssLister;
use super::writer::OssWriter;
use crate::raw::*;
-use crate::services::oss::reader::OssReader;
use crate::services::oss::writer::OssWriters;
use crate::*;
@@ -375,7 +375,7 @@ pub struct OssBackend {
}
impl Access for OssBackend {
- type Reader = OssReader;
+ type Reader = HttpBody;
type Writer = OssWriters;
type Lister = oio::PageLister<OssLister>;
type BlockingReader = ();
@@ -454,10 +454,20 @@ impl Access for OssBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- OssReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.oss_get_object(path, args.range(), &args).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/oss/core.rs b/core/src/services/oss/core.rs
index 585b8dbea1..229ef050d0 100644
--- a/core/src/services/oss/core.rs
+++ b/core/src/services/oss/core.rs
@@ -363,10 +363,10 @@ impl OssCore {
path: &str,
range: BytesRange,
args: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let mut req = self.oss_get_object_request(path, range, false, args)?;
self.sign(&mut req).await?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub async fn oss_head_object(
diff --git a/core/src/services/oss/mod.rs b/core/src/services/oss/mod.rs
index 03b2f9612a..3954a9885c 100644
--- a/core/src/services/oss/mod.rs
+++ b/core/src/services/oss/mod.rs
@@ -21,5 +21,4 @@ pub use backend::OssBuilder as Oss;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/oss/reader.rs b/core/src/services/oss/reader.rs
deleted file mode 100644
index eceb1d1684..0000000000
--- a/core/src/services/oss/reader.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::OssCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct OssReader {
- core: Arc<OssCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl OssReader {
- pub fn new(core: Arc<OssCore>, path: &str, op: OpRead) -> Self {
- OssReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for OssReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self
- .core
- .oss_get_object(&self.path, range, &self.op)
- .await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/pcloud/backend.rs b/core/src/services/pcloud/backend.rs
index c53147229a..d924453929 100644
--- a/core/src/services/pcloud/backend.rs
+++ b/core/src/services/pcloud/backend.rs
@@ -21,6 +21,7 @@ use std::fmt::Formatter;
use std::sync::Arc;
use bytes::Buf;
+use http::Response;
use http::StatusCode;
use log::debug;
use serde::Deserialize;
@@ -32,7 +33,6 @@ use super::lister::PcloudLister;
use super::writer::PcloudWriter;
use super::writer::PcloudWriters;
use crate::raw::*;
-use crate::services::pcloud::reader::PcloudReader;
use crate::*;
/// Config for backblaze Pcloud services support.
@@ -228,7 +228,7 @@ pub struct PcloudBackend {
}
impl Access for PcloudBackend {
- type Reader = PcloudReader;
+ type Reader = HttpBody;
type Writer = PcloudWriters;
type Lister = oio::PageLister<PcloudLister>;
type BlockingReader = ();
@@ -297,10 +297,20 @@ impl Access for PcloudBackend {
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
let link = self.core.get_file_link(path).await?;
- Ok((
- RpRead::default(),
- PcloudReader::new(self.core.clone(), &link, args),
- ))
+ let resp = self.core.download(&link, args.range()).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, _args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/pcloud/core.rs b/core/src/services/pcloud/core.rs
index 4a146d3697..91c32b09ea 100644
--- a/core/src/services/pcloud/core.rs
+++ b/core/src/services/pcloud/core.rs
@@ -107,7 +107,7 @@ impl PcloudCore {
}
}
- pub async fn download(&self, url: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn download(&self, url: &str, range: BytesRange) -> Result<Response<HttpBody>> {
let req = Request::get(url);
// set body
@@ -116,7 +116,7 @@ impl PcloudCore {
.body(Buffer::new())
.map_err(new_request_build_error)?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub async fn ensure_dir_exists(&self, path: &str) -> Result<()> {
diff --git a/core/src/services/pcloud/mod.rs b/core/src/services/pcloud/mod.rs
index 5ec313a14f..50fb5e5f53 100644
--- a/core/src/services/pcloud/mod.rs
+++ b/core/src/services/pcloud/mod.rs
@@ -22,5 +22,4 @@ pub use backend::PcloudConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/pcloud/reader.rs b/core/src/services/pcloud/reader.rs
deleted file mode 100644
index 81288ab2f2..0000000000
--- a/core/src/services/pcloud/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::PcloudCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct PcloudReader {
- core: Arc<PcloudCore>,
-
- link: String,
- _op: OpRead,
-}
-
-impl PcloudReader {
- pub fn new(core: Arc<PcloudCore>, link: &str, op: OpRead) -> Self {
- PcloudReader {
- core,
- link: link.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for PcloudReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.download(&self.link, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 2f3a49b1f2..c9a1846435 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -26,7 +26,7 @@ use std::sync::Arc;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use bytes::Buf;
-use http::StatusCode;
+use http::{Response, StatusCode};
use log::debug;
use log::warn;
use md5::Digest;
@@ -44,7 +44,6 @@ use super::core::*;
use super::error::parse_error;
use super::error::parse_s3_error_code;
use super::lister::S3Lister;
-use super::reader::S3Reader;
use super::writer::S3Writer;
use super::writer::S3Writers;
use crate::raw::*;
@@ -1019,7 +1018,7 @@ pub struct S3Backend {
}
impl Access for S3Backend {
- type Reader = S3Reader;
+ type Reader = HttpBody;
type Writer = S3Writers;
type Lister = oio::PageLister<S3Lister>;
type BlockingReader = ();
@@ -1099,10 +1098,19 @@ impl Access for S3Backend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- S3Reader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.s3_get_object(path, args.range(), &args).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs
index be4f7c34f0..708554e451 100644
--- a/core/src/services/s3/core.rs
+++ b/core/src/services/s3/core.rs
@@ -398,12 +398,12 @@ impl S3Core {
path: &str,
range: BytesRange,
args: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let mut req = self.s3_get_object_request(path, range, args)?;
self.sign(&mut req).await?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub fn s3_put_object_request(
diff --git a/core/src/services/s3/mod.rs b/core/src/services/s3/mod.rs
index 01a986b6bb..4e2f283cb2 100644
--- a/core/src/services/s3/mod.rs
+++ b/core/src/services/s3/mod.rs
@@ -22,5 +22,4 @@ pub use backend::S3Config;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/s3/reader.rs b/core/src/services/s3/reader.rs
deleted file mode 100644
index db0bacb294..0000000000
--- a/core/src/services/s3/reader.rs
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::*;
-use super::error::*;
-use crate::raw::*;
-use crate::*;
-
-pub struct S3Reader {
- core: Arc<S3Core>,
-
- path: String,
- op: OpRead,
-}
-
-impl S3Reader {
- pub fn new(core: Arc<S3Core>, path: &str, op: OpRead) -> Self {
- S3Reader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for S3Reader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
- let resp = self.core.s3_get_object(&self.path, range, &self.op).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/seafile/backend.rs b/core/src/services/seafile/backend.rs
index e9f4a9ac3e..ca6e3f381a 100644
--- a/core/src/services/seafile/backend.rs
+++ b/core/src/services/seafile/backend.rs
@@ -20,6 +20,8 @@ use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
+use http::Response;
+use http::StatusCode;
use log::debug;
use serde::Deserialize;
use tokio::sync::RwLock;
@@ -27,12 +29,12 @@ use tokio::sync::RwLock;
use super::core::parse_dir_detail;
use super::core::parse_file_detail;
use super::core::SeafileCore;
+use super::error::parse_error;
use super::lister::SeafileLister;
use super::writer::SeafileWriter;
use super::writer::SeafileWriters;
use crate::raw::*;
use crate::services::seafile::core::SeafileSigner;
-use crate::services::seafile::reader::SeafileReader;
use crate::*;
/// Config for backblaze seafile services support.
@@ -253,7 +255,7 @@ pub struct SeafileBackend {
}
impl Access for SeafileBackend {
- type Reader = SeafileReader;
+ type Reader = HttpBody;
type Writer = SeafileWriters;
type Lister = oio::PageLister<SeafileLister>;
type BlockingReader = ();
@@ -300,10 +302,20 @@ impl Access for SeafileBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- SeafileReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.download_file(path, args.range()).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/seafile/core.rs b/core/src/services/seafile/core.rs
index 9962cde98f..db712cc170 100644
--- a/core/src/services/seafile/core.rs
+++ b/core/src/services/seafile/core.rs
@@ -218,7 +218,7 @@ impl SeafileCore {
}
/// download file
- pub async fn download_file(&self, path: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn download_file(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
let download_url = self.get_download_url(path).await?;
let req = Request::get(download_url);
@@ -228,7 +228,7 @@ impl SeafileCore {
.body(Buffer::new())
.map_err(new_request_build_error)?;
- self.send(req).await
+ self.client.fetch(req).await
}
/// file detail
diff --git a/core/src/services/seafile/mod.rs b/core/src/services/seafile/mod.rs
index 63a112ee7b..dc9e1ccce0 100644
--- a/core/src/services/seafile/mod.rs
+++ b/core/src/services/seafile/mod.rs
@@ -22,5 +22,4 @@ pub use backend::SeafileConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/seafile/reader.rs b/core/src/services/seafile/reader.rs
deleted file mode 100644
index bb125e3069..0000000000
--- a/core/src/services/seafile/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::SeafileCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct SeafileReader {
- core: Arc<SeafileCore>,
-
- path: String,
- _op: OpRead,
-}
-
-impl SeafileReader {
- pub fn new(core: Arc<SeafileCore>, path: &str, op: OpRead) -> Self {
- SeafileReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for SeafileReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.download_file(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs
index d5012e8b85..24276207e8 100644
--- a/core/src/services/sftp/backend.rs
+++ b/core/src/services/sftp/backend.rs
@@ -18,6 +18,7 @@
use std::collections::HashMap;
use std::fmt::Debug;
use std::fmt::Formatter;
+use std::io::SeekFrom;
use std::path::Path;
use std::path::PathBuf;
@@ -29,6 +30,7 @@ use openssh::SessionBuilder;
use openssh_sftp_client::Sftp;
use openssh_sftp_client::SftpOptions;
use serde::Deserialize;
+use tokio::io::AsyncSeekExt;
use tokio::sync::OnceCell;
use super::error::is_not_found;
@@ -409,10 +411,28 @@ impl Access for SftpBackend {
Ok(RpStat::new(meta))
}
- async fn read(&self, path: &str, _: OpRead) -> Result<(RpRead, Self::Reader)> {
+ async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+ let client = self.connect().await?;
+
+ let mut fs = client.fs();
+ fs.set_cwd(&self.root);
+
+ let path = fs.canonicalize(path).await.map_err(parse_sftp_error)?;
+
+ let mut f = client
+ .open(path.as_path())
+ .await
+ .map_err(parse_sftp_error)?;
+
+ if args.range().offset() != 0 {
+ f.seek(SeekFrom::Start(args.range().offset()))
+ .await
+ .map_err(new_std_io_error)?;
+ }
+
Ok((
RpRead::default(),
- SftpReader::new(self.clone(), self.root.clone(), path.to_owned()),
+ SftpReader::new(client, f, args.range().size().unwrap_or(u64::MAX) as _),
))
}
diff --git a/core/src/services/sftp/reader.rs b/core/src/services/sftp/reader.rs
index bdbe34a0c5..e64dfeac03 100644
--- a/core/src/services/sftp/reader.rs
+++ b/core/src/services/sftp/reader.rs
@@ -15,68 +15,86 @@
// specific language governing permissions and limitations
// under the License.
-use std::io::SeekFrom;
+use bb8::PooledConnection;
use bytes::BytesMut;
-use tokio::io::AsyncSeekExt;
+use openssh_sftp_client::file::File;
-use super::backend::SftpBackend;
+use super::backend::Manager;
use super::error::parse_sftp_error;
use crate::raw::*;
use crate::*;
pub struct SftpReader {
- inner: SftpBackend,
- root: String,
- path: String,
+ /// Keep the connection alive while data stream is alive.
+ _conn: PooledConnection<'static, Manager>,
+
+ file: File,
+ chunk: usize,
+ size: usize,
+ read: usize,
+ buf: BytesMut,
}
impl SftpReader {
- pub fn new(inner: SftpBackend, root: String, path: String) -> Self {
- Self { inner, root, path }
+ pub fn new(conn: PooledConnection<'static, Manager>, file: File, size: usize) -> Self {
+ Self {
+ _conn: conn,
+ file,
+ size,
+ chunk: 2 * 1024 * 1024,
+ read: 0,
+ buf: BytesMut::new(),
+ }
}
}
impl oio::Read for SftpReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let client = self.inner.connect().await?;
-
- let mut fs = client.fs();
- fs.set_cwd(&self.root);
+ async fn read(&mut self) -> Result<Buffer> {
+ // let client = self.inner.connect().await?;
+ //
+ // let mut fs = client.fs();
+ // fs.set_cwd(&self.root);
+ //
+ // let path = fs
+ // .canonicalize(&self.path)
+ // .await
+ // .map_err(parse_sftp_error)?;
+ //
+ // let mut f = client
+ // .open(path.as_path())
+ // .await
+ // .map_err(parse_sftp_error)?;
- let path = fs
- .canonicalize(&self.path)
- .await
- .map_err(parse_sftp_error)?;
+ // f.seek(SeekFrom::Start(offset))
+ // .await
+ // .map_err(new_std_io_error)?;
- let mut f = client
- .open(path.as_path())
- .await
- .map_err(parse_sftp_error)?;
+ // let mut size = size;
+ // if size == 0 {
+ // return Ok(Buffer::new());
+ // }
- f.seek(SeekFrom::Start(offset))
- .await
- .map_err(new_std_io_error)?;
-
- let mut size = size;
- if size == 0 {
+ if self.read >= self.size {
return Ok(Buffer::new());
}
- let mut buf = BytesMut::with_capacity(size);
- while size > 0 {
- let len = buf.len();
- if let Some(bytes) = f
- .read(size as u32, buf.split_off(len))
- .await
- .map_err(parse_sftp_error)?
- {
- size -= bytes.len();
- buf.unsplit(bytes);
- } else {
- break;
- }
- }
- Ok(Buffer::from(buf.freeze()))
+ let size = (self.size - self.read).min(self.chunk);
+ self.buf.reserve(size);
+
+ let Some(bytes) = self
+ .file
+ .read(size as u32, self.buf.split_off(size))
+ .await
+ .map_err(parse_sftp_error)?
+ else {
+ return Err(Error::new(
+ ErrorKind::RangeNotSatisfied,
+ "sftp read file reaching EoF",
+ ));
+ };
+
+ self.read += bytes.len();
+ Ok(Buffer::from(bytes.freeze()))
}
}
diff --git a/core/src/services/supabase/backend.rs b/core/src/services/supabase/backend.rs
index 90ca76d30e..e38b46b8f8 100644
--- a/core/src/services/supabase/backend.rs
+++ b/core/src/services/supabase/backend.rs
@@ -18,14 +18,13 @@
use std::fmt::Debug;
use std::sync::Arc;
-use http::StatusCode;
+use http::{Response, StatusCode};
use log::debug;
use super::core::*;
use super::error::parse_error;
use super::writer::*;
use crate::raw::*;
-use crate::services::supabase::reader::SupabaseReader;
use crate::*;
/// [Supabase](https://supabase.com/) service support
@@ -155,7 +154,7 @@ pub struct SupabaseBackend {
}
impl Access for SupabaseBackend {
- type Reader = SupabaseReader;
+ type Reader = HttpBody;
type Writer = oio::OneShotWriter<SupabaseWriter>;
// todo: implement Lister to support list
type Lister = ();
@@ -203,10 +202,18 @@ impl Access for SupabaseBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- SupabaseReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.supabase_get_object(path, args.range()).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/supabase/core.rs b/core/src/services/supabase/core.rs
index 9be0d0c9f1..41e95f7a29 100644
--- a/core/src/services/supabase/core.rs
+++ b/core/src/services/supabase/core.rs
@@ -226,14 +226,14 @@ impl SupabaseCore {
&self,
path: &str,
range: BytesRange,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let mut req = if self.key.is_some() {
self.supabase_get_object_auth_request(path, range)?
} else {
self.supabase_get_object_public_request(path, range)?
};
self.sign(&mut req)?;
- self.send(req).await
+ self.http_client.fetch(req).await
}
pub async fn supabase_head_object(&self, path: &str) -> Result<Response<Buffer>> {
diff --git a/core/src/services/supabase/mod.rs b/core/src/services/supabase/mod.rs
index 5abaac6b85..89aeada889 100644
--- a/core/src/services/supabase/mod.rs
+++ b/core/src/services/supabase/mod.rs
@@ -19,5 +19,4 @@ mod backend;
pub use backend::SupabaseBuilder as Supabase;
mod core;
mod error;
-mod reader;
mod writer;
diff --git a/core/src/services/supabase/reader.rs b/core/src/services/supabase/reader.rs
deleted file mode 100644
index d7e662a350..0000000000
--- a/core/src/services/supabase/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::SupabaseCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct SupabaseReader {
- core: Arc<SupabaseCore>,
-
- path: String,
- _op: OpRead,
-}
-
-impl SupabaseReader {
- pub fn new(core: Arc<SupabaseCore>, path: &str, op: OpRead) -> Self {
- SupabaseReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for SupabaseReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.supabase_get_object(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/swift/backend.rs b/core/src/services/swift/backend.rs
index 5b8707c20c..f5e9d51b66 100644
--- a/core/src/services/swift/backend.rs
+++ b/core/src/services/swift/backend.rs
@@ -20,6 +20,7 @@ use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
+use http::Response;
use http::StatusCode;
use log::debug;
@@ -28,7 +29,6 @@ use super::error::parse_error;
use super::lister::SwiftLister;
use super::writer::SwiftWriter;
use crate::raw::*;
-use crate::services::swift::reader::SwiftReader;
use crate::*;
/// [OpenStack Swift](https://docs.openstack.org/api-ref/object-store/#)'s REST API support.
@@ -189,7 +189,7 @@ pub struct SwiftBackend {
}
impl Access for SwiftBackend {
- type Reader = SwiftReader;
+ type Reader = HttpBody;
type Writer = oio::OneShotWriter<SwiftWriter>;
type Lister = oio::PageLister<SwiftLister>;
type BlockingReader = ();
@@ -232,10 +232,18 @@ impl Access for SwiftBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- SwiftReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.swift_read(path, args.range(), &args).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/swift/core.rs b/core/src/services/swift/core.rs
index 18009e504e..7e59e90465 100644
--- a/core/src/services/swift/core.rs
+++ b/core/src/services/swift/core.rs
@@ -130,7 +130,7 @@ impl SwiftCore {
path: &str,
range: BytesRange,
_arg: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let p = build_abs_path(&self.root, path)
.trim_end_matches('/')
.to_string();
@@ -152,7 +152,7 @@ impl SwiftCore {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
pub async fn swift_copy(&self, src_p: &str, dst_p: &str) -> Result<Response<Buffer>> {
diff --git a/core/src/services/swift/mod.rs b/core/src/services/swift/mod.rs
index c4d08c8c44..9cfeb92e07 100644
--- a/core/src/services/swift/mod.rs
+++ b/core/src/services/swift/mod.rs
@@ -21,5 +21,4 @@ pub use backend::SwiftBuilder as Swift;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/swift/reader.rs b/core/src/services/swift/reader.rs
deleted file mode 100644
index 2c83f77408..0000000000
--- a/core/src/services/swift/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::SwiftCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct SwiftReader {
- core: Arc<SwiftCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl SwiftReader {
- pub fn new(core: Arc<SwiftCore>, path: &str, op: OpRead) -> Self {
- SwiftReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for SwiftReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.swift_read(&self.path, range, &self.op).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/upyun/backend.rs b/core/src/services/upyun/backend.rs
index 4603964665..aaffb18e63 100644
--- a/core/src/services/upyun/backend.rs
+++ b/core/src/services/upyun/backend.rs
@@ -20,6 +20,7 @@ use std::fmt::Debug;
use std::fmt::Formatter;
use std::sync::Arc;
+use http::Response;
use http::StatusCode;
use log::debug;
use serde::Deserialize;
@@ -32,7 +33,6 @@ use super::writer::UpyunWriter;
use super::writer::UpyunWriters;
use crate::raw::*;
use crate::services::upyun::core::UpyunSigner;
-use crate::services::upyun::reader::UpyunReader;
use crate::*;
/// Config for backblaze upyun services support.
@@ -232,7 +232,7 @@ pub struct UpyunBackend {
}
impl Access for UpyunBackend {
- type Reader = UpyunReader;
+ type Reader = HttpBody;
type Writer = UpyunWriters;
type Lister = oio::PageLister<UpyunLister>;
type BlockingReader = ();
@@ -296,10 +296,20 @@ impl Access for UpyunBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- UpyunReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.download_file(path, args.range()).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/upyun/core.rs b/core/src/services/upyun/core.rs
index 9c0442fa19..8f6f9f905d 100644
--- a/core/src/services/upyun/core.rs
+++ b/core/src/services/upyun/core.rs
@@ -104,7 +104,7 @@ impl UpyunCore {
}
impl UpyunCore {
- pub async fn download_file(&self, path: &str, range: BytesRange) -> Result<Response<Buffer>> {
+ pub async fn download_file(&self, path: &str, range: BytesRange) -> Result<Response<HttpBody>> {
let path = build_abs_path(&self.root, path);
let url = format!(
@@ -122,7 +122,7 @@ impl UpyunCore {
self.sign(&mut req).await?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub async fn info(&self, path: &str) -> Result<Response<Buffer>> {
diff --git a/core/src/services/upyun/mod.rs b/core/src/services/upyun/mod.rs
index a390359a71..039f2aa229 100644
--- a/core/src/services/upyun/mod.rs
+++ b/core/src/services/upyun/mod.rs
@@ -22,5 +22,4 @@ pub use backend::UpyunConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/upyun/reader.rs b/core/src/services/upyun/reader.rs
deleted file mode 100644
index 9a5ec37c46..0000000000
--- a/core/src/services/upyun/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::UpyunCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct UpyunReader {
- core: Arc<UpyunCore>,
-
- path: String,
- _op: OpRead,
-}
-
-impl UpyunReader {
- pub fn new(core: Arc<UpyunCore>, path: &str, op: OpRead) -> Self {
- UpyunReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for UpyunReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.download_file(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/vercel_artifacts/backend.rs b/core/src/services/vercel_artifacts/backend.rs
index 91530b9542..1967bab671 100644
--- a/core/src/services/vercel_artifacts/backend.rs
+++ b/core/src/services/vercel_artifacts/backend.rs
@@ -25,7 +25,6 @@ use http::StatusCode;
use super::error::parse_error;
use super::writer::VercelArtifactsWriter;
use crate::raw::*;
-use crate::services::vercel_artifacts::reader::VercelArtifactsReader;
use crate::*;
#[doc = include_str!("docs.md")]
@@ -44,7 +43,7 @@ impl Debug for VercelArtifactsBackend {
}
impl Access for VercelArtifactsBackend {
- type Reader = VercelArtifactsReader;
+ type Reader = HttpBody;
type Writer = oio::OneShotWriter<VercelArtifactsWriter>;
type Lister = ();
type BlockingReader = ();
@@ -83,10 +82,18 @@ impl Access for VercelArtifactsBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- VercelArtifactsReader::new(self.clone(), path, args),
- ))
+ let resp = self.vercel_artifacts_get(path, args.range(), &args).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
@@ -107,7 +114,7 @@ impl VercelArtifactsBackend {
hash: &str,
range: BytesRange,
_: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let url: String = format!(
"https://api.vercel.com/v8/artifacts/{}",
percent_encode_path(hash)
@@ -124,7 +131,7 @@ impl VercelArtifactsBackend {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
pub async fn vercel_artifacts_put(
diff --git a/core/src/services/vercel_artifacts/mod.rs b/core/src/services/vercel_artifacts/mod.rs
index 656bbfdc5d..3bfddabca7 100644
--- a/core/src/services/vercel_artifacts/mod.rs
+++ b/core/src/services/vercel_artifacts/mod.rs
@@ -18,7 +18,6 @@
mod backend;
mod builder;
mod error;
-mod reader;
mod writer;
pub use builder::VercelArtifactsBuilder as VercelArtifacts;
diff --git a/core/src/services/vercel_artifacts/reader.rs b/core/src/services/vercel_artifacts/reader.rs
deleted file mode 100644
index de6dac1cdb..0000000000
--- a/core/src/services/vercel_artifacts/reader.rs
+++ /dev/null
@@ -1,58 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use http::StatusCode;
-
-use super::error::parse_error;
-use crate::raw::*;
-use crate::services::vercel_artifacts::backend::VercelArtifactsBackend;
-use crate::*;
-
-pub struct VercelArtifactsReader {
- core: VercelArtifactsBackend,
-
- path: String,
- op: OpRead,
-}
-
-impl VercelArtifactsReader {
- pub fn new(core: VercelArtifactsBackend, path: &str, op: OpRead) -> Self {
- VercelArtifactsReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for VercelArtifactsReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self
- .core
- .vercel_artifacts_get(&self.path, range, &self.op)
- .await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/vercel_blob/backend.rs b/core/src/services/vercel_blob/backend.rs
index 51489a702d..9edeb651ac 100644
--- a/core/src/services/vercel_blob/backend.rs
+++ b/core/src/services/vercel_blob/backend.rs
@@ -21,6 +21,7 @@ use std::fmt::Formatter;
use std::sync::Arc;
use bytes::Buf;
+use http::Response;
use http::StatusCode;
use log::debug;
use serde::Deserialize;
@@ -33,7 +34,6 @@ use super::lister::VercelBlobLister;
use super::writer::VercelBlobWriter;
use super::writer::VercelBlobWriters;
use crate::raw::*;
-use crate::services::vercel_blob::reader::VercelBlobReader;
use crate::*;
/// Config for backblaze VercelBlob services support.
@@ -178,7 +178,7 @@ pub struct VercelBlobBackend {
}
impl Access for VercelBlobBackend {
- type Reader = VercelBlobReader;
+ type Reader = HttpBody;
type Writer = VercelBlobWriters;
type Lister = oio::PageLister<VercelBlobLister>;
type BlockingReader = ();
@@ -230,10 +230,20 @@ impl Access for VercelBlobBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- VercelBlobReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.download(path, args.range(), &args).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/vercel_blob/core.rs b/core/src/services/vercel_blob/core.rs
index 2c1135615d..58b77b480d 100644
--- a/core/src/services/vercel_blob/core.rs
+++ b/core/src/services/vercel_blob/core.rs
@@ -89,7 +89,7 @@ impl VercelBlobCore {
path: &str,
range: BytesRange,
_: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let p = build_abs_path(&self.root, path);
// Vercel blob use an unguessable random id url to download the file
// So we use list to get the url of the file and then use it to download the file
@@ -111,7 +111,7 @@ impl VercelBlobCore {
// Set body
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.send(req).await
+ self.client.fetch(req).await
}
pub async fn get_put_request(
diff --git a/core/src/services/vercel_blob/mod.rs b/core/src/services/vercel_blob/mod.rs
index 050f922abb..2804d3fe73 100644
--- a/core/src/services/vercel_blob/mod.rs
+++ b/core/src/services/vercel_blob/mod.rs
@@ -22,5 +22,4 @@ pub use backend::VercelBlobConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/vercel_blob/reader.rs b/core/src/services/vercel_blob/reader.rs
deleted file mode 100644
index a76c651922..0000000000
--- a/core/src/services/vercel_blob/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::VercelBlobCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct VercelBlobReader {
- core: Arc<VercelBlobCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl VercelBlobReader {
- pub fn new(core: Arc<VercelBlobCore>, path: &str, op: OpRead) -> Self {
- VercelBlobReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for VercelBlobReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.download(&self.path, range, &self.op).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs
index 037566914e..3d70ac813a 100644
--- a/core/src/services/webdav/backend.rs
+++ b/core/src/services/webdav/backend.rs
@@ -21,6 +21,7 @@ use std::fmt::Formatter;
use std::str::FromStr;
use std::sync::Arc;
+use http::Response;
use http::StatusCode;
use log::debug;
use serde::Deserialize;
@@ -30,7 +31,6 @@ use super::error::parse_error;
use super::lister::WebdavLister;
use super::writer::WebdavWriter;
use crate::raw::*;
-use crate::services::webdav::reader::WebdavReader;
use crate::*;
/// Config for [WebDAV](https://datatracker.ietf.org/doc/html/rfc4918) backend support.
@@ -236,7 +236,7 @@ impl Debug for WebdavBackend {
}
impl Access for WebdavBackend {
- type Reader = WebdavReader;
+ type Reader = HttpBody;
type Writer = oio::OneShotWriter<WebdavWriter>;
type Lister = oio::PageLister<WebdavLister>;
type BlockingReader = ();
@@ -282,10 +282,20 @@ impl Access for WebdavBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- WebdavReader::new(self.core.clone(), path, args),
- ))
+ let resp = self.core.webdav_get(path, args.range(), &args).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/webdav/core.rs b/core/src/services/webdav/core.rs
index 737e1af3c5..204f1fa3c1 100644
--- a/core/src/services/webdav/core.rs
+++ b/core/src/services/webdav/core.rs
@@ -135,7 +135,7 @@ impl WebdavCore {
path: &str,
range: BytesRange,
_: &OpRead,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let path = build_rooted_abs_path(&self.root, path);
let url: String = format!("{}{}", self.endpoint, percent_encode_path(&path));
@@ -151,7 +151,7 @@ impl WebdavCore {
let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
pub async fn webdav_put(
diff --git a/core/src/services/webdav/mod.rs b/core/src/services/webdav/mod.rs
index 455fbe3d2d..1e18871a98 100644
--- a/core/src/services/webdav/mod.rs
+++ b/core/src/services/webdav/mod.rs
@@ -22,5 +22,4 @@ pub use backend::WebdavConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/webdav/reader.rs b/core/src/services/webdav/reader.rs
deleted file mode 100644
index 3af6a95b77..0000000000
--- a/core/src/services/webdav/reader.rs
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::StatusCode;
-
-use super::core::WebdavCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct WebdavReader {
- core: Arc<WebdavCore>,
-
- path: String,
- op: OpRead,
-}
-
-impl WebdavReader {
- pub fn new(core: Arc<WebdavCore>, path: &str, op: OpRead) -> Self {
- WebdavReader {
- core,
- path: path.to_string(),
- op,
- }
- }
-}
-
-impl oio::Read for WebdavReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.webdav_get(&self.path, range, &self.op).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/webhdfs/backend.rs b/core/src/services/webhdfs/backend.rs
index f5798b08cd..b118ee7bdd 100644
--- a/core/src/services/webhdfs/backend.rs
+++ b/core/src/services/webhdfs/backend.rs
@@ -36,7 +36,6 @@ use super::message::FileStatusWrapper;
use super::writer::WebhdfsWriter;
use super::writer::WebhdfsWriters;
use crate::raw::*;
-use crate::services::webhdfs::reader::WebhdfsReader;
use crate::*;
const WEBHDFS_DEFAULT_ENDPOINT: &str = "http://127.0.0.1:9870";
@@ -461,9 +460,9 @@ impl WebhdfsBackend {
&self,
path: &str,
range: BytesRange,
- ) -> Result<Response<Buffer>> {
+ ) -> Result<Response<HttpBody>> {
let req = self.webhdfs_open_request(path, &range).await?;
- self.client.send(req).await
+ self.client.fetch(req).await
}
pub(super) async fn webhdfs_get_file_status(&self, path: &str) -> Result<Response<Buffer>> {
@@ -530,7 +529,7 @@ impl WebhdfsBackend {
}
impl Access for WebhdfsBackend {
- type Reader = WebhdfsReader;
+ type Reader = HttpBody;
type Writer = WebhdfsWriters;
type Lister = oio::PageLister<WebhdfsLister>;
type BlockingReader = ();
@@ -625,10 +624,20 @@ impl Access for WebhdfsBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- WebhdfsReader::new(self.clone(), path, args),
- ))
+ let resp = self.webhdfs_read_file(path, args.range()).await?;
+
+ let status = resp.status();
+
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+ Ok((RpRead::default(), resp.into_body()))
+ }
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
diff --git a/core/src/services/webhdfs/mod.rs b/core/src/services/webhdfs/mod.rs
index 5118af2681..3bfbbc77dd 100644
--- a/core/src/services/webhdfs/mod.rs
+++ b/core/src/services/webhdfs/mod.rs
@@ -21,5 +21,4 @@ pub use backend::WebhdfsBuilder as Webhdfs;
mod error;
mod lister;
mod message;
-mod reader;
mod writer;
diff --git a/core/src/services/webhdfs/reader.rs b/core/src/services/webhdfs/reader.rs
deleted file mode 100644
index 3ecb79d873..0000000000
--- a/core/src/services/webhdfs/reader.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use http::StatusCode;
-
-use super::error::parse_error;
-use crate::raw::*;
-use crate::services::webhdfs::backend::WebhdfsBackend;
-use crate::*;
-
-pub struct WebhdfsReader {
- core: WebhdfsBackend,
-
- path: String,
- _op: OpRead,
-}
-
-impl WebhdfsReader {
- pub fn new(core: WebhdfsBackend, path: &str, op: OpRead) -> Self {
- WebhdfsReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for WebhdfsReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- let resp = self.core.webhdfs_read_file(&self.path, range).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/services/yandex_disk/backend.rs b/core/src/services/yandex_disk/backend.rs
index c04d6843a1..f4d1a6fd0b 100644
--- a/core/src/services/yandex_disk/backend.rs
+++ b/core/src/services/yandex_disk/backend.rs
@@ -21,6 +21,9 @@ use std::fmt::Formatter;
use std::sync::Arc;
use bytes::Buf;
+use http::header;
+use http::Request;
+use http::Response;
use http::StatusCode;
use log::debug;
use serde::Deserialize;
@@ -31,7 +34,6 @@ use super::lister::YandexDiskLister;
use super::writer::YandexDiskWriter;
use super::writer::YandexDiskWriters;
use crate::raw::*;
-use crate::services::yandex_disk::reader::YandexDiskReader;
use crate::*;
/// Config for backblaze YandexDisk services support.
@@ -178,7 +180,7 @@ pub struct YandexDiskBackend {
}
impl Access for YandexDiskBackend {
- type Reader = YandexDiskReader;
+ type Reader = HttpBody;
type Writer = YandexDiskWriters;
type Lister = oio::PageLister<YandexDiskLister>;
type BlockingReader = ();
@@ -245,10 +247,24 @@ impl Access for YandexDiskBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- Ok((
- RpRead::default(),
- YandexDiskReader::new(self.core.clone(), path, args),
- ))
+ // TODO: move this out of reader.
+ let download_url = self.core.get_download_url(path).await?;
+
+ let req = Request::get(download_url)
+ .header(header::RANGE, args.range().to_header())
+ .body(Buffer::new())
+ .map_err(new_request_build_error)?;
+ let resp = self.core.client.fetch(req).await?;
+
+ let status = resp.status();
+ match status {
+ StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+ _ => {
+ let (part, mut body) = resp.into_parts();
+ let buf = body.to_buffer().await?;
+ Err(parse_error(Response::from_parts(part, buf)).await?)
+ }
+ }
}
async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
diff --git a/core/src/services/yandex_disk/mod.rs b/core/src/services/yandex_disk/mod.rs
index 606f8f8161..e2f2aff448 100644
--- a/core/src/services/yandex_disk/mod.rs
+++ b/core/src/services/yandex_disk/mod.rs
@@ -22,5 +22,4 @@ pub use backend::YandexDiskConfig;
mod core;
mod error;
mod lister;
-mod reader;
mod writer;
diff --git a/core/src/services/yandex_disk/reader.rs b/core/src/services/yandex_disk/reader.rs
deleted file mode 100644
index 0dbca67500..0000000000
--- a/core/src/services/yandex_disk/reader.rs
+++ /dev/null
@@ -1,66 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::sync::Arc;
-
-use http::header;
-use http::Request;
-use http::StatusCode;
-
-use super::core::YandexDiskCore;
-use super::error::parse_error;
-use crate::raw::*;
-use crate::*;
-
-pub struct YandexDiskReader {
- core: Arc<YandexDiskCore>,
-
- path: String,
- _op: OpRead,
-}
-
-impl YandexDiskReader {
- pub fn new(core: Arc<YandexDiskCore>, path: &str, op: OpRead) -> Self {
- YandexDiskReader {
- core,
- path: path.to_string(),
- _op: op,
- }
- }
-}
-
-impl oio::Read for YandexDiskReader {
- async fn read_at(&self, offset: u64, size: usize) -> Result<Buffer> {
- let range = BytesRange::new(offset, Some(size as u64));
-
- // TODO: move this out of reader.
- let download_url = self.core.get_download_url(&self.path).await?;
-
- let req = Request::get(download_url)
- .header(header::RANGE, range.to_header())
- .body(Buffer::new())
- .map_err(new_request_build_error)?;
- let resp = self.core.send(req).await?;
-
- let status = resp.status();
-
- match status {
- StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(resp.into_body()),
- _ => Err(parse_error(resp).await?),
- }
- }
-}
diff --git a/core/src/types/blocking_read/blocking_reader.rs b/core/src/types/blocking_read/blocking_reader.rs
index 13264ebf52..857bb9a9d2 100644
--- a/core/src/types/blocking_read/blocking_reader.rs
+++ b/core/src/types/blocking_read/blocking_reader.rs
@@ -20,7 +20,6 @@ use std::ops::Range;
use std::ops::RangeBounds;
use std::sync::Arc;
-use bytes::Buf;
use bytes::BufMut;
use crate::raw::*;
@@ -28,11 +27,9 @@ use crate::*;
/// BlockingReader is designed to read data from given path in an blocking
/// manner.
+#[derive(Clone)]
pub struct BlockingReader {
- acc: Accessor,
- path: Arc<String>,
-
- pub(crate) inner: oio::BlockingReader,
+ ctx: Arc<ReadContext>,
/// Total size of the reader.
size: Arc<AtomicContentLength>,
@@ -46,15 +43,11 @@ impl BlockingReader {
///
/// We don't want to expose those details to users so keep this function
/// in crate only.
- pub(crate) fn create(acc: Accessor, path: Arc<String>, op: OpRead) -> crate::Result<Self> {
- let (_, r) = acc.blocking_read(&path, op)?;
-
- Ok(BlockingReader {
- acc,
- path,
- inner: r,
+ pub(crate) fn new(ctx: ReadContext) -> Self {
+ BlockingReader {
+ ctx: Arc::new(ctx),
size: Arc::new(AtomicContentLength::new()),
- })
+ }
}
/// Parse users input range bounds into valid `Range<u64>`.
@@ -74,8 +67,9 @@ impl BlockingReader {
Some(v) => v,
None => {
let size = self
- .acc
- .blocking_stat(&self.path, OpStat::new())?
+ .ctx
+ .accessor()
+ .blocking_stat(self.ctx.path(), OpStat::new())?
.into_metadata()
.content_length();
self.size.store(size);
@@ -96,28 +90,11 @@ impl BlockingReader {
///
/// - Buffer length smaller than range means we have reached the end of file.
pub fn read(&self, range: impl RangeBounds<u64>) -> Result<Buffer> {
- let range = self.parse_range(range)?;
- let start = range.start;
- let end = range.end;
-
- let mut bufs = Vec::new();
- let mut offset = start;
-
- loop {
- // TODO: use service preferred io size instead.
- let size = (end - offset) as usize;
- let bs = self.inner.read_at(offset, size)?;
- let n = bs.remaining();
- bufs.push(bs);
- if n < size {
- return Ok(bufs.into_iter().flatten().collect());
- }
-
- offset += n as u64;
- if offset == end {
- return Ok(bufs.into_iter().flatten().collect());
- }
+ let mut bufs = vec![];
+ for buf in self.clone().into_iterator(range)? {
+ bufs.push(buf?)
}
+ Ok(bufs.into_iter().flatten().collect())
}
///
@@ -128,39 +105,36 @@ impl BlockingReader {
///
/// - Returning length smaller than range means we have reached the end of file.
pub fn read_into(&self, buf: &mut impl BufMut, range: impl RangeBounds<u64>) -> Result<usize> {
- let range = self.parse_range(range)?;
- let start = range.start;
- let end = range.end;
+ let mut iter = self.clone().into_iterator(range)?;
- let mut offset = start;
let mut read = 0;
-
loop {
- // TODO: use service preferred io size instead.
- let size = (end - offset) as usize;
- let bs = self.inner.read_at(offset, size)?;
- let n = bs.remaining();
+ let Some(bs) = iter.next().transpose()? else {
+ return Ok(read);
+ };
+ read += bs.len();
buf.put(bs);
- read += n as u64;
- offset += n as u64;
- if offset == end {
- return Ok(read as _);
- }
}
}
+ /// Create a buffer iterator to read specific range from given reader.
+ fn into_iterator(self, range: impl RangeBounds<u64>) -> Result<BufferIterator> {
+ let range = self.parse_range(range)?;
+ Ok(BufferIterator::new(self.ctx, range))
+ }
+
/// Convert reader into [`StdReader`] which implements [`futures::AsyncRead`],
/// [`futures::AsyncSeek`] and [`futures::AsyncBufRead`].
#[inline]
pub fn into_std_read(self, range: impl RangeBounds<u64>) -> Result<StdReader> {
let range = self.parse_range(range)?;
- Ok(StdReader::new(self.inner, range))
+ Ok(StdReader::new(self.ctx, range))
}
/// Convert reader into [`StdBytesIterator`] which implements [`Iterator`].
#[inline]
pub fn into_bytes_iterator(self, range: impl RangeBounds<u64>) -> Result<StdBytesIterator> {
let range = self.parse_range(range)?;
- Ok(StdBytesIterator::new(self.inner, range))
+ Ok(StdBytesIterator::new(self.ctx, range))
}
}
diff --git a/core/src/types/blocking_read/buffer_iterator.rs b/core/src/types/blocking_read/buffer_iterator.rs
new file mode 100644
index 0000000000..8f5c770855
--- /dev/null
+++ b/core/src/types/blocking_read/buffer_iterator.rs
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::*;
+use crate::Buffer;
+use crate::*;
+use std::ops::Range;
+use std::sync::Arc;
+
+struct IteratingReader {
+ generator: ReadGenerator,
+ reader: Option<oio::BlockingReader>,
+}
+
+impl IteratingReader {
+ /// Create a new iterating reader.
+ #[inline]
+ fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
+ let generator = ReadGenerator::new(ctx.clone(), range);
+ Self {
+ generator,
+ reader: None,
+ }
+ }
+}
+
+impl oio::BlockingRead for IteratingReader {
+ fn read(&mut self) -> Result<Buffer> {
+ loop {
+ if self.reader.is_none() {
+ self.reader = self.generator.next_blocking_reader()?;
+ }
+ let Some(r) = self.reader.as_mut() else {
+ return Ok(Buffer::new());
+ };
+
+ let buf = r.read()?;
+ // Reset reader to None if this reader returns empty buffer.
+ if buf.is_empty() {
+ self.reader = None;
+ continue;
+ } else {
+ return Ok(buf);
+ }
+ }
+ }
+}
+
+/// BufferIterator is an iterator of buffers.
+///
+/// # TODO
+///
+/// We can support chunked reader for concurrent read in the future.
+pub struct BufferIterator {
+ inner: IteratingReader,
+}
+
+impl BufferIterator {
+ /// Create a new buffer iterator.
+ #[inline]
+ pub fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
+ Self {
+ inner: IteratingReader::new(ctx, range),
+ }
+ }
+}
+
+impl Iterator for BufferIterator {
+ type Item = Result<Buffer>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ use oio::BlockingRead;
+
+ match self.inner.read() {
+ Ok(buf) if buf.is_empty() => None,
+ Ok(buf) => Some(Ok(buf)),
+ Err(err) => Some(Err(err)),
+ }
+ }
+}
diff --git a/core/src/types/blocking_read/mod.rs b/core/src/types/blocking_read/mod.rs
index ff7631cd99..334f784f58 100644
--- a/core/src/types/blocking_read/mod.rs
+++ b/core/src/types/blocking_read/mod.rs
@@ -19,6 +19,9 @@
mod blocking_reader;
pub use blocking_reader::BlockingReader;
+mod buffer_iterator;
+pub use buffer_iterator::BufferIterator;
+
mod std_bytes_iterator;
pub use std_bytes_iterator::StdBytesIterator;
mod std_reader;
diff --git a/core/src/types/blocking_read/std_bytes_iterator.rs b/core/src/types/blocking_read/std_bytes_iterator.rs
index ea29cb1eca..9aacb690ec 100644
--- a/core/src/types/blocking_read/std_bytes_iterator.rs
+++ b/core/src/types/blocking_read/std_bytes_iterator.rs
@@ -17,11 +17,13 @@
use std::io;
use std::ops::Range;
+use std::sync::Arc;
-use bytes::Buf;
+use crate::{Buffer, BufferIterator};
use bytes::Bytes;
use crate::raw::*;
+use crate::*;
/// StdIterator is the adapter of [`Iterator`] for [`BlockingReader`][crate::BlockingReader].
///
@@ -29,52 +31,38 @@ use crate::raw::*;
///
/// StdIterator also implements [`Send`] and [`Sync`].
pub struct StdBytesIterator {
- inner: oio::BlockingReader,
- offset: u64,
- end: u64,
- cap: usize,
-
- cur: u64,
+ iter: BufferIterator,
+ buf: Buffer,
}
impl StdBytesIterator {
/// NOTE: don't allow users to create StdIterator directly.
#[inline]
- pub(crate) fn new(r: oio::BlockingReader, range: Range<u64>) -> Self {
+ pub(crate) fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
+ let iter = BufferIterator::new(ctx, range);
+
StdBytesIterator {
- inner: r,
- offset: range.start,
- end: range.end,
- // TODO: should use services preferred io size.
- cap: 4 * 1024 * 1024,
- cur: 0,
+ iter,
+ buf: Buffer::new(),
}
}
-
- /// Set the capacity of this reader to control the IO size.
- pub fn with_capacity(mut self, cap: usize) -> Self {
- self.cap = cap;
- self
- }
}
impl Iterator for StdBytesIterator {
type Item = io::Result<Bytes>;
fn next(&mut self) -> Option<Self::Item> {
- if self.offset + self.cur >= self.end {
- return None;
- }
-
- let next_offset = self.offset + self.cur;
- let next_size = (self.end - self.offset - self.cur).min(self.cap as u64) as usize;
- match self.inner.read_at(next_offset, next_size) {
- Ok(buf) if !buf.has_remaining() => None,
- Ok(mut buf) => {
- self.cur += buf.remaining() as u64;
- Some(Ok(buf.copy_to_bytes(buf.remaining())))
+ loop {
+ // Consume current buffer
+ if let Some(bs) = Iterator::next(&mut self.buf) {
+ return Some(Ok(bs));
}
- Err(err) => Some(Err(format_std_io_error(err))),
+
+ self.buf = match self.iter.next() {
+ Some(Ok(buf)) => buf,
+ Some(Err(err)) => return Some(Err(format_std_io_error(err))),
+ None => return None,
+ };
}
}
}
diff --git a/core/src/types/blocking_read/std_reader.rs b/core/src/types/blocking_read/std_reader.rs
index 191eeb1fbb..85eda248a1 100644
--- a/core/src/types/blocking_read/std_reader.rs
+++ b/core/src/types/blocking_read/std_reader.rs
@@ -21,6 +21,7 @@ use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::ops::Range;
+use std::sync::Arc;
use bytes::Buf;
@@ -33,72 +34,76 @@ use crate::*;
///
/// StdReader also implements [`Send`] and [`Sync`].
pub struct StdReader {
- inner: oio::BlockingReader,
- offset: u64,
- size: u64,
- cap: usize,
+ ctx: Arc<ReadContext>,
- cur: u64,
+ iter: BufferIterator,
buf: Buffer,
+ start: u64,
+ end: u64,
+ pos: u64,
}
impl StdReader {
/// NOTE: don't allow users to create StdReader directly.
#[inline]
- pub(super) fn new(r: oio::BlockingReader, range: Range<u64>) -> Self {
- StdReader {
- inner: r,
- offset: range.start,
- size: range.end - range.start,
- // TODO: should use services preferred io size.
- cap: 4 * 1024 * 1024,
-
- cur: 0,
+ pub(super) fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
+ let (start, end) = (range.start, range.end);
+ let iter = BufferIterator::new(ctx.clone(), range);
+
+ Self {
+ ctx,
+ iter,
buf: Buffer::new(),
+ start,
+ end,
+ pos: 0,
}
}
-
- /// Set the capacity of this reader to control the IO size.
- pub fn with_capacity(mut self, cap: usize) -> Self {
- self.cap = cap;
- self
- }
}
impl BufRead for StdReader {
fn fill_buf(&mut self) -> io::Result<&[u8]> {
- if self.buf.has_remaining() {
- return Ok(self.buf.chunk());
- }
-
- // Make sure cur didn't exceed size.
- if self.cur >= self.size {
- return Ok(&[]);
+ loop {
+ if self.buf.has_remaining() {
+ return Ok(self.buf.chunk());
+ }
+
+ self.buf = match self.iter.next().transpose().map_err(format_std_io_error)? {
+ Some(buf) => buf,
+ None => return Ok(&[]),
+ };
}
-
- let next_offset = self.offset + self.cur;
- let next_size = (self.size - self.cur).min(self.cap as u64) as usize;
- self.buf = self
- .inner
- .read_at(next_offset, next_size)
- .map_err(format_std_io_error)?;
- Ok(self.buf.chunk())
}
fn consume(&mut self, amt: usize) {
self.buf.advance(amt);
- self.cur += amt as u64;
+ // Make sure buf has been dropped before starting new request.
+ // Otherwise, we will hold those bytes in memory until next
+ // buffer reaching.
+ if self.buf.is_empty() {
+ self.buf = Buffer::new();
+ }
+ self.pos += amt as u64;
}
}
impl Read for StdReader {
#[inline]
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
- let bs = self.fill_buf()?;
- let n = bs.len().min(buf.len());
- buf[..n].copy_from_slice(&bs[..n]);
- self.consume(n);
- Ok(n)
+ loop {
+ if self.buf.remaining() > 0 {
+ let size = self.buf.remaining().min(buf.len());
+ self.buf.copy_to_slice(&mut buf[..size]);
+ self.pos += size as u64;
+ return Ok(size);
+ }
+
+ self.buf = match self.iter.next() {
+ Some(Ok(buf)) => buf,
+ Some(Err(err)) => return Err(format_std_io_error(err)),
+ None => return Ok(0),
+ };
+ }
}
}
@@ -107,10 +112,11 @@ impl Seek for StdReader {
fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
let new_pos = match pos {
SeekFrom::Start(pos) => pos as i64,
- SeekFrom::End(pos) => self.size as i64 + pos,
- SeekFrom::Current(pos) => self.cur as i64 + pos,
+ SeekFrom::End(pos) => self.end as i64 - self.start as i64 + pos,
+ SeekFrom::Current(pos) => self.pos as i64 + pos,
};
+ // Check if new_pos is negative.
if new_pos < 0 {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
@@ -120,14 +126,15 @@ impl Seek for StdReader {
let new_pos = new_pos as u64;
- if (self.cur..self.cur + self.buf.remaining() as u64).contains(&new_pos) {
- let cnt = new_pos - self.cur;
+ if (self.pos..self.pos + self.buf.remaining() as u64).contains(&new_pos) {
+ let cnt = new_pos - self.pos;
self.buf.advance(cnt as _);
} else {
- self.buf = Buffer::new()
+ self.buf = Buffer::new();
+ self.iter = BufferIterator::new(self.ctx.clone(), new_pos + self.start..self.end);
}
- self.cur = new_pos;
- Ok(self.cur)
+ self.pos = new_pos;
+ Ok(self.pos)
}
}
diff --git a/core/src/types/buffer.rs b/core/src/types/buffer.rs
index f72e9f1af1..27e7c8eed1 100644
--- a/core/src/types/buffer.rs
+++ b/core/src/types/buffer.rs
@@ -34,6 +34,8 @@ use bytes::Bytes;
use bytes::BytesMut;
use futures::Stream;
+use crate::*;
+
/// Buffer is a wrapper of contiguous `Bytes` and non-contiguous `[Bytes]`.
///
/// We designed buffer to allow underlying storage to return non-contiguous bytes. For example,
diff --git a/core/src/types/context.rs b/core/src/types/context.rs
new file mode 100644
index 0000000000..f5c03c76f0
--- /dev/null
+++ b/core/src/types/context.rs
@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::*;
+use crate::*;
+use std::ops::Range;
+use std::sync::Arc;
+
+/// ReadContext holds the immutable context for give read operation.
+pub struct ReadContext {
+ /// The accessor to the storage services.
+ acc: Accessor,
+ /// Path to the file.
+ path: String,
+ /// Arguments for the read operation.
+ args: OpRead,
+ /// Options for the reader.
+ options: OpReader,
+}
+
+impl ReadContext {
+ /// Create a new ReadContext.
+ #[inline]
+ pub fn new(acc: Accessor, path: String, args: OpRead, options: OpReader) -> Self {
+ Self {
+ acc,
+ path,
+ args,
+ options,
+ }
+ }
+
+ /// Get the accessor.
+ #[inline]
+ pub fn accessor(&self) -> &Accessor {
+ &self.acc
+ }
+
+ /// Get the path.
+ #[inline]
+ pub fn path(&self) -> &str {
+ &self.path
+ }
+
+ /// Get the arguments.
+ #[inline]
+ pub fn args(&self) -> &OpRead {
+ &self.args
+ }
+
+ /// Get the options.
+ #[inline]
+ pub fn options(&self) -> &OpReader {
+ &self.options
+ }
+}
+
+/// ReadGenerator is used to generate new readers.
+///
+/// If chunk is None, ReaderGenerator will only return one reader.
+/// Otherwise, ReaderGenerator will return multiple readers, each with size
+/// of chunk.
+///
+/// It's design that we didn't implement the generator as a stream, because
+/// we don't expose the generator to the user. Instead, we use the async method
+/// directly to keep it simple and easy to understand.
+pub struct ReadGenerator {
+ ctx: Arc<ReadContext>,
+
+ offset: u64,
+ end: u64,
+}
+
+impl ReadGenerator {
+ /// Create a new ReadGenerator.
+ #[inline]
+ pub fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
+ Self {
+ ctx,
+ offset: range.start,
+ end: range.end,
+ }
+ }
+
+ /// Generate next reader.
+ pub async fn next_reader(&mut self) -> Result<Option<oio::Reader>> {
+ if self.offset >= self.end {
+ return Ok(None);
+ }
+
+ let offset = self.offset;
+ let mut size = (self.end - self.offset) as usize;
+ if let Some(chunk) = self.ctx.options.chunk() {
+ size = size.min(chunk)
+ }
+
+ // Update self.offset before building future.
+ self.offset += size as u64;
+ let args = self
+ .ctx
+ .args
+ .clone()
+ .with_range(BytesRange::new(offset, Some(size as u64)));
+ let (_, r) = self.ctx.acc.read(&self.ctx.path, args).await?;
+ Ok(Some(r))
+ }
+
+ /// Generate next blocking reader.
+ pub fn next_blocking_reader(&mut self) -> Result<Option<oio::BlockingReader>> {
+ if self.offset >= self.end {
+ return Ok(None);
+ }
+
+ let offset = self.offset;
+ let mut size = (self.end - self.offset) as usize;
+ if let Some(chunk) = self.ctx.options.chunk() {
+ size = size.min(chunk)
+ }
+
+ // Update self.offset before building future.
+ self.offset += size as u64;
+ let args = self
+ .ctx
+ .args
+ .clone()
+ .with_range(BytesRange::new(offset, Some(size as u64)));
+ let (_, r) = self.ctx.acc.blocking_read(&self.ctx.path, args)?;
+ Ok(Some(r))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use bytes::Bytes;
+ use std::collections::HashMap;
+
+ #[tokio::test]
+ async fn test_next_reader() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
+ op.write(
+ "test",
+ Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+ )
+ .await?;
+
+ let acc = op.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new().with_chunk(3),
+ ));
+ let mut generator = ReadGenerator::new(ctx, 0..10);
+ let mut readers = vec![];
+ while let Some(r) = generator.next_reader().await? {
+ readers.push(r);
+ }
+
+ pretty_assertions::assert_eq!(readers.len(), 4);
+ Ok(())
+ }
+
+ #[test]
+ fn test_next_blocking_reader() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
+ op.blocking().write(
+ "test",
+ Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+ )?;
+
+ let acc = op.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new().with_chunk(3),
+ ));
+ let mut generator = ReadGenerator::new(ctx, 0..10);
+ let mut readers = vec![];
+ while let Some(r) = generator.next_blocking_reader()? {
+ readers.push(r);
+ }
+
+ pretty_assertions::assert_eq!(readers.len(), 4);
+ Ok(())
+ }
+}
diff --git a/core/src/types/mod.rs b/core/src/types/mod.rs
index 7aefe43841..07a0b4e865 100644
--- a/core/src/types/mod.rs
+++ b/core/src/types/mod.rs
@@ -68,3 +68,7 @@ pub use scheme::Scheme;
mod capability;
pub use capability::Capability;
+
+mod context;
+pub(crate) use context::ReadContext;
+pub(crate) use context::ReadGenerator;
diff --git a/core/src/types/operator/blocking_operator.rs b/core/src/types/operator/blocking_operator.rs
index 8fe16d8353..8d2344778b 100644
--- a/core/src/types/operator/blocking_operator.rs
+++ b/core/src/types/operator/blocking_operator.rs
@@ -16,8 +16,6 @@
// under the License.
use bytes::Buf;
-use bytes::Bytes;
-use std::sync::Arc;
use super::operator_functions::*;
use crate::raw::*;
@@ -397,8 +395,8 @@ impl BlockingOperator {
);
}
- let path = Arc::new(path);
- let r = BlockingReader::create(inner, path, args)?;
+ let context = ReadContext::new(inner, path, args, OpReader::default());
+ let r = BlockingReader::new(context);
let buf = r.read(range.to_range())?;
Ok(buf)
},
@@ -456,8 +454,8 @@ impl BlockingOperator {
);
}
- let path = Arc::new(path);
- BlockingReader::create(inner.clone(), path, args)
+ let context = ReadContext::new(inner, path, args, OpReader::default());
+ Ok(BlockingReader::new(context))
},
))
}
@@ -482,7 +480,7 @@ impl BlockingOperator {
/// # Ok(())
/// # }
/// ```
- pub fn write(&self, path: &str, bs: impl Into<Bytes>) -> Result<()> {
+ pub fn write(&self, path: &str, bs: impl Into<Buffer>) -> Result<()> {
self.write_with(path, bs).call()
}
@@ -623,7 +621,7 @@ impl BlockingOperator {
/// # Ok(())
/// # }
/// ```
- pub fn write_with(&self, path: &str, bs: impl Into<Bytes>) -> FunctionWrite {
+ pub fn write_with(&self, path: &str, bs: impl Into<Buffer>) -> FunctionWrite {
let path = normalize_path(path);
let bs = bs.into();
@@ -644,7 +642,7 @@ impl BlockingOperator {
let (_, mut w) = inner.blocking_write(&path, args)?;
while !bs.is_empty() {
- let n = w.write(bs.clone().into())?;
+ let n = w.write(bs.clone())?;
bs.advance(n);
}
w.close()?;
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index 59a8d86b8b..0a9c56abd5 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -16,7 +16,6 @@
// under the License.
use std::future::Future;
-use std::sync::Arc;
use std::time::Duration;
use futures::stream;
@@ -73,7 +72,7 @@ pub struct Operator {
/// # Operator basic API.
impl Operator {
- pub(super) fn inner(&self) -> &Accessor {
+ pub(crate) fn inner(&self) -> &Accessor {
&self.accessor
}
@@ -90,7 +89,7 @@ impl Operator {
}
}
- pub(super) fn into_inner(self) -> Accessor {
+ pub(crate) fn into_inner(self) -> Accessor {
self.accessor
}
@@ -569,9 +568,9 @@ impl Operator {
);
}
- let path = Arc::new(path);
- let range = options.range();
- let r = Reader::create(inner, path, args, options).await?;
+ let range = args.range();
+ let context = ReadContext::new(inner, path, args, options);
+ let r = Reader::new(context);
let buf = r.read(range.to_range()).await?;
Ok(buf)
},
@@ -682,8 +681,8 @@ impl Operator {
);
}
- let path = Arc::new(path);
- Reader::create(inner.clone(), path, args, options).await
+ let context = ReadContext::new(inner, path, args, options);
+ Ok(Reader::new(context))
},
)
}
diff --git a/core/src/types/operator/operator_functions.rs b/core/src/types/operator/operator_functions.rs
index dbb7200f0f..1788561cdf 100644
--- a/core/src/types/operator/operator_functions.rs
+++ b/core/src/types/operator/operator_functions.rs
@@ -21,7 +21,6 @@
use std::ops::RangeBounds;
-use bytes::Bytes;
use flagset::FlagSet;
use crate::raw::*;
@@ -72,7 +71,7 @@ impl<T, R> OperatorFunction<T, R> {
pub struct FunctionWrite(
/// The args for FunctionWrite is a bit special because we also
/// need to move the bytes input this function.
- pub(crate) OperatorFunction<(OpWrite, Bytes), ()>,
+ pub(crate) OperatorFunction<(OpWrite, Buffer), ()>,
);
impl FunctionWrite {
diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs
index 832cae0194..4d16680087 100644
--- a/core/src/types/operator/operator_futures.rs
+++ b/core/src/types/operator/operator_futures.rs
@@ -224,7 +224,7 @@ impl<F: Future<Output = Result<Buffer>>> FutureRead<F> {
/// Set the range header for this operation.
pub fn range(self, range: impl RangeBounds<u64>) -> Self {
- self.map(|(args, op_reader)| (args, op_reader.with_range(range.into())))
+ self.map(|(args, op_reader)| (args.with_range(range.into()), op_reader))
}
/// Set the concurrent read task amount.
diff --git a/core/src/types/read/buffer_stream.rs b/core/src/types/read/buffer_stream.rs
index 21297367aa..63494fb3bd 100644
--- a/core/src/types/read/buffer_stream.rs
+++ b/core/src/types/read/buffer_stream.rs
@@ -17,100 +17,168 @@
use std::ops::Range;
use std::pin::Pin;
+use std::sync::Arc;
use std::task::Context;
use std::task::Poll;
-use futures::stream::Buffered;
-use futures::stream::FusedStream;
-use futures::stream::Iter;
-use futures::stream::{self};
-use futures::Stream;
-use futures::StreamExt;
+use crate::raw::oio::Read;
+use futures::{ready, Stream};
use crate::raw::*;
use crate::*;
-/// FutureIterator is an iterator that returns future generated by `Reader`'s
-/// read.
-struct FutureIterator {
- r: oio::Reader,
- chunk: Option<usize>,
-
- offset: u64,
- end: u64,
+/// StreamingReader will stream the content of the file without reading into
+/// memory first.
+///
+/// StreamingReader is good for small memory footprint and optimized for latency.
+pub struct StreamingReader {
+ generator: ReadGenerator,
+ reader: Option<oio::Reader>,
}
-impl FutureIterator {
+impl StreamingReader {
+ /// Create a new streaming reader.
#[inline]
- fn new(r: oio::Reader, chunk: Option<usize>, range: Range<u64>) -> Self {
- FutureIterator {
- r,
- chunk,
- offset: range.start,
- end: range.end,
+ fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
+ let generator = ReadGenerator::new(ctx.clone(), range);
+ Self {
+ generator,
+ reader: None,
}
}
}
-impl Iterator for FutureIterator {
- type Item = BoxedFuture<'static, Result<Buffer>>;
-
- fn next(&mut self) -> Option<Self::Item> {
- if self.offset >= self.end {
- return None;
+impl oio::Read for StreamingReader {
+ async fn read(&mut self) -> Result<Buffer> {
+ loop {
+ if self.reader.is_none() {
+ self.reader = self.generator.next_reader().await?;
+ }
+ let Some(r) = self.reader.as_mut() else {
+ return Ok(Buffer::new());
+ };
+
+ let buf = r.read().await?;
+ // Reset reader to None if this reader returns empty buffer.
+ if buf.is_empty() {
+ self.reader = None;
+ continue;
+ } else {
+ return Ok(buf);
+ }
}
+ }
+}
- let offset = self.offset;
- let mut limit = (self.end - self.offset) as usize;
- if let Some(chunk) = self.chunk {
- limit = limit.min(chunk)
- }
+/// ChunkedReader will read the file in chunks.
+///
+/// ChunkedReader is good for concurrent read and optimized for throughput.
+pub struct ChunkedReader {
+ generator: ReadGenerator,
+ tasks: ConcurrentTasks<oio::Reader, Buffer>,
+ done: bool,
+}
- // Update self.offset before building future.
- self.offset += limit as u64;
- let r = self.r.clone();
- let fut = async move {
- let buf = r.read_at_dyn(offset, limit).await?;
- Ok(buf)
- };
+impl ChunkedReader {
+ /// Create a new chunked reader.
+ fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
+ let tasks = ConcurrentTasks::new(
+ ctx.args().executor().cloned(),
+ ctx.options().concurrent(),
+ |mut r: oio::Reader| {
+ Box::pin(async {
+ match r.read_all().await {
+ Ok(buf) => (r, Ok(buf)),
+ Err(err) => (r, Err(err)),
+ }
+ })
+ },
+ );
+ let generator = ReadGenerator::new(ctx, range);
+ Self {
+ generator,
+ tasks,
+ done: false,
+ }
+ }
+}
- Some(Box::pin(fut))
+impl oio::Read for ChunkedReader {
+ async fn read(&mut self) -> Result<Buffer> {
+ while self.tasks.has_remaining() && !self.done {
+ if let Some(r) = self.generator.next_reader().await? {
+ self.tasks.execute(r).await?;
+ } else {
+ self.done = true;
+ break;
+ }
+ if self.tasks.has_result() {
+ break;
+ }
+ }
+ Ok(self.tasks.next().await.transpose()?.unwrap_or_default())
}
}
-/// BufferStream is a stream that returns [`Buffer`] from [`Reader`].
-///
-/// This stream will use concurrent read to fetch data from underlying storage.
+/// BufferStream is a stream of buffers.
///
/// # Notes
///
-/// BufferStream uses `Buffered<Iter<ReadFutureIterator>>` internally,
-/// but we want to hide those details from users.
-pub struct BufferStream(Buffered<Iter<FutureIterator>>);
+/// The underlying reader is either a StreamingReader or a ChunkedReader.
+///
+/// - If chunk is None, BufferStream will use StreamingReader to iterate
+/// data in streaming way.
+/// - Otherwise, BufferStream will use ChunkedReader to read data in chunks.
+pub struct BufferStream {
+ state: State,
+}
-impl BufferStream {
- /// Create a new buffer stream from given reader.
- #[inline]
- pub(crate) fn new(r: oio::Reader, options: OpReader, range: Range<u64>) -> Self {
- let iter = FutureIterator::new(r, options.chunk(), range);
- let stream = stream::iter(iter).buffered(options.concurrent());
+enum State {
+ Idle(Option<TwoWays<StreamingReader, ChunkedReader>>),
+ Reading(BoxedStaticFuture<(TwoWays<StreamingReader, ChunkedReader>, Result<Buffer>)>),
+}
- BufferStream(stream)
+impl BufferStream {
+ /// Create a new buffer stream.
+ pub fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
+ let reader = if ctx.options().chunk().is_some() {
+ TwoWays::Two(ChunkedReader::new(ctx, range))
+ } else {
+ TwoWays::One(StreamingReader::new(ctx, range))
+ };
+ Self {
+ state: State::Idle(Some(reader)),
+ }
}
}
impl Stream for BufferStream {
type Item = Result<Buffer>;
- #[inline]
- fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
- self.0.poll_next_unpin(cx)
- }
-}
-
-impl FusedStream for BufferStream {
- fn is_terminated(&self) -> bool {
- self.0.is_terminated()
+ fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+ let this = self.get_mut();
+ loop {
+ match &mut this.state {
+ State::Idle(reader) => {
+ let mut reader = reader.take().unwrap();
+ let fut = async move {
+ let ret = reader.read().await;
+ (reader, ret)
+ };
+ this.state = State::Reading(Box::pin(fut));
+ }
+ State::Reading(fut) => {
+ let fut = fut.as_mut();
+ let (reader, buf) = ready!(fut.poll(cx));
+ this.state = State::Idle(Some(reader));
+ return match buf {
+ Ok(buf) if buf.is_empty() => Poll::Ready(None),
+ Ok(buf) => Poll::Ready(Some(Ok(buf))),
+ Err(err) => Poll::Ready(Some(Err(err))),
+ };
+ }
+ }
+ }
}
}
@@ -120,34 +188,45 @@ mod tests {
use bytes::Bytes;
use futures::TryStreamExt;
use pretty_assertions::assert_eq;
+ use std::collections::HashMap;
use std::sync::Arc;
use super::*;
#[test]
- fn test_trait() {
- let v = BufferStream::new(Arc::new(Buffer::new()), OpReader::new(), 4..8);
+ fn test_trait() -> Result<()> {
+ let acc = Operator::via_map(Scheme::Memory, HashMap::default())?.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new(),
+ ));
+ let v = BufferStream::new(ctx, 4..8);
let _: Box<dyn Unpin + MaybeSend + 'static> = Box::new(v);
- }
- #[test]
- fn test_future_iterator() {
- let r: oio::Reader = Arc::new(Buffer::new());
-
- let it = FutureIterator::new(r.clone(), Some(1), 1..3);
- let futs: Vec<_> = it.collect();
- assert_eq!(futs.len(), 2);
+ Ok(())
}
#[tokio::test]
- async fn test_buffer_stream() {
- let r: oio::Reader = Arc::new(Buffer::from(vec![
- Bytes::from("Hello"),
- Bytes::from("World"),
- ]));
-
- let s = BufferStream::new(r, OpReader::new(), 4..8);
+ async fn test_buffer_stream() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
+ op.write(
+ "test",
+ Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+ )
+ .await?;
+
+ let acc = op.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new(),
+ ));
+
+ let s = BufferStream::new(ctx, 4..8);
let bufs: Vec<_> = s.try_collect().await.unwrap();
assert_eq!(bufs.len(), 1);
assert_eq!(bufs[0].chunk(), "o".as_bytes());
@@ -155,5 +234,7 @@ mod tests {
let buf: Buffer = bufs.into_iter().flatten().collect();
assert_eq!(buf.len(), 4);
assert_eq!(&buf.to_vec(), "oWor".as_bytes());
+
+ Ok(())
}
}
diff --git a/core/src/types/read/futures_async_reader.rs b/core/src/types/read/futures_async_reader.rs
index 2156fc36c0..ddbcd6f89e 100644
--- a/core/src/types/read/futures_async_reader.rs
+++ b/core/src/types/read/futures_async_reader.rs
@@ -19,6 +19,7 @@ use std::io;
use std::io::SeekFrom;
use std::ops::Range;
use std::pin::Pin;
+use std::sync::Arc;
use std::task::ready;
use std::task::Context;
use std::task::Poll;
@@ -41,8 +42,7 @@ use crate::*;
///
/// FuturesAsyncReader also implements [`Unpin`], [`Send`] and [`Sync`]
pub struct FuturesAsyncReader {
- r: oio::Reader,
- options: OpReader,
+ ctx: Arc<ReadContext>,
stream: BufferStream,
buf: Buffer,
@@ -61,13 +61,12 @@ impl FuturesAsyncReader {
///
/// Extend this API to accept `impl RangeBounds`.
#[inline]
- pub(super) fn new(r: oio::Reader, options: OpReader, range: Range<u64>) -> Self {
+ pub(super) fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
let (start, end) = (range.start, range.end);
- let stream = BufferStream::new(r.clone(), options.clone(), range);
+ let stream = BufferStream::new(ctx.clone(), range);
FuturesAsyncReader {
- r,
- options,
+ ctx,
stream,
buf: Buffer::new(),
start,
@@ -158,11 +157,7 @@ impl AsyncSeek for FuturesAsyncReader {
self.buf.advance(cnt as _);
} else {
self.buf = Buffer::new();
- self.stream = BufferStream::new(
- self.r.clone(),
- self.options.clone(),
- new_pos + self.start..self.end,
- );
+ self.stream = BufferStream::new(self.ctx.clone(), new_pos + self.start..self.end);
}
self.pos = new_pos;
@@ -172,6 +167,7 @@ impl AsyncSeek for FuturesAsyncReader {
#[cfg(test)]
mod tests {
+ use std::collections::HashMap;
use std::sync::Arc;
use bytes::Bytes;
@@ -183,20 +179,39 @@ mod tests {
use super::*;
#[test]
- fn test_trait() {
- let v = FuturesAsyncReader::new(Arc::new(Buffer::new()), OpReader::new(), 4..8);
+ fn test_trait() -> Result<()> {
+ let acc = Operator::via_map(Scheme::Memory, HashMap::default())?.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new(),
+ ));
+
+ let v = FuturesAsyncReader::new(ctx, 4..8);
let _: Box<dyn Unpin + MaybeSend + Sync + 'static> = Box::new(v);
+ Ok(())
}
#[tokio::test]
- async fn test_futures_async_read() {
- let r: oio::Reader = Arc::new(Buffer::from(vec![
- Bytes::from("Hello"),
- Bytes::from("World"),
- ]));
-
- let mut fr = FuturesAsyncReader::new(r, OpReader::new(), 4..8);
+ async fn test_futures_async_read() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
+ op.write(
+ "test",
+ Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+ )
+ .await?;
+
+ let acc = op.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new(),
+ ));
+
+ let mut fr = FuturesAsyncReader::new(ctx, 4..8);
let mut bs = vec![];
fr.read_to_end(&mut bs).await.unwrap();
assert_eq!(&bs, "oWor".as_bytes());
@@ -206,41 +221,66 @@ mod tests {
let mut bs = vec![];
fr.read_to_end(&mut bs).await.unwrap();
assert_eq!(&bs, "or".as_bytes());
+
+ Ok(())
}
#[tokio::test]
- async fn test_futures_async_read_with_concurrent() {
- let r: oio::Reader = Arc::new(Buffer::from(vec![
- Bytes::from("Hello"),
- Bytes::from("World"),
- ]));
-
- let mut fr =
- FuturesAsyncReader::new(r, OpReader::new().with_concurrent(3).with_chunk(1), 4..8);
+ async fn test_futures_async_read_with_concurrent() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
+ op.write(
+ "test",
+ Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+ )
+ .await?;
+
+ let acc = op.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new().with_concurrent(3).with_chunk(1),
+ ));
+
+ let mut fr = FuturesAsyncReader::new(ctx, 4..8);
let mut bs = vec![];
fr.read_to_end(&mut bs).await.unwrap();
assert_eq!(&bs, "oWor".as_bytes());
- let pos = fr.seek(SeekFrom::Current(-2)).await.unwrap();
- assert_eq!(pos, 2);
- let mut bs = vec![];
- fr.read_to_end(&mut bs).await.unwrap();
- assert_eq!(&bs, "or".as_bytes());
+ // let pos = fr.seek(SeekFrom::Current(-2)).await.unwrap();
+ // assert_eq!(pos, 2);
+ // let mut bs = vec![];
+ // fr.read_to_end(&mut bs).await.unwrap();
+ // assert_eq!(&bs, "or".as_bytes());
+
+ Ok(())
}
#[tokio::test]
- async fn test_futures_async_buf_read() {
- let r: oio::Reader = Arc::new(Buffer::from(vec![
- Bytes::from("Hello"),
- Bytes::from("World"),
- ]));
-
- let mut fr = FuturesAsyncReader::new(r, OpReader::new(), 4..8);
+ async fn test_futures_async_buf_read() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
+ op.write(
+ "test",
+ Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+ )
+ .await?;
+
+ let acc = op.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new().with_concurrent(3).with_chunk(1),
+ ));
+
+ let mut fr = FuturesAsyncReader::new(ctx, 4..8);
let chunk = fr.fill_buf().await.unwrap();
assert_eq!(chunk, "o".as_bytes());
fr.consume_unpin(1);
let chunk = fr.fill_buf().await.unwrap();
- assert_eq!(chunk, "Wor".as_bytes());
+ assert_eq!(chunk, "W".as_bytes());
+
+ Ok(())
}
}
diff --git a/core/src/types/read/futures_bytes_stream.rs b/core/src/types/read/futures_bytes_stream.rs
index 7fc98007ba..ddc1dc2e5a 100644
--- a/core/src/types/read/futures_bytes_stream.rs
+++ b/core/src/types/read/futures_bytes_stream.rs
@@ -18,6 +18,7 @@
use std::io;
use std::ops::Range;
use std::pin::Pin;
+use std::sync::Arc;
use std::task::ready;
use std::task::Context;
use std::task::Poll;
@@ -32,8 +33,8 @@ use crate::*;
/// FuturesBytesStream is the adapter of [`Stream`] generated by [`Reader::into_bytes_stream`].
///
/// Users can use this adapter in cases where they need to use [`Stream`] trait. FuturesBytesStream
-/// reuses the same concurrent and chunk settings from [`Reader`].
-///
+/// reuses the same concurrent adand chunk settings from [`Reader`].
+///ad
/// FuturesStream also implements [`Unpin`], [`Send`] and [`Sync`].
pub struct FuturesBytesStream {
stream: BufferStream,
@@ -46,8 +47,8 @@ unsafe impl Sync for FuturesBytesStream {}
impl FuturesBytesStream {
/// NOTE: don't allow users to create FuturesStream directly.
#[inline]
- pub(crate) fn new(r: oio::Reader, options: OpReader, range: Range<u64>) -> Self {
- let stream = BufferStream::new(r, options, range);
+ pub(crate) fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
+ let stream = BufferStream::new(ctx, range);
FuturesBytesStream {
stream,
@@ -79,6 +80,7 @@ impl Stream for FuturesBytesStream {
#[cfg(test)]
mod tests {
+ use std::collections::HashMap;
use std::sync::Arc;
use bytes::Bytes;
@@ -88,37 +90,70 @@ mod tests {
use super::*;
#[test]
- fn test_trait() {
- let v = FuturesBytesStream::new(Arc::new(Buffer::new()), OpReader::new(), 4..8);
+ fn test_trait() -> Result<()> {
+ let acc = Operator::via_map(Scheme::Memory, HashMap::default())?.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new(),
+ ));
+ let v = FuturesBytesStream::new(ctx, 4..8);
let _: Box<dyn Unpin + MaybeSend + Sync + 'static> = Box::new(v);
+
+ Ok(())
}
#[tokio::test]
- async fn test_futures_bytes_stream() {
- let r: oio::Reader = Arc::new(Buffer::from(vec![
- Bytes::from("Hello"),
- Bytes::from("World"),
- ]));
-
- let s = FuturesBytesStream::new(r, OpReader::new(), 4..8);
+ async fn test_futures_bytes_stream() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
+ op.write(
+ "test",
+ Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+ )
+ .await?;
+
+ let acc = op.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new(),
+ ));
+
+ let s = FuturesBytesStream::new(ctx, 4..8);
let bufs: Vec<Bytes> = s.try_collect().await.unwrap();
assert_eq!(&bufs[0], "o".as_bytes());
assert_eq!(&bufs[1], "Wor".as_bytes());
+
+ Ok(())
}
#[tokio::test]
- async fn test_futures_bytes_stream_with_concurrent() {
- let r: oio::Reader = Arc::new(Buffer::from(vec![
- Bytes::from("Hello"),
- Bytes::from("World"),
- ]));
-
- let s = FuturesBytesStream::new(r, OpReader::new().with_concurrent(3).with_chunk(1), 4..8);
+ async fn test_futures_bytes_stream_with_concurrent() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
+ op.write(
+ "test",
+ Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+ )
+ .await?;
+
+ let acc = op.into_inner();
+ let ctx = Arc::new(ReadContext::new(
+ acc,
+ "test".to_string(),
+ OpRead::new(),
+ OpReader::new().with_concurrent(3).with_chunk(1),
+ ));
+
+ let s = FuturesBytesStream::new(ctx, 4..8);
let bufs: Vec<Bytes> = s.try_collect().await.unwrap();
assert_eq!(&bufs[0], "o".as_bytes());
assert_eq!(&bufs[1], "W".as_bytes());
assert_eq!(&bufs[2], "o".as_bytes());
assert_eq!(&bufs[3], "r".as_bytes());
+
+ Ok(())
}
}
diff --git a/core/src/types/read/mod.rs b/core/src/types/read/mod.rs
index e4b7c80b7f..5995d6ffa6 100644
--- a/core/src/types/read/mod.rs
+++ b/core/src/types/read/mod.rs
@@ -21,7 +21,9 @@ pub use reader::Reader;
mod buffer_stream;
pub(crate) use buffer_stream::BufferStream;
+
mod futures_async_reader;
pub use futures_async_reader::FuturesAsyncReader;
+
mod futures_bytes_stream;
pub use futures_bytes_stream::FuturesBytesStream;
diff --git a/core/src/types/read/reader.rs b/core/src/types/read/reader.rs
index 867d09e6b7..9ecf5f8324 100644
--- a/core/src/types/read/reader.rs
+++ b/core/src/types/read/reader.rs
@@ -40,11 +40,7 @@ use crate::*;
/// [`Reader`] provides public API including [`Reader::read`], [`Reader:read_range`], and [`Reader::read_to_end`]. You can use those APIs directly without extra copy.
#[derive(Clone)]
pub struct Reader {
- acc: Accessor,
- path: Arc<String>,
-
- inner: oio::Reader,
- options: OpReader,
+ ctx: Arc<ReadContext>,
/// Total size of the reader.
size: Arc<AtomicContentLength>,
@@ -58,21 +54,11 @@ impl Reader {
///
/// We don't want to expose those details to users so keep this function
/// in crate only.
- pub(crate) async fn create(
- acc: Accessor,
- path: Arc<String>,
- args: OpRead,
- options: OpReader,
- ) -> Result<Self> {
- let (_, r) = acc.read(&path, args).await?;
-
- Ok(Reader {
- acc,
- path,
- inner: r,
- options,
+ pub(crate) fn new(ctx: ReadContext) -> Self {
+ Reader {
+ ctx: Arc::new(ctx),
size: Arc::new(AtomicContentLength::new()),
- })
+ }
}
/// Parse users input range bounds into valid `Range<u64>`.
@@ -92,8 +78,9 @@ impl Reader {
Some(v) => v,
None => {
let size = self
- .acc
- .stat(&self.path, OpStat::new())
+ .ctx
+ .accessor()
+ .stat(self.ctx.path(), OpStat::new())
.await?
.into_metadata()
.content_length();
@@ -149,7 +136,7 @@ impl Reader {
let merged_bufs: Vec<_> =
stream::iter(merged_ranges.clone().into_iter().map(|v| self.read(v)))
- .buffered(self.options.concurrent())
+ .buffered(self.ctx.options().concurrent())
.try_collect()
.await?;
@@ -166,7 +153,7 @@ impl Reader {
/// Merge given ranges into a list of non-overlapping ranges.
fn merge_ranges(&self, mut ranges: Vec<Range<u64>>) -> Vec<Range<u64>> {
- let gap = self.options.gap().unwrap_or(1024 * 1024) as u64;
+ let gap = self.ctx.options().gap().unwrap_or(1024 * 1024) as u64;
// We don't care about the order of range with same start, they
// will be merged in the next step.
ranges.sort_unstable_by(|a, b| a.start.cmp(&b.start));
@@ -201,7 +188,7 @@ impl Reader {
/// Let's keep it inside for now.
async fn into_stream(self, range: impl RangeBounds<u64>) -> Result<BufferStream> {
let range = self.parse_range(range).await?;
- Ok(BufferStream::new(self.inner, self.options, range))
+ Ok(BufferStream::new(self.ctx, range))
}
/// Convert reader into [`FuturesAsyncReader`] which implements [`futures::AsyncRead`],
@@ -267,7 +254,7 @@ impl Reader {
range: impl RangeBounds<u64>,
) -> Result<FuturesAsyncReader> {
let range = self.parse_range(range).await?;
- Ok(FuturesAsyncReader::new(self.inner, self.options, range))
+ Ok(FuturesAsyncReader::new(self.ctx, range))
}
/// Convert reader into [`FuturesBytesStream`] which implements [`futures::Stream`].
@@ -322,34 +309,38 @@ impl Reader {
range: impl RangeBounds<u64>,
) -> Result<FuturesBytesStream> {
let range = self.parse_range(range).await?;
- Ok(FuturesBytesStream::new(self.inner, self.options, range))
+ Ok(FuturesBytesStream::new(self.ctx, range))
}
}
#[cfg(test)]
mod tests {
- use crate::layers::TypeEraseLayer;
+ use bytes::Bytes;
use rand::rngs::ThreadRng;
use rand::Rng;
use rand::RngCore;
- use std::sync::Arc;
+ use std::collections::HashMap;
use super::*;
use crate::raw::MaybeSend;
use crate::services;
use crate::Operator;
- #[test]
- fn test_trait() {
- let v = Reader {
- acc: Arc::new(TypeEraseLayer.layer(())),
- path: Arc::new("hello".to_string()),
- inner: Arc::new(Buffer::new()),
- options: OpReader::new(),
- size: Arc::new(AtomicContentLength::new()),
- };
+ #[tokio::test]
+ async fn test_trait() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
+ op.write(
+ "test",
+ Buffer::from(vec![Bytes::from("Hello"), Bytes::from("World")]),
+ )
+ .await?;
+
+ let acc = op.into_inner();
+ let ctx = ReadContext::new(acc, "test".to_string(), OpRead::new(), OpReader::new());
- let _: Box<dyn Unpin + MaybeSend + Sync + 'static> = Box::new(v);
+ let _: Box<dyn Unpin + MaybeSend + Sync + 'static> = Box::new(Reader::new(ctx));
+
+ Ok(())
}
fn gen_random_bytes() -> Vec<u8> {
@@ -369,8 +360,8 @@ mod tests {
}
#[tokio::test]
- async fn test_reader_read() {
- let op = Operator::new(services::Memory::default()).unwrap().finish();
+ async fn test_reader_read() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
let path = "test_file";
let content = gen_random_bytes();
@@ -382,11 +373,12 @@ mod tests {
let buf = reader.read(..).await.expect("read to end must succeed");
assert_eq!(buf.to_bytes(), content);
+ Ok(())
}
#[tokio::test]
- async fn test_reader_read_with_chunk() {
- let op = Operator::new(services::Memory::default()).unwrap().finish();
+ async fn test_reader_read_with_chunk() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
let path = "test_file";
let content = gen_random_bytes();
@@ -398,11 +390,12 @@ mod tests {
let buf = reader.read(..).await.expect("read to end must succeed");
assert_eq!(buf.to_bytes(), content);
+ Ok(())
}
#[tokio::test]
- async fn test_reader_read_with_concurrent() {
- let op = Operator::new(services::Memory::default()).unwrap().finish();
+ async fn test_reader_read_with_concurrent() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
let path = "test_file";
let content = gen_random_bytes();
@@ -419,11 +412,12 @@ mod tests {
let buf = reader.read(..).await.expect("read to end must succeed");
assert_eq!(buf.to_bytes(), content);
+ Ok(())
}
#[tokio::test]
- async fn test_reader_read_into() {
- let op = Operator::new(services::Memory::default()).unwrap().finish();
+ async fn test_reader_read_into() -> Result<()> {
+ let op = Operator::via_map(Scheme::Memory, HashMap::default())?;
let path = "test_file";
let content = gen_random_bytes();
@@ -439,10 +433,11 @@ mod tests {
.expect("read to end must succeed");
assert_eq!(buf, content);
+ Ok(())
}
#[tokio::test]
- async fn test_merge_ranges() {
+ async fn test_merge_ranges() -> Result<()> {
let op = Operator::new(services::Memory::default()).unwrap().finish();
let path = "test_file";
@@ -456,10 +451,11 @@ mod tests {
let ranges = vec![0..10, 10..20, 21..30, 40..50, 40..60, 45..59];
let merged = reader.merge_ranges(ranges.clone());
assert_eq!(merged, vec![0..30, 40..60]);
+ Ok(())
}
#[tokio::test]
- async fn test_fetch() {
+ async fn test_fetch() -> Result<()> {
let op = Operator::new(services::Memory::default()).unwrap().finish();
let path = "test_file";
@@ -491,5 +487,6 @@ mod tests {
content[range.start as usize..range.end as usize]
);
}
+ Ok(())
}
}