You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by ju...@apache.org on 2023/08/22 04:57:46 UTC
[incubator-opendal] 01/04: feat: remove operator rangge_read and range_reader API
This is an automated email from the ASF dual-hosted git repository.
junouyang pushed a commit to branch feat/merge_range_read
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 797885398580dfdc61ec24a4e25586d12d51177b
Author: owl <ou...@gmail.com>
AuthorDate: Tue Aug 22 01:15:04 2023 +0800
feat: remove operator rangge_read and range_reader API
---
bin/oay/src/services/webdav/webdav_file.rs | 2 +-
core/benches/ops/read.rs | 6 ++--
core/fuzz/fuzz_reader.rs | 2 +-
core/src/types/operator/operator.rs | 55 +++---------------------------
core/tests/behavior/fuzz.rs | 2 +-
core/tests/behavior/read_only.rs | 36 ++-----------------
core/tests/behavior/write.rs | 20 +++++------
integrations/object_store/src/lib.rs | 2 +-
8 files changed, 23 insertions(+), 102 deletions(-)
diff --git a/bin/oay/src/services/webdav/webdav_file.rs b/bin/oay/src/services/webdav/webdav_file.rs
index e3a49bf86..5f9b07131 100644
--- a/bin/oay/src/services/webdav/webdav_file.rs
+++ b/bin/oay/src/services/webdav/webdav_file.rs
@@ -40,7 +40,7 @@ impl DavFile for WebdavFile {
async move {
let file_path = self.path.as_url_string();
self.op
- .range_read(&file_path, 0..count as u64)
+ .read_with(&file_path).range(0..count as u64)
.await
.map(Bytes::from)
.map_err(convert_error)
diff --git a/core/benches/ops/read.rs b/core/benches/ops/read.rs
index ef8315b95..ab591fc7b 100644
--- a/core/benches/ops/read.rs
+++ b/core/benches/ops/read.rs
@@ -58,7 +58,7 @@ fn bench_read_full(c: &mut Criterion, name: &str, op: Operator) {
group.bench_with_input(size.to_string(), &(op.clone(), &path), |b, (op, path)| {
b.to_async(&*TOKIO).iter(|| async {
let r = op
- .range_reader(path, 0..=size.bytes() as u64)
+ .reader_with(path).range(0..=size.bytes() as u64)
.await
.unwrap();
io::copy(r, &mut io::sink()).await.unwrap();
@@ -91,7 +91,7 @@ fn bench_read_part(c: &mut Criterion, name: &str, op: Operator) {
group.throughput(criterion::Throughput::Bytes(size.bytes() as u64));
group.bench_with_input(size.to_string(), &(op.clone(), &path), |b, (op, path)| {
b.to_async(&*TOKIO).iter(|| async {
- let r = op.range_reader(path, offset..).await.unwrap();
+ let r = op.reader_with(path).range(offset..).await.unwrap();
io::copy(r, &mut io::sink()).await.unwrap();
})
});
@@ -130,7 +130,7 @@ fn bench_read_parallel(c: &mut Criterion, name: &str, op: Operator) {
.map(|_| async {
let mut buf = buf.clone();
let mut r = op
- .range_reader(path, offset..=offset + size.bytes() as u64)
+ .reader_with(path).range(offset..=offset + size.bytes() as u64)
.await
.unwrap();
r.read_exact(&mut buf).await.unwrap();
diff --git a/core/fuzz/fuzz_reader.rs b/core/fuzz/fuzz_reader.rs
index 25b9db37e..7b13207bf 100644
--- a/core/fuzz/fuzz_reader.rs
+++ b/core/fuzz/fuzz_reader.rs
@@ -226,7 +226,7 @@ async fn fuzz_reader(op: Operator, input: FuzzInput) -> Result<()> {
let mut checker = ReadChecker::new(input.size, input.range);
op.write(&path, checker.raw_data.clone()).await?;
- let mut o = op.range_reader(&path, input.range.to_range()).await?;
+ let mut o = op.reader_with(&path).range(input.range.to_range()).await?;
for action in input.actions {
match action {
diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs
index b65eb8698..70c276c2d 100644
--- a/core/src/types/operator/operator.rs
+++ b/core/src/types/operator/operator.rs
@@ -327,7 +327,7 @@ impl Operator {
/// # }
/// ```
pub async fn read(&self, path: &str) -> Result<Vec<u8>> {
- self.range_read(path, ..).await
+ self.read_with(path).await
}
/// Read the whole path into a bytes with extra options.
@@ -361,7 +361,7 @@ impl Operator {
ErrorKind::IsADirectory,
"read path is a directory",
)
- .with_operation("range_read")
+ .with_operation("read")
.with_context("service", inner.info().scheme())
.with_context("path", &path));
}
@@ -381,7 +381,7 @@ impl Operator {
// TODO: use native read api
s.read_exact(buf.initialized_mut()).await.map_err(|err| {
Error::new(ErrorKind::Unexpected, "read from storage")
- .with_operation("range_read")
+ .with_operation("read")
.with_context("service", inner.info().scheme().into_static())
.with_context("path", &path)
.with_context("range", br.to_string())
@@ -401,31 +401,6 @@ impl Operator {
fut
}
- /// Read the specified range of path into a bytes.
- ///
- /// This function will allocate a new bytes internally. For more precise memory control or
- /// reading data lazily, please use [`Operator::range_reader`]
- ///
- /// # Notes
- ///
- /// - The returning content's length may be smaller than the range specified.
- ///
- /// # Examples
- ///
- /// ```
- /// # use std::io::Result;
- /// # use opendal::Operator;
- /// # use futures::TryStreamExt;
- /// # #[tokio::main]
- /// # async fn test(op: Operator) -> Result<()> {
- /// let bs = op.range_read("path/to/file", 1024..2048).await?;
- /// # Ok(())
- /// # }
- /// ```
- pub async fn range_read(&self, path: &str, range: impl RangeBounds<u64>) -> Result<Vec<u8>> {
- self.read_with(path).range(range).await
- }
-
/// Create a new reader which can read the whole path.
///
/// # Examples
@@ -445,28 +420,6 @@ impl Operator {
self.reader_with(path).await
}
- /// Create a new reader which can read the specified range.
- ///
- /// # Notes
- ///
- /// - The returning content's length may be smaller than the range specified.
- ///
- /// # Examples
- ///
- /// ```no_run
- /// # use std::io::Result;
- /// # use opendal::Operator;
- /// # use futures::TryStreamExt;
- /// # #[tokio::main]
- /// # async fn test(op: Operator) -> Result<()> {
- /// let r = op.range_reader("path/to/file", 1024..2048).await?;
- /// # Ok(())
- /// # }
- /// ```
- pub async fn range_reader(&self, path: &str, range: impl RangeBounds<u64>) -> Result<Reader> {
- self.reader_with(path).range(range).await
- }
-
/// Create a new reader with extra options
///
/// # Examples
@@ -496,7 +449,7 @@ impl Operator {
ErrorKind::IsADirectory,
"read path is a directory",
)
- .with_operation("Operator::range_reader")
+ .with_operation("Operator::reader")
.with_context("service", inner.info().scheme())
.with_context("path", path));
}
diff --git a/core/tests/behavior/fuzz.rs b/core/tests/behavior/fuzz.rs
index 8a7b8187f..b8f47fdfb 100644
--- a/core/tests/behavior/fuzz.rs
+++ b/core/tests/behavior/fuzz.rs
@@ -81,7 +81,7 @@ pub async fn test_fuzz_issue_2717(op: Operator) -> Result<()> {
.await
.expect("write must succeed");
- let mut r = op.range_reader(&path, 1..2).await?;
+ let mut r = op.reader_with(&path).range(1..2).await?;
// Perform a seek
let result = r.seek(SeekFrom::End(-2)).await;
diff --git a/core/tests/behavior/read_only.rs b/core/tests/behavior/read_only.rs
index 85d897aaf..2edaea846 100644
--- a/core/tests/behavior/read_only.rs
+++ b/core/tests/behavior/read_only.rs
@@ -40,8 +40,6 @@ pub fn behavior_read_only_tests(op: &Operator) -> Vec<Trial> {
test_read_only_stat_root,
test_read_only_read_full,
test_read_only_read_full_with_special_chars,
- test_read_only_read_range,
- test_read_only_reader_range,
test_read_only_reader_from,
test_read_only_reader_tail,
test_read_only_read_not_exist,
@@ -183,39 +181,9 @@ pub async fn test_read_only_read_full_with_special_chars(op: Operator) -> Result
Ok(())
}
-/// Read full content should match.
-pub async fn test_read_only_read_range(op: Operator) -> Result<()> {
- let bs = op.range_read("normal_file", 1024..2048).await?;
- assert_eq!(bs.len(), 1024, "read size");
- assert_eq!(
- format!("{:x}", Sha256::digest(&bs)),
- "28786fb63abfe5545479e4f50da853652d1d67b88be5553c265ede4022774913",
- "read content"
- );
-
- Ok(())
-}
-
-/// Read range should match.
-pub async fn test_read_only_reader_range(op: Operator) -> Result<()> {
- let mut r = op.range_reader("normal_file", 1024..2048).await?;
-
- let mut bs = Vec::new();
- r.read_to_end(&mut bs).await?;
-
- assert_eq!(bs.len(), 1024, "read size");
- assert_eq!(
- format!("{:x}", Sha256::digest(&bs)),
- "28786fb63abfe5545479e4f50da853652d1d67b88be5553c265ede4022774913",
- "read content"
- );
-
- Ok(())
-}
-
/// Read from should match.
pub async fn test_read_only_reader_from(op: Operator) -> Result<()> {
- let mut r = op.range_reader("normal_file", 261120..).await?;
+ let mut r = op.reader_with("normal_file").range(261120..).await?;
let mut bs = Vec::new();
r.read_to_end(&mut bs).await?;
@@ -232,7 +200,7 @@ pub async fn test_read_only_reader_from(op: Operator) -> Result<()> {
/// Read tail should match.
pub async fn test_read_only_reader_tail(op: Operator) -> Result<()> {
- let mut r = op.range_reader("normal_file", ..1024).await?;
+ let mut r = op.reader_with("normal_file").range(..1024).await?;
let mut bs = Vec::new();
r.read_to_end(&mut bs).await?;
diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs
index f60b6673b..a1effab43 100644
--- a/core/tests/behavior/write.rs
+++ b/core/tests/behavior/write.rs
@@ -68,7 +68,7 @@ pub fn behavior_write_tests(op: &Operator) -> Vec<Trial> {
test_read_not_exist,
test_read_with_if_match,
test_read_with_if_none_match,
- test_fuzz_range_reader,
+ test_fuzz_reader_with_range,
test_fuzz_offset_reader,
test_fuzz_part_reader,
test_read_with_dir_path,
@@ -438,7 +438,7 @@ pub async fn test_read_range(op: Operator) -> Result<()> {
.await
.expect("write must succeed");
- let bs = op.range_read(&path, offset..offset + length).await?;
+ let bs = op.read_with(&path).range(offset..offset + length).await?;
assert_eq!(bs.len() as u64, length, "read size");
assert_eq!(
format!("{:x}", Sha256::digest(&bs)),
@@ -468,7 +468,7 @@ pub async fn test_read_large_range(op: Operator) -> Result<()> {
.await
.expect("write must succeed");
- let bs = op.range_read(&path, offset..u32::MAX as u64).await?;
+ let bs = op.read_with(&path).range(offset..u32::MAX as u64).await?;
assert_eq!(
bs.len() as u64,
size as u64 - offset,
@@ -499,7 +499,7 @@ pub async fn test_reader_range(op: Operator) -> Result<()> {
.await
.expect("write must succeed");
- let mut r = op.range_reader(&path, offset..offset + length).await?;
+ let mut r = op.reader_with(&path).range(offset..offset + length).await?;
let mut bs = Vec::new();
r.read_to_end(&mut bs).await?;
@@ -532,7 +532,7 @@ pub async fn test_reader_from(op: Operator) -> Result<()> {
.await
.expect("write must succeed");
- let mut r = op.range_reader(&path, offset..).await?;
+ let mut r = op.reader_with(&path).range(offset..).await?;
let mut bs = Vec::new();
r.read_to_end(&mut bs).await?;
@@ -563,7 +563,7 @@ pub async fn test_reader_tail(op: Operator) -> Result<()> {
.await
.expect("write must succeed");
- let mut r = match op.range_reader(&path, ..length).await {
+ let mut r = match op.reader_with(&path).range(..length).await {
Ok(r) => r,
// Not all services support range with tail range, let's tolerate this.
Err(err) if err.kind() == ErrorKind::Unsupported => {
@@ -663,7 +663,7 @@ pub async fn test_read_with_if_none_match(op: Operator) -> Result<()> {
Ok(())
}
-pub async fn test_fuzz_range_reader(op: Operator) -> Result<()> {
+pub async fn test_fuzz_reader_with_range(op: Operator) -> Result<()> {
if !op.info().full_capability().read_with_range {
return Ok(());
}
@@ -677,7 +677,7 @@ pub async fn test_fuzz_range_reader(op: Operator) -> Result<()> {
.expect("write must succeed");
let mut fuzzer = ObjectReaderFuzzer::new(&path, content.clone(), 0, content.len());
- let mut o = op.range_reader(&path, 0..content.len() as u64).await?;
+ let mut o = op.reader_with(&path).range(0..content.len() as u64).await?;
for _ in 0..100 {
match fuzzer.fuzz() {
@@ -718,7 +718,7 @@ pub async fn test_fuzz_offset_reader(op: Operator) -> Result<()> {
.expect("write must succeed");
let mut fuzzer = ObjectReaderFuzzer::new(&path, content.clone(), 0, content.len());
- let mut o = op.range_reader(&path, 0..).await?;
+ let mut o = op.reader_with(&path).range(0..).await?;
for _ in 0..100 {
match fuzzer.fuzz() {
@@ -761,7 +761,7 @@ pub async fn test_fuzz_part_reader(op: Operator) -> Result<()> {
let mut fuzzer =
ObjectReaderFuzzer::new(&path, content.clone(), offset as usize, length as usize);
- let mut o = op.range_reader(&path, offset..offset + length).await?;
+ let mut o = op.reader_with(&path).range(offset..offset + length).await?;
for _ in 0..100 {
match fuzzer.fuzz() {
diff --git a/integrations/object_store/src/lib.rs b/integrations/object_store/src/lib.rs
index c05aaa231..00ebdf0d3 100644
--- a/integrations/object_store/src/lib.rs
+++ b/integrations/object_store/src/lib.rs
@@ -111,7 +111,7 @@ impl ObjectStore for OpendalStore {
async fn get_range(&self, location: &Path, range: Range<usize>) -> Result<Bytes> {
let bs = self
.inner
- .range_read(location.as_ref(), range.start as u64..range.end as u64)
+ .read_with(location.as_ref()).range(range.start as u64..range.end as u64)
.await
.map_err(|err| format_object_store_error(err, location.as_ref()))?;