You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/05/23 10:45:00 UTC
(opendal) 03/18: Remove blocking APIs
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch remove-blocking-api-in-oio-access
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 4bb00fc5d33ddc6eb1c3ab27298b27896d235410
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri May 9 22:29:10 2025 +0800
Remove blocking APIs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/layers/async_backtrace.rs | 6 ---
core/src/layers/await_tree.rs | 16 -------
core/src/layers/concurrent_limit.rs | 16 -------
core/src/layers/fastrace.rs | 8 ----
core/src/layers/retry.rs | 13 ------
core/src/raw/oio/list/flat_list.rs | 3 --
core/src/raw/oio/list/hierarchy_list.rs | 24 ----------
core/src/services/dashmap/backend.rs | 16 -------
core/src/services/fs/lister.rs | 36 ---------------
core/src/services/fs/writer.rs | 35 --------------
core/src/services/hdfs/lister.rs | 29 ------------
core/src/services/hdfs/writer.rs | 34 --------------
core/src/services/rocksdb/backend.rs | 2 +-
core/src/services/sled/backend.rs | 3 +-
core/src/types/context/read.rs | 11 -----
core/src/types/context/write.rs | 81 ---------------------------------
16 files changed, 3 insertions(+), 330 deletions(-)
diff --git a/core/src/layers/async_backtrace.rs b/core/src/layers/async_backtrace.rs
index 04f23327d..47a761185 100644
--- a/core/src/layers/async_backtrace.rs
+++ b/core/src/layers/async_backtrace.rs
@@ -162,12 +162,6 @@ impl<R: oio::List> oio::List for AsyncBacktraceWrapper<R> {
}
}
-impl<R: oio::BlockingList> oio::BlockingList for AsyncBacktraceWrapper<R> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.inner.next()
- }
-}
-
impl<R: oio::Delete> oio::Delete for AsyncBacktraceWrapper<R> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
self.inner.delete(path, args)
diff --git a/core/src/layers/await_tree.rs b/core/src/layers/await_tree.rs
index 02722fbfd..8d6f66a80 100644
--- a/core/src/layers/await_tree.rs
+++ b/core/src/layers/await_tree.rs
@@ -179,16 +179,6 @@ impl<R: oio::Write> oio::Write for AwaitTreeWrapper<R> {
}
}
-impl<R: oio::BlockingWrite> oio::BlockingWrite for AwaitTreeWrapper<R> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- self.inner.write(bs)
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.inner.close()
- }
-}
-
impl<R: oio::List> oio::List for AwaitTreeWrapper<R> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
self.inner
@@ -198,12 +188,6 @@ impl<R: oio::List> oio::List for AwaitTreeWrapper<R> {
}
}
-impl<R: oio::BlockingList> oio::BlockingList for AwaitTreeWrapper<R> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.inner.next()
- }
-}
-
impl<R: oio::Delete> oio::Delete for AwaitTreeWrapper<R> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
self.inner.delete(path, args)
diff --git a/core/src/layers/concurrent_limit.rs b/core/src/layers/concurrent_limit.rs
index 64cabde58..435149190 100644
--- a/core/src/layers/concurrent_limit.rs
+++ b/core/src/layers/concurrent_limit.rs
@@ -305,28 +305,12 @@ impl<R: oio::Write> oio::Write for ConcurrentLimitWrapper<R> {
}
}
-impl<R: oio::BlockingWrite> oio::BlockingWrite for ConcurrentLimitWrapper<R> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- self.inner.write(bs)
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.inner.close()
- }
-}
-
impl<R: oio::List> oio::List for ConcurrentLimitWrapper<R> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
self.inner.next().await
}
}
-impl<R: oio::BlockingList> oio::BlockingList for ConcurrentLimitWrapper<R> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.inner.next()
- }
-}
-
impl<R: oio::Delete> oio::Delete for ConcurrentLimitWrapper<R> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
self.inner.delete(path, args)
diff --git a/core/src/layers/fastrace.rs b/core/src/layers/fastrace.rs
index 57eaf257e..18dd50ca5 100644
--- a/core/src/layers/fastrace.rs
+++ b/core/src/layers/fastrace.rs
@@ -257,14 +257,6 @@ impl<R: oio::List> oio::List for FastraceWrapper<R> {
}
}
-impl<R: oio::BlockingList> oio::BlockingList for FastraceWrapper<R> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- let _g = self.span.set_local_parent();
- let _span = LocalSpan::enter_with_local_parent(Operation::List.into_static());
- self.inner.next()
- }
-}
-
impl<R: oio::Delete> oio::Delete for FastraceWrapper<R> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
let _g = self.span.set_local_parent();
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index 3781e0157..fd64c4079 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -564,19 +564,6 @@ impl<P: oio::List, I: RetryInterceptor> oio::List for RetryWrapper<P, I> {
}
}
-impl<P: oio::BlockingList, I: RetryInterceptor> oio::BlockingList for RetryWrapper<P, I> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- { || self.inner.as_mut().unwrap().next() }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| {
- self.notify.intercept(err, dur);
- })
- .call()
- .map_err(|e| e.set_persistent())
- }
-}
-
impl<P: oio::Delete, I: RetryInterceptor> oio::Delete for RetryWrapper<P, I> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
{ || self.inner.as_mut().unwrap().delete(path, args.clone()) }
diff --git a/core/src/raw/oio/list/flat_list.rs b/core/src/raw/oio/list/flat_list.rs
index dbc2b14d8..0c044a916 100644
--- a/core/src/raw/oio/list/flat_list.rs
+++ b/core/src/raw/oio/list/flat_list.rs
@@ -143,9 +143,6 @@ mod tests {
use std::vec;
use std::vec::IntoIter;
- use log::debug;
- use oio::BlockingList;
-
use super::*;
#[derive(Debug)]
diff --git a/core/src/raw/oio/list/hierarchy_list.rs b/core/src/raw/oio/list/hierarchy_list.rs
index 435a53e15..7795cb2f6 100644
--- a/core/src/raw/oio/list/hierarchy_list.rs
+++ b/core/src/raw/oio/list/hierarchy_list.rs
@@ -133,27 +133,3 @@ impl<P: oio::List> oio::List for HierarchyLister<P> {
}
}
}
-
-#[cfg(test)]
-mod tests {
-
- use std::collections::HashSet;
- use std::vec::IntoIter;
-
- use log::debug;
- use oio::BlockingList;
-
- use super::*;
-
- struct MockLister {
- inner: IntoIter<&'static str>,
- }
-
- impl MockLister {
- fn new(inner: Vec<&'static str>) -> Self {
- Self {
- inner: inner.into_iter(),
- }
- }
- }
-}
diff --git a/core/src/services/dashmap/backend.rs b/core/src/services/dashmap/backend.rs
index 91398aac5..7f56126e9 100644
--- a/core/src/services/dashmap/backend.rs
+++ b/core/src/services/dashmap/backend.rs
@@ -99,10 +99,6 @@ impl typed_kv::Adapter for Adapter {
}
async fn get(&self, path: &str) -> Result<Option<typed_kv::Value>> {
- self.blocking_get(path)
- }
-
- fn blocking_get(&self, path: &str) -> Result<Option<typed_kv::Value>> {
match self.inner.get(path) {
None => Ok(None),
Some(bs) => Ok(Some(bs.value().to_owned())),
@@ -110,30 +106,18 @@ impl typed_kv::Adapter for Adapter {
}
async fn set(&self, path: &str, value: typed_kv::Value) -> Result<()> {
- self.blocking_set(path, value)
- }
-
- fn blocking_set(&self, path: &str, value: typed_kv::Value) -> Result<()> {
self.inner.insert(path.to_string(), value);
Ok(())
}
async fn delete(&self, path: &str) -> Result<()> {
- self.blocking_delete(path)
- }
-
- fn blocking_delete(&self, path: &str) -> Result<()> {
self.inner.remove(path);
Ok(())
}
async fn scan(&self, path: &str) -> Result<Vec<String>> {
- self.blocking_scan(path)
- }
-
- fn blocking_scan(&self, path: &str) -> Result<Vec<String>> {
let keys = self.inner.iter().map(|kv| kv.key().to_string());
if path.is_empty() {
Ok(keys.collect())
diff --git a/core/src/services/fs/lister.rs b/core/src/services/fs/lister.rs
index fe9e02499..7120ccdf0 100644
--- a/core/src/services/fs/lister.rs
+++ b/core/src/services/fs/lister.rs
@@ -79,39 +79,3 @@ impl oio::List for FsLister<tokio::fs::ReadDir> {
Ok(Some(entry))
}
}
-
-impl oio::BlockingList for FsLister<std::fs::ReadDir> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- // since list should return path itself, we return it first
- if let Some(path) = self.current_path.take() {
- let e = oio::Entry::new(path.as_str(), Metadata::new(EntryMode::DIR));
- return Ok(Some(e));
- }
-
- let de = match self.rd.next() {
- Some(de) => de.map_err(new_std_io_error)?,
- None => return Ok(None),
- };
-
- let entry_path = de.path();
- let rel_path = normalize_path(
- &entry_path
- .strip_prefix(&self.root)
- .expect("cannot fail because the prefix is iterated")
- .to_string_lossy()
- .replace('\\', "/"),
- );
-
- let ft = de.file_type().map_err(new_std_io_error)?;
- let entry = if ft.is_dir() {
- // Make sure we are returning the correct path.
- oio::Entry::new(&format!("{rel_path}/"), Metadata::new(EntryMode::DIR))
- } else if ft.is_file() {
- oio::Entry::new(&rel_path, Metadata::new(EntryMode::FILE))
- } else {
- oio::Entry::new(&rel_path, Metadata::new(EntryMode::Unknown))
- };
-
- Ok(Some(entry))
- }
-}
diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs
index d3a641f7c..2b02ced16 100644
--- a/core/src/services/fs/writer.rs
+++ b/core/src/services/fs/writer.rs
@@ -102,41 +102,6 @@ impl oio::Write for FsWriter<tokio::fs::File> {
}
}
-impl oio::BlockingWrite for FsWriter<std::fs::File> {
- fn write(&mut self, mut bs: Buffer) -> Result<()> {
- let f = self.f.as_mut().expect("FsWriter must be initialized");
-
- while bs.has_remaining() {
- let n = f.write(bs.chunk()).map_err(new_std_io_error)?;
- bs.advance(n);
- }
-
- Ok(())
- }
-
- fn close(&mut self) -> Result<Metadata> {
- let f = self.f.as_mut().expect("FsWriter must be initialized");
- f.sync_all().map_err(new_std_io_error)?;
-
- if let Some(tmp_path) = &self.tmp_path {
- std::fs::rename(tmp_path, &self.target_path).map_err(new_std_io_error)?;
- }
-
- let file_meta = f.metadata().map_err(new_std_io_error)?;
- let mode = if file_meta.is_file() {
- EntryMode::FILE
- } else if file_meta.is_dir() {
- EntryMode::DIR
- } else {
- EntryMode::Unknown
- };
- let meta = Metadata::new(mode)
- .with_content_length(file_meta.len())
- .with_last_modified(file_meta.modified().map_err(new_std_io_error)?.into());
- Ok(meta)
- }
-}
-
impl oio::PositionWrite for FsWriter<tokio::fs::File> {
async fn write_all_at(&self, offset: u64, buf: Buffer) -> Result<()> {
let f = self.f.as_ref().expect("FsWriter must be initialized");
diff --git a/core/src/services/hdfs/lister.rs b/core/src/services/hdfs/lister.rs
index 35be04ee4..a097639ad 100644
--- a/core/src/services/hdfs/lister.rs
+++ b/core/src/services/hdfs/lister.rs
@@ -68,32 +68,3 @@ impl oio::List for HdfsLister {
Ok(Some(entry))
}
}
-
-impl oio::BlockingList for HdfsLister {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- if let Some(path) = self.current_path.take() {
- return Ok(Some(oio::Entry::new(&path, Metadata::new(EntryMode::DIR))));
- }
-
- let de = match self.rd.next() {
- Some(de) => de,
- None => return Ok(None),
- };
-
- let path = build_rel_path(&self.root, de.path());
-
- let entry = if de.is_file() {
- let meta = Metadata::new(EntryMode::FILE)
- .with_content_length(de.len())
- .with_last_modified(de.modified().into());
- oio::Entry::new(&path, meta)
- } else if de.is_dir() {
- // Make sure we are returning the correct path.
- oio::Entry::new(&format!("{path}/"), Metadata::new(EntryMode::DIR))
- } else {
- oio::Entry::new(&path, Metadata::new(EntryMode::Unknown))
- };
-
- Ok(Some(entry))
- }
-}
diff --git a/core/src/services/hdfs/writer.rs b/core/src/services/hdfs/writer.rs
index 8b6faa8dc..11c48cf1b 100644
--- a/core/src/services/hdfs/writer.rs
+++ b/core/src/services/hdfs/writer.rs
@@ -99,37 +99,3 @@ impl oio::Write for HdfsWriter<hdrs::AsyncFile> {
))
}
}
-
-impl oio::BlockingWrite for HdfsWriter<hdrs::File> {
- fn write(&mut self, mut bs: Buffer) -> Result<()> {
- let len = bs.len() as u64;
-
- let f = self.f.as_mut().expect("HdfsWriter must be initialized");
- while bs.has_remaining() {
- let n = f.write(bs.chunk()).map_err(new_std_io_error)?;
- bs.advance(n);
- }
-
- self.size += len;
- Ok(())
- }
-
- fn close(&mut self) -> Result<Metadata> {
- let f = self.f.as_mut().expect("HdfsWriter must be initialized");
- f.flush().map_err(new_std_io_error)?;
-
- if let Some(tmp_path) = &self.tmp_path {
- // we must delete the target_path, otherwise the rename_file operation will fail
- if self.target_path_exists {
- self.client
- .remove_file(&self.target_path)
- .map_err(new_std_io_error)?;
- }
- self.client
- .rename_file(tmp_path, &self.target_path)
- .map_err(new_std_io_error)?;
- }
-
- Ok(Metadata::default().with_content_length(self.size))
- }
-}
diff --git a/core/src/services/rocksdb/backend.rs b/core/src/services/rocksdb/backend.rs
index a7fbc62be..1251e64db 100644
--- a/core/src/services/rocksdb/backend.rs
+++ b/core/src/services/rocksdb/backend.rs
@@ -152,7 +152,7 @@ impl kv::Adapter for Adapter {
res.push(key.to_string());
}
- Ok(res)
+ Ok(Box::new(kv::ScanStdIter::new(res.into_iter().map(Ok))))
}
}
diff --git a/core/src/services/sled/backend.rs b/core/src/services/sled/backend.rs
index cc0e44e50..891e54d9e 100644
--- a/core/src/services/sled/backend.rs
+++ b/core/src/services/sled/backend.rs
@@ -22,6 +22,7 @@ use std::str;
use tokio::task;
use crate::raw::adapters::kv;
+use crate::raw::adapters::kv::ScanStdIter;
use crate::raw::*;
use crate::services::SledConfig;
use crate::Builder;
@@ -188,7 +189,7 @@ impl kv::Adapter for Adapter {
res.push(v);
}
- Ok(res)
+ Ok(Box::new(kv::ScanStdIter::new(res.into_iter().map(Ok))))
}
}
diff --git a/core/src/types/context/read.rs b/core/src/types/context/read.rs
index d3cf432a3..b6bab726f 100644
--- a/core/src/types/context/read.rs
+++ b/core/src/types/context/read.rs
@@ -166,17 +166,6 @@ impl ReadGenerator {
let (_, r) = self.ctx.acc.read(&self.ctx.path, args).await?;
Ok(Some(r))
}
-
- /// Generate next blocking reader.
- pub fn next_blocking_reader(&mut self) -> Result<Option<oio::BlockingReader>> {
- let Some(range) = self.next_range() else {
- return Ok(None);
- };
-
- let args = self.ctx.args.clone().with_range(range);
- let (_, r) = self.ctx.acc.blocking_read(&self.ctx.path, args)?;
- Ok(Some(r))
- }
}
#[cfg(test)]
diff --git a/core/src/types/context/write.rs b/core/src/types/context/write.rs
index d426db5a8..038a4e4bb 100644
--- a/core/src/types/context/write.rs
+++ b/core/src/types/context/write.rs
@@ -204,87 +204,6 @@ impl WriteGenerator<oio::Writer> {
}
}
-impl WriteGenerator<oio::BlockingWriter> {
- /// Create a new exact buf writer.
- pub fn blocking_create(ctx: Arc<WriteContext>) -> Result<Self> {
- let (chunk_size, exact) = ctx.calculate_chunk_size();
- let (_, w) = ctx.acc.blocking_write(ctx.path(), ctx.args().clone())?;
-
- Ok(Self {
- w,
- chunk_size,
- exact,
- buffer: oio::QueueBuf::new(),
- })
- }
-}
-
-impl WriteGenerator<oio::BlockingWriter> {
- /// Write the entire buffer into writer.
- pub fn write(&mut self, mut bs: Buffer) -> Result<usize> {
- let Some(chunk_size) = self.chunk_size else {
- let size = bs.len();
- self.w.write(bs)?;
- return Ok(size);
- };
-
- if self.buffer.len() + bs.len() < chunk_size {
- let size = bs.len();
- self.buffer.push(bs);
- return Ok(size);
- }
-
- // Condition:
- // - exact is false
- // - buffer + bs is larger than chunk_size.
- // Action:
- // - write buffer + bs directly.
- if !self.exact {
- let fill_size = bs.len();
- self.buffer.push(bs);
- let buf = self.buffer.take().collect();
- self.w.write(buf)?;
- return Ok(fill_size);
- }
-
- // Condition:
- // - exact is true: we need write buffer in exact chunk size.
- // - buffer is larger than chunk_size
- // - in exact mode, the size must be chunk_size, use `>=` just for safe coding.
- // Action:
- // - write existing buffer in chunk_size to make more rooms for writing data.
- if self.buffer.len() >= chunk_size {
- let buf = self.buffer.take().collect();
- self.w.write(buf)?;
- }
-
- // Condition
- // - exact is true.
- // - buffer size must lower than chunk_size.
- // Action:
- // - write bs to buffer with remaining size.
- let remaining = chunk_size - self.buffer.len();
- bs.truncate(remaining);
- let n = bs.len();
- self.buffer.push(bs);
- Ok(n)
- }
-
- /// Finish the write process.
- pub fn close(&mut self) -> Result<Metadata> {
- loop {
- if self.buffer.is_empty() {
- break;
- }
-
- let buf = self.buffer.take().collect();
- self.w.write(buf)?;
- }
-
- self.w.close()
- }
-}
-
#[cfg(test)]
mod tests {
use bytes::Buf;