You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/05/23 10:45:00 UTC

(opendal) 03/18: Remove blocking APIs

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch remove-blocking-api-in-oio-access
in repository https://gitbox.apache.org/repos/asf/opendal.git

commit 4bb00fc5d33ddc6eb1c3ab27298b27896d235410
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri May 9 22:29:10 2025 +0800

    Remove blocking APIs
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 core/src/layers/async_backtrace.rs      |  6 ---
 core/src/layers/await_tree.rs           | 16 -------
 core/src/layers/concurrent_limit.rs     | 16 -------
 core/src/layers/fastrace.rs             |  8 ----
 core/src/layers/retry.rs                | 13 ------
 core/src/raw/oio/list/flat_list.rs      |  3 --
 core/src/raw/oio/list/hierarchy_list.rs | 24 ----------
 core/src/services/dashmap/backend.rs    | 16 -------
 core/src/services/fs/lister.rs          | 36 ---------------
 core/src/services/fs/writer.rs          | 35 --------------
 core/src/services/hdfs/lister.rs        | 29 ------------
 core/src/services/hdfs/writer.rs        | 34 --------------
 core/src/services/rocksdb/backend.rs    |  2 +-
 core/src/services/sled/backend.rs       |  3 +-
 core/src/types/context/read.rs          | 11 -----
 core/src/types/context/write.rs         | 81 ---------------------------------
 16 files changed, 3 insertions(+), 330 deletions(-)

diff --git a/core/src/layers/async_backtrace.rs b/core/src/layers/async_backtrace.rs
index 04f23327d..47a761185 100644
--- a/core/src/layers/async_backtrace.rs
+++ b/core/src/layers/async_backtrace.rs
@@ -162,12 +162,6 @@ impl<R: oio::List> oio::List for AsyncBacktraceWrapper<R> {
     }
 }
 
-impl<R: oio::BlockingList> oio::BlockingList for AsyncBacktraceWrapper<R> {
-    fn next(&mut self) -> Result<Option<oio::Entry>> {
-        self.inner.next()
-    }
-}
-
 impl<R: oio::Delete> oio::Delete for AsyncBacktraceWrapper<R> {
     fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
         self.inner.delete(path, args)
diff --git a/core/src/layers/await_tree.rs b/core/src/layers/await_tree.rs
index 02722fbfd..8d6f66a80 100644
--- a/core/src/layers/await_tree.rs
+++ b/core/src/layers/await_tree.rs
@@ -179,16 +179,6 @@ impl<R: oio::Write> oio::Write for AwaitTreeWrapper<R> {
     }
 }
 
-impl<R: oio::BlockingWrite> oio::BlockingWrite for AwaitTreeWrapper<R> {
-    fn write(&mut self, bs: Buffer) -> Result<()> {
-        self.inner.write(bs)
-    }
-
-    fn close(&mut self) -> Result<Metadata> {
-        self.inner.close()
-    }
-}
-
 impl<R: oio::List> oio::List for AwaitTreeWrapper<R> {
     async fn next(&mut self) -> Result<Option<oio::Entry>> {
         self.inner
@@ -198,12 +188,6 @@ impl<R: oio::List> oio::List for AwaitTreeWrapper<R> {
     }
 }
 
-impl<R: oio::BlockingList> oio::BlockingList for AwaitTreeWrapper<R> {
-    fn next(&mut self) -> Result<Option<oio::Entry>> {
-        self.inner.next()
-    }
-}
-
 impl<R: oio::Delete> oio::Delete for AwaitTreeWrapper<R> {
     fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
         self.inner.delete(path, args)
diff --git a/core/src/layers/concurrent_limit.rs b/core/src/layers/concurrent_limit.rs
index 64cabde58..435149190 100644
--- a/core/src/layers/concurrent_limit.rs
+++ b/core/src/layers/concurrent_limit.rs
@@ -305,28 +305,12 @@ impl<R: oio::Write> oio::Write for ConcurrentLimitWrapper<R> {
     }
 }
 
-impl<R: oio::BlockingWrite> oio::BlockingWrite for ConcurrentLimitWrapper<R> {
-    fn write(&mut self, bs: Buffer) -> Result<()> {
-        self.inner.write(bs)
-    }
-
-    fn close(&mut self) -> Result<Metadata> {
-        self.inner.close()
-    }
-}
-
 impl<R: oio::List> oio::List for ConcurrentLimitWrapper<R> {
     async fn next(&mut self) -> Result<Option<oio::Entry>> {
         self.inner.next().await
     }
 }
 
-impl<R: oio::BlockingList> oio::BlockingList for ConcurrentLimitWrapper<R> {
-    fn next(&mut self) -> Result<Option<oio::Entry>> {
-        self.inner.next()
-    }
-}
-
 impl<R: oio::Delete> oio::Delete for ConcurrentLimitWrapper<R> {
     fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
         self.inner.delete(path, args)
diff --git a/core/src/layers/fastrace.rs b/core/src/layers/fastrace.rs
index 57eaf257e..18dd50ca5 100644
--- a/core/src/layers/fastrace.rs
+++ b/core/src/layers/fastrace.rs
@@ -257,14 +257,6 @@ impl<R: oio::List> oio::List for FastraceWrapper<R> {
     }
 }
 
-impl<R: oio::BlockingList> oio::BlockingList for FastraceWrapper<R> {
-    fn next(&mut self) -> Result<Option<oio::Entry>> {
-        let _g = self.span.set_local_parent();
-        let _span = LocalSpan::enter_with_local_parent(Operation::List.into_static());
-        self.inner.next()
-    }
-}
-
 impl<R: oio::Delete> oio::Delete for FastraceWrapper<R> {
     fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
         let _g = self.span.set_local_parent();
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index 3781e0157..fd64c4079 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -564,19 +564,6 @@ impl<P: oio::List, I: RetryInterceptor> oio::List for RetryWrapper<P, I> {
     }
 }
 
-impl<P: oio::BlockingList, I: RetryInterceptor> oio::BlockingList for RetryWrapper<P, I> {
-    fn next(&mut self) -> Result<Option<oio::Entry>> {
-        { || self.inner.as_mut().unwrap().next() }
-            .retry(self.builder)
-            .when(|e| e.is_temporary())
-            .notify(|err, dur| {
-                self.notify.intercept(err, dur);
-            })
-            .call()
-            .map_err(|e| e.set_persistent())
-    }
-}
-
 impl<P: oio::Delete, I: RetryInterceptor> oio::Delete for RetryWrapper<P, I> {
     fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
         { || self.inner.as_mut().unwrap().delete(path, args.clone()) }
diff --git a/core/src/raw/oio/list/flat_list.rs b/core/src/raw/oio/list/flat_list.rs
index dbc2b14d8..0c044a916 100644
--- a/core/src/raw/oio/list/flat_list.rs
+++ b/core/src/raw/oio/list/flat_list.rs
@@ -143,9 +143,6 @@ mod tests {
     use std::vec;
     use std::vec::IntoIter;
 
-    use log::debug;
-    use oio::BlockingList;
-
     use super::*;
 
     #[derive(Debug)]
diff --git a/core/src/raw/oio/list/hierarchy_list.rs b/core/src/raw/oio/list/hierarchy_list.rs
index 435a53e15..7795cb2f6 100644
--- a/core/src/raw/oio/list/hierarchy_list.rs
+++ b/core/src/raw/oio/list/hierarchy_list.rs
@@ -133,27 +133,3 @@ impl<P: oio::List> oio::List for HierarchyLister<P> {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-
-    use std::collections::HashSet;
-    use std::vec::IntoIter;
-
-    use log::debug;
-    use oio::BlockingList;
-
-    use super::*;
-
-    struct MockLister {
-        inner: IntoIter<&'static str>,
-    }
-
-    impl MockLister {
-        fn new(inner: Vec<&'static str>) -> Self {
-            Self {
-                inner: inner.into_iter(),
-            }
-        }
-    }
-}
diff --git a/core/src/services/dashmap/backend.rs b/core/src/services/dashmap/backend.rs
index 91398aac5..7f56126e9 100644
--- a/core/src/services/dashmap/backend.rs
+++ b/core/src/services/dashmap/backend.rs
@@ -99,10 +99,6 @@ impl typed_kv::Adapter for Adapter {
     }
 
     async fn get(&self, path: &str) -> Result<Option<typed_kv::Value>> {
-        self.blocking_get(path)
-    }
-
-    fn blocking_get(&self, path: &str) -> Result<Option<typed_kv::Value>> {
         match self.inner.get(path) {
             None => Ok(None),
             Some(bs) => Ok(Some(bs.value().to_owned())),
@@ -110,30 +106,18 @@ impl typed_kv::Adapter for Adapter {
     }
 
     async fn set(&self, path: &str, value: typed_kv::Value) -> Result<()> {
-        self.blocking_set(path, value)
-    }
-
-    fn blocking_set(&self, path: &str, value: typed_kv::Value) -> Result<()> {
         self.inner.insert(path.to_string(), value);
 
         Ok(())
     }
 
     async fn delete(&self, path: &str) -> Result<()> {
-        self.blocking_delete(path)
-    }
-
-    fn blocking_delete(&self, path: &str) -> Result<()> {
         self.inner.remove(path);
 
         Ok(())
     }
 
     async fn scan(&self, path: &str) -> Result<Vec<String>> {
-        self.blocking_scan(path)
-    }
-
-    fn blocking_scan(&self, path: &str) -> Result<Vec<String>> {
         let keys = self.inner.iter().map(|kv| kv.key().to_string());
         if path.is_empty() {
             Ok(keys.collect())
diff --git a/core/src/services/fs/lister.rs b/core/src/services/fs/lister.rs
index fe9e02499..7120ccdf0 100644
--- a/core/src/services/fs/lister.rs
+++ b/core/src/services/fs/lister.rs
@@ -79,39 +79,3 @@ impl oio::List for FsLister<tokio::fs::ReadDir> {
         Ok(Some(entry))
     }
 }
-
-impl oio::BlockingList for FsLister<std::fs::ReadDir> {
-    fn next(&mut self) -> Result<Option<oio::Entry>> {
-        // since list should return path itself, we return it first
-        if let Some(path) = self.current_path.take() {
-            let e = oio::Entry::new(path.as_str(), Metadata::new(EntryMode::DIR));
-            return Ok(Some(e));
-        }
-
-        let de = match self.rd.next() {
-            Some(de) => de.map_err(new_std_io_error)?,
-            None => return Ok(None),
-        };
-
-        let entry_path = de.path();
-        let rel_path = normalize_path(
-            &entry_path
-                .strip_prefix(&self.root)
-                .expect("cannot fail because the prefix is iterated")
-                .to_string_lossy()
-                .replace('\\', "/"),
-        );
-
-        let ft = de.file_type().map_err(new_std_io_error)?;
-        let entry = if ft.is_dir() {
-            // Make sure we are returning the correct path.
-            oio::Entry::new(&format!("{rel_path}/"), Metadata::new(EntryMode::DIR))
-        } else if ft.is_file() {
-            oio::Entry::new(&rel_path, Metadata::new(EntryMode::FILE))
-        } else {
-            oio::Entry::new(&rel_path, Metadata::new(EntryMode::Unknown))
-        };
-
-        Ok(Some(entry))
-    }
-}
diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs
index d3a641f7c..2b02ced16 100644
--- a/core/src/services/fs/writer.rs
+++ b/core/src/services/fs/writer.rs
@@ -102,41 +102,6 @@ impl oio::Write for FsWriter<tokio::fs::File> {
     }
 }
 
-impl oio::BlockingWrite for FsWriter<std::fs::File> {
-    fn write(&mut self, mut bs: Buffer) -> Result<()> {
-        let f = self.f.as_mut().expect("FsWriter must be initialized");
-
-        while bs.has_remaining() {
-            let n = f.write(bs.chunk()).map_err(new_std_io_error)?;
-            bs.advance(n);
-        }
-
-        Ok(())
-    }
-
-    fn close(&mut self) -> Result<Metadata> {
-        let f = self.f.as_mut().expect("FsWriter must be initialized");
-        f.sync_all().map_err(new_std_io_error)?;
-
-        if let Some(tmp_path) = &self.tmp_path {
-            std::fs::rename(tmp_path, &self.target_path).map_err(new_std_io_error)?;
-        }
-
-        let file_meta = f.metadata().map_err(new_std_io_error)?;
-        let mode = if file_meta.is_file() {
-            EntryMode::FILE
-        } else if file_meta.is_dir() {
-            EntryMode::DIR
-        } else {
-            EntryMode::Unknown
-        };
-        let meta = Metadata::new(mode)
-            .with_content_length(file_meta.len())
-            .with_last_modified(file_meta.modified().map_err(new_std_io_error)?.into());
-        Ok(meta)
-    }
-}
-
 impl oio::PositionWrite for FsWriter<tokio::fs::File> {
     async fn write_all_at(&self, offset: u64, buf: Buffer) -> Result<()> {
         let f = self.f.as_ref().expect("FsWriter must be initialized");
diff --git a/core/src/services/hdfs/lister.rs b/core/src/services/hdfs/lister.rs
index 35be04ee4..a097639ad 100644
--- a/core/src/services/hdfs/lister.rs
+++ b/core/src/services/hdfs/lister.rs
@@ -68,32 +68,3 @@ impl oio::List for HdfsLister {
         Ok(Some(entry))
     }
 }
-
-impl oio::BlockingList for HdfsLister {
-    fn next(&mut self) -> Result<Option<oio::Entry>> {
-        if let Some(path) = self.current_path.take() {
-            return Ok(Some(oio::Entry::new(&path, Metadata::new(EntryMode::DIR))));
-        }
-
-        let de = match self.rd.next() {
-            Some(de) => de,
-            None => return Ok(None),
-        };
-
-        let path = build_rel_path(&self.root, de.path());
-
-        let entry = if de.is_file() {
-            let meta = Metadata::new(EntryMode::FILE)
-                .with_content_length(de.len())
-                .with_last_modified(de.modified().into());
-            oio::Entry::new(&path, meta)
-        } else if de.is_dir() {
-            // Make sure we are returning the correct path.
-            oio::Entry::new(&format!("{path}/"), Metadata::new(EntryMode::DIR))
-        } else {
-            oio::Entry::new(&path, Metadata::new(EntryMode::Unknown))
-        };
-
-        Ok(Some(entry))
-    }
-}
diff --git a/core/src/services/hdfs/writer.rs b/core/src/services/hdfs/writer.rs
index 8b6faa8dc..11c48cf1b 100644
--- a/core/src/services/hdfs/writer.rs
+++ b/core/src/services/hdfs/writer.rs
@@ -99,37 +99,3 @@ impl oio::Write for HdfsWriter<hdrs::AsyncFile> {
         ))
     }
 }
-
-impl oio::BlockingWrite for HdfsWriter<hdrs::File> {
-    fn write(&mut self, mut bs: Buffer) -> Result<()> {
-        let len = bs.len() as u64;
-
-        let f = self.f.as_mut().expect("HdfsWriter must be initialized");
-        while bs.has_remaining() {
-            let n = f.write(bs.chunk()).map_err(new_std_io_error)?;
-            bs.advance(n);
-        }
-
-        self.size += len;
-        Ok(())
-    }
-
-    fn close(&mut self) -> Result<Metadata> {
-        let f = self.f.as_mut().expect("HdfsWriter must be initialized");
-        f.flush().map_err(new_std_io_error)?;
-
-        if let Some(tmp_path) = &self.tmp_path {
-            // we must delete the target_path, otherwise the rename_file operation will fail
-            if self.target_path_exists {
-                self.client
-                    .remove_file(&self.target_path)
-                    .map_err(new_std_io_error)?;
-            }
-            self.client
-                .rename_file(tmp_path, &self.target_path)
-                .map_err(new_std_io_error)?;
-        }
-
-        Ok(Metadata::default().with_content_length(self.size))
-    }
-}
diff --git a/core/src/services/rocksdb/backend.rs b/core/src/services/rocksdb/backend.rs
index a7fbc62be..1251e64db 100644
--- a/core/src/services/rocksdb/backend.rs
+++ b/core/src/services/rocksdb/backend.rs
@@ -152,7 +152,7 @@ impl kv::Adapter for Adapter {
             res.push(key.to_string());
         }
 
-        Ok(res)
+        Ok(Box::new(kv::ScanStdIter::new(res.into_iter().map(Ok))))
     }
 }
 
diff --git a/core/src/services/sled/backend.rs b/core/src/services/sled/backend.rs
index cc0e44e50..891e54d9e 100644
--- a/core/src/services/sled/backend.rs
+++ b/core/src/services/sled/backend.rs
@@ -22,6 +22,7 @@ use std::str;
 use tokio::task;
 
 use crate::raw::adapters::kv;
+use crate::raw::adapters::kv::ScanStdIter;
 use crate::raw::*;
 use crate::services::SledConfig;
 use crate::Builder;
@@ -188,7 +189,7 @@ impl kv::Adapter for Adapter {
             res.push(v);
         }
 
-        Ok(res)
+        Ok(Box::new(kv::ScanStdIter::new(res.into_iter().map(Ok))))
     }
 }
 
diff --git a/core/src/types/context/read.rs b/core/src/types/context/read.rs
index d3cf432a3..b6bab726f 100644
--- a/core/src/types/context/read.rs
+++ b/core/src/types/context/read.rs
@@ -166,17 +166,6 @@ impl ReadGenerator {
         let (_, r) = self.ctx.acc.read(&self.ctx.path, args).await?;
         Ok(Some(r))
     }
-
-    /// Generate next blocking reader.
-    pub fn next_blocking_reader(&mut self) -> Result<Option<oio::BlockingReader>> {
-        let Some(range) = self.next_range() else {
-            return Ok(None);
-        };
-
-        let args = self.ctx.args.clone().with_range(range);
-        let (_, r) = self.ctx.acc.blocking_read(&self.ctx.path, args)?;
-        Ok(Some(r))
-    }
 }
 
 #[cfg(test)]
diff --git a/core/src/types/context/write.rs b/core/src/types/context/write.rs
index d426db5a8..038a4e4bb 100644
--- a/core/src/types/context/write.rs
+++ b/core/src/types/context/write.rs
@@ -204,87 +204,6 @@ impl WriteGenerator<oio::Writer> {
     }
 }
 
-impl WriteGenerator<oio::BlockingWriter> {
-    /// Create a new exact buf writer.
-    pub fn blocking_create(ctx: Arc<WriteContext>) -> Result<Self> {
-        let (chunk_size, exact) = ctx.calculate_chunk_size();
-        let (_, w) = ctx.acc.blocking_write(ctx.path(), ctx.args().clone())?;
-
-        Ok(Self {
-            w,
-            chunk_size,
-            exact,
-            buffer: oio::QueueBuf::new(),
-        })
-    }
-}
-
-impl WriteGenerator<oio::BlockingWriter> {
-    /// Write the entire buffer into writer.
-    pub fn write(&mut self, mut bs: Buffer) -> Result<usize> {
-        let Some(chunk_size) = self.chunk_size else {
-            let size = bs.len();
-            self.w.write(bs)?;
-            return Ok(size);
-        };
-
-        if self.buffer.len() + bs.len() < chunk_size {
-            let size = bs.len();
-            self.buffer.push(bs);
-            return Ok(size);
-        }
-
-        // Condition:
-        // - exact is false
-        // - buffer + bs is larger than chunk_size.
-        // Action:
-        // - write buffer + bs directly.
-        if !self.exact {
-            let fill_size = bs.len();
-            self.buffer.push(bs);
-            let buf = self.buffer.take().collect();
-            self.w.write(buf)?;
-            return Ok(fill_size);
-        }
-
-        // Condition:
-        // - exact is true: we need write buffer in exact chunk size.
-        // - buffer is larger than chunk_size
-        //   - in exact mode, the size must be chunk_size, use `>=` just for safe coding.
-        // Action:
-        // - write existing buffer in chunk_size to make more rooms for writing data.
-        if self.buffer.len() >= chunk_size {
-            let buf = self.buffer.take().collect();
-            self.w.write(buf)?;
-        }
-
-        // Condition
-        // - exact is true.
-        // - buffer size must lower than chunk_size.
-        // Action:
-        // - write bs to buffer with remaining size.
-        let remaining = chunk_size - self.buffer.len();
-        bs.truncate(remaining);
-        let n = bs.len();
-        self.buffer.push(bs);
-        Ok(n)
-    }
-
-    /// Finish the write process.
-    pub fn close(&mut self) -> Result<Metadata> {
-        loop {
-            if self.buffer.is_empty() {
-                break;
-            }
-
-            let buf = self.buffer.take().collect();
-            self.w.write(buf)?;
-        }
-
-        self.w.close()
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use bytes::Buf;