You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/05/23 12:14:43 UTC
(opendal) branch main updated: refactor!: Remove native blocking api in opendal (#6216)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new d0c5748a3 refactor!: Remove native blocking api in opendal (#6216)
d0c5748a3 is described below
commit d0c5748a317056a009f284004087722c52562658
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Fri May 23 20:14:37 2025 +0800
refactor!: Remove native blocking api in opendal (#6216)
* remove blocking
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Remove blocking APIs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Save work
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix build
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Rename
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix build for bindings c
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix build for cpp
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix java build
* Fix python
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix lua support
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix build for nodejs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix build for ruby
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Format code
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* We don't have blocking anymore
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix java
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Make sure blocking structs drop inside async context
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Remove blocking
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix unit tests
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix dart
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix dotnet
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Format
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix haskell
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Re-gen for nodejs
Signed-off-by: Xuanwo <gi...@xuanwo.io>
* Fix ocamle build
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---------
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
bin/oli/Cargo.toml | 8 +-
bindings/c/Cargo.toml | 2 +-
bindings/c/include/opendal.h | 12 +-
bindings/c/src/lister.rs | 8 +-
bindings/c/src/operator.rs | 28 +-
bindings/c/src/operator_info.rs | 4 -
bindings/c/src/reader.rs | 10 +-
bindings/c/src/writer.rs | 12 +-
bindings/c/tests/bdd.cpp | 2 +-
bindings/c/tests/opinfo.cpp | 2 -
bindings/c/tests/reader.cpp | 22 +-
bindings/cpp/Cargo.toml | 2 +
bindings/cpp/src/async.rs | 8 +-
bindings/cpp/src/lib.rs | 21 +-
bindings/cpp/src/lister.rs | 2 +-
bindings/cpp/src/reader.rs | 2 +-
bindings/dart/rust/Cargo.toml | 3 +
bindings/dart/rust/src/api/opendal_api.rs | 45 +-
bindings/dotnet/Cargo.toml | 3 +
bindings/dotnet/src/lib.rs | 28 +-
bindings/haskell/Cargo.toml | 2 +
bindings/haskell/src/lib.rs | 78 +-
bindings/java/Cargo.toml | 2 +-
bindings/java/src/async_operator.rs | 36 +-
bindings/java/src/lib.rs | 3 +-
.../java/org/apache/opendal/AsyncOperator.java | 8 +-
.../main/java/org/apache/opendal/Capability.java | 8 +-
bindings/java/src/operator.rs | 94 +-
bindings/java/src/operator_input_stream.rs | 8 +-
bindings/java/src/operator_output_stream.rs | 15 +-
.../apache/opendal/test/OperatorDuplicateTest.java | 1 +
.../org/apache/opendal/test/OperatorInfoTest.java | 1 +
.../opendal/test/behavior/BlockingListTest.java | 8 +-
.../test/behavior/BlockingReadOnlyTest.java | 2 +-
.../opendal/test/behavior/BlockingRenameTest.java | 7 +-
.../opendal/test/behavior/BlockingWriteTest.java | 3 +-
bindings/lua/Cargo.toml | 2 +
bindings/lua/src/lib.rs | 16 +-
bindings/nodejs/Cargo.toml | 2 +-
bindings/nodejs/generated.d.ts | 35 +-
bindings/nodejs/src/capability.rs | 6 -
bindings/nodejs/src/lib.rs | 279 +++--
bindings/ocaml/Cargo.toml | 2 +
bindings/ocaml/src/lib.rs | 8 +
bindings/ocaml/src/operator/_type.rs | 4 +-
bindings/ocaml/src/operator/mod.rs | 7 +-
bindings/python/Cargo.toml | 4 +-
bindings/python/python/opendal/__init__.pyi | 8 +-
bindings/python/src/capability.rs | 4 -
bindings/python/src/file.rs | 8 +-
bindings/python/src/lister.rs | 4 +-
bindings/python/src/operator.rs | 87 +-
bindings/python/src/options.rs | 15 +
bindings/python/tests/test_sync_delete.py | 2 +-
bindings/python/tests/test_sync_exists.py | 2 +-
bindings/ruby/Cargo.toml | 2 +
bindings/ruby/src/capability.rs | 2 -
bindings/ruby/src/io.rs | 6 +-
bindings/ruby/src/lib.rs | 9 +
bindings/ruby/src/lister.rs | 4 +-
bindings/ruby/src/middlewares.rs | 2 +-
bindings/ruby/src/operator.rs | 66 +-
core/Cargo.lock | 1 +
core/Cargo.toml | 5 +-
core/benches/vs_fs/Cargo.toml | 1 +
core/benches/vs_fs/src/main.rs | 18 +-
.../blocking_deleter.rs => blocking/delete.rs} | 62 +-
core/src/blocking/list.rs | 71 ++
core/src/{types/blocking_write => blocking}/mod.rs | 19 +-
core/src/blocking/operator.rs | 720 ++++++++++++
core/src/blocking/read/buffer_iterator.rs | 66 ++
.../{types/blocking_read => blocking/read}/mod.rs | 5 +-
core/src/blocking/read/reader.rs | 125 +++
.../read}/std_bytes_iterator.rs | 44 +-
core/src/blocking/read/std_reader.rs | 95 ++
.../blocking_write => blocking/write}/mod.rs | 4 +-
.../write}/std_writer.rs | 63 +-
.../write/writer.rs} | 67 +-
core/src/docs/concepts.rs | 6 +-
core/src/docs/rfcs/1477_remove_object_concept.md | 2 +-
core/src/docs/rfcs/6189_remove_native_blocking.md | 8 +-
core/src/layers/async_backtrace.rs | 60 -
core/src/layers/await_tree.rs | 60 -
core/src/layers/blocking.rs | 362 -------
core/src/layers/capability_check.rs | 68 --
core/src/layers/chaos.rs | 32 -
core/src/layers/complete.rs | 226 ----
core/src/layers/concurrent_limit.rs | 102 --
core/src/layers/correctness_check.rs | 83 --
core/src/layers/dtrace.rs | 130 +--
core/src/layers/error_context.rs | 188 ----
core/src/layers/fastrace.rs | 120 --
core/src/layers/immutable_index.rs | 37 -
core/src/layers/logging.rs | 520 ---------
core/src/layers/mime_guess.rs | 61 --
core/src/layers/mod.rs | 5 -
core/src/layers/observe/metrics.rs | 297 -----
core/src/layers/oteltrace.rs | 102 --
core/src/layers/retry.rs | 196 ----
core/src/layers/throttle.rs | 63 --
core/src/layers/timeout.rs | 24 -
core/src/layers/tracing.rs | 112 --
core/src/layers/type_eraser.rs | 28 -
core/src/lib.rs | 10 +-
core/src/raw/accessor.rs | 259 -----
core/src/raw/adapters/kv/api.rs | 60 -
core/src/raw/adapters/kv/backend.rs | 112 --
core/src/raw/adapters/typed_kv/api.rs | 21 -
core/src/raw/adapters/typed_kv/backend.rs | 94 --
core/src/raw/enum_utils.rs | 55 -
core/src/raw/layer.rs | 101 --
core/src/raw/oio/delete/api.rs | 52 -
core/src/raw/oio/delete/mod.rs | 3 -
core/src/raw/oio/delete/one_shot_delete.rs | 26 -
core/src/raw/oio/list/api.rs | 33 -
core/src/raw/oio/list/flat_list.rs | 150 ---
core/src/raw/oio/list/hierarchy_list.rs | 100 --
core/src/raw/oio/list/mod.rs | 2 -
core/src/raw/oio/list/prefix_list.rs | 14 -
core/src/raw/oio/read/api.rs | 35 -
core/src/raw/oio/read/mod.rs | 2 -
core/src/raw/oio/write/api.rs | 48 -
core/src/raw/oio/write/mod.rs | 2 -
core/src/raw/ops.rs | 12 +
core/src/raw/tests/read.rs | 16 -
core/src/raw/tests/utils.rs | 11 +-
core/src/services/aliyun_drive/backend.rs | 4 -
core/src/services/alluxio/backend.rs | 4 -
core/src/services/azblob/backend.rs | 4 -
core/src/services/azdls/backend.rs | 4 -
core/src/services/azfile/backend.rs | 4 -
core/src/services/b2/backend.rs | 4 -
core/src/services/cacache/backend.rs | 17 -
core/src/services/compfs/backend.rs | 4 -
core/src/services/cos/backend.rs | 4 -
core/src/services/dashmap/backend.rs | 16 -
core/src/services/dbfs/backend.rs | 4 -
core/src/services/dropbox/backend.rs | 4 -
core/src/services/fs/backend.rs | 199 ----
core/src/services/fs/core.rs | 26 -
core/src/services/fs/delete.rs | 22 -
core/src/services/fs/lister.rs | 36 -
core/src/services/fs/reader.rs | 38 -
core/src/services/fs/writer.rs | 35 -
core/src/services/ftp/backend.rs | 4 -
core/src/services/gcs/backend.rs | 4 -
core/src/services/gdrive/backend.rs | 4 -
core/src/services/ghac/backend.rs | 4 -
core/src/services/github/backend.rs | 4 -
core/src/services/hdfs/backend.rs | 187 ----
core/src/services/hdfs/delete.rs | 29 -
core/src/services/hdfs/lister.rs | 29 -
core/src/services/hdfs/reader.rs | 36 -
core/src/services/hdfs/writer.rs | 35 -
core/src/services/hdfs_native/backend.rs | 4 -
core/src/services/http/backend.rs | 4 -
core/src/services/huggingface/backend.rs | 4 -
core/src/services/icloud/backend.rs | 4 -
core/src/services/ipfs/backend.rs | 4 -
core/src/services/ipmfs/backend.rs | 4 -
core/src/services/koofr/backend.rs | 4 -
core/src/services/lakefs/backend.rs | 4 -
core/src/services/memory/backend.rs | 38 -
core/src/services/mini_moka/backend.rs | 16 -
core/src/services/moka/backend.rs | 16 -
core/src/services/monoiofs/backend.rs | 4 -
core/src/services/obs/backend.rs | 4 -
core/src/services/onedrive/backend.rs | 4 -
core/src/services/oss/backend.rs | 4 -
core/src/services/pcloud/backend.rs | 4 -
core/src/services/persy/backend.rs | 31 -
core/src/services/redb/backend.rs | 33 -
core/src/services/rocksdb/backend.rs | 43 +-
core/src/services/s3/backend.rs | 4 -
core/src/services/seafile/backend.rs | 4 -
core/src/services/sftp/backend.rs | 4 -
core/src/services/sled/backend.rs | 43 +-
core/src/services/swift/backend.rs | 4 -
core/src/services/tikv/backend.rs | 1 -
core/src/services/upyun/backend.rs | 4 -
core/src/services/vercel_artifacts/backend.rs | 4 -
core/src/services/vercel_blob/backend.rs | 4 -
core/src/services/webdav/backend.rs | 4 -
core/src/services/webhdfs/backend.rs | 4 -
core/src/services/yandex_disk/backend.rs | 4 -
core/src/types/blocking_read/blocking_reader.rs | 140 ---
core/src/types/blocking_read/buffer_iterator.rs | 95 --
core/src/types/blocking_read/std_reader.rs | 140 ---
core/src/types/capability.rs | 7 +-
core/src/types/context/read.rs | 36 -
core/src/types/context/write.rs | 81 --
core/src/types/delete/mod.rs | 3 -
core/src/types/list.rs | 49 -
core/src/types/mod.rs | 9 -
core/src/types/operator/blocking_operator.rs | 1142 --------------------
core/src/types/operator/mod.rs | 4 -
core/src/types/operator/operator.rs | 74 +-
core/src/types/operator/operator_functions.rs | 437 --------
core/src/types/operator/operator_futures.rs | 80 +-
core/src/types/options.rs | 7 +-
core/tests/behavior/blocking_copy.rs | 163 ---
core/tests/behavior/blocking_create_dir.rs | 63 --
core/tests/behavior/blocking_delete.rs | 113 --
core/tests/behavior/blocking_list.rs | 247 -----
core/tests/behavior/blocking_read.rs | 177 ---
core/tests/behavior/blocking_rename.rs | 177 ---
core/tests/behavior/blocking_stat.rs | 94 --
core/tests/behavior/blocking_write.rs | 219 ----
core/tests/behavior/main.rs | 19 -
core/tests/behavior/utils.rs | 21 -
210 files changed, 1894 insertions(+), 9343 deletions(-)
diff --git a/bin/oli/Cargo.toml b/bin/oli/Cargo.toml
index 98a24e942..b544628f5 100644
--- a/bin/oli/Cargo.toml
+++ b/bin/oli/Cargo.toml
@@ -64,9 +64,9 @@ uuid = { version = "1.12" }
[dev-dependencies]
assert_cmd = { version = "2.0" }
-tempfile = { version = "3.14" }
-insta-cmd = { version = "0.6" }
-insta = { version = "1.43", features = ["filters"] }
-walkdir = "2.5.0"
comfy-table = "7"
+insta = { version = "1.43", features = ["filters"] }
+insta-cmd = { version = "0.6" }
regex = "1"
+tempfile = { version = "3.14" }
+walkdir = "2.5.0"
diff --git a/bindings/c/Cargo.toml b/bindings/c/Cargo.toml
index 7ce6a2c38..09eec6946 100644
--- a/bindings/c/Cargo.toml
+++ b/bindings/c/Cargo.toml
@@ -36,7 +36,7 @@ cbindgen = "0.26.0"
[dependencies]
bytes = "1.4.0"
# this crate won't be published, we always use the local version
-opendal = { version = ">=0", path = "../../core", features = ["layers-blocking"] }
+opendal = { version = ">=0", path = "../../core", features = ["blocking"] }
tokio = { version = "1.27", features = ["fs", "macros", "rt-multi-thread"] }
[features]
diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h
index 6ca9ea859..fa5523d7f 100644
--- a/bindings/c/include/opendal.h
+++ b/bindings/c/include/opendal.h
@@ -213,14 +213,14 @@ typedef struct opendal_metadata {
* @see opendal_operator_free This function frees the heap memory of the operator
*
* \note The opendal_operator actually owns a pointer to
- * an opendal::BlockingOperator, which is inside the Rust core code.
+ * an opendal::blocking::Operator, which is inside the Rust core code.
*
* \remark You may use the field `ptr` to check whether this is a NULL
* operator.
*/
typedef struct opendal_operator {
/**
- * The pointer to the opendal::BlockingOperator in the Rust code.
+ * The pointer to the opendal::blocking::Operator in the Rust code.
* Only touch this on judging whether it is NULL.
*/
void *inner;
@@ -320,11 +320,11 @@ typedef struct opendal_result_operator_reader {
/**
* \brief The result type returned by opendal's writer operation.
* \note The opendal_writer actually owns a pointer to
- * an opendal::BlockingWriter, which is inside the Rust core code.
+ * an opendal::blocking::Writer, which is inside the Rust core code.
*/
typedef struct opendal_writer {
/**
- * The pointer to the opendal::BlockingWriter in the Rust code.
+ * The pointer to the opendal::blocking::Writer in the Rust code.
* Only touch this on judging whether it is NULL.
*/
void *inner;
@@ -581,10 +581,6 @@ typedef struct opendal_capability {
* If operator supports shared.
*/
bool shared;
- /**
- * If operator supports blocking.
- */
- bool blocking;
} opendal_capability;
/**
diff --git a/bindings/c/src/lister.rs b/bindings/c/src/lister.rs
index b6a2a0e77..c6508621e 100644
--- a/bindings/c/src/lister.rs
+++ b/bindings/c/src/lister.rs
@@ -35,15 +35,15 @@ pub struct opendal_lister {
}
impl opendal_lister {
- fn deref_mut(&mut self) -> &mut core::BlockingLister {
+ fn deref_mut(&mut self) -> &mut core::blocking::Lister {
// Safety: the inner should never be null once constructed
// The use-after-free is undefined behavior
- unsafe { &mut *(self.inner as *mut core::BlockingLister) }
+ unsafe { &mut *(self.inner as *mut core::blocking::Lister) }
}
}
impl opendal_lister {
- pub(crate) fn new(lister: core::BlockingLister) -> Self {
+ pub(crate) fn new(lister: core::blocking::Lister) -> Self {
Self {
inner: Box::into_raw(Box::new(lister)) as _,
}
@@ -85,7 +85,7 @@ impl opendal_lister {
#[no_mangle]
pub unsafe extern "C" fn opendal_lister_free(ptr: *mut opendal_lister) {
if !ptr.is_null() {
- drop(Box::from_raw((*ptr).inner as *mut core::BlockingLister));
+ drop(Box::from_raw((*ptr).inner as *mut core::blocking::Lister));
drop(Box::from_raw(ptr));
}
}
diff --git a/bindings/c/src/operator.rs b/bindings/c/src/operator.rs
index 48ab4a707..79395e3d3 100644
--- a/bindings/c/src/operator.rs
+++ b/bindings/c/src/operator.rs
@@ -39,22 +39,22 @@ static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
/// @see opendal_operator_free This function frees the heap memory of the operator
///
/// \note The opendal_operator actually owns a pointer to
-/// an opendal::BlockingOperator, which is inside the Rust core code.
+/// an opendal::blocking::Operator, which is inside the Rust core code.
///
/// \remark You may use the field `ptr` to check whether this is a NULL
/// operator.
#[repr(C)]
pub struct opendal_operator {
- /// The pointer to the opendal::BlockingOperator in the Rust code.
+ /// The pointer to the opendal::blocking::Operator in the Rust code.
/// Only touch this on judging whether it is NULL.
inner: *mut c_void,
}
impl opendal_operator {
- pub(crate) fn deref(&self) -> &core::BlockingOperator {
+ pub(crate) fn deref(&self) -> &core::blocking::Operator {
// Safety: the inner should never be null once constructed
// The use-after-free is undefined behavior
- unsafe { &*(self.inner as *mut core::BlockingOperator) }
+ unsafe { &*(self.inner as *mut core::blocking::Operator) }
}
}
@@ -77,7 +77,7 @@ impl opendal_operator {
#[no_mangle]
pub unsafe extern "C" fn opendal_operator_free(ptr: *const opendal_operator) {
if !ptr.is_null() {
- drop(Box::from_raw((*ptr).inner as *mut core::BlockingOperator));
+ drop(Box::from_raw((*ptr).inner as *mut core::blocking::Operator));
drop(Box::from_raw(ptr as *mut opendal_operator));
}
}
@@ -86,15 +86,13 @@ impl opendal_operator {
fn build_operator(
schema: core::Scheme,
map: HashMap<String, String>,
-) -> core::Result<core::Operator> {
- let mut op = core::Operator::via_iter(schema, map)?.layer(core::layers::RetryLayer::new());
- if !op.info().full_capability().blocking {
- let runtime =
- tokio::runtime::Handle::try_current().unwrap_or_else(|_| RUNTIME.handle().clone());
- let _guard = runtime.enter();
- op = op
- .layer(core::layers::BlockingLayer::create().expect("blocking layer must be created"));
- }
+) -> core::Result<core::blocking::Operator> {
+ let op = core::Operator::via_iter(schema, map)?.layer(core::layers::RetryLayer::new());
+
+ let runtime =
+ tokio::runtime::Handle::try_current().unwrap_or_else(|_| RUNTIME.handle().clone());
+ let _guard = runtime.enter();
+ let op = core::blocking::Operator::new(op)?;
Ok(op)
}
@@ -164,7 +162,7 @@ pub unsafe extern "C" fn opendal_operator_new(
match build_operator(scheme, map) {
Ok(op) => opendal_result_operator_new {
op: Box::into_raw(Box::new(opendal_operator {
- inner: Box::into_raw(Box::new(op.blocking())) as _,
+ inner: Box::into_raw(Box::new(op)) as _,
})),
error: std::ptr::null_mut(),
},
diff --git a/bindings/c/src/operator_info.rs b/bindings/c/src/operator_info.rs
index 02a12eb91..c9e9bb5cd 100644
--- a/bindings/c/src/operator_info.rs
+++ b/bindings/c/src/operator_info.rs
@@ -128,9 +128,6 @@ pub struct opendal_capability {
/// If operator supports shared.
pub shared: bool,
-
- /// If operator supports blocking.
- pub blocking: bool,
}
impl opendal_operator_info {
@@ -255,7 +252,6 @@ impl From<core::Capability> for opendal_capability {
presign_stat: value.presign_stat,
presign_write: value.presign_write,
shared: value.shared,
- blocking: value.blocking,
}
}
}
diff --git a/bindings/c/src/reader.rs b/bindings/c/src/reader.rs
index 63be5389a..f11ebc54b 100644
--- a/bindings/c/src/reader.rs
+++ b/bindings/c/src/reader.rs
@@ -40,15 +40,15 @@ pub struct opendal_reader {
}
impl opendal_reader {
- fn deref_mut(&mut self) -> &mut core::StdReader {
+ fn deref_mut(&mut self) -> &mut core::blocking::StdReader {
// Safety: the inner should never be null once constructed
// The use-after-free is undefined behavior
- unsafe { &mut *(self.inner as *mut core::StdReader) }
+ unsafe { &mut *(self.inner as *mut core::blocking::StdReader) }
}
}
impl opendal_reader {
- pub(crate) fn new(reader: core::StdReader) -> Self {
+ pub(crate) fn new(reader: core::blocking::StdReader) -> Self {
Self {
inner: Box::into_raw(Box::new(reader)) as _,
}
@@ -119,7 +119,9 @@ impl opendal_reader {
#[no_mangle]
pub unsafe extern "C" fn opendal_reader_free(ptr: *mut opendal_reader) {
if !ptr.is_null() {
- drop(Box::from_raw((*ptr).inner as *mut core::StdReader));
+ drop(Box::from_raw(
+ (*ptr).inner as *mut core::blocking::StdReader,
+ ));
drop(Box::from_raw(ptr));
}
}
diff --git a/bindings/c/src/writer.rs b/bindings/c/src/writer.rs
index c3c72eb8f..5e27617c8 100644
--- a/bindings/c/src/writer.rs
+++ b/bindings/c/src/writer.rs
@@ -22,24 +22,24 @@ use super::*;
/// \brief The result type returned by opendal's writer operation.
/// \note The opendal_writer actually owns a pointer to
-/// an opendal::BlockingWriter, which is inside the Rust core code.
+/// an opendal::blocking::Writer, which is inside the Rust core code.
#[repr(C)]
pub struct opendal_writer {
- /// The pointer to the opendal::BlockingWriter in the Rust code.
+ /// The pointer to the opendal::blocking::Writer in the Rust code.
/// Only touch this on judging whether it is NULL.
inner: *mut c_void,
}
impl opendal_writer {
- fn deref_mut(&mut self) -> &mut core::BlockingWriter {
+ fn deref_mut(&mut self) -> &mut core::blocking::Writer {
// Safety: the inner should never be null once constructed
// The use-after-free is undefined behavior
- unsafe { &mut *(self.inner as *mut core::BlockingWriter) }
+ unsafe { &mut *(self.inner as *mut core::blocking::Writer) }
}
}
impl opendal_writer {
- pub(crate) fn new(writer: core::BlockingWriter) -> Self {
+ pub(crate) fn new(writer: core::blocking::Writer) -> Self {
Self {
inner: Box::into_raw(Box::new(writer)) as _,
}
@@ -85,7 +85,7 @@ impl opendal_writer {
#[no_mangle]
pub unsafe extern "C" fn opendal_writer_free(ptr: *mut opendal_writer) {
if !ptr.is_null() {
- drop(Box::from_raw((*ptr).inner as *mut core::BlockingWriter));
+ drop(Box::from_raw((*ptr).inner as *mut core::blocking::Writer));
drop(Box::from_raw(ptr));
}
}
diff --git a/bindings/c/tests/bdd.cpp b/bindings/c/tests/bdd.cpp
index 1ca2bde5c..946df0c86 100644
--- a/bindings/c/tests/bdd.cpp
+++ b/bindings/c/tests/bdd.cpp
@@ -105,7 +105,7 @@ TEST_F(OpendalBddTest, FeatureTest)
opendal_result_writer_write w = opendal_writer_write(writer.writer, &data);
EXPECT_EQ(w.error, nullptr);
EXPECT_EQ(w.size, this->content.length());
- opendal_error *close_err = opendal_writer_close(writer.writer);
+ opendal_error* close_err = opendal_writer_close(writer.writer);
EXPECT_EQ(close_err, nullptr);
opendal_writer_free(writer.writer);
diff --git a/bindings/c/tests/opinfo.cpp b/bindings/c/tests/opinfo.cpp
index 4312a152a..398608ec4 100644
--- a/bindings/c/tests/opinfo.cpp
+++ b/bindings/c/tests/opinfo.cpp
@@ -63,7 +63,6 @@ TEST_F(OpendalOperatorInfoTest, CapabilityTest)
opendal_capability full_cap = opendal_operator_info_get_full_capability(this->info);
opendal_capability native_cap = opendal_operator_info_get_native_capability(this->info);
- EXPECT_TRUE(full_cap.blocking);
EXPECT_TRUE(full_cap.read);
EXPECT_TRUE(full_cap.stat);
EXPECT_TRUE(full_cap.write);
@@ -73,7 +72,6 @@ TEST_F(OpendalOperatorInfoTest, CapabilityTest)
EXPECT_TRUE(full_cap.list);
EXPECT_TRUE(full_cap.list_with_recursive);
- EXPECT_TRUE(native_cap.blocking);
EXPECT_TRUE(native_cap.read);
EXPECT_TRUE(native_cap.stat);
EXPECT_TRUE(native_cap.write);
diff --git a/bindings/c/tests/reader.cpp b/bindings/c/tests/reader.cpp
index 216ffc3cc..2cbde7f9b 100644
--- a/bindings/c/tests/reader.cpp
+++ b/bindings/c/tests/reader.cpp
@@ -18,20 +18,18 @@
*/
#include "gtest/gtest.h"
-extern "C"
-{
+extern "C" {
#include "opendal.h"
}
-class OpendalReaderTest : public ::testing::Test
-{
+class OpendalReaderTest : public ::testing::Test {
protected:
- const opendal_operator *p;
+ const opendal_operator* p;
// set up a brand new operator
void SetUp() override
{
- opendal_operator_options *options = opendal_operator_options_new();
+ opendal_operator_options* options = opendal_operator_options_new();
opendal_operator_options_set(options, "root", "/myroot");
opendal_result_operator_new result = opendal_operator_new("memory", options);
@@ -51,12 +49,12 @@ TEST_F(OpendalReaderTest, SeekTest)
{
std::string payload = "ABCDEFGabcdefg1234567";
opendal_bytes data = {
- .data = (uint8_t *)(payload.c_str()),
+ .data = (uint8_t*)(payload.c_str()),
.len = payload.length(),
};
// Prepare the file to be read immediately
- opendal_error *err = opendal_operator_write(this->p, "/testseek", &data);
+ opendal_error* err = opendal_operator_write(this->p, "/testseek", &data);
EXPECT_EQ(err, nullptr);
opendal_result_operator_reader r = opendal_operator_reader(this->p, "/testseek");
@@ -67,8 +65,8 @@ TEST_F(OpendalReaderTest, SeekTest)
EXPECT_EQ(seek_result.pos, 6);
EXPECT_EQ(seek_result.error, nullptr);
- char buf1[64] = {0};
- opendal_result_reader_read read_result = opendal_reader_read(r.reader, (uint8_t *)buf1, 7);
+ char buf1[64] = { 0 };
+ opendal_result_reader_read read_result = opendal_reader_read(r.reader, (uint8_t*)buf1, 7);
EXPECT_EQ(read_result.error, nullptr);
EXPECT_EQ(read_result.size, 7);
EXPECT_EQ(std::string(buf1), "Gabcdef");
@@ -78,7 +76,7 @@ TEST_F(OpendalReaderTest, SeekTest)
EXPECT_EQ(seek_result.pos, 16);
EXPECT_EQ(seek_result.error, nullptr);
- char buf2[64] = {0};
+ char buf2[64] = { 0 };
read_result = opendal_reader_read(r.reader, (uint8_t*)buf2, 32 /* no more 32 bytes*/);
EXPECT_EQ(read_result.error, nullptr);
EXPECT_EQ(read_result.size, 5);
@@ -89,7 +87,7 @@ TEST_F(OpendalReaderTest, SeekTest)
EXPECT_EQ(seek_result.pos, 13);
EXPECT_EQ(seek_result.error, nullptr);
- char buf3[64] = {0};
+ char buf3[64] = { 0 };
read_result = opendal_reader_read(r.reader, (uint8_t*)buf3, 32 /* no more 32 bytes*/);
EXPECT_EQ(read_result.error, nullptr);
EXPECT_EQ(read_result.size, 8);
diff --git a/bindings/cpp/Cargo.toml b/bindings/cpp/Cargo.toml
index 4ee8d3996..c2861de61 100644
--- a/bindings/cpp/Cargo.toml
+++ b/bindings/cpp/Cargo.toml
@@ -36,6 +36,7 @@ cxx = "1.0"
cxx-async = { version = "0.1.2", optional = true }
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
+ "blocking",
# These are default features before v0.46. TODO: change to optional features
"services-azblob",
"services-azdls",
@@ -53,6 +54,7 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-webhdfs",
"services-azfile",
] }
+tokio = { version = "1.27", features = ["fs", "macros", "rt-multi-thread"] }
[build-dependencies]
cxx-build = "1.0"
diff --git a/bindings/cpp/src/async.rs b/bindings/cpp/src/async.rs
index 30f25e1b0..b4f94899e 100644
--- a/bindings/cpp/src/async.rs
+++ b/bindings/cpp/src/async.rs
@@ -88,7 +88,7 @@ unsafe impl Send for ffi::OperatorPtr {}
unsafe fn operator_read(op: ffi::OperatorPtr, path: String) -> RustFutureRead {
RustFutureRead::fallible(async move {
- Ok((*op)
+ Ok(op
.0
.read(&path)
.await
@@ -99,11 +99,9 @@ unsafe fn operator_read(op: ffi::OperatorPtr, path: String) -> RustFutureRead {
unsafe fn operator_write(op: ffi::OperatorPtr, path: String, bs: Vec<u8>) -> RustFutureWrite {
RustFutureWrite::fallible(async move {
- Ok((*op)
- .0
- .write(&path, bs)
+ op.0.write(&path, bs)
.await
.map(|_| ())
- .map_err(|e| CxxAsyncException::new(e.to_string().into_boxed_str()))?)
+ .map_err(|e| CxxAsyncException::new(e.to_string().into_boxed_str()))
})
}
diff --git a/bindings/cpp/src/lib.rs b/bindings/cpp/src/lib.rs
index a679e0d24..c220ecfc4 100644
--- a/bindings/cpp/src/lib.rs
+++ b/bindings/cpp/src/lib.rs
@@ -23,12 +23,20 @@ mod types;
use std::collections::HashMap;
use std::str::FromStr;
+use std::sync::LazyLock;
use anyhow::Result;
use lister::Lister;
use opendal as od;
use reader::Reader;
+static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+});
+
#[cxx::bridge(namespace = "opendal::ffi")]
mod ffi {
struct HashMapValue {
@@ -104,7 +112,7 @@ mod ffi {
}
}
-pub struct Operator(od::BlockingOperator);
+pub struct Operator(od::blocking::Operator);
fn new_operator(scheme: &str, configs: Vec<ffi::HashMapValue>) -> Result<*mut Operator> {
let scheme = od::Scheme::from_str(scheme)?;
@@ -114,9 +122,14 @@ fn new_operator(scheme: &str, configs: Vec<ffi::HashMapValue>) -> Result<*mut Op
.map(|value| (value.key, value.value))
.collect();
- let op = Box::into_raw(Box::new(Operator(
- od::Operator::via_iter(scheme, map)?.blocking(),
- )));
+ let runtime =
+ tokio::runtime::Handle::try_current().unwrap_or_else(|_| RUNTIME.handle().clone());
+ let _guard = runtime.enter();
+
+ let op = od::Operator::via_iter(scheme, map)?;
+ let op = od::blocking::Operator::new(op)?;
+
+ let op = Box::into_raw(Box::new(Operator(op)));
Ok(op)
}
diff --git a/bindings/cpp/src/lister.rs b/bindings/cpp/src/lister.rs
index 51b3007eb..b420cb5f8 100644
--- a/bindings/cpp/src/lister.rs
+++ b/bindings/cpp/src/lister.rs
@@ -20,7 +20,7 @@ use opendal as od;
use super::ffi;
-pub struct Lister(pub od::BlockingLister);
+pub struct Lister(pub od::blocking::Lister);
impl Lister {
pub fn next(&mut self) -> Result<ffi::OptionalEntry> {
diff --git a/bindings/cpp/src/reader.rs b/bindings/cpp/src/reader.rs
index 00d4ad862..ed7d48fff 100644
--- a/bindings/cpp/src/reader.rs
+++ b/bindings/cpp/src/reader.rs
@@ -23,7 +23,7 @@ use opendal as od;
use super::ffi;
-pub struct Reader(pub od::StdReader);
+pub struct Reader(pub od::blocking::StdReader);
impl Reader {
pub fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
diff --git a/bindings/dart/rust/Cargo.toml b/bindings/dart/rust/Cargo.toml
index d873126f8..b68ad0673 100644
--- a/bindings/dart/rust/Cargo.toml
+++ b/bindings/dart/rust/Cargo.toml
@@ -23,6 +23,8 @@ version = "0.1.0"
[dependencies]
flutter_rust_bridge = "=2.8.0"
opendal = { path = "../../../core", features = [
+ "blocking",
+
# These are default features before v0.46. TODO: change to optional features
"services-azblob",
"services-azdls",
@@ -40,6 +42,7 @@ opendal = { path = "../../../core", features = [
"services-webhdfs",
"services-azfile",
] }
+tokio = { version = "1.0", features = ["full"] }
[lib]
crate-type = ["staticlib", "cdylib"]
diff --git a/bindings/dart/rust/src/api/opendal_api.rs b/bindings/dart/rust/src/api/opendal_api.rs
index 3d1017341..d9a7fe636 100644
--- a/bindings/dart/rust/src/api/opendal_api.rs
+++ b/bindings/dart/rust/src/api/opendal_api.rs
@@ -21,57 +21,76 @@ use ::opendal as od;
use std::collections::HashMap;
use std::str::FromStr;
+use std::sync::LazyLock;
+
+static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+});
#[frb(opaque)]
-pub struct Operator(opendal::Operator);
+pub struct Operator {
+ async_op: opendal::Operator,
+ blocking_op: opendal::blocking::Operator,
+}
impl Operator {
#[frb(sync)]
pub fn new(scheme_str: String, map: HashMap<String, String>) -> Operator {
let scheme: od::Scheme = od::Scheme::from_str(&scheme_str).unwrap();
- Self(od::Operator::via_iter(scheme, map).unwrap())
+ let async_op = od::Operator::via_iter(scheme, map).unwrap();
+ let handle = RUNTIME.handle();
+ let _enter = handle.enter();
+ let blocking_op = opendal::blocking::Operator::new(async_op.clone()).unwrap();
+
+ Self {
+ async_op,
+ blocking_op,
+ }
}
pub async fn stat(&self, path: String) -> Metadata {
- let meta = self.0.stat(&path).await.unwrap();
+ let meta = self.async_op.stat(&path).await.unwrap();
Metadata(meta)
}
#[frb(sync)]
pub fn stat_sync(&self, path: String) -> Metadata {
- let meta = self.0.blocking().stat(&path).unwrap();
+ let meta = self.blocking_op.stat(&path).unwrap();
Metadata(meta)
}
pub async fn check(&self) {
- self.0.check().await.unwrap()
+ self.async_op.check().await.unwrap()
}
pub async fn exists(&self, path: String) -> bool {
- self.0.exists(&path).await.unwrap()
+ self.async_op.exists(&path).await.unwrap()
}
pub async fn delete(&self, path: String) {
- self.0.delete(&path).await.unwrap()
+ self.async_op.delete(&path).await.unwrap()
}
#[frb(sync)]
pub fn delete_sync(&self, path: String) -> () {
- self.0.blocking().delete(&path).unwrap()
+ self.blocking_op.delete(&path).unwrap()
}
#[frb(sync)]
pub fn exists_sync(&self, path: String) -> bool {
- self.0.blocking().exists(&path).unwrap()
+ self.blocking_op.exists(&path).unwrap()
}
pub async fn create_dir(&self, path: String) {
- self.0.create_dir(&path).await.unwrap()
+ self.async_op.create_dir(&path).await.unwrap()
}
#[frb(sync)]
pub fn create_dir_sync(&self, path: String) -> () {
- self.0.blocking().create_dir(&path).unwrap()
+ self.blocking_op.create_dir(&path).unwrap()
}
pub async fn rename(&self, from: String, to: String) {
- self.0.rename(&from, &to).await.unwrap()
+ self.async_op.rename(&from, &to).await.unwrap()
}
#[frb(sync)]
pub fn rename_sync(&self, from: String, to: String) -> () {
- self.0.blocking().rename(&from, &to).unwrap()
+ self.blocking_op.rename(&from, &to).unwrap()
}
}
diff --git a/bindings/dotnet/Cargo.toml b/bindings/dotnet/Cargo.toml
index 382abe7f6..1fd75f5a2 100644
--- a/bindings/dotnet/Cargo.toml
+++ b/bindings/dotnet/Cargo.toml
@@ -33,6 +33,8 @@ doc = false
[dependencies]
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
+ "blocking",
+
# These are default features before v0.46. TODO: change to optional features
"services-azblob",
"services-azdls",
@@ -50,3 +52,4 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-webhdfs",
"services-azfile",
] }
+tokio = { version = "1", features = ["full"] }
diff --git a/bindings/dotnet/src/lib.rs b/bindings/dotnet/src/lib.rs
index 617874d80..52be936af 100644
--- a/bindings/dotnet/src/lib.rs
+++ b/bindings/dotnet/src/lib.rs
@@ -18,6 +18,14 @@
use std::collections::HashMap;
use std::os::raw::c_char;
use std::str::FromStr;
+use std::sync::LazyLock;
+
+static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+});
/// # Safety
///
@@ -25,7 +33,7 @@ use std::str::FromStr;
#[no_mangle]
pub unsafe extern "C" fn blocking_operator_construct(
scheme: *const c_char,
-) -> *const opendal::BlockingOperator {
+) -> *const opendal::blocking::Operator {
if scheme.is_null() {
return std::ptr::null();
}
@@ -39,14 +47,24 @@ pub unsafe extern "C" fn blocking_operator_construct(
let mut map = HashMap::<String, String>::default();
map.insert("root".to_string(), "/tmp".to_string());
let op = match opendal::Operator::via_iter(scheme, map) {
- Ok(op) => op.blocking(),
+ Ok(op) => op,
+ Err(err) => {
+ println!("err={err:?}");
+ return std::ptr::null();
+ }
+ };
+
+ let handle = RUNTIME.handle();
+ let _enter = handle.enter();
+ let blocking_op = match opendal::blocking::Operator::new(op) {
+ Ok(op) => op,
Err(err) => {
println!("err={err:?}");
return std::ptr::null();
}
};
- Box::leak(Box::new(op))
+ Box::leak(Box::new(blocking_op))
}
/// # Safety
@@ -54,7 +72,7 @@ pub unsafe extern "C" fn blocking_operator_construct(
/// Not yet.
#[no_mangle]
pub unsafe extern "C" fn blocking_operator_write(
- op: *const opendal::BlockingOperator,
+ op: *const opendal::blocking::Operator,
path: *const c_char,
content: *const c_char,
) {
@@ -69,7 +87,7 @@ pub unsafe extern "C" fn blocking_operator_write(
/// Not yet.
#[no_mangle]
pub unsafe extern "C" fn blocking_operator_read(
- op: *const opendal::BlockingOperator,
+ op: *const opendal::blocking::Operator,
path: *const c_char,
) -> *const c_char {
let op = &*(op);
diff --git a/bindings/haskell/Cargo.toml b/bindings/haskell/Cargo.toml
index afa89e023..d4a9f27ff 100644
--- a/bindings/haskell/Cargo.toml
+++ b/bindings/haskell/Cargo.toml
@@ -35,6 +35,7 @@ chrono = "0.4"
log = { version = "0.4", features = ["std"] }
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
+ "blocking",
# These are default features before v0.46. TODO: change to optional features
"services-azblob",
"services-azdls",
@@ -52,3 +53,4 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-webhdfs",
"services-azfile",
] }
+tokio = { version = "1", features = ["full"] }
diff --git a/bindings/haskell/src/lib.rs b/bindings/haskell/src/lib.rs
index 350e34706..b4ada218e 100644
--- a/bindings/haskell/src/lib.rs
+++ b/bindings/haskell/src/lib.rs
@@ -24,16 +24,25 @@ use std::ffi::CStr;
use std::mem;
use std::os::raw::c_char;
use std::str::FromStr;
+use std::sync::LazyLock;
use ::opendal as od;
use logger::HsLogger;
use od::layers::LoggingLayer;
use od::layers::RetryLayer;
-use od::BlockingLister;
+use od::options;
+use opendal::blocking;
use result::FFIResult;
use types::ByteSlice;
use types::Metadata;
+static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+});
+
/// # Safety
///
/// * The `keys`, `values`, `len` are valid from `HashMap`.
@@ -55,7 +64,7 @@ pub unsafe extern "C" fn via_map_ffi(
values: *const *const c_char,
len: usize,
callback: Option<extern "C" fn(u32, *const c_char)>,
- result: *mut FFIResult<od::BlockingOperator>,
+ result: *mut FFIResult<od::blocking::Operator>,
) {
let scheme_str = match CStr::from_ptr(scheme).to_str() {
Ok(s) => s,
@@ -106,7 +115,10 @@ pub unsafe extern "C" fn via_map_ffi(
operator = operator.layer(LoggingLayer::default());
}
- FFIResult::ok(operator.blocking())
+ let handle = RUNTIME.handle();
+ let _enter = handle.enter();
+ let op = od::blocking::Operator::new(operator).unwrap();
+ FFIResult::ok(op)
}
Err(e) => FFIResult::err_with_source("Failed to create Operator", e),
};
@@ -116,13 +128,13 @@ pub unsafe extern "C" fn via_map_ffi(
/// # Safety
///
-/// * `operator` is a valid pointer to a `BlockingOperator`.
+/// * `operator` is a valid pointer to a `blocking::Operator`.
///
/// # Panics
///
/// * If `operator` is not a valid pointer.
#[no_mangle]
-pub unsafe extern "C" fn free_operator(operator: *mut od::BlockingOperator) {
+pub unsafe extern "C" fn free_operator(operator: *mut od::blocking::Operator) {
if !operator.is_null() {
drop(Box::from_raw(operator));
}
@@ -130,7 +142,7 @@ pub unsafe extern "C" fn free_operator(operator: *mut od::BlockingOperator) {
/// # Safety
///
-/// * `op` is a valid pointer to a `BlockingOperator`.
+/// * `op` is a valid pointer to a `blocking::Operator`.
/// * `path` is a valid pointer to a nul terminated string.
/// * `result` is a valid pointer, and has available memory to write to
///
@@ -140,7 +152,7 @@ pub unsafe extern "C" fn free_operator(operator: *mut od::BlockingOperator) {
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn blocking_read(
- op: *mut od::BlockingOperator,
+ op: *mut od::blocking::Operator,
path: *const c_char,
result: *mut FFIResult<ByteSlice>,
) {
@@ -169,7 +181,7 @@ pub unsafe extern "C" fn blocking_read(
/// # Safety
///
-/// * `op` is a valid pointer to a `BlockingOperator`.
+/// * `op` is a valid pointer to a `blocking::Operator`.
/// * `path` is a valid pointer to a nul terminated string.
/// * `bytes` is a valid pointer to a byte array.
/// * `len` is the length of `bytes`.
@@ -182,7 +194,7 @@ pub unsafe extern "C" fn blocking_read(
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn blocking_write(
- op: *mut od::BlockingOperator,
+ op: *mut od::blocking::Operator,
path: *const c_char,
bytes: *const c_char,
len: usize,
@@ -218,7 +230,7 @@ pub unsafe extern "C" fn blocking_write(
/// # Safety
///
-/// * `op` is a valid pointer to a `BlockingOperator`.
+/// * `op` is a valid pointer to a `blocking::Operator`.
/// * `path` is a valid pointer to a nul terminated string.
/// * `result` is a valid pointer, and has available memory to write to
///
@@ -228,7 +240,7 @@ pub unsafe extern "C" fn blocking_write(
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn blocking_is_exist(
- op: *mut od::BlockingOperator,
+ op: *mut od::blocking::Operator,
path: *const c_char,
result: *mut FFIResult<bool>,
) {
@@ -257,7 +269,7 @@ pub unsafe extern "C" fn blocking_is_exist(
/// # Safety
///
-/// * `op` is a valid pointer to a `BlockingOperator`.
+/// * `op` is a valid pointer to a `blocking::Operator`.
/// * `path` is a valid pointer to a nul terminated string.
/// * `result` is a valid pointer, and has available memory to write to
///
@@ -267,7 +279,7 @@ pub unsafe extern "C" fn blocking_is_exist(
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn blocking_create_dir(
- op: *mut od::BlockingOperator,
+ op: *mut od::blocking::Operator,
path: *const c_char,
result: *mut FFIResult<()>,
) {
@@ -296,7 +308,7 @@ pub unsafe extern "C" fn blocking_create_dir(
/// # Safety
///
-/// * `op` is a valid pointer to a `BlockingOperator`.
+/// * `op` is a valid pointer to a `blocking::Operator`.
/// * `path_from` is a valid pointer to a nul terminated string.
/// * `path_to` is a valid pointer to a nul terminated string.
/// * `result` is a valid pointer, and has available memory to write to
@@ -307,7 +319,7 @@ pub unsafe extern "C" fn blocking_create_dir(
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn blocking_copy(
- op: *mut od::BlockingOperator,
+ op: *mut od::blocking::Operator,
path_from: *const c_char,
path_to: *const c_char,
result: *mut FFIResult<()>,
@@ -345,7 +357,7 @@ pub unsafe extern "C" fn blocking_copy(
/// # Safety
///
-/// * `op` is a valid pointer to a `BlockingOperator`.
+/// * `op` is a valid pointer to a `blocking::Operator`.
/// * `path_from` is a valid pointer to a nul terminated string.
/// * `path_to` is a valid pointer to a nul terminated string.
/// * `result` is a valid pointer, and has available memory to write to
@@ -356,7 +368,7 @@ pub unsafe extern "C" fn blocking_copy(
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn blocking_rename(
- op: *mut od::BlockingOperator,
+ op: *mut od::blocking::Operator,
path_from: *const c_char,
path_to: *const c_char,
result: *mut FFIResult<()>,
@@ -394,7 +406,7 @@ pub unsafe extern "C" fn blocking_rename(
/// # Safety
///
-/// * `op` is a valid pointer to a `BlockingOperator`.
+/// * `op` is a valid pointer to a `blocking::Operator`.
/// * `path` is a valid pointer to a nul terminated string.
/// * `result` is a valid pointer, and has available memory to write to
///
@@ -404,7 +416,7 @@ pub unsafe extern "C" fn blocking_rename(
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn blocking_delete(
- op: *mut od::BlockingOperator,
+ op: *mut od::blocking::Operator,
path: *const c_char,
result: *mut FFIResult<()>,
) {
@@ -433,7 +445,7 @@ pub unsafe extern "C" fn blocking_delete(
/// # Safety
///
-/// * `op` is a valid pointer to a `BlockingOperator`.
+/// * `op` is a valid pointer to a `blocking::Operator`.
/// * `path` is a valid pointer to a nul terminated string.
/// * `result` is a valid pointer, and has available memory to write to
///
@@ -443,7 +455,7 @@ pub unsafe extern "C" fn blocking_delete(
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn blocking_stat(
- op: *mut od::BlockingOperator,
+ op: *mut od::blocking::Operator,
path: *const c_char,
result: *mut FFIResult<Metadata>,
) {
@@ -472,7 +484,7 @@ pub unsafe extern "C" fn blocking_stat(
/// # Safety
///
-/// * `op` is a valid pointer to a `BlockingOperator`.
+/// * `op` is a valid pointer to a `blocking::Operator`.
/// * `path` is a valid pointer to a nul terminated string.
/// * `result` is a valid pointer, and has available memory to write to
///
@@ -482,9 +494,9 @@ pub unsafe extern "C" fn blocking_stat(
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn blocking_list(
- op: *mut od::BlockingOperator,
+ op: *mut od::blocking::Operator,
path: *const c_char,
- result: *mut FFIResult<*mut BlockingLister>,
+ result: *mut FFIResult<*mut blocking::Lister>,
) {
let op = if op.is_null() {
*result = FFIResult::err("Operator is null");
@@ -511,7 +523,7 @@ pub unsafe extern "C" fn blocking_list(
/// # Safety
///
-/// * `op` is a valid pointer to a `BlockingOperator`.
+/// * `op` is a valid pointer to a `blocking::Operator`.
/// * `path` is a valid pointer to a nul terminated string.
/// * `result` is a valid pointer, and has available memory to write to
///
@@ -521,9 +533,9 @@ pub unsafe extern "C" fn blocking_list(
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn blocking_scan(
- op: *mut od::BlockingOperator,
+ op: *mut od::blocking::Operator,
path: *const c_char,
- result: *mut FFIResult<*mut BlockingLister>,
+ result: *mut FFIResult<*mut blocking::Lister>,
) {
let op = if op.is_null() {
*result = FFIResult::err("Operator is null");
@@ -540,7 +552,13 @@ pub unsafe extern "C" fn blocking_scan(
}
};
- let res = match op.lister_with(path_str).recursive(true).call() {
+ let res = match op.lister_options(
+ path_str,
+ options::ListOptions {
+ recursive: true,
+ ..Default::default()
+ },
+ ) {
Ok(lister) => FFIResult::ok(Box::into_raw(Box::new(lister))),
Err(e) => FFIResult::err_with_source("Failed to scan", e),
};
@@ -559,7 +577,7 @@ pub unsafe extern "C" fn blocking_scan(
/// * If `result` is not a valid pointer, or does not have available memory to write to.
#[no_mangle]
pub unsafe extern "C" fn lister_next(
- lister: *mut BlockingLister,
+ lister: *mut blocking::Lister,
result: *mut FFIResult<*const c_char>,
) {
let lister = if lister.is_null() {
@@ -589,7 +607,7 @@ pub unsafe extern "C" fn lister_next(
///
/// * If `lister` is not a valid pointer.
#[no_mangle]
-pub unsafe extern "C" fn free_lister(lister: *mut BlockingLister) {
+pub unsafe extern "C" fn free_lister(lister: *mut blocking::Lister) {
if !lister.is_null() {
drop(Box::from_raw(lister));
}
diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml
index 4ae7df212..75f4e4ad4 100644
--- a/bindings/java/Cargo.toml
+++ b/bindings/java/Cargo.toml
@@ -153,7 +153,7 @@ anyhow = "1.0.71"
jni = "0.21.1"
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
- "layers-blocking",
+ "blocking",
] }
tokio = { version = "1.28.1", features = ["full"] }
diff --git a/bindings/java/src/async_operator.rs b/bindings/java/src/async_operator.rs
index 92734291c..9c1e943e0 100644
--- a/bindings/java/src/async_operator.rs
+++ b/bindings/java/src/async_operator.rs
@@ -28,7 +28,7 @@ use jni::sys::jlong;
use jni::sys::jobject;
use jni::sys::jsize;
use jni::JNIEnv;
-use opendal::layers::BlockingLayer;
+use opendal::blocking;
use opendal::Entry;
use opendal::Operator;
use opendal::Scheme;
@@ -51,29 +51,19 @@ use crate::Result;
pub extern "system" fn Java_org_apache_opendal_AsyncOperator_constructor(
mut env: JNIEnv,
_: JClass,
- executor: *const Executor,
scheme: JString,
map: JObject,
) -> jlong {
- intern_constructor(&mut env, executor, scheme, map).unwrap_or_else(|e| {
+ intern_constructor(&mut env, scheme, map).unwrap_or_else(|e| {
e.throw(&mut env);
0
})
}
-fn intern_constructor(
- env: &mut JNIEnv,
- executor: *const Executor,
- scheme: JString,
- map: JObject,
-) -> Result<jlong> {
+fn intern_constructor(env: &mut JNIEnv, scheme: JString, map: JObject) -> Result<jlong> {
let scheme = Scheme::from_str(jstring_to_string(env, &scheme)?.as_str())?;
let map = jmap_to_hashmap(env, &map)?;
- let mut op = Operator::via_iter(scheme, map)?;
- if !op.info().full_capability().blocking {
- let layer = executor_or_default(env, executor)?.enter_with(BlockingLayer::create)?;
- op = op.layer(layer);
- }
+ let op = Operator::via_iter(scheme, map)?;
Ok(Box::into_raw(Box::new(op)) as jlong)
}
@@ -310,12 +300,26 @@ fn intern_delete(
/// This function should not be called before the Operator is ready.
#[no_mangle]
pub unsafe extern "system" fn Java_org_apache_opendal_AsyncOperator_makeBlockingOp(
- _: JNIEnv,
+ mut env: JNIEnv,
_: JClass,
op: *mut Operator,
+ executor: *const Executor,
) -> jlong {
+ intern_make_blocking_op(&mut env, op, executor).unwrap_or_else(|e| {
+ e.throw(&mut env);
+ 0
+ })
+}
+
+fn intern_make_blocking_op(
+ env: &mut JNIEnv,
+ op: *mut Operator,
+ executor: *const Executor,
+) -> Result<jlong> {
let op = unsafe { &mut *op };
- Box::into_raw(Box::new(op.blocking())) as jlong
+ let op = executor_or_default(env, executor)?
+ .enter_with(move || blocking::Operator::new(op.clone()))?;
+ Ok(Box::into_raw(Box::new(op)) as jlong)
}
/// # Safety
diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs
index e67d7f479..683baefb8 100644
--- a/bindings/java/src/lib.rs
+++ b/bindings/java/src/lib.rs
@@ -94,7 +94,7 @@ fn make_operator_info<'a>(env: &mut JNIEnv<'a>, info: OperatorInfo) -> Result<JO
fn make_capability<'a>(env: &mut JNIEnv<'a>, cap: Capability) -> Result<JObject<'a>> {
let capability = env.new_object(
"org/apache/opendal/Capability",
- "(ZZZZZZZZZZZZZZZZZZZJJZZZZZZZZZZZZZZ)V",
+ "(ZZZZZZZZZZZZZZZZZZZJJZZZZZZZZZZZZZ)V",
&[
JValue::Bool(cap.stat as jboolean),
JValue::Bool(cap.stat_with_if_match as jboolean),
@@ -130,7 +130,6 @@ fn make_capability<'a>(env: &mut JNIEnv<'a>, cap: Capability) -> Result<JObject<
JValue::Bool(cap.presign_stat as jboolean),
JValue::Bool(cap.presign_write as jboolean),
JValue::Bool(cap.shared as jboolean),
- JValue::Bool(cap.blocking as jboolean),
],
)?;
Ok(capability)
diff --git a/bindings/java/src/main/java/org/apache/opendal/AsyncOperator.java b/bindings/java/src/main/java/org/apache/opendal/AsyncOperator.java
index 36b2d6c65..c0e0d3c00 100644
--- a/bindings/java/src/main/java/org/apache/opendal/AsyncOperator.java
+++ b/bindings/java/src/main/java/org/apache/opendal/AsyncOperator.java
@@ -153,7 +153,7 @@ public class AsyncOperator extends NativeObject {
*/
public static AsyncOperator of(String scheme, Map<String, String> map, AsyncExecutor executor) {
final long executorHandle = executor != null ? executor.nativeHandle : 0;
- final long nativeHandle = constructor(executorHandle, scheme, map);
+ final long nativeHandle = constructor(scheme, map);
final OperatorInfo info = makeOperatorInfo(nativeHandle);
return new AsyncOperator(nativeHandle, executorHandle, info);
}
@@ -202,7 +202,7 @@ public class AsyncOperator extends NativeObject {
* @return the blocking new operator.
*/
public Operator blocking() {
- final long nativeHandle = makeBlockingOp(this.nativeHandle);
+ final long nativeHandle = makeBlockingOp(this.nativeHandle, this.executorHandle);
final OperatorInfo info = this.info;
return new Operator(nativeHandle, info);
}
@@ -300,7 +300,7 @@ public class AsyncOperator extends NativeObject {
private static native long duplicate(long nativeHandle);
- private static native long constructor(long executorHandle, String scheme, Map<String, String> map);
+ private static native long constructor(String scheme, Map<String, String> map);
private static native long read(long nativeHandle, long executorHandle, String path, ReadOptions options);
@@ -321,7 +321,7 @@ public class AsyncOperator extends NativeObject {
private static native OperatorInfo makeOperatorInfo(long nativeHandle);
- private static native long makeBlockingOp(long nativeHandle);
+ private static native long makeBlockingOp(long nativeHandle, long executorHandle);
private static native long createDir(long nativeHandle, long executorHandle, String path);
diff --git a/bindings/java/src/main/java/org/apache/opendal/Capability.java b/bindings/java/src/main/java/org/apache/opendal/Capability.java
index 668156c89..0e3ad0420 100644
--- a/bindings/java/src/main/java/org/apache/opendal/Capability.java
+++ b/bindings/java/src/main/java/org/apache/opendal/Capability.java
@@ -23,6 +23,7 @@ import lombok.Data;
@Data
public class Capability {
+
/**
* If operator supports stat.
*/
@@ -196,11 +197,6 @@ public class Capability {
*/
public final boolean shared;
- /**
- * If operator supports blocking.
- */
- public final boolean blocking;
-
public Capability(
boolean stat,
boolean statWithIfMatch,
@@ -235,7 +231,6 @@ public class Capability {
boolean presignRead,
boolean presignStat,
boolean presignWrite,
- boolean blocking,
boolean shared) {
this.stat = stat;
this.statWithIfMatch = statWithIfMatch;
@@ -270,7 +265,6 @@ public class Capability {
this.presignRead = presignRead;
this.presignStat = presignStat;
this.presignWrite = presignWrite;
- this.blocking = blocking;
this.shared = shared;
}
}
diff --git a/bindings/java/src/operator.rs b/bindings/java/src/operator.rs
index 2b9ba001a..4e7689a1a 100644
--- a/bindings/java/src/operator.rs
+++ b/bindings/java/src/operator.rs
@@ -25,7 +25,8 @@ use jni::sys::jobject;
use jni::sys::jobjectArray;
use jni::sys::jsize;
use jni::JNIEnv;
-use opendal::BlockingOperator;
+use opendal::blocking;
+use opendal::options;
use crate::convert::{
bytes_to_jbytearray, jstring_to_string, offset_length_to_range, read_bool_field,
@@ -42,7 +43,7 @@ use crate::Result;
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_disposeInternal(
_: JNIEnv,
_: JObject,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
) {
drop(Box::from_raw(op));
}
@@ -54,7 +55,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_disposeInternal(
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_duplicate(
_: JNIEnv,
_: JObject,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
) -> jlong {
let op = &mut *op;
Box::into_raw(Box::new(op.clone())) as jlong
@@ -67,7 +68,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_duplicate(
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_read(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
path: JString,
read_options: JObject,
) -> jbyteArray {
@@ -79,7 +80,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_read(
fn intern_read(
env: &mut JNIEnv,
- op: &mut BlockingOperator,
+ op: &mut blocking::Operator,
path: JString,
options: JObject,
) -> Result<jbyteArray> {
@@ -88,9 +89,13 @@ fn intern_read(
let offset = read_int64_field(env, &options, "offset")?;
let length = read_int64_field(env, &options, "length")?;
- let mut read_op = op.read_with(&path);
- read_op = read_op.range(offset_length_to_range(offset, length)?);
- let content = read_op.call()?;
+ let content = op.read_options(
+ &path,
+ options::ReadOptions {
+ range: offset_length_to_range(offset, length)?.into(),
+ ..Default::default()
+ },
+ )?;
let result = bytes_to_jbytearray(env, content.to_bytes())?;
Ok(result.into_raw())
@@ -103,7 +108,7 @@ fn intern_read(
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_write(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
path: JString,
content: JByteArray,
write_options: JObject,
@@ -115,7 +120,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_write(
fn intern_write(
env: &mut JNIEnv,
- op: &mut BlockingOperator,
+ op: &mut blocking::Operator,
path: JString,
content: JByteArray,
options: JObject,
@@ -123,27 +128,25 @@ fn intern_write(
let path = jstring_to_string(env, &path)?;
let content = env.convert_byte_array(content)?;
+ let append = read_bool_field(env, &options, "append")?;
let content_type = read_string_field(env, &options, "contentType")?;
let content_disposition = read_string_field(env, &options, "contentDisposition")?;
let cache_control = read_string_field(env, &options, "cacheControl")?;
let user_metadata = read_map_field(env, &options, "userMetadata")?;
- let append = read_bool_field(env, &options, "append")?;
- let mut write_op = op.write_with(&path, content);
- if let Some(content_type) = content_type {
- write_op = write_op.content_type(&content_type);
- }
- if let Some(content_disposition) = content_disposition {
- write_op = write_op.content_disposition(&content_disposition);
- }
- if let Some(cache_control) = cache_control {
- write_op = write_op.cache_control(&cache_control);
- }
- if let Some(user_metadata) = user_metadata {
- write_op = write_op.user_metadata(user_metadata);
- }
- write_op = write_op.append(append);
- Ok(write_op.call().map(|_| ())?)
+ let _ = op.write_options(
+ &path,
+ content,
+ options::WriteOptions {
+ append,
+ content_type,
+ content_disposition,
+ cache_control,
+ user_metadata,
+ ..Default::default()
+ },
+ )?;
+ Ok(())
}
/// # Safety
@@ -153,7 +156,7 @@ fn intern_write(
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_stat(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
path: JString,
) -> jobject {
intern_stat(&mut env, &mut *op, path).unwrap_or_else(|e| {
@@ -162,7 +165,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_stat(
})
}
-fn intern_stat(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString) -> Result<jobject> {
+fn intern_stat(env: &mut JNIEnv, op: &mut blocking::Operator, path: JString) -> Result<jobject> {
let path = jstring_to_string(env, &path)?;
let metadata = op.stat(&path)?;
Ok(make_metadata(env, metadata)?.into_raw())
@@ -175,7 +178,7 @@ fn intern_stat(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString) -> Re
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_delete(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
path: JString,
) {
intern_delete(&mut env, &mut *op, path).unwrap_or_else(|e| {
@@ -183,7 +186,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_delete(
})
}
-fn intern_delete(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString) -> Result<()> {
+fn intern_delete(env: &mut JNIEnv, op: &mut blocking::Operator, path: JString) -> Result<()> {
let path = jstring_to_string(env, &path)?;
Ok(op.delete(&path)?)
}
@@ -195,7 +198,7 @@ fn intern_delete(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString) ->
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_createDir(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
path: JString,
) {
intern_create_dir(&mut env, &mut *op, path).unwrap_or_else(|e| {
@@ -203,7 +206,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_createDir(
})
}
-fn intern_create_dir(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString) -> Result<()> {
+fn intern_create_dir(env: &mut JNIEnv, op: &mut blocking::Operator, path: JString) -> Result<()> {
let path = jstring_to_string(env, &path)?;
Ok(op.create_dir(&path)?)
}
@@ -215,7 +218,7 @@ fn intern_create_dir(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString)
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_copy(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
source_path: JString,
target_path: JString,
) {
@@ -226,7 +229,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_copy(
fn intern_copy(
env: &mut JNIEnv,
- op: &mut BlockingOperator,
+ op: &mut blocking::Operator,
source_path: JString,
target_path: JString,
) -> Result<()> {
@@ -243,7 +246,7 @@ fn intern_copy(
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_rename(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
source_path: JString,
target_path: JString,
) {
@@ -254,7 +257,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_rename(
fn intern_rename(
env: &mut JNIEnv,
- op: &mut BlockingOperator,
+ op: &mut blocking::Operator,
source_path: JString,
target_path: JString,
) -> Result<()> {
@@ -271,7 +274,7 @@ fn intern_rename(
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_removeAll(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
path: JString,
) {
intern_remove_all(&mut env, &mut *op, path).unwrap_or_else(|e| {
@@ -279,7 +282,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_removeAll(
})
}
-fn intern_remove_all(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString) -> Result<()> {
+fn intern_remove_all(env: &mut JNIEnv, op: &mut blocking::Operator, path: JString) -> Result<()> {
let path = jstring_to_string(env, &path)?;
Ok(op.remove_all(&path)?)
@@ -292,7 +295,7 @@ fn intern_remove_all(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString)
pub unsafe extern "system" fn Java_org_apache_opendal_Operator_list(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
path: JString,
options: JObject,
) -> jobjectArray {
@@ -304,17 +307,20 @@ pub unsafe extern "system" fn Java_org_apache_opendal_Operator_list(
fn intern_list(
env: &mut JNIEnv,
- op: &mut BlockingOperator,
+ op: &mut blocking::Operator,
path: JString,
options: JObject,
) -> Result<jobjectArray> {
let path = jstring_to_string(env, &path)?;
let recursive = read_bool_field(env, &options, "recursive")?;
- let mut list_op = op.list_with(&path);
- list_op = list_op.recursive(recursive);
-
- let entries = list_op.call()?;
+ let entries = op.list_options(
+ &path,
+ options::ListOptions {
+ recursive,
+ ..Default::default()
+ },
+ )?;
let jarray = env.new_object_array(
entries.len() as jsize,
diff --git a/bindings/java/src/operator_input_stream.rs b/bindings/java/src/operator_input_stream.rs
index 105d15154..846cc71f9 100644
--- a/bindings/java/src/operator_input_stream.rs
+++ b/bindings/java/src/operator_input_stream.rs
@@ -22,8 +22,8 @@ use jni::objects::JString;
use jni::sys::jbyteArray;
use jni::sys::jlong;
use jni::JNIEnv;
-use opendal::BlockingOperator;
-use opendal::StdBytesIterator;
+use opendal::blocking;
+use opendal::blocking::StdBytesIterator;
use crate::convert::jstring_to_string;
@@ -34,7 +34,7 @@ use crate::convert::jstring_to_string;
pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_constructReader(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
path: JString,
) -> jlong {
intern_construct_reader(&mut env, &mut *op, path).unwrap_or_else(|e| {
@@ -45,7 +45,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_constr
fn intern_construct_reader(
env: &mut JNIEnv,
- op: &mut BlockingOperator,
+ op: &mut blocking::Operator,
path: JString,
) -> crate::Result<jlong> {
let path = jstring_to_string(env, &path)?;
diff --git a/bindings/java/src/operator_output_stream.rs b/bindings/java/src/operator_output_stream.rs
index f0bc8ad20..3ff9722f0 100644
--- a/bindings/java/src/operator_output_stream.rs
+++ b/bindings/java/src/operator_output_stream.rs
@@ -20,8 +20,7 @@ use jni::objects::JClass;
use jni::objects::JString;
use jni::sys::jlong;
use jni::JNIEnv;
-use opendal::BlockingOperator;
-use opendal::BlockingWriter;
+use opendal::blocking;
use crate::convert::jstring_to_string;
@@ -32,7 +31,7 @@ use crate::convert::jstring_to_string;
pub unsafe extern "system" fn Java_org_apache_opendal_OperatorOutputStream_constructWriter(
mut env: JNIEnv,
_: JClass,
- op: *mut BlockingOperator,
+ op: *mut blocking::Operator,
path: JString,
) -> jlong {
intern_construct_write(&mut env, &mut *op, path).unwrap_or_else(|e| {
@@ -43,7 +42,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorOutputStream_const
fn intern_construct_write(
env: &mut JNIEnv,
- op: &mut BlockingOperator,
+ op: &mut blocking::Operator,
path: JString,
) -> crate::Result<jlong> {
let path = jstring_to_string(env, &path)?;
@@ -58,7 +57,7 @@ fn intern_construct_write(
pub unsafe extern "system" fn Java_org_apache_opendal_OperatorOutputStream_disposeWriter(
mut env: JNIEnv,
_: JClass,
- writer: *mut BlockingWriter,
+ writer: *mut blocking::Writer,
) {
let mut writer = Box::from_raw(writer);
intern_dispose_write(&mut writer).unwrap_or_else(|e| {
@@ -66,7 +65,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorOutputStream_dispo
})
}
-fn intern_dispose_write(writer: &mut BlockingWriter) -> crate::Result<()> {
+fn intern_dispose_write(writer: &mut blocking::Writer) -> crate::Result<()> {
writer.close()?;
Ok(())
}
@@ -78,7 +77,7 @@ fn intern_dispose_write(writer: &mut BlockingWriter) -> crate::Result<()> {
pub unsafe extern "system" fn Java_org_apache_opendal_OperatorOutputStream_writeBytes(
mut env: JNIEnv,
_: JClass,
- writer: *mut BlockingWriter,
+ writer: *mut blocking::Writer,
content: JByteArray,
) {
intern_write_bytes(&mut env, &mut *writer, content).unwrap_or_else(|e| {
@@ -88,7 +87,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorOutputStream_write
fn intern_write_bytes(
env: &mut JNIEnv,
- writer: &mut BlockingWriter,
+ writer: &mut blocking::Writer,
content: JByteArray,
) -> crate::Result<()> {
let content = env.convert_byte_array(content)?;
diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorDuplicateTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorDuplicateTest.java
index 65bbf722d..03dbf8fbf 100644
--- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorDuplicateTest.java
+++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorDuplicateTest.java
@@ -28,6 +28,7 @@ import org.apache.opendal.Operator;
import org.junit.jupiter.api.Test;
public class OperatorDuplicateTest {
+
@Test
public void testDuplicateOperator() {
final Map<String, String> conf = new HashMap<>();
diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorInfoTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorInfoTest.java
index 511c998b4..a2d9521b7 100644
--- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorInfoTest.java
+++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorInfoTest.java
@@ -29,6 +29,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
public class OperatorInfoTest {
+
@TempDir
private static Path tempDir;
diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingListTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingListTest.java
index e77af8ab6..cbb00ea55 100644
--- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingListTest.java
+++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingListTest.java
@@ -37,15 +37,11 @@ import org.junit.jupiter.api.TestInstance;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class BlockingListTest extends BehaviorTestBase {
+
@BeforeAll
public void precondition() {
final Capability capability = op().info.fullCapability;
- assumeTrue(capability.read
- && capability.write
- && capability.copy
- && capability.blocking
- && capability.list
- && capability.createDir);
+ assumeTrue(capability.read && capability.write && capability.copy && capability.list && capability.createDir);
}
@Test
diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingReadOnlyTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingReadOnlyTest.java
index 42c543857..a924e35f2 100644
--- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingReadOnlyTest.java
+++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingReadOnlyTest.java
@@ -47,7 +47,7 @@ public class BlockingReadOnlyTest extends BehaviorTestBase {
@BeforeAll
public void precondition() {
final Capability capability = op().info.fullCapability;
- assumeTrue(capability.read && !capability.write && capability.blocking);
+ assumeTrue(capability.read && !capability.write);
}
/**
diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingRenameTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingRenameTest.java
index e72f7732c..3d7417141 100644
--- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingRenameTest.java
+++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingRenameTest.java
@@ -33,14 +33,11 @@ import org.junit.jupiter.api.TestInstance;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class BlockingRenameTest extends BehaviorTestBase {
+
@BeforeAll
public void precondition() {
final Capability capability = op().info.fullCapability;
- assumeTrue(capability.read
- && capability.write
- && capability.blocking
- && capability.rename
- && capability.createDir);
+ assumeTrue(capability.read && capability.write && capability.rename && capability.createDir);
}
/**
diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java
index ce0be0f54..f9c24378b 100644
--- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java
+++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java
@@ -33,10 +33,11 @@ import org.junit.jupiter.api.TestInstance;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class BlockingWriteTest extends BehaviorTestBase {
+
@BeforeAll
public void precondition() {
final Capability capability = op().info.fullCapability;
- assumeTrue(capability.read && capability.write && capability.blocking);
+ assumeTrue(capability.read && capability.write);
}
/**
diff --git a/bindings/lua/Cargo.toml b/bindings/lua/Cargo.toml
index 17b140ee1..fb076cb3a 100644
--- a/bindings/lua/Cargo.toml
+++ b/bindings/lua/Cargo.toml
@@ -40,6 +40,7 @@ mlua = { version = "0.9", features = [
], default-features = false, optional = true }
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
+ "blocking",
# These are default features before v0.46. TODO: change to optional features
"services-azblob",
"services-azdls",
@@ -57,3 +58,4 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-webhdfs",
"services-azfile",
] }
+tokio = { version = "1.28.1", features = ["full"] }
diff --git a/bindings/lua/src/lib.rs b/bindings/lua/src/lib.rs
index 21a359b2d..2ee8bd7cc 100644
--- a/bindings/lua/src/lib.rs
+++ b/bindings/lua/src/lib.rs
@@ -17,14 +17,22 @@
use std::collections::HashMap;
use std::str::FromStr;
+use std::sync::LazyLock;
use ::opendal as od;
use mlua::prelude::*;
use mlua::UserData;
+static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+});
+
#[derive(Clone, mlua::FromLua)]
struct ODOperator {
- operator: od::BlockingOperator,
+ operator: od::blocking::Operator,
}
impl UserData for ODOperator {}
@@ -56,10 +64,14 @@ fn operator_new<'a>(
};
let op = match od::Operator::via_iter(od_schema, map) {
- Ok(o) => o.blocking(),
+ Ok(o) => o,
Err(e) => return Err(LuaError::external(e)),
};
+ let handle = RUNTIME.handle();
+ let _guard = handle.enter();
+ let op = od::blocking::Operator::new(op).map_err(LuaError::external)?;
+
// this prevents the operator memory from being dropped by the Box
let op = ODOperator { operator: op };
diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml
index 7145fdc07..95b5ab52c 100644
--- a/bindings/nodejs/Cargo.toml
+++ b/bindings/nodejs/Cargo.toml
@@ -159,7 +159,7 @@ napi = { version = "2.11.3", default-features = false, features = [
napi-derive = "2.14.6"
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
- "layers-blocking",
+ "blocking",
] }
tokio = "1"
diff --git a/bindings/nodejs/generated.d.ts b/bindings/nodejs/generated.d.ts
index 9394c9e55..62e61e970 100644
--- a/bindings/nodejs/generated.d.ts
+++ b/bindings/nodejs/generated.d.ts
@@ -61,35 +61,6 @@ export interface WriteOptions {
/** Set the [Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) of op. */
cacheControl?: string
}
-export interface WriterOptions {
- /**
- * Append bytes into a path.
- *
- * ### Notes
- *
- * - It always appends content to the end of the file.
- * - It will create file if the path does not exist.
- */
- append?: boolean
- /**
- * Set the chunk of op.
- *
- * If chunk is set, the data will be chunked by the underlying writer.
- *
- * ## NOTE
- *
- * A service could have their own minimum chunk size while perform write
- * operations like multipart uploads. So the chunk size may be larger than
- * the given buffer size.
- */
- chunk?: bigint
- /** Set the [Content-Type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type) of op. */
- contentType?: string
- /** Set the [Content-Disposition](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition) of op. */
- contentDisposition?: string
- /** Set the [Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) of op. */
- cacheControl?: string
-}
/** PresignedRequest is a presigned request return by `presign`. */
export interface PresignedRequest {
/** HTTP method of this request. */
@@ -188,8 +159,6 @@ export class Capability {
get presignWrite(): boolean
/** If operator supports shared. */
get shared(): boolean
- /** If operator supports blocking. */
- get blocking(): boolean
}
export class Operator {
/**
@@ -337,13 +306,13 @@ export class Operator {
*
* It could be used to write large file in a streaming way.
*/
- writer(path: string, options?: WriterOptions | undefined | null): Promise<Writer>
+ writer(path: string, options?: WriteOptions | undefined | null): Promise<Writer>
/**
* Write multiple bytes into a path synchronously.
*
* It could be used to write large file in a streaming way.
*/
- writerSync(path: string, options?: WriterOptions | undefined | null): BlockingWriter
+ writerSync(path: string, options?: WriteOptions | undefined | null): BlockingWriter
/**
* Write bytes into a path synchronously.
*
diff --git a/bindings/nodejs/src/capability.rs b/bindings/nodejs/src/capability.rs
index afa01c0df..72fbb2c17 100644
--- a/bindings/nodejs/src/capability.rs
+++ b/bindings/nodejs/src/capability.rs
@@ -232,10 +232,4 @@ impl Capability {
pub fn shared(&self) -> bool {
self.0.shared
}
-
- /// If operator supports blocking.
- #[napi(getter)]
- pub fn blocking(&self) -> bool {
- self.0.blocking
- }
}
diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs
index d6e8ab06c..2aa1e023d 100644
--- a/bindings/nodejs/src/lib.rs
+++ b/bindings/nodejs/src/lib.rs
@@ -31,7 +31,10 @@ use napi::bindgen_prelude::*;
mod capability;
#[napi]
-pub struct Operator(opendal::Operator);
+pub struct Operator {
+ async_op: opendal::Operator,
+ blocking_op: opendal::blocking::Operator,
+}
#[napi]
impl Operator {
@@ -49,25 +52,27 @@ impl Operator {
.map_err(format_napi_error)?;
let options = options.unwrap_or_default();
- let mut op = opendal::Operator::via_iter(scheme, options).map_err(format_napi_error)?;
+ let async_op = opendal::Operator::via_iter(scheme, options).map_err(format_napi_error)?;
- if !op.info().full_capability().blocking {
- if let Ok(handle) = tokio::runtime::Handle::try_current() {
- let _guard = handle.enter();
- op = op.layer(
- opendal::layers::BlockingLayer::create()
- .expect("blocking layer must be created"),
- );
- }
- }
+ let blocking_op = {
+ let handle = tokio::runtime::Handle::current();
+ let _guard = handle.enter();
+
+ opendal::blocking::Operator::new(async_op.clone()).map_err(format_napi_error)?
+ };
- Ok(Operator(op))
+ Ok(Operator {
+ async_op,
+ blocking_op,
+ })
}
/// Get current operator(service)'s full capability.
#[napi]
pub fn capability(&self) -> Result<capability::Capability> {
- Ok(capability::Capability::new(self.0.info().full_capability()))
+ Ok(capability::Capability::new(
+ self.async_op.info().full_capability(),
+ ))
}
/// Get current path's metadata **without cache** directly.
@@ -89,7 +94,7 @@ impl Operator {
/// ```
#[napi]
pub async fn stat(&self, path: String) -> Result<Metadata> {
- let meta = self.0.stat(&path).await.map_err(format_napi_error)?;
+ let meta = self.async_op.stat(&path).await.map_err(format_napi_error)?;
Ok(Metadata(meta))
}
@@ -105,7 +110,7 @@ impl Operator {
/// ```
#[napi]
pub fn stat_sync(&self, path: String) -> Result<Metadata> {
- let meta = self.0.blocking().stat(&path).map_err(format_napi_error)?;
+ let meta = self.blocking_op.stat(&path).map_err(format_napi_error)?;
Ok(Metadata(meta))
}
@@ -120,7 +125,7 @@ impl Operator {
/// ```
#[napi]
pub async fn check(&self) -> Result<()> {
- self.0.check().await.map_err(format_napi_error)
+ self.async_op.check().await.map_err(format_napi_error)
}
/// Check the op synchronously.
@@ -131,7 +136,7 @@ impl Operator {
/// ```
#[napi]
pub fn check_sync(&self) -> Result<()> {
- self.0.blocking().check().map_err(format_napi_error)
+ self.blocking_op.check().map_err(format_napi_error)
}
/// Check if this path exists or not.
@@ -142,7 +147,7 @@ impl Operator {
/// ```
#[napi]
pub async fn exists(&self, path: String) -> Result<bool> {
- self.0.exists(&path).await.map_err(format_napi_error)
+ self.async_op.exists(&path).await.map_err(format_napi_error)
}
/// Check if this path exists or not synchronously.
@@ -153,7 +158,7 @@ impl Operator {
/// ```
#[napi]
pub fn exists_sync(&self, path: String) -> Result<bool> {
- self.0.blocking().exists(&path).map_err(format_napi_error)
+ self.blocking_op.exists(&path).map_err(format_napi_error)
}
/// Create dir with a given path.
@@ -164,7 +169,10 @@ impl Operator {
/// ```
#[napi]
pub async fn create_dir(&self, path: String) -> Result<()> {
- self.0.create_dir(&path).await.map_err(format_napi_error)
+ self.async_op
+ .create_dir(&path)
+ .await
+ .map_err(format_napi_error)
}
/// Create dir with a given path synchronously.
@@ -175,8 +183,7 @@ impl Operator {
/// ```
#[napi]
pub fn create_dir_sync(&self, path: String) -> Result<()> {
- self.0
- .blocking()
+ self.blocking_op
.create_dir(&path)
.map_err(format_napi_error)
}
@@ -190,7 +197,7 @@ impl Operator {
#[napi]
pub async fn read(&self, path: String) -> Result<Buffer> {
let res = self
- .0
+ .async_op
.read(&path)
.await
.map_err(format_napi_error)?
@@ -203,7 +210,11 @@ impl Operator {
/// It could be used to read large file in a streaming way.
#[napi]
pub async fn reader(&self, path: String) -> Result<Reader> {
- let r = self.0.reader(&path).await.map_err(format_napi_error)?;
+ let r = self
+ .async_op
+ .reader(&path)
+ .await
+ .map_err(format_napi_error)?;
Ok(Reader {
inner: r
.into_futures_async_read(..)
@@ -221,8 +232,7 @@ impl Operator {
#[napi]
pub fn read_sync(&self, path: String) -> Result<Buffer> {
let res = self
- .0
- .blocking()
+ .blocking_op
.read(&path)
.map_err(format_napi_error)?
.to_vec();
@@ -234,7 +244,7 @@ impl Operator {
/// It could be used to read large file in a streaming way.
#[napi]
pub fn reader_sync(&self, path: String) -> Result<BlockingReader> {
- let r = self.0.blocking().reader(&path).map_err(format_napi_error)?;
+ let r = self.blocking_op.reader(&path).map_err(format_napi_error)?;
Ok(BlockingReader {
inner: r.into_std_read(..).map_err(format_napi_error)?,
})
@@ -262,7 +272,7 @@ impl Operator {
Either::A(buf) => buf.as_ref().to_owned(),
Either::B(s) => s.into_bytes(),
};
- let mut writer = self.0.write_with(&path, c);
+ let mut writer = self.async_op.write_with(&path, c);
if let Some(options) = options {
if let Some(append) = options.append {
writer = writer.append(append);
@@ -288,27 +298,14 @@ impl Operator {
///
/// It could be used to write large file in a streaming way.
#[napi]
- pub async fn writer(&self, path: String, options: Option<WriterOptions>) -> Result<Writer> {
- let mut writer = self.0.writer_with(&path);
- if let Some(options) = options {
- if let Some(append) = options.append {
- writer = writer.append(append);
- }
- if let Some(chunk) = options.chunk {
- writer = writer.chunk(chunk.get_u64().1 as usize);
- }
- if let Some(ref content_type) = options.content_type {
- writer = writer.content_type(content_type);
- }
- if let Some(ref content_disposition) = options.content_disposition {
- writer = writer.content_disposition(content_disposition);
- }
- if let Some(ref cache_control) = options.cache_control {
- writer = writer.cache_control(cache_control);
- }
- }
- let w = writer.await.map_err(format_napi_error)?;
- Ok(Writer(w))
+ pub async fn writer(&self, path: String, options: Option<WriteOptions>) -> Result<Writer> {
+ let options = options.unwrap_or_default();
+ let writer = self
+ .async_op
+ .writer_options(&path, options.into())
+ .await
+ .map_err(format_napi_error)?;
+ Ok(Writer(writer))
}
/// Write multiple bytes into a path synchronously.
@@ -318,28 +315,14 @@ impl Operator {
pub fn writer_sync(
&self,
path: String,
- options: Option<WriterOptions>,
+ options: Option<WriteOptions>,
) -> Result<BlockingWriter> {
- let mut writer = self.0.blocking().writer_with(&path);
- if let Some(options) = options {
- if let Some(append) = options.append {
- writer = writer.append(append);
- }
- if let Some(chunk) = options.chunk {
- writer = writer.chunk(chunk.get_u64().1 as usize);
- }
- if let Some(ref content_type) = options.content_type {
- writer = writer.content_type(content_type);
- }
- if let Some(ref content_disposition) = options.content_disposition {
- writer = writer.content_disposition(content_disposition);
- }
- if let Some(ref cache_control) = options.cache_control {
- writer = writer.cache_control(cache_control);
- }
- }
- let w = writer.call().map_err(format_napi_error)?;
- Ok(BlockingWriter(w))
+ let options = options.unwrap_or_default();
+ let writer = self
+ .blocking_op
+ .writer_options(&path, options.into())
+ .map_err(format_napi_error)?;
+ Ok(BlockingWriter(writer))
}
//noinspection DuplicatedCode
@@ -364,25 +347,11 @@ impl Operator {
Either::A(buf) => buf.as_ref().to_owned(),
Either::B(s) => s.into_bytes(),
};
- let mut writer = self.0.blocking().write_with(&path, c);
- if let Some(options) = options {
- if let Some(append) = options.append {
- writer = writer.append(append);
- }
- if let Some(chunk) = options.chunk {
- writer = writer.chunk(chunk.get_u64().1 as usize);
- }
- if let Some(ref content_type) = options.content_type {
- writer = writer.content_type(content_type);
- }
- if let Some(ref content_disposition) = options.content_disposition {
- writer = writer.content_disposition(content_disposition);
- }
- if let Some(ref cache_control) = options.cache_control {
- writer = writer.cache_control(cache_control);
- }
- }
- writer.call().map(|_| ()).map_err(format_napi_error)
+ let options = options.unwrap_or_default();
+ self.blocking_op
+ .write_options(&path, c, options.into())
+ .map_err(format_napi_error)?;
+ Ok(())
}
/// Copy file according to given `from` and `to` path.
@@ -393,7 +362,10 @@ impl Operator {
/// ```
#[napi]
pub async fn copy(&self, from: String, to: String) -> Result<()> {
- self.0.copy(&from, &to).await.map_err(format_napi_error)
+ self.async_op
+ .copy(&from, &to)
+ .await
+ .map_err(format_napi_error)
}
/// Copy file according to given `from` and `to` path synchronously.
@@ -404,10 +376,7 @@ impl Operator {
/// ```
#[napi]
pub fn copy_sync(&self, from: String, to: String) -> Result<()> {
- self.0
- .blocking()
- .copy(&from, &to)
- .map_err(format_napi_error)
+ self.blocking_op.copy(&from, &to).map_err(format_napi_error)
}
/// Rename file according to given `from` and `to` path.
@@ -420,7 +389,10 @@ impl Operator {
/// ```
#[napi]
pub async fn rename(&self, from: String, to: String) -> Result<()> {
- self.0.rename(&from, &to).await.map_err(format_napi_error)
+ self.async_op
+ .rename(&from, &to)
+ .await
+ .map_err(format_napi_error)
}
/// Rename file according to given `from` and `to` path synchronously.
@@ -433,8 +405,7 @@ impl Operator {
/// ```
#[napi]
pub fn rename_sync(&self, from: String, to: String) -> Result<()> {
- self.0
- .blocking()
+ self.blocking_op
.rename(&from, &to)
.map_err(format_napi_error)
}
@@ -450,7 +421,7 @@ impl Operator {
/// ```
#[napi]
pub async fn delete(&self, path: String) -> Result<()> {
- self.0.delete(&path).await.map_err(format_napi_error)
+ self.async_op.delete(&path).await.map_err(format_napi_error)
}
/// Delete the given path synchronously.
@@ -461,7 +432,7 @@ impl Operator {
/// ```
#[napi]
pub fn delete_sync(&self, path: String) -> Result<()> {
- self.0.blocking().delete(&path).map_err(format_napi_error)
+ self.blocking_op.delete(&path).map_err(format_napi_error)
}
/// Remove given paths.
@@ -475,7 +446,10 @@ impl Operator {
/// ```
#[napi]
pub async fn remove(&self, paths: Vec<String>) -> Result<()> {
- self.0.delete_iter(paths).await.map_err(format_napi_error)
+ self.async_op
+ .delete_iter(paths)
+ .await
+ .map_err(format_napi_error)
}
/// Remove the path and all nested dirs and files recursively.
@@ -489,7 +463,10 @@ impl Operator {
/// ```
#[napi]
pub async fn remove_all(&self, path: String) -> Result<()> {
- self.0.remove_all(&path).await.map_err(format_napi_error)
+ self.async_op
+ .remove_all(&path)
+ .await
+ .map_err(format_napi_error)
}
/// List the given path.
@@ -525,7 +502,7 @@ impl Operator {
/// ```
#[napi]
pub async fn list(&self, path: String, options: Option<ListOptions>) -> Result<Vec<Entry>> {
- let mut l = self.0.list_with(&path);
+ let mut l = self.async_op.list_with(&path);
if let Some(options) = options {
if let Some(limit) = options.limit {
l = l.limit(limit as usize);
@@ -575,21 +552,13 @@ impl Operator {
/// ```
#[napi]
pub fn list_sync(&self, path: String, options: Option<ListOptions>) -> Result<Vec<Entry>> {
- let mut l = self.0.blocking().list_with(&path);
- if let Some(options) = options {
- if let Some(limit) = options.limit {
- l = l.limit(limit as usize);
- }
- if let Some(recursive) = options.recursive {
- l = l.recursive(recursive);
- }
- }
+ let options = options.unwrap_or_default();
+ let l = self
+ .blocking_op
+ .list_options(&path, options.into())
+ .map_err(format_napi_error)?;
- Ok(l.call()
- .map_err(format_napi_error)?
- .iter()
- .map(|e| Entry(e.to_owned()))
- .collect())
+ Ok(l.into_iter().map(Entry).collect())
}
/// Get a presigned request for read.
@@ -608,7 +577,7 @@ impl Operator {
#[napi]
pub async fn presign_read(&self, path: String, expires: u32) -> Result<PresignedRequest> {
let res = self
- .0
+ .async_op
.presign_read(&path, Duration::from_secs(expires as u64))
.await
.map_err(format_napi_error)?;
@@ -631,7 +600,7 @@ impl Operator {
#[napi]
pub async fn presign_write(&self, path: String, expires: u32) -> Result<PresignedRequest> {
let res = self
- .0
+ .async_op
.presign_write(&path, Duration::from_secs(expires as u64))
.await
.map_err(format_napi_error)?;
@@ -654,7 +623,7 @@ impl Operator {
#[napi]
pub async fn presign_stat(&self, path: String, expires: u32) -> Result<PresignedRequest> {
let res = self
- .0
+ .async_op
.presign_stat(&path, Duration::from_secs(expires as u64))
.await
.map_err(format_napi_error)?;
@@ -733,16 +702,27 @@ impl Metadata {
}
#[napi(object)]
+#[derive(Default)]
pub struct ListOptions {
pub limit: Option<u32>,
pub recursive: Option<bool>,
}
+impl From<ListOptions> for opendal::options::ListOptions {
+ fn from(value: ListOptions) -> Self {
+ Self {
+ limit: value.limit.map(|v| v as usize),
+ recursive: value.recursive.unwrap_or_default(),
+ ..Default::default()
+ }
+ }
+}
+
/// BlockingReader is designed to read data from a given path in a blocking
/// manner.
#[napi]
pub struct BlockingReader {
- inner: opendal::StdReader,
+ inner: opendal::blocking::StdReader,
}
#[napi]
@@ -780,7 +760,7 @@ impl Reader {
/// BlockingWriter is designed to write data into a given path in a blocking
/// manner.
#[napi]
-pub struct BlockingWriter(opendal::BlockingWriter);
+pub struct BlockingWriter(opendal::blocking::Writer);
#[napi]
impl BlockingWriter {
@@ -902,36 +882,17 @@ pub struct WriteOptions {
pub cache_control: Option<String>,
}
-#[napi(object)]
-#[derive(Default)]
-pub struct WriterOptions {
- /// Append bytes into a path.
- ///
- /// ### Notes
- ///
- /// - It always appends content to the end of the file.
- /// - It will create file if the path does not exist.
- pub append: Option<bool>,
-
- /// Set the chunk of op.
- ///
- /// If chunk is set, the data will be chunked by the underlying writer.
- ///
- /// ## NOTE
- ///
- /// A service could have their own minimum chunk size while perform write
- /// operations like multipart uploads. So the chunk size may be larger than
- /// the given buffer size.
- pub chunk: Option<BigInt>,
-
- /// Set the [Content-Type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type) of op.
- pub content_type: Option<String>,
-
- /// Set the [Content-Disposition](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition) of op.
- pub content_disposition: Option<String>,
-
- /// Set the [Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) of op.
- pub cache_control: Option<String>,
+impl From<WriteOptions> for opendal::options::WriteOptions {
+ fn from(value: WriteOptions) -> Self {
+ Self {
+ append: value.append.unwrap_or_default(),
+ chunk: value.chunk.map(|v| v.get_u64().1 as usize),
+ content_type: value.content_type,
+ content_disposition: value.content_disposition,
+ cache_control: value.cache_control,
+ ..Default::default()
+ }
+ }
}
/// Lister is designed to list entries at a given path in an asynchronous
@@ -961,7 +922,7 @@ impl Lister {
/// BlockingLister is designed to list entries at a given path in a blocking
/// manner.
#[napi]
-pub struct BlockingLister(opendal::BlockingLister);
+pub struct BlockingLister(opendal::blocking::Lister);
/// Method `next` can be confused for the standard trait method `std::iter::Iterator::next`.
/// But in JavaScript, it is also customary to use the next method directly to get the next element.
@@ -1031,7 +992,19 @@ impl Operator {
/// Add a layer to this operator.
#[napi]
pub fn layer(&self, layer: External<Layer>) -> Result<Self> {
- Ok(Self(layer.inner.layer(self.0.clone())))
+ let async_op = layer.inner.layer(self.async_op.clone());
+
+ let blocking_op = {
+ let handle = tokio::runtime::Handle::current();
+ let _guard = handle.enter();
+
+ opendal::blocking::Operator::new(async_op.clone()).map_err(format_napi_error)?
+ };
+
+ Ok(Self {
+ async_op,
+ blocking_op,
+ })
}
}
diff --git a/bindings/ocaml/Cargo.toml b/bindings/ocaml/Cargo.toml
index 13ccd9d4b..8cb11945d 100644
--- a/bindings/ocaml/Cargo.toml
+++ b/bindings/ocaml/Cargo.toml
@@ -34,6 +34,7 @@ doc = false
ocaml = { version = "1.2.0" }
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
+ "blocking",
# These are default features before v0.46. TODO: change to optional features
"services-azblob",
"services-azdls",
@@ -51,6 +52,7 @@ opendal = { version = ">=0", path = "../../core", features = [
"services-webhdfs",
"services-azfile",
] }
+tokio = { version = "1.0", features = ["full"] }
[build-dependencies]
ocaml-build = { version = "1.0.0" }
diff --git a/bindings/ocaml/src/lib.rs b/bindings/ocaml/src/lib.rs
index 9a6ee6139..4ea8e6000 100644
--- a/bindings/ocaml/src/lib.rs
+++ b/bindings/ocaml/src/lib.rs
@@ -18,12 +18,20 @@
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::str::FromStr;
+use std::sync::LazyLock;
use ::opendal as od;
mod operator;
mod seek_from;
+static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+});
+
pub fn new_operator(
scheme_str: String,
map: BTreeMap<String, String>,
diff --git a/bindings/ocaml/src/operator/_type.rs b/bindings/ocaml/src/operator/_type.rs
index e79a3dde4..50bb7a5b1 100644
--- a/bindings/ocaml/src/operator/_type.rs
+++ b/bindings/ocaml/src/operator/_type.rs
@@ -20,11 +20,11 @@
use super::*;
#[ocaml::sig]
-pub struct Operator(pub(crate) od::BlockingOperator);
+pub struct Operator(pub(crate) od::blocking::Operator);
ocaml::custom!(Operator);
#[ocaml::sig]
-pub struct Reader(pub(crate) od::BlockingReader);
+pub struct Reader(pub(crate) od::blocking::Reader);
ocaml::custom!(Reader);
#[ocaml::sig]
diff --git a/bindings/ocaml/src/operator/mod.rs b/bindings/ocaml/src/operator/mod.rs
index b2bb876c0..9cbb51b40 100644
--- a/bindings/ocaml/src/operator/mod.rs
+++ b/bindings/ocaml/src/operator/mod.rs
@@ -31,7 +31,12 @@ pub fn operator(
map: BTreeMap<String, String>,
) -> Result<ocaml::Pointer<Operator>, String> {
let op = map_res_error(new_operator(scheme_str, map))?;
- Ok(Operator(op.blocking()).into())
+
+ let handle = RUNTIME.handle();
+ let _enter = handle.enter();
+ let blocking_op = od::blocking::Operator::new(op.clone()).unwrap();
+
+ Ok(Operator(blocking_op).into())
}
#[ocaml::func]
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
index 1f53cf20e..02121afca 100644
--- a/bindings/python/Cargo.toml
+++ b/bindings/python/Cargo.toml
@@ -162,7 +162,7 @@ dict_derive = "0.6.0"
futures = "0.3.28"
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
- "layers-blocking",
+ "blocking",
"layers-mime-guess"
] }
pyo3 = { version = "0.24.1", features = ["generate-import-lib", "chrono"] }
@@ -171,7 +171,7 @@ tokio = "1"
[target.'cfg(unix)'.dependencies.opendal]
features = [
- "layers-blocking",
+ "blocking",
"layers-mime-guess",
# Depend on "openssh" which depends on "tokio-pipe" that is unavailable on Windows.
"services-sftp",
diff --git a/bindings/python/python/opendal/__init__.pyi b/bindings/python/python/opendal/__init__.pyi
index 816c0b5c8..c6bfaca25 100644
--- a/bindings/python/python/opendal/__init__.pyi
+++ b/bindings/python/python/opendal/__init__.pyi
@@ -175,10 +175,7 @@ class Operator(_Base):
target (str|Path): The target path.
"""
def remove_all(self, path: PathBuf) -> None:
- """Remove all objects at the given path.
-
- Args:
- path (str|Path): The path to the directory.
+ """Convert into an async operator
"""
def to_async_operator(self) -> AsyncOperator: ...
@@ -656,6 +653,3 @@ class Capability:
shared: bool
"""If operator supports shared"""
-
- blocking: bool
- """If operator supports blocking"""
diff --git a/bindings/python/src/capability.rs b/bindings/python/src/capability.rs
index d2890cda8..c828a605d 100644
--- a/bindings/python/src/capability.rs
+++ b/bindings/python/src/capability.rs
@@ -102,9 +102,6 @@ pub struct Capability {
/// If operator supports shared.
pub shared: bool,
-
- /// If operator supports blocking.
- pub blocking: bool,
}
impl Capability {
@@ -144,7 +141,6 @@ impl Capability {
presign_write: capability.presign_write,
presign_delete: capability.presign_delete,
shared: capability.shared,
- blocking: capability.blocking,
}
}
}
diff --git a/bindings/python/src/file.rs b/bindings/python/src/file.rs
index d8946fa3a..26ec4c606 100644
--- a/bindings/python/src/file.rs
+++ b/bindings/python/src/file.rs
@@ -41,17 +41,17 @@ use crate::*;
pub struct File(FileState);
enum FileState {
- Reader(ocore::StdReader),
- Writer(ocore::StdWriter),
+ Reader(ocore::blocking::StdReader),
+ Writer(ocore::blocking::StdWriter),
Closed,
}
impl File {
- pub fn new_reader(reader: ocore::StdReader) -> Self {
+ pub fn new_reader(reader: ocore::blocking::StdReader) -> Self {
Self(FileState::Reader(reader))
}
- pub fn new_writer(writer: ocore::BlockingWriter) -> Self {
+ pub fn new_writer(writer: ocore::blocking::Writer) -> Self {
Self(FileState::Writer(writer.into_std_write()))
}
}
diff --git a/bindings/python/src/lister.rs b/bindings/python/src/lister.rs
index 6019689de..b6ddb7c10 100644
--- a/bindings/python/src/lister.rs
+++ b/bindings/python/src/lister.rs
@@ -26,11 +26,11 @@ use tokio::sync::Mutex;
use crate::*;
#[pyclass(unsendable, module = "opendal")]
-pub struct BlockingLister(ocore::BlockingLister);
+pub struct BlockingLister(ocore::blocking::Lister);
impl BlockingLister {
/// Create a new blocking lister.
- pub fn new(inner: ocore::BlockingLister) -> Self {
+ pub fn new(inner: ocore::blocking::Lister) -> Self {
Self(inner)
}
}
diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs
index 64c1fdbee..38d372d8c 100644
--- a/bindings/python/src/operator.rs
+++ b/bindings/python/src/operator.rs
@@ -33,14 +33,19 @@ fn build_operator(
scheme: ocore::Scheme,
map: HashMap<String, String>,
) -> PyResult<ocore::Operator> {
- let mut op = ocore::Operator::via_iter(scheme, map).map_err(format_pyerr)?;
- if !op.info().full_capability().blocking {
- let runtime = pyo3_async_runtimes::tokio::get_runtime();
- let _guard = runtime.enter();
- op = op
- .layer(ocore::layers::BlockingLayer::create().expect("blocking layer must be created"));
- }
+ let op = ocore::Operator::via_iter(scheme, map).map_err(format_pyerr)?;
+ Ok(op)
+}
+
+fn build_blocking_operator(
+ scheme: ocore::Scheme,
+ map: HashMap<String, String>,
+) -> PyResult<ocore::blocking::Operator> {
+ let op = ocore::Operator::via_iter(scheme, map).map_err(format_pyerr)?;
+ let runtime = pyo3_async_runtimes::tokio::get_runtime();
+ let _guard = runtime.enter();
+ let op = ocore::blocking::Operator::new(op).map_err(format_pyerr)?;
Ok(op)
}
@@ -49,7 +54,7 @@ fn build_operator(
/// Create a new blocking `Operator` with the given `scheme` and options(`**kwargs`).
#[pyclass(module = "opendal")]
pub struct Operator {
- core: ocore::BlockingOperator,
+ core: ocore::blocking::Operator,
__scheme: ocore::Scheme,
__map: HashMap<String, String>,
}
@@ -73,7 +78,7 @@ impl Operator {
.unwrap_or_default();
Ok(Operator {
- core: build_operator(scheme, map.clone())?.blocking(),
+ core: build_blocking_operator(scheme, map.clone())?,
__scheme: scheme,
__map: map,
})
@@ -82,8 +87,12 @@ impl Operator {
/// Add new layers upon existing operator
pub fn layer(&self, layer: &layers::Layer) -> PyResult<Self> {
let op = layer.0.layer(self.core.clone().into());
+
+ let runtime = pyo3_async_runtimes::tokio::get_runtime();
+ let _guard = runtime.enter();
+ let op = ocore::blocking::Operator::new(op).map_err(format_pyerr)?;
Ok(Self {
- core: op.blocking(),
+ core: op,
__scheme: self.__scheme,
__map: self.__map.clone(),
})
@@ -121,28 +130,11 @@ impl Operator {
#[pyo3(signature = (path, bs, **kwargs))]
pub fn write(&self, path: PathBuf, bs: Vec<u8>, kwargs: Option<WriteOptions>) -> PyResult<()> {
let path = path.to_string_lossy().to_string();
- let mut kwargs = kwargs.unwrap_or_default();
- let mut write = self
- .core
- .write_with(&path, bs)
- .append(kwargs.append.unwrap_or(false));
- if let Some(chunk) = kwargs.chunk {
- write = write.chunk(chunk);
- }
- if let Some(content_type) = &kwargs.content_type {
- write = write.content_type(content_type);
- }
- if let Some(content_disposition) = &kwargs.content_disposition {
- write = write.content_disposition(content_disposition);
- }
- if let Some(cache_control) = &kwargs.cache_control {
- write = write.cache_control(cache_control);
- }
- if let Some(user_metadata) = kwargs.user_metadata.take() {
- write = write.user_metadata(user_metadata);
- }
-
- write.call().map(|_| ()).map_err(format_pyerr)
+ let kwargs = kwargs.unwrap_or_default();
+ self.core
+ .write_options(&path, bs, kwargs.into())
+ .map(|_| ())
+ .map_err(format_pyerr)
}
/// Get current path's metadata **without cache** directly.
@@ -215,11 +207,16 @@ impl Operator {
#[pyo3(signature = (path, *, start_after=None))]
pub fn list(&self, path: PathBuf, start_after: Option<String>) -> PyResult<BlockingLister> {
let path = path.to_string_lossy().to_string();
- let mut builder = self.core.lister_with(&path);
- if let Some(start_after) = start_after {
- builder = builder.start_after(&start_after);
- }
- let l = builder.call().map_err(format_pyerr)?;
+ let l = self
+ .core
+ .lister_options(
+ &path,
+ ocore::options::ListOptions {
+ start_after,
+ ..Default::default()
+ },
+ )
+ .map_err(format_pyerr)?;
Ok(BlockingLister::new(l))
}
@@ -228,9 +225,13 @@ impl Operator {
let path = path.to_string_lossy().to_string();
let l = self
.core
- .lister_with(&path)
- .recursive(true)
- .call()
+ .lister_options(
+ &path,
+ ocore::options::ListOptions {
+ recursive: true,
+ ..Default::default()
+ },
+ )
.map_err(format_pyerr)?;
Ok(BlockingLister::new(l))
}
@@ -629,8 +630,12 @@ impl AsyncOperator {
}
pub fn to_operator(&self) -> PyResult<Operator> {
+ let runtime = pyo3_async_runtimes::tokio::get_runtime();
+ let _guard = runtime.enter();
+ let op = ocore::blocking::Operator::new(self.core.clone()).map_err(format_pyerr)?;
+
Ok(Operator {
- core: self.core.clone().blocking(),
+ core: op,
__scheme: self.__scheme,
__map: self.__map.clone(),
})
diff --git a/bindings/python/src/options.rs b/bindings/python/src/options.rs
index aabee7833..a1c17b676 100644
--- a/bindings/python/src/options.rs
+++ b/bindings/python/src/options.rs
@@ -16,6 +16,7 @@
// under the License.
use dict_derive::FromPyObject;
+use opendal as ocore;
use pyo3::pyclass;
use std::collections::HashMap;
@@ -29,3 +30,17 @@ pub struct WriteOptions {
pub cache_control: Option<String>,
pub user_metadata: Option<HashMap<String, String>>,
}
+
+impl From<WriteOptions> for ocore::options::WriteOptions {
+ fn from(opts: WriteOptions) -> Self {
+ Self {
+ append: opts.append.unwrap_or(false),
+ chunk: opts.chunk,
+ content_type: opts.content_type,
+ content_disposition: opts.content_disposition,
+ cache_control: opts.cache_control,
+ user_metadata: opts.user_metadata,
+ ..Default::default()
+ }
+ }
+}
diff --git a/bindings/python/tests/test_sync_delete.py b/bindings/python/tests/test_sync_delete.py
index bb63f45e6..25e9d77d2 100644
--- a/bindings/python/tests/test_sync_delete.py
+++ b/bindings/python/tests/test_sync_delete.py
@@ -24,7 +24,7 @@ from opendal.exceptions import NotFound
@pytest.mark.need_capability(
- "read", "write", "delete", "list", "blocking", "create_dir"
+ "read", "write", "delete", "list", "create_dir"
)
def test_sync_remove_all(service_name, operator, async_operator):
parent = f"random_dir_{str(uuid4())}"
diff --git a/bindings/python/tests/test_sync_exists.py b/bindings/python/tests/test_sync_exists.py
index 6e4b3913b..f1e5a6f81 100644
--- a/bindings/python/tests/test_sync_exists.py
+++ b/bindings/python/tests/test_sync_exists.py
@@ -22,7 +22,7 @@ import pytest
@pytest.mark.need_capability(
- "read", "write", "delete", "list", "blocking", "create_dir"
+ "read", "write", "delete", "list", "create_dir"
)
def test_sync_exists(service_name, operator, async_operator):
content = os.urandom(1024)
diff --git a/bindings/ruby/Cargo.toml b/bindings/ruby/Cargo.toml
index 83e35a4bd..f51ecb2c3 100644
--- a/bindings/ruby/Cargo.toml
+++ b/bindings/ruby/Cargo.toml
@@ -35,6 +35,7 @@ name = "opendal_ruby"
magnus = { version = "0.7.1", features = ["bytes"] }
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
+ "blocking",
"layers-throttle",
# These are default features before v0.46. TODO: change to optional features
"services-azblob",
@@ -56,6 +57,7 @@ opendal = { version = ">=0", path = "../../core", features = [
rb-sys = { version = "0.9.110", default-features = false }
# should be in sync with opendal
bytes = "1.6"
+tokio = { version = "1", features = ["full"] }
[build-dependencies]
rb-sys-env = "0.1.2"
diff --git a/bindings/ruby/src/capability.rs b/bindings/ruby/src/capability.rs
index 969655f79..1df438cf1 100644
--- a/bindings/ruby/src/capability.rs
+++ b/bindings/ruby/src/capability.rs
@@ -95,7 +95,6 @@ define_accessors!(Capability, {
presign_read: bool,
presign_stat: bool,
presign_write: bool,
- blocking: bool,
});
// includes class into the Ruby module
@@ -142,7 +141,6 @@ pub fn include(gem_module: &RModule) -> Result<(), Error> {
presign_read,
presign_stat,
presign_write,
- blocking
});
Ok(())
diff --git a/bindings/ruby/src/io.rs b/bindings/ruby/src/io.rs
index 22cb6c50b..ca04c2545 100644
--- a/bindings/ruby/src/io.rs
+++ b/bindings/ruby/src/io.rs
@@ -53,8 +53,8 @@ use crate::*;
pub struct Io(RefCell<FileState>);
enum FileState {
- Reader(ocore::StdReader, bool), // bool indicates binary mode
- Writer(ocore::StdWriter, bool), // bool indicates binary mode
+ Reader(ocore::blocking::StdReader, bool), // bool indicates binary mode
+ Writer(ocore::blocking::StdWriter, bool), // bool indicates binary mode
Closed,
}
@@ -68,7 +68,7 @@ impl Io {
/// See [`Operator::open`] for more information.
pub fn new(
ruby: &Ruby,
- operator: ocore::BlockingOperator,
+ operator: ocore::blocking::Operator,
path: String,
mode: String,
) -> Result<Self, Error> {
diff --git a/bindings/ruby/src/lib.rs b/bindings/ruby/src/lib.rs
index d2a3f8d67..481153b0f 100644
--- a/bindings/ruby/src/lib.rs
+++ b/bindings/ruby/src/lib.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+use std::sync::LazyLock;
+
use magnus::exception;
use magnus::function;
use magnus::Error;
@@ -32,6 +34,13 @@ mod middlewares;
mod operator;
mod operator_info;
+static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+});
+
pub fn format_magnus_error(err: ocore::Error) -> Error {
Error::new(exception::runtime_error(), err.to_string())
}
diff --git a/bindings/ruby/src/lister.rs b/bindings/ruby/src/lister.rs
index bc4d30e9a..fc3885e36 100644
--- a/bindings/ruby/src/lister.rs
+++ b/bindings/ruby/src/lister.rs
@@ -84,7 +84,7 @@ impl Entry {
///
/// `Lister` is thread-safe.
#[magnus::wrap(class = "OpenDAL::Lister", free_immediately, size)]
-pub struct Lister(Arc<Mutex<ocore::BlockingLister>>);
+pub struct Lister(Arc<Mutex<ocore::blocking::Lister>>);
impl Iterator for Lister {
type Item = Entry;
@@ -104,7 +104,7 @@ impl Iterator for Lister {
impl Lister {
/// Creates a new blocking Lister.
- pub fn new(inner: ocore::BlockingLister) -> Self {
+ pub fn new(inner: ocore::blocking::Lister) -> Self {
Self(Arc::new(Mutex::new(inner)))
}
diff --git a/bindings/ruby/src/middlewares.rs b/bindings/ruby/src/middlewares.rs
index 0b271aece..8060c1368 100644
--- a/bindings/ruby/src/middlewares.rs
+++ b/bindings/ruby/src/middlewares.rs
@@ -56,7 +56,7 @@ where
)
})?;
- let layered = operator.operator.clone().layer(guard.clone());
+ let layered = operator.async_op.clone().layer(guard.clone());
Ok(Operator::from_operator(layered))
}
diff --git a/bindings/ruby/src/operator.rs b/bindings/ruby/src/operator.rs
index 75bc89d04..a75082bbc 100644
--- a/bindings/ruby/src/operator.rs
+++ b/bindings/ruby/src/operator.rs
@@ -57,23 +57,28 @@ pub struct Operator {
// 3. Majority of use cases are still blocking.
//
// In practice, we will not use operator directly because of the async support.
- pub operator: ocore::Operator,
+ pub async_op: ocore::Operator,
// The cached blocking operator coming from `Operator::blocking`.
// Important to keep in sync after making changes to the `Operator`.
//
// We declare this `BlockingOperator` to state the intent of assumptions instead of
// getting a `BlockingOperator` for an operation dynamically every time.
- pub blocking: ocore::BlockingOperator,
+ pub blocking_op: ocore::blocking::Operator,
}
impl Operator {
// Convenience helper to construct operator
#[inline]
pub(crate) fn from_operator(operator: ocore::Operator) -> Operator {
- // doesn't change `operator`'s state, but creates a new pointer to the inner accessor
- let blocking = operator.blocking();
+ let handle = RUNTIME.handle();
+ let _enter = handle.enter();
+ let blocking_op = ocore::blocking::Operator::new(operator.clone())
+ .expect("initiate blocking operator must succeed");
- Operator { operator, blocking }
+ Operator {
+ async_op: operator,
+ blocking_op,
+ }
}
}
@@ -104,7 +109,7 @@ impl Operator {
/// @return [String]
fn read(ruby: &Ruby, rb_self: &Self, path: String) -> Result<bytes::Bytes, Error> {
let buffer = rb_self
- .blocking
+ .blocking_op
.read(&path)
.map_err(|err| Error::new(ruby.exception_runtime_error(), err.to_string()))?;
Ok(buffer.to_bytes())
@@ -118,7 +123,7 @@ impl Operator {
/// @return [nil]
fn write(ruby: &Ruby, rb_self: &Self, path: String, bs: RString) -> Result<(), Error> {
rb_self
- .blocking
+ .blocking_op
.write(&path, bs.to_bytes())
.map(|_| ())
.map_err(|err| Error::new(ruby.exception_runtime_error(), err.to_string()))
@@ -131,7 +136,7 @@ impl Operator {
/// @return [Metadata]
fn stat(ruby: &Ruby, rb_self: &Self, path: String) -> Result<Metadata, Error> {
rb_self
- .blocking
+ .blocking_op
.stat(&path)
.map_err(|err| Error::new(ruby.exception_runtime_error(), err.to_string()))
.map(Metadata::new)
@@ -142,7 +147,7 @@ impl Operator {
/// Gets capabilities of the current operator
/// @return [Capability]
fn capability(&self) -> Result<Capability, Error> {
- let capability = self.blocking.info().full_capability();
+ let capability = self.blocking_op.info().full_capability();
Ok(Capability::new(capability))
}
@@ -154,7 +159,7 @@ impl Operator {
/// @return [nil]
fn create_dir(ruby: &Ruby, rb_self: &Self, path: String) -> Result<(), Error> {
rb_self
- .blocking
+ .blocking_op
.create_dir(&path)
.map_err(|err| Error::new(ruby.exception_runtime_error(), err.to_string()))
}
@@ -166,7 +171,7 @@ impl Operator {
/// @return [nil]
fn delete(ruby: &Ruby, rb_self: &Self, path: String) -> Result<(), Error> {
rb_self
- .blocking
+ .blocking_op
.delete(&path)
.map_err(|err| Error::new(ruby.exception_runtime_error(), err.to_string()))
}
@@ -178,7 +183,7 @@ impl Operator {
/// @return [Boolean]
fn exists(ruby: &Ruby, rb_self: &Self, path: String) -> Result<bool, Error> {
rb_self
- .blocking
+ .blocking_op
.exists(&path)
.map_err(|err| Error::new(ruby.exception_runtime_error(), err.to_string()))
}
@@ -191,7 +196,7 @@ impl Operator {
/// @return [nil]
fn rename(ruby: &Ruby, rb_self: &Self, from: String, to: String) -> Result<(), Error> {
rb_self
- .blocking
+ .blocking_op
.rename(&from, &to)
.map_err(|err| Error::new(ruby.exception_runtime_error(), err.to_string()))
}
@@ -203,7 +208,7 @@ impl Operator {
/// @return [nil]
fn remove_all(ruby: &Ruby, rb_self: &Self, path: String) -> Result<(), Error> {
rb_self
- .blocking
+ .blocking_op
.remove_all(&path)
.map_err(|err| Error::new(ruby.exception_runtime_error(), err.to_string()))
}
@@ -216,7 +221,7 @@ impl Operator {
/// @return [nil]
fn copy(ruby: &Ruby, rb_self: &Self, from: String, to: String) -> Result<(), Error> {
rb_self
- .blocking
+ .blocking_op
.copy(&from, &to)
.map_err(|err| Error::new(ruby.exception_runtime_error(), err.to_string()))
}
@@ -229,7 +234,7 @@ impl Operator {
/// @raise [ArgumentError] invalid mode, or when the mode is not unique
/// @return [OpenDAL::IO]
fn open(ruby: &Ruby, rb_self: &Self, path: String, mode: String) -> Result<Io, Error> {
- let operator = rb_self.blocking.clone();
+ let operator = rb_self.blocking_op.clone();
Io::new(ruby, operator, path, mode)
}
@@ -250,22 +255,17 @@ impl Operator {
)?;
let (limit, start_after, recursive) = kwargs.optional;
- let mut builder = rb_self.blocking.lister_with(&path);
-
- if let Some(limit) = limit {
- builder = builder.limit(limit);
- }
-
- if let Some(start_after) = start_after {
- builder = builder.start_after(start_after.as_str());
- }
-
- if let Some(true) = recursive {
- builder = builder.recursive(true);
- }
-
- let lister = builder
- .call()
+ let lister = rb_self
+ .blocking_op
+ .lister_options(
+ &path,
+ ocore::options::ListOptions {
+ limit,
+ start_after,
+ recursive: recursive.unwrap_or(false),
+ ..Default::default()
+ },
+ )
.map_err(|err| Error::new(ruby.exception_runtime_error(), err.to_string()))?;
Ok(Lister::new(lister))
@@ -276,7 +276,7 @@ impl Operator {
/// Gets meta information of the underlying accessor.
/// @return [OperatorInfo]
fn info(&self) -> Result<OperatorInfo, Error> {
- Ok(OperatorInfo(self.blocking.info()))
+ Ok(OperatorInfo(self.blocking_op.info()))
}
}
diff --git a/core/Cargo.lock b/core/Cargo.lock
index 98d6ddc1c..a7d7fe7b1 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -5529,6 +5529,7 @@ dependencies = [
"criterion",
"opendal",
"rand 0.8.5",
+ "tokio",
"uuid",
]
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 2b18e382c..87b6069ef 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -49,6 +49,9 @@ version = "0.53.0"
[features]
default = ["reqwest/rustls-tls", "executors-tokio", "services-memory"]
+# Enable opendal's blocking support.
+blocking = ["internal-tokio-rt"]
+
# Build test utils or not.
#
# These features are used to control whether to build opendal's test utils.
@@ -59,7 +62,6 @@ tests = [
"dep:rand",
"dep:sha2",
"dep:dotenvy",
- "layers-blocking",
"services-azblob",
"services-fs",
"services-http",
@@ -102,7 +104,6 @@ layers-await-tree = ["dep:await-tree"]
# Enable layers async-backtrace support.
layers-async-backtrace = ["dep:async-backtrace"]
# Enable dtrace support.
-layers-blocking = ["internal-tokio-rt"]
layers-dtrace = ["dep:probe"]
services-aliyun-drive = []
diff --git a/core/benches/vs_fs/Cargo.toml b/core/benches/vs_fs/Cargo.toml
index 18febb6ba..c3f0ef9f1 100644
--- a/core/benches/vs_fs/Cargo.toml
+++ b/core/benches/vs_fs/Cargo.toml
@@ -28,4 +28,5 @@ version = "0.0.0"
criterion = { version = "0.5", features = ["async", "async_tokio"] }
opendal = { path = "../..", features = ["tests"] }
rand = "0.8"
+tokio = { version = "1", features = ["full"] }
uuid = { version = "1", features = ["v4"] }
diff --git a/core/benches/vs_fs/src/main.rs b/core/benches/vs_fs/src/main.rs
index 7db0b2497..78b8a7c80 100644
--- a/core/benches/vs_fs/src/main.rs
+++ b/core/benches/vs_fs/src/main.rs
@@ -16,6 +16,8 @@
// under the License.
use criterion::Criterion;
+use opendal::blocking;
+use opendal::options;
use opendal::services;
use opendal::Operator;
use rand::prelude::*;
@@ -29,7 +31,11 @@ fn main() {
fn bench_vs_fs(c: &mut Criterion) {
let cfg = services::Fs::default().root("/tmp/opendal/");
- let op = Operator::new(cfg).unwrap().finish().blocking();
+ let op = Operator::new(cfg).unwrap().finish();
+
+ let runtime = tokio::runtime::Runtime::new().unwrap();
+ let _guard = runtime.enter();
+ let op = blocking::Operator::new(op).unwrap();
let mut group = c.benchmark_group("read");
group.throughput(criterion::Throughput::Bytes(16 * 1024 * 1024));
@@ -50,9 +56,13 @@ fn bench_vs_fs(c: &mut Criterion) {
let path = prepare();
b.iter(|| {
let _ = op
- .read_with(&path)
- .range(0..16 * 1024 * 1024)
- .call()
+ .read_options(
+ &path,
+ options::ReadOptions {
+ range: (0..16 * 1024 * 1024).into(),
+ ..Default::default()
+ },
+ )
.unwrap();
});
});
diff --git a/core/src/types/delete/blocking_deleter.rs b/core/src/blocking/delete.rs
similarity index 61%
rename from core/src/types/delete/blocking_deleter.rs
rename to core/src/blocking/delete.rs
index 53a5f2619..41d091d63 100644
--- a/core/src/types/delete/blocking_deleter.rs
+++ b/core/src/blocking/delete.rs
@@ -15,47 +15,25 @@
// specific language governing permissions and limitations
// under the License.
-use crate::raw::*;
+use crate::Deleter as AsyncDeleter;
use crate::*;
/// BlockingDeleter is designed to continuously remove content from storage.
///
/// It leverages batch deletion capabilities provided by storage services for efficient removal.
-pub struct BlockingDeleter {
- deleter: oio::BlockingDeleter,
-
- max_size: usize,
- cur_size: usize,
+pub struct Deleter {
+ handle: tokio::runtime::Handle,
+ inner: AsyncDeleter,
}
-impl BlockingDeleter {
- pub(crate) fn create(acc: Accessor) -> Result<Self> {
- let max_size = acc.info().full_capability().delete_max_size.unwrap_or(1);
- let (_, deleter) = acc.blocking_delete()?;
-
- Ok(Self {
- deleter,
- max_size,
- cur_size: 0,
- })
+impl Deleter {
+ pub(crate) fn create(handle: tokio::runtime::Handle, inner: AsyncDeleter) -> Result<Self> {
+ Ok(Self { handle, inner })
}
/// Delete a path.
pub fn delete(&mut self, input: impl IntoDeleteInput) -> Result<()> {
- if self.cur_size >= self.max_size {
- let deleted = self.deleter.flush()?;
- self.cur_size -= deleted;
- }
-
- let input = input.into_delete_input();
- let mut op = OpDelete::default();
- if let Some(version) = &input.version {
- op = op.with_version(version);
- }
-
- self.deleter.delete(&input.path, op)?;
- self.cur_size += 1;
- Ok(())
+ self.handle.block_on(self.inner.delete(input))
}
/// Delete an infallible iterator of paths.
@@ -68,11 +46,7 @@ impl BlockingDeleter {
I: IntoIterator<Item = D>,
D: IntoDeleteInput,
{
- for entry in iter {
- self.delete(entry)?;
- }
-
- Ok(())
+ self.handle.block_on(self.inner.delete_iter(iter))
}
/// Delete an fallible iterator of paths.
@@ -85,28 +59,16 @@ impl BlockingDeleter {
I: IntoIterator<Item = Result<D>>,
D: IntoDeleteInput,
{
- for entry in try_iter {
- self.delete(entry?)?;
- }
-
- Ok(())
+ self.handle.block_on(self.inner.delete_try_iter(try_iter))
}
/// Flush the deleter, returns the number of deleted paths.
pub fn flush(&mut self) -> Result<usize> {
- let deleted = self.deleter.flush()?;
- self.cur_size -= deleted;
- Ok(deleted)
+ self.handle.block_on(self.inner.flush())
}
/// Close the deleter, this will flush the deleter and wait until all paths are deleted.
pub fn close(&mut self) -> Result<()> {
- loop {
- self.flush()?;
- if self.cur_size == 0 {
- break;
- }
- }
- Ok(())
+ self.handle.block_on(self.inner.close())
}
}
diff --git a/core/src/blocking/list.rs b/core/src/blocking/list.rs
new file mode 100644
index 000000000..85bea84ab
--- /dev/null
+++ b/core/src/blocking/list.rs
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use futures::StreamExt;
+
+use crate::Lister as AsyncLister;
+use crate::*;
+
+/// BlockingLister is designed to list entries at given path in a blocking
+/// manner.
+///
+/// Users can construct Lister by [`blocking::Operator::lister`] or [`blocking::Operator::lister_with`].
+///
+/// - Lister implements `Iterator<Item = Result<Entry>>`.
+/// - Lister will return `None` if there is no more entries or error has been returned.
+pub struct Lister {
+ handle: tokio::runtime::Handle,
+ lister: Option<AsyncLister>,
+}
+
+/// # Safety
+///
+/// BlockingLister will only be accessed by `&mut Self`
+unsafe impl Sync for Lister {}
+
+impl Lister {
+ /// Create a new lister.
+ pub(crate) fn new(handle: tokio::runtime::Handle, lister: AsyncLister) -> Self {
+ Self {
+ handle,
+ lister: Some(lister),
+ }
+ }
+}
+
+impl Iterator for Lister {
+ type Item = Result<Entry>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let Some(lister) = self.lister.as_mut() else {
+ return Some(Err(Error::new(
+ ErrorKind::Unexpected,
+ "lister has been dropped",
+ )));
+ };
+
+ self.handle.block_on(lister.next())
+ }
+}
+
+impl Drop for Lister {
+ fn drop(&mut self) {
+ if let Some(v) = self.lister.take() {
+ self.handle.block_on(async move { drop(v) });
+ }
+ }
+}
diff --git a/core/src/types/blocking_write/mod.rs b/core/src/blocking/mod.rs
similarity index 79%
copy from core/src/types/blocking_write/mod.rs
copy to core/src/blocking/mod.rs
index ec7419477..122497a53 100644
--- a/core/src/types/blocking_write/mod.rs
+++ b/core/src/blocking/mod.rs
@@ -15,8 +15,19 @@
// specific language governing permissions and limitations
// under the License.
-mod blocking_writer;
-pub use blocking_writer::BlockingWriter;
+//! blocking module provides blocking APIs for OpenDAL.
-mod std_writer;
-pub use std_writer::StdWriter;
+mod delete;
+pub use delete::*;
+
+mod list;
+pub use list::*;
+
+mod read;
+pub use read::*;
+
+mod write;
+pub use write::*;
+
+mod operator;
+pub use operator::*;
diff --git a/core/src/blocking/operator.rs b/core/src/blocking/operator.rs
new file mode 100644
index 000000000..ef5c3202d
--- /dev/null
+++ b/core/src/blocking/operator.rs
@@ -0,0 +1,720 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use tokio::runtime::Handle;
+
+use crate::Operator as AsyncOperator;
+use crate::*;
+
+/// Use OpenDAL in blocking context.
+///
+/// # Notes
+///
+/// blocking::Operator is a wrapper around [`AsyncOperator`]. It calls async runtimes' `block_on` API to spawn blocking tasks.
+/// Please avoid using blocking::Operator in async context.
+///
+/// # Examples
+///
+/// ## Init in async context
+///
+/// blocking::Operator will use current async context's runtime to handle the async calls.
+///
+/// This is just for initialization. You must use `blocking::Operator` in blocking context.
+///
+/// ```rust,no_run
+/// # use opendal::services;
+/// # use opendal::blocking;
+/// # use opendal::Operator;
+/// # use opendal::Result;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<()> {
+/// // Create fs backend builder.
+/// let mut builder = services::S3::default().bucket("test").region("us-east-1");
+/// let op = Operator::new(builder)?.finish();
+///
+/// // Build an `blocking::Operator` with blocking layer to start operating the storage.
+/// let _: blocking::Operator = blocking::Operator::new(op)?;
+///
+/// Ok(())
+/// }
+/// ```
+///
+/// ## In async context with blocking functions
+///
+/// If `blocking::Operator` is called in blocking function, please fetch a [`tokio::runtime::EnterGuard`]
+/// first. You can use [`Handle::try_current`] first to get the handle and then call [`Handle::enter`].
+/// This often happens in the case that async function calls blocking function.
+///
+/// ```rust,no_run
+/// # use opendal::services;
+/// # use opendal::blocking;
+/// # use opendal::Operator;
+/// # use opendal::Result;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<()> {
+/// let _ = blocking_fn()?;
+/// Ok(())
+/// }
+///
+/// fn blocking_fn() -> Result<blocking::Operator> {
+/// // Create fs backend builder.
+/// let mut builder = services::S3::default().bucket("test").region("us-east-1");
+/// let op = Operator::new(builder)?.finish();
+///
+/// let handle = tokio::runtime::Handle::try_current().unwrap();
+/// let _guard = handle.enter();
+/// // Build an `blocking::Operator` to start operating the storage.
+/// let op: blocking::Operator = blocking::Operator::new(op)?;
+/// Ok(op)
+/// }
+/// ```
+///
+/// ## In blocking context
+///
+/// In a pure blocking context, we can create a runtime and use it to create the `blocking::Operator`.
+///
+/// > The following code uses a global statically created runtime as an example, please manage the
+/// > runtime on demand.
+///
+/// ```rust,no_run
+/// # use std::sync::LazyLock;
+/// # use opendal::services;
+/// # use opendal::blocking;
+/// # use opendal::Operator;
+/// # use opendal::Result;
+///
+/// static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
+/// tokio::runtime::Builder::new_multi_thread()
+/// .enable_all()
+/// .build()
+/// .unwrap()
+/// });
+///
+/// fn main() -> Result<()> {
+/// // Create fs backend builder.
+/// let mut builder = services::S3::default().bucket("test").region("us-east-1");
+/// let op = Operator::new(builder)?.finish();
+///
+/// // Fetch the `EnterGuard` from global runtime.
+/// let _guard = RUNTIME.enter();
+/// // Build an `blocking::Operator` with blocking layer to start operating the storage.
+/// let _: blocking::Operator = blocking::Operator::new(op)?;
+///
+/// Ok(())
+/// }
+/// ```
+#[derive(Clone, Debug)]
+pub struct Operator {
+ handle: tokio::runtime::Handle,
+ op: AsyncOperator,
+}
+
+impl Operator {
+ /// Create a new `BlockingLayer` with the current runtime's handle
+ pub fn new(op: AsyncOperator) -> Result<Self> {
+ Ok(Self {
+ handle: Handle::try_current()
+ .map_err(|_| Error::new(ErrorKind::Unexpected, "failed to get current handle"))?,
+ op,
+ })
+ }
+
+ /// Get information of underlying accessor.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// # use std::sync::Arc;
+ /// use opendal::blocking;
+ /// # use anyhow::Result;
+ /// use opendal::blocking::Operator;
+ ///
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// let info = op.info();
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn info(&self) -> OperatorInfo {
+ self.op.info()
+ }
+}
+
+/// # Operator blocking API.
+impl Operator {
+ /// Get given path's metadata.
+ ///
+ /// # Behavior
+ ///
+ /// ## Services that support `create_dir`
+ ///
+ /// `test` and `test/` may vary in some services such as S3. However, on a local file system,
+ /// they're identical. Therefore, the behavior of `stat("test")` and `stat("test/")` might differ
+ /// in certain edge cases. Always use `stat("test/")` when you need to access a directory if possible.
+ ///
+ /// Here are the behavior list:
+ ///
+ /// | Case | Path | Result |
+ /// |------------------------|-----------------|--------------------------------------------|
+ /// | stat existing dir | `abc/` | Metadata with dir mode |
+ /// | stat existing file | `abc/def_file` | Metadata with file mode |
+ /// | stat dir without `/` | `abc/def_dir` | Error `NotFound` or metadata with dir mode |
+ /// | stat file with `/` | `abc/def_file/` | Error `NotFound` |
+ /// | stat not existing path | `xyz` | Error `NotFound` |
+ ///
+ /// Refer to [RFC: List Prefix][crate::docs::rfcs::rfc_3243_list_prefix] for more details.
+ ///
+ /// ## Services that not support `create_dir`
+ ///
+ /// For services that not support `create_dir`, `stat("test/")` will return `NotFound` even
+ /// when `test/abc` exists since the service won't have the concept of dir. There is nothing
+ /// we can do about this.
+ ///
+ /// # Examples
+ ///
+ /// ## Check if file exists
+ ///
+ /// ```
+ /// # use anyhow::Result;
+ /// # use futures::io;
+ /// use opendal::blocking;
+ /// # use opendal::blocking::Operator;
+ /// use opendal::ErrorKind;
+ /// #
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// if let Err(e) = op.stat("test") {
+ /// if e.kind() == ErrorKind::NotFound {
+ /// println!("file not exist")
+ /// }
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn stat(&self, path: &str) -> Result<Metadata> {
+ self.stat_options(path, options::StatOptions::default())
+ }
+
+ /// Get given path's metadata with extra options.
+ ///
+ /// # Behavior
+ ///
+ /// ## Services that support `create_dir`
+ ///
+ /// `test` and `test/` may vary in some services such as S3. However, on a local file system,
+ /// they're identical. Therefore, the behavior of `stat("test")` and `stat("test/")` might differ
+ /// in certain edge cases. Always use `stat("test/")` when you need to access a directory if possible.
+ ///
+ /// Here are the behavior list:
+ ///
+ /// | Case | Path | Result |
+ /// |------------------------|-----------------|--------------------------------------------|
+ /// | stat existing dir | `abc/` | Metadata with dir mode |
+ /// | stat existing file | `abc/def_file` | Metadata with file mode |
+ /// | stat dir without `/` | `abc/def_dir` | Error `NotFound` or metadata with dir mode |
+ /// | stat file with `/` | `abc/def_file/` | Error `NotFound` |
+ /// | stat not existing path | `xyz` | Error `NotFound` |
+ ///
+ /// Refer to [RFC: List Prefix][crate::docs::rfcs::rfc_3243_list_prefix] for more details.
+ ///
+ /// ## Services that not support `create_dir`
+ ///
+ /// For services that not support `create_dir`, `stat("test/")` will return `NotFound` even
+ /// when `test/abc` exists since the service won't have the concept of dir. There is nothing
+ /// we can do about this.
+ pub fn stat_options(&self, path: &str, opts: options::StatOptions) -> Result<Metadata> {
+ self.handle.block_on(self.op.stat_options(path, opts))
+ }
+
+ /// Check if this path exists or not.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use anyhow::Result;
+ /// use opendal::blocking;
+ /// use opendal::blocking::Operator;
+ /// fn test(op: blocking::Operator) -> Result<()> {
+ /// let _ = op.exists("test")?;
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub fn exists(&self, path: &str) -> Result<bool> {
+ let r = self.stat(path);
+ match r {
+ Ok(_) => Ok(true),
+ Err(err) => match err.kind() {
+ ErrorKind::NotFound => Ok(false),
+ _ => Err(err),
+ },
+ }
+ }
+
+ /// Create a dir at given path.
+ ///
+ /// # Notes
+ ///
+ /// To indicate that a path is a directory, it is compulsory to include
+ /// a trailing / in the path. Failure to do so may result in
+ /// `NotADirectory` error being returned by OpenDAL.
+ ///
+ /// # Behavior
+ ///
+ /// - Create on existing dir will succeed.
+ /// - Create dir is always recursive, works like `mkdir -p`
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use opendal::Result;
+ /// use opendal::blocking;
+ /// # use opendal::blocking::Operator;
+ /// # use futures::TryStreamExt;
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// op.create_dir("path/to/dir/")?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn create_dir(&self, path: &str) -> Result<()> {
+ self.handle.block_on(self.op.create_dir(path))
+ }
+
+ /// Read the whole path into a bytes.
+ ///
+ /// This function will allocate a new bytes internally. For more precise memory control or
+ /// reading data lazily, please use [`blocking::Operator::reader`]
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use opendal::Result;
+ /// use opendal::blocking;
+ /// # use opendal::blocking::Operator;
+ /// #
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// let bs = op.read("path/to/file")?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn read(&self, path: &str) -> Result<Buffer> {
+ self.read_options(path, options::ReadOptions::default())
+ }
+
+ /// Read the whole path into a bytes with extra options.
+ ///
+ /// This function will allocate a new bytes internally. For more precise memory control or
+ /// reading data lazily, please use [`blocking::Operator::reader`]
+ pub fn read_options(&self, path: &str, opts: options::ReadOptions) -> Result<Buffer> {
+ self.handle.block_on(self.op.read_options(path, opts))
+ }
+
+ /// Create a new reader which can read the whole path.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use opendal::Result;
+ /// use opendal::blocking;
+ /// # use opendal::blocking::Operator;
+ /// # use futures::TryStreamExt;
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// let r = op.reader("path/to/file")?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn reader(&self, path: &str) -> Result<blocking::Reader> {
+ self.reader_options(path, options::ReaderOptions::default())
+ }
+
+ /// Create a new reader with extra options
+ pub fn reader_options(
+ &self,
+ path: &str,
+ opts: options::ReaderOptions,
+ ) -> Result<blocking::Reader> {
+ let r = self.handle.block_on(self.op.reader_options(path, opts))?;
+ Ok(blocking::Reader::new(self.handle.clone(), r))
+ }
+
+ /// Write bytes into given path.
+ ///
+ /// # Notes
+ ///
+ /// - Write will make sure all bytes has been written, or an error will be returned.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use opendal::Result;
+ /// use opendal::blocking;
+ /// # use opendal::blocking::Operator;
+ /// # use futures::StreamExt;
+ /// # use futures::SinkExt;
+ /// use bytes::Bytes;
+ ///
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// op.write("path/to/file", vec![0; 4096])?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn write(&self, path: &str, bs: impl Into<Buffer>) -> Result<Metadata> {
+ self.write_options(path, bs, options::WriteOptions::default())
+ }
+
+ /// Write data with options.
+ ///
+ /// # Notes
+ ///
+ /// - Write will make sure all bytes has been written, or an error will be returned.
+ pub fn write_options(
+ &self,
+ path: &str,
+ bs: impl Into<Buffer>,
+ opts: options::WriteOptions,
+ ) -> Result<Metadata> {
+ self.handle.block_on(self.op.write_options(path, bs, opts))
+ }
+
+ /// Write multiple bytes into given path.
+ ///
+ /// # Notes
+ ///
+ /// - Write will make sure all bytes has been written, or an error will be returned.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use opendal::Result;
+ /// # use opendal::blocking;
+ /// # use opendal::blocking::Operator;
+ /// # use futures::StreamExt;
+ /// # use futures::SinkExt;
+ /// use bytes::Bytes;
+ ///
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// let mut w = op.writer("path/to/file")?;
+ /// w.write(vec![0; 4096])?;
+ /// w.write(vec![1; 4096])?;
+ /// w.close()?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn writer(&self, path: &str) -> Result<blocking::Writer> {
+ self.writer_options(path, options::WriteOptions::default())
+ }
+
+ /// Create a new writer with extra options
+ pub fn writer_options(
+ &self,
+ path: &str,
+ opts: options::WriteOptions,
+ ) -> Result<blocking::Writer> {
+ let w = self.handle.block_on(self.op.writer_options(path, opts))?;
+ Ok(blocking::Writer::new(self.handle.clone(), w))
+ }
+
+ /// Copy a file from `from` to `to`.
+ ///
+ /// # Notes
+ ///
+ /// - `from` and `to` must be a file.
+ /// - `to` will be overwritten if it exists.
+ /// - If `from` and `to` are the same, nothing will happen.
+ /// - `copy` is idempotent. For same `from` and `to` input, the result will be the same.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// # use opendal::Result;
+ /// use opendal::blocking;
+ /// # use opendal::blocking::Operator;
+ ///
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// op.copy("path/to/file", "path/to/file2")?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn copy(&self, from: &str, to: &str) -> Result<()> {
+ self.handle.block_on(self.op.copy(from, to))
+ }
+
+ /// Rename a file from `from` to `to`.
+ ///
+ /// # Notes
+ ///
+ /// - `from` and `to` must be a file.
+ /// - `to` will be overwritten if it exists.
+ /// - If `from` and `to` are the same, a `IsSameFile` error will occur.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// # use opendal::Result;
+ /// use opendal::blocking;
+ /// # use opendal::blocking::Operator;
+ ///
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// op.rename("path/to/file", "path/to/file2")?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn rename(&self, from: &str, to: &str) -> Result<()> {
+ self.handle.block_on(self.op.rename(from, to))
+ }
+
+ /// Delete given path.
+ ///
+ /// # Notes
+ ///
+ /// - Delete not existing error won't return errors.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use anyhow::Result;
+ /// # use futures::io;
+ /// use opendal::blocking;
+ /// # use opendal::blocking::Operator;
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// op.delete("path/to/file")?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn delete(&self, path: &str) -> Result<()> {
+ self.delete_options(path, options::DeleteOptions::default())
+ }
+
+ /// Delete given path with options.
+ ///
+ /// # Notes
+ ///
+ /// - Delete not existing error won't return errors.
+ pub fn delete_options(&self, path: &str, opts: options::DeleteOptions) -> Result<()> {
+ self.handle.block_on(self.op.delete_options(path, opts))
+ }
+
+ /// Delete an infallible iterator of paths.
+ ///
+ /// Also see:
+ ///
+ /// - [`blocking::Operator::delete_try_iter`]: delete an fallible iterator of paths.
+ pub fn delete_iter<I, D>(&self, iter: I) -> Result<()>
+ where
+ I: IntoIterator<Item = D>,
+ D: IntoDeleteInput,
+ {
+ self.handle.block_on(self.op.delete_iter(iter))
+ }
+
+ /// Delete a fallible iterator of paths.
+ ///
+ /// Also see:
+ ///
+ /// - [`blocking::Operator::delete_iter`]: delete an infallible iterator of paths.
+ pub fn delete_try_iter<I, D>(&self, try_iter: I) -> Result<()>
+ where
+ I: IntoIterator<Item = Result<D>>,
+ D: IntoDeleteInput,
+ {
+ self.handle.block_on(self.op.delete_try_iter(try_iter))
+ }
+
+ /// Create a [`BlockingDeleter`] to continuously remove content from storage.
+ ///
+ /// It leverages batch deletion capabilities provided by storage services for efficient removal.
+ ///
+ /// Users can have more control over the deletion process by using [`BlockingDeleter`] directly.
+ pub fn deleter(&self) -> Result<blocking::Deleter> {
+ blocking::Deleter::create(
+ self.handle.clone(),
+ self.handle.block_on(self.op.deleter())?,
+ )
+ }
+
+ /// Remove the path and all nested dirs and files recursively.
+ ///
+ /// # Notes
+ ///
+ /// We don't support batch delete now.
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// # use anyhow::Result;
+ /// # use futures::io;
+ /// use opendal::blocking;
+ /// # use opendal::blocking::Operator;
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// op.remove_all("path/to/dir")?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn remove_all(&self, path: &str) -> Result<()> {
+ self.handle.block_on(self.op.remove_all(path))
+ }
+
+ /// List entries that starts with given `path` in parent dir.
+ ///
+ /// # Notes
+ ///
+ /// ## Recursively List
+ ///
+ /// This function only read the children of the given directory. To read
+ /// all entries recursively, use `blocking::Operator::list_options("path", opts)`
+ /// instead.
+ ///
+ /// ## Streaming List
+ ///
+ /// This function will read all entries in the given directory. It could
+ /// take very long time and consume a lot of memory if the directory
+ /// contains a lot of entries.
+ ///
+ /// In order to avoid this, you can use [`blocking::Operator::lister`] to list entries in
+ /// a streaming way.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use anyhow::Result;
+ /// use opendal::blocking;
+ /// use opendal::blocking::Operator;
+ /// use opendal::EntryMode;
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// let mut entries = op.list("path/to/dir/")?;
+ /// for entry in entries {
+ /// match entry.metadata().mode() {
+ /// EntryMode::FILE => {
+ /// println!("Handling file")
+ /// }
+ /// EntryMode::DIR => {
+ /// println!("Handling dir {}", entry.path())
+ /// }
+ /// EntryMode::Unknown => continue,
+ /// }
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn list(&self, path: &str) -> Result<Vec<Entry>> {
+ self.list_options(path, options::ListOptions::default())
+ }
+
+ /// List entries that starts with given `path` in parent dir. with options.
+ ///
+ /// # Notes
+ ///
+ /// ## Streaming List
+ ///
+ /// This function will read all entries in the given directory. It could
+ /// take very long time and consume a lot of memory if the directory
+ /// contains a lot of entries.
+ ///
+ /// In order to avoid this, you can use [`blocking::Operator::lister`] to list entries in
+ /// a streaming way.
+ pub fn list_options(&self, path: &str, opts: options::ListOptions) -> Result<Vec<Entry>> {
+ self.handle.block_on(self.op.list_options(path, opts))
+ }
+
+ /// List entries that starts with given `path` in parent dir.
+ ///
+ /// This function will create a new [`BlockingLister`] to list entries. Users can stop listing
+ /// via dropping this [`Lister`].
+ ///
+ /// # Notes
+ ///
+ /// ## Recursively List
+ ///
+ /// This function only read the children of the given directory. To read
+ /// all entries recursively, use [`blocking::Operator::lister_with`] and `delimiter("")`
+ /// instead.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// # use anyhow::Result;
+ /// # use futures::io;
+ /// use futures::TryStreamExt;
+ /// use opendal::blocking;
+ /// use opendal::blocking::Operator;
+ /// use opendal::EntryMode;
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// let mut ds = op.lister("path/to/dir/")?;
+ /// for de in ds {
+ /// let de = de?;
+ /// match de.metadata().mode() {
+ /// EntryMode::FILE => {
+ /// println!("Handling file")
+ /// }
+ /// EntryMode::DIR => {
+ /// println!("Handling dir like start a new list via meta.path()")
+ /// }
+ /// EntryMode::Unknown => continue,
+ /// }
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn lister(&self, path: &str) -> Result<blocking::Lister> {
+ self.lister_options(path, options::ListOptions::default())
+ }
+
+ /// List entries within a given directory as an iterator with options.
+ ///
+ /// This function will create a new handle to list entries.
+ ///
+ /// An error will be returned if given path doesn't end with `/`.
+ pub fn lister_options(
+ &self,
+ path: &str,
+ opts: options::ListOptions,
+ ) -> Result<blocking::Lister> {
+ let l = self.handle.block_on(self.op.lister_options(path, opts))?;
+ Ok(blocking::Lister::new(self.handle.clone(), l))
+ }
+
+ /// Check if this operator can work correctly.
+ ///
+ /// We will send a `list` request to path and return any errors we met.
+ ///
+ /// ```
+ /// # use std::sync::Arc;
+ /// # use anyhow::Result;
+ /// use opendal::blocking;
+ /// use opendal::blocking::Operator;
+ /// use opendal::ErrorKind;
+ ///
+ /// # fn test(op: blocking::Operator) -> Result<()> {
+ /// op.check()?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn check(&self) -> Result<()> {
+ let mut ds = self.lister("/")?;
+
+ match ds.next() {
+ Some(Err(e)) if e.kind() != ErrorKind::NotFound => Err(e),
+ _ => Ok(()),
+ }
+ }
+}
+
+impl From<Operator> for AsyncOperator {
+ fn from(val: Operator) -> Self {
+ val.op
+ }
+}
diff --git a/core/src/blocking/read/buffer_iterator.rs b/core/src/blocking/read/buffer_iterator.rs
new file mode 100644
index 000000000..15bd98ddf
--- /dev/null
+++ b/core/src/blocking/read/buffer_iterator.rs
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use futures::StreamExt;
+
+use crate::Buffer;
+use crate::*;
+
+/// BufferIterator is an iterator of buffers.
+///
+/// # TODO
+///
+/// We can support chunked reader for concurrent read in the future.
+pub struct BufferIterator {
+ handle: tokio::runtime::Handle,
+ inner: Option<BufferStream>,
+}
+
+impl BufferIterator {
+ /// Create a new buffer iterator.
+ #[inline]
+ pub(crate) fn new(handle: tokio::runtime::Handle, inner: BufferStream) -> Self {
+ Self {
+ handle,
+ inner: Some(inner),
+ }
+ }
+}
+
+impl Iterator for BufferIterator {
+ type Item = Result<Buffer>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let Some(inner) = self.inner.as_mut() else {
+ return Some(Err(Error::new(
+ ErrorKind::Unexpected,
+ "reader has been dropped",
+ )));
+ };
+
+ self.handle.block_on(inner.next())
+ }
+}
+
+/// Make sure the inner reader is dropped in async context.
+impl Drop for BufferIterator {
+ fn drop(&mut self) {
+ if let Some(v) = self.inner.take() {
+ self.handle.block_on(async move { drop(v) });
+ }
+ }
+}
diff --git a/core/src/types/blocking_read/mod.rs b/core/src/blocking/read/mod.rs
similarity index 91%
rename from core/src/types/blocking_read/mod.rs
rename to core/src/blocking/read/mod.rs
index 334f784f5..57083a521 100644
--- a/core/src/types/blocking_read/mod.rs
+++ b/core/src/blocking/read/mod.rs
@@ -15,9 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-#[allow(clippy::module_inception)]
-mod blocking_reader;
-pub use blocking_reader::BlockingReader;
+mod reader;
+pub use reader::Reader;
mod buffer_iterator;
pub use buffer_iterator::BufferIterator;
diff --git a/core/src/blocking/read/reader.rs b/core/src/blocking/read/reader.rs
new file mode 100644
index 000000000..e4f5942c9
--- /dev/null
+++ b/core/src/blocking/read/reader.rs
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::ops::RangeBounds;
+
+use bytes::BufMut;
+
+use crate::Reader as AsyncReader;
+use crate::*;
+
+use super::BufferIterator;
+use super::StdBytesIterator;
+use super::StdReader;
+
+/// BlockingReader is designed to read data from given path in an blocking
+/// manner.
+#[derive(Clone)]
+pub struct Reader {
+ handle: tokio::runtime::Handle,
+ inner: Option<AsyncReader>,
+}
+
+impl Reader {
+ /// Create a new blocking reader.
+ ///
+ /// We don't want to expose those details to users so keep this function
+ /// in crate only.
+ pub(crate) fn new(handle: tokio::runtime::Handle, inner: AsyncReader) -> Self {
+ Reader {
+ handle,
+ inner: Some(inner),
+ }
+ }
+
+ /// Read give range from reader into [`Buffer`].
+ ///
+ /// This operation is zero-copy, which means it keeps the [`bytes::Bytes`] returned by underlying
+ /// storage services without any extra copy or intensive memory allocations.
+ ///
+ /// # Notes
+ ///
+ /// - Buffer length smaller than range means we have reached the end of file.
+ pub fn read(&self, range: impl RangeBounds<u64>) -> Result<Buffer> {
+ let inner = self
+ .inner
+ .as_ref()
+ .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?;
+ self.handle.block_on(inner.read(range))
+ }
+
+ ///
+ /// This operation will copy and write bytes into given [`BufMut`]. Allocation happens while
+ /// [`BufMut`] doesn't have enough space.
+ ///
+ /// # Notes
+ ///
+ /// - Returning length smaller than range means we have reached the end of file.
+ pub fn read_into(&self, buf: &mut impl BufMut, range: impl RangeBounds<u64>) -> Result<usize> {
+ let inner = self
+ .inner
+ .as_ref()
+ .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?;
+ self.handle.block_on(inner.read_into(buf, range))
+ }
+
+ /// Create a buffer iterator to read specific range from given reader.
+ pub fn into_iterator(mut self, range: impl RangeBounds<u64>) -> Result<BufferIterator> {
+ let inner = self
+ .inner
+ .take()
+ .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?;
+ let iter = self.handle.block_on(inner.into_stream(range))?;
+
+ Ok(BufferIterator::new(self.handle.clone(), iter))
+ }
+
+ /// Convert reader into [`StdReader`] which implements [`futures::AsyncRead`],
+ /// [`futures::AsyncSeek`] and [`futures::AsyncBufRead`].
+ #[inline]
+ pub fn into_std_read(mut self, range: impl RangeBounds<u64>) -> Result<StdReader> {
+ let inner = self
+ .inner
+ .take()
+ .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?;
+
+ let r = self.handle.block_on(inner.into_futures_async_read(range))?;
+
+ Ok(StdReader::new(self.handle.clone(), r))
+ }
+
+ /// Convert reader into [`StdBytesIterator`] which implements [`Iterator`].
+ #[inline]
+ pub fn into_bytes_iterator(mut self, range: impl RangeBounds<u64>) -> Result<StdBytesIterator> {
+ let inner = self
+ .inner
+ .take()
+ .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?;
+
+ let iter = self.handle.block_on(inner.into_bytes_stream(range))?;
+ Ok(StdBytesIterator::new(self.handle.clone(), iter))
+ }
+}
+
+/// Make sure the inner reader is dropped in async context.
+impl Drop for Reader {
+ fn drop(&mut self) {
+ if let Some(v) = self.inner.take() {
+ self.handle.block_on(async move { drop(v) });
+ }
+ }
+}
diff --git a/core/src/types/blocking_read/std_bytes_iterator.rs b/core/src/blocking/read/std_bytes_iterator.rs
similarity index 65%
rename from core/src/types/blocking_read/std_bytes_iterator.rs
rename to core/src/blocking/read/std_bytes_iterator.rs
index 23049dd6e..193270a9d 100644
--- a/core/src/types/blocking_read/std_bytes_iterator.rs
+++ b/core/src/blocking/read/std_bytes_iterator.rs
@@ -16,14 +16,10 @@
// under the License.
use std::io;
-use std::ops::Range;
-use std::sync::Arc;
use bytes::Bytes;
+use futures::StreamExt;
-use crate::raw::*;
-use crate::Buffer;
-use crate::BufferIterator;
use crate::*;
/// StdIterator is the adapter of [`Iterator`] for [`BlockingReader`][crate::BlockingReader].
@@ -32,19 +28,17 @@ use crate::*;
///
/// StdIterator also implements [`Send`] and [`Sync`].
pub struct StdBytesIterator {
- iter: BufferIterator,
- buf: Buffer,
+ handle: tokio::runtime::Handle,
+ inner: Option<FuturesBytesStream>,
}
impl StdBytesIterator {
/// NOTE: don't allow users to create StdIterator directly.
#[inline]
- pub(crate) fn new(ctx: Arc<ReadContext>, range: Range<u64>) -> Self {
- let iter = BufferIterator::new(ctx, range);
-
+ pub(crate) fn new(handle: tokio::runtime::Handle, inner: FuturesBytesStream) -> Self {
StdBytesIterator {
- iter,
- buf: Buffer::new(),
+ handle,
+ inner: Some(inner),
}
}
}
@@ -53,17 +47,23 @@ impl Iterator for StdBytesIterator {
type Item = io::Result<Bytes>;
fn next(&mut self) -> Option<Self::Item> {
- loop {
- // Consume current buffer
- if let Some(bs) = Iterator::next(&mut self.buf) {
- return Some(Ok(bs));
- }
+ let Some(inner) = self.inner.as_mut() else {
+ return Some(Err(Error::new(
+ ErrorKind::Unexpected,
+ "reader has been dropped",
+ )
+ .into()));
+ };
+
+ self.handle.block_on(inner.next())
+ }
+}
- self.buf = match self.iter.next() {
- Some(Ok(buf)) => buf,
- Some(Err(err)) => return Some(Err(format_std_io_error(err))),
- None => return None,
- };
+/// Make sure the inner reader is dropped in async context.
+impl Drop for StdBytesIterator {
+ fn drop(&mut self) {
+ if let Some(v) = self.inner.take() {
+ self.handle.block_on(async move { drop(v) });
}
}
}
diff --git a/core/src/blocking/read/std_reader.rs b/core/src/blocking/read/std_reader.rs
new file mode 100644
index 000000000..9cdea17e3
--- /dev/null
+++ b/core/src/blocking/read/std_reader.rs
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::io::BufRead;
+use std::io::Read;
+use std::io::Seek;
+use std::io::SeekFrom;
+
+use futures::AsyncBufReadExt;
+use futures::AsyncReadExt;
+use futures::AsyncSeekExt;
+
+use crate::*;
+
+/// StdReader is the adapter of [`Read`], [`Seek`] and [`BufRead`] for [`BlockingReader`][crate::BlockingReader].
+///
+/// Users can use this adapter in cases where they need to use [`Read`] or [`BufRead`] trait.
+///
+/// StdReader also implements [`Send`] and [`Sync`].
+pub struct StdReader {
+ handle: tokio::runtime::Handle,
+ r: Option<FuturesAsyncReader>,
+}
+
+impl StdReader {
+ /// NOTE: don't allow users to create StdReader directly.
+ #[inline]
+ pub(super) fn new(handle: tokio::runtime::Handle, r: FuturesAsyncReader) -> Self {
+ Self { handle, r: Some(r) }
+ }
+}
+
+impl BufRead for StdReader {
+ fn fill_buf(&mut self) -> io::Result<&[u8]> {
+ let Some(r) = self.r.as_mut() else {
+ return Err(Error::new(ErrorKind::Unexpected, "reader has been dropped").into());
+ };
+
+ self.handle.block_on(r.fill_buf())
+ }
+
+ fn consume(&mut self, amt: usize) {
+ let Some(r) = self.r.as_mut() else {
+ return;
+ };
+
+ r.consume_unpin(amt);
+ }
+}
+
+impl Read for StdReader {
+ #[inline]
+ fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+ let Some(r) = self.r.as_mut() else {
+ return Err(Error::new(ErrorKind::Unexpected, "reader has been dropped").into());
+ };
+
+ self.handle.block_on(r.read(buf))
+ }
+}
+
+impl Seek for StdReader {
+ #[inline]
+ fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
+ let Some(r) = self.r.as_mut() else {
+ return Err(Error::new(ErrorKind::Unexpected, "reader has been dropped").into());
+ };
+
+ self.handle.block_on(r.seek(pos))
+ }
+}
+
+/// Make sure the inner reader is dropped in async context.
+impl Drop for StdReader {
+ fn drop(&mut self) {
+ if let Some(v) = self.r.take() {
+ self.handle.block_on(async move { drop(v) });
+ }
+ }
+}
diff --git a/core/src/types/blocking_write/mod.rs b/core/src/blocking/write/mod.rs
similarity index 93%
rename from core/src/types/blocking_write/mod.rs
rename to core/src/blocking/write/mod.rs
index ec7419477..68813d14a 100644
--- a/core/src/types/blocking_write/mod.rs
+++ b/core/src/blocking/write/mod.rs
@@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-mod blocking_writer;
-pub use blocking_writer::BlockingWriter;
+mod writer;
+pub use writer::Writer;
mod std_writer;
pub use std_writer::StdWriter;
diff --git a/core/src/types/blocking_write/std_writer.rs b/core/src/blocking/write/std_writer.rs
similarity index 53%
rename from core/src/types/blocking_write/std_writer.rs
rename to core/src/blocking/write/std_writer.rs
index 2a53b97bb..41e5c3507 100644
--- a/core/src/types/blocking_write/std_writer.rs
+++ b/core/src/blocking/write/std_writer.rs
@@ -17,7 +17,8 @@
use std::io::Write;
-use crate::raw::*;
+use futures::AsyncWriteExt;
+
use crate::*;
/// StdWriter is the adapter of [`std::io::Write`] for [`BlockingWriter`].
@@ -29,79 +30,53 @@ use crate::*;
/// Files are automatically closed when they go out of scope. Errors detected on closing are ignored
/// by the implementation of Drop. Use the method `close` if these errors must be manually handled.
pub struct StdWriter {
- w: Option<WriteGenerator<oio::BlockingWriter>>,
- buf: oio::FlexBuf,
+ handle: tokio::runtime::Handle,
+ w: Option<FuturesAsyncWriter>,
}
impl StdWriter {
/// NOTE: don't allow users to create directly.
#[inline]
- pub(crate) fn new(w: WriteGenerator<oio::BlockingWriter>) -> Self {
+ pub(crate) fn new(handle: tokio::runtime::Handle, w: Writer) -> Self {
StdWriter {
- w: Some(w),
- buf: oio::FlexBuf::new(256 * 1024),
+ handle,
+ w: Some(w.into_futures_async_write()),
}
}
/// Close the internal writer and make sure all data have been stored.
pub fn close(&mut self) -> std::io::Result<()> {
- // Make sure all cache has been flushed.
- self.flush()?;
-
- let Some(w) = &mut self.w else {
- return Err(std::io::Error::other("writer has been closed"));
+ let Some(w) = self.w.as_mut() else {
+ return Err(Error::new(ErrorKind::Unexpected, "writer has been dropped").into());
};
- w.close().map_err(std::io::Error::other)?;
-
- // Drop writer after close succeed;
- self.w = None;
-
- Ok(())
+ self.handle.block_on(w.close())
}
}
impl Write for StdWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
- let Some(w) = &mut self.w else {
- return Err(std::io::Error::other("writer has been closed"));
+ let Some(w) = self.w.as_mut() else {
+ return Err(Error::new(ErrorKind::Unexpected, "writer has been dropped").into());
};
- loop {
- let n = self.buf.put(buf);
- if n > 0 {
- return Ok(n);
- }
-
- let bs = self.buf.get().expect("frozen buffer must be valid");
- let n = w.write(Buffer::from(bs)).map_err(std::io::Error::other)?;
- self.buf.advance(n);
- }
+ self.handle.block_on(w.write(buf))
}
fn flush(&mut self) -> std::io::Result<()> {
- let Some(w) = &mut self.w else {
- return Err(std::io::Error::other("writer has been closed"));
+ let Some(w) = self.w.as_mut() else {
+ return Err(Error::new(ErrorKind::Unexpected, "writer has been dropped").into());
};
- loop {
- // Make sure buf has been frozen.
- self.buf.freeze();
- let Some(bs) = self.buf.get() else {
- return Ok(());
- };
-
- w.write(Buffer::from(bs)).map_err(std::io::Error::other)?;
- self.buf.clean();
- }
+ self.handle.block_on(w.flush())
}
}
+/// Make sure the inner writer is dropped in async context.
impl Drop for StdWriter {
fn drop(&mut self) {
- if let Some(mut w) = self.w.take() {
- // Ignore error happens in close.
- let _ = w.close();
+ if let Some(v) = self.w.take() {
+ self.handle.block_on(async move { drop(v) });
}
}
}
diff --git a/core/src/types/blocking_write/blocking_writer.rs b/core/src/blocking/write/writer.rs
similarity index 62%
rename from core/src/types/blocking_write/blocking_writer.rs
rename to core/src/blocking/write/writer.rs
index 5d59d9ca5..cdb617b49 100644
--- a/core/src/types/blocking_write/blocking_writer.rs
+++ b/core/src/blocking/write/writer.rs
@@ -15,22 +15,18 @@
// specific language governing permissions and limitations
// under the License.
-use std::sync::Arc;
-
-use bytes::Buf;
-
-use crate::raw::*;
+use super::std_writer::StdWriter;
+use crate::Writer as AsyncWriter;
use crate::*;
/// BlockingWriter is designed to write data into given path in an blocking
/// manner.
-pub struct BlockingWriter {
- /// Keep a reference to write context in writer.
- _ctx: Arc<WriteContext>,
- inner: WriteGenerator<oio::BlockingWriter>,
+pub struct Writer {
+ handle: tokio::runtime::Handle,
+ inner: Option<AsyncWriter>,
}
-impl BlockingWriter {
+impl Writer {
/// Create a new writer.
///
/// Create will use internal information to decide the most suitable
@@ -38,11 +34,11 @@ impl BlockingWriter {
///
/// We don't want to expose those details to users so keep this function
/// in crate only.
- pub(crate) fn new(ctx: WriteContext) -> Result<Self> {
- let ctx = Arc::new(ctx);
- let inner = WriteGenerator::blocking_create(ctx.clone())?;
-
- Ok(Self { _ctx: ctx, inner })
+ pub(crate) fn new(handle: tokio::runtime::Handle, inner: AsyncWriter) -> Self {
+ Self {
+ handle,
+ inner: Some(inner),
+ }
}
/// Write [`Buffer`] into writer.
@@ -53,10 +49,11 @@ impl BlockingWriter {
///
/// ```
/// use bytes::Bytes;
- /// use opendal::BlockingOperator;
+ /// use opendal::blocking;
+ /// use opendal::blocking::Operator;
/// use opendal::Result;
///
- /// async fn test(op: BlockingOperator) -> Result<()> {
+ /// fn test(op: blocking::Operator) -> Result<()> {
/// let mut w = op.writer("hello.txt")?;
/// // Buffer can be created from continues bytes.
/// w.write("hello, world")?;
@@ -69,12 +66,11 @@ impl BlockingWriter {
/// }
/// ```
pub fn write(&mut self, bs: impl Into<Buffer>) -> Result<()> {
- let mut bs = bs.into();
- while !bs.is_empty() {
- let n = self.inner.write(bs.clone())?;
- bs.advance(n);
- }
- Ok(())
+ let Some(inner) = self.inner.as_mut() else {
+ return Err(Error::new(ErrorKind::Unexpected, "writer has been dropped"));
+ };
+
+ self.handle.block_on(inner.write(bs))
}
/// Close the writer and make sure all data have been committed.
@@ -84,11 +80,30 @@ impl BlockingWriter {
/// Close should only be called when the writer is not closed or
/// aborted, otherwise an unexpected error could be returned.
pub fn close(&mut self) -> Result<Metadata> {
- self.inner.close()
+ let Some(inner) = self.inner.as_mut() else {
+ return Err(Error::new(ErrorKind::Unexpected, "writer has been dropped"));
+ };
+
+ self.handle.block_on(inner.close())
}
/// Convert writer into [`StdWriter`] which implements [`std::io::Write`],
- pub fn into_std_write(self) -> StdWriter {
- StdWriter::new(self.inner)
+ pub fn into_std_write(mut self) -> StdWriter {
+ let inner = self
+ .inner
+ .take()
+ .ok_or_else(|| Error::new(ErrorKind::Unexpected, "writer has been dropped"))
+ .expect("writer has been dropped");
+
+ StdWriter::new(self.handle.clone(), inner)
+ }
+}
+
+/// Make sure the inner writer is dropped in async context.
+impl Drop for Writer {
+ fn drop(&mut self) {
+ if let Some(v) = self.inner.take() {
+ self.handle.block_on(async move { drop(v) });
+ }
}
}
diff --git a/core/src/docs/concepts.rs b/core/src/docs/concepts.rs
index fc6ecc7ce..e512ba603 100644
--- a/core/src/docs/concepts.rs
+++ b/core/src/docs/concepts.rs
@@ -22,10 +22,10 @@
//! There are two core concepts in OpenDAL:
//!
//! - [`Builder`]: Builder accepts a series of parameters to set up an instance of underlying services.
-//! You can adjust the behaviour of underlying services with these parameters.
+//! You can adjust the behaviour of underlying services with these parameters.
//! - [`Operator`]: Developer can access underlying storage services with manipulating one Operator.
-//! The Operator is a delegate for underlying implementation detail, and provides one unified access interface,
-//! including `read`, `write`, `list` and so on.
+//! The Operator is a delegate for underlying implementation detail, and provides one unified access interface,
+//! including `read`, `write`, `list` and so on.
//!
//! If you are interested in internal implementation details, please have a look at [`internals`][super::internals].
//!
diff --git a/core/src/docs/rfcs/1477_remove_object_concept.md b/core/src/docs/rfcs/1477_remove_object_concept.md
index 71585d3fb..7d790ca1e 100644
--- a/core/src/docs/rfcs/1477_remove_object_concept.md
+++ b/core/src/docs/rfcs/1477_remove_object_concept.md
@@ -99,7 +99,7 @@ op.list("dir/").await?;
op.scan("dir/").await?;
```
-We will include the `BlockingOperator` for enhanced ease of use while performing blocking operations.
+We will include the `blocking::Operator` for enhanced ease of use while performing blocking operations.
```rust
# this is a cheap call without allocation
diff --git a/core/src/docs/rfcs/6189_remove_native_blocking.md b/core/src/docs/rfcs/6189_remove_native_blocking.md
index 8c4543477..bb561ae75 100644
--- a/core/src/docs/rfcs/6189_remove_native_blocking.md
+++ b/core/src/docs/rfcs/6189_remove_native_blocking.md
@@ -20,15 +20,15 @@ However, this approach has several drawbacks:
Apart from the above drawbacks, we also find that the following facts:
- Only a few services have native blocking support, such as `fs`, `memory` and `moka`.
-- Only a few users are using `BlockingOperator` (based on [search results from github](https://github.com/search?q=opendal%3A%3ABlockingOperator&type=code))
+- Only a few users are using `blocking::Operator` (based on [search results from github](https://github.com/search?q=opendal%3A%3Ablocking::Operator&type=code))
So I propose to remove the native blocking support from OpenDAL and use `block_on` in the async runtime instead. This will simplify the codebase, reduce maintenance overhead, and provide a more consistent API for users.
# Guide-level explanation
-I plan to move `BlockingOperator` to the `opendal::blocking` module, which will offer a blocking interface for users. The `BlockingOperator` will utilize `block_on` within an async runtime to execute asynchronous operations in a blocking way.
+I plan to move `blocking::Operator` to the `opendal::blocking` module, which will offer a blocking interface for users. The `blocking::Operator` will utilize `block_on` within an async runtime to execute asynchronous operations in a blocking way.
-- `opendal::BlockingOperator` will become `opendal::blocking::Operator`
+- `opendal::blocking::Operator` will become `opendal::blocking::Operator`
- `opendal::BlockingReader` will become `opendal::blocking::Reader`
- `opendal::BlockingWriter` will become `opendal::blocking::Writer`
- `opendal::BlockingLister` will become `opendal::blocking::Lister`
@@ -81,7 +81,7 @@ pub trait Access: Send + Sync + Debug + Unpin + 'static {
}
```
-Due to this change, we will remove all blocking implementations in services and layers. All existing public APIs that start with `Blocking` will be moved into the `blocking` module. For example, `opendal::BlockingOperator` will be relocated to `opendal::blocking::Operator`.
+Due to this change, we will remove all blocking implementations in services and layers. All existing public APIs that start with `Blocking` will be moved into the `blocking` module. For example, `opendal::blocking::Operator` will be relocated to `opendal::blocking::Operator`.
# Drawbacks
diff --git a/core/src/layers/async_backtrace.rs b/core/src/layers/async_backtrace.rs
index f22172c29..47a761185 100644
--- a/core/src/layers/async_backtrace.rs
+++ b/core/src/layers/async_backtrace.rs
@@ -60,13 +60,9 @@ pub struct AsyncBacktraceAccessor<A: Access> {
impl<A: Access> LayeredAccess for AsyncBacktraceAccessor<A> {
type Inner = A;
type Reader = AsyncBacktraceWrapper<A::Reader>;
- type BlockingReader = AsyncBacktraceWrapper<A::BlockingReader>;
type Writer = AsyncBacktraceWrapper<A::Writer>;
- type BlockingWriter = AsyncBacktraceWrapper<A::BlockingWriter>;
type Lister = AsyncBacktraceWrapper<A::Lister>;
- type BlockingLister = AsyncBacktraceWrapper<A::BlockingLister>;
type Deleter = AsyncBacktraceWrapper<A::Deleter>;
- type BlockingDeleter = AsyncBacktraceWrapper<A::BlockingDeleter>;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -123,30 +119,6 @@ impl<A: Access> LayeredAccess for AsyncBacktraceAccessor<A> {
async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
self.inner.presign(path, args).await
}
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.inner
- .blocking_read(path, args)
- .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.inner
- .blocking_write(path, args)
- .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.inner
- .blocking_list(path, args)
- .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner
- .blocking_delete()
- .map(|(rp, r)| (rp, AsyncBacktraceWrapper::new(r)))
- }
}
pub struct AsyncBacktraceWrapper<R> {
@@ -166,12 +138,6 @@ impl<R: oio::Read> oio::Read for AsyncBacktraceWrapper<R> {
}
}
-impl<R: oio::BlockingRead> oio::BlockingRead for AsyncBacktraceWrapper<R> {
- fn read(&mut self) -> Result<Buffer> {
- self.inner.read()
- }
-}
-
impl<R: oio::Write> oio::Write for AsyncBacktraceWrapper<R> {
#[async_backtrace::framed]
async fn write(&mut self, bs: Buffer) -> Result<()> {
@@ -189,16 +155,6 @@ impl<R: oio::Write> oio::Write for AsyncBacktraceWrapper<R> {
}
}
-impl<R: oio::BlockingWrite> oio::BlockingWrite for AsyncBacktraceWrapper<R> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- self.inner.write(bs)
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.inner.close()
- }
-}
-
impl<R: oio::List> oio::List for AsyncBacktraceWrapper<R> {
#[async_backtrace::framed]
async fn next(&mut self) -> Result<Option<oio::Entry>> {
@@ -206,12 +162,6 @@ impl<R: oio::List> oio::List for AsyncBacktraceWrapper<R> {
}
}
-impl<R: oio::BlockingList> oio::BlockingList for AsyncBacktraceWrapper<R> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.inner.next()
- }
-}
-
impl<R: oio::Delete> oio::Delete for AsyncBacktraceWrapper<R> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
self.inner.delete(path, args)
@@ -222,13 +172,3 @@ impl<R: oio::Delete> oio::Delete for AsyncBacktraceWrapper<R> {
self.inner.flush().await
}
}
-
-impl<R: oio::BlockingDelete> oio::BlockingDelete for AsyncBacktraceWrapper<R> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- self.inner.delete(path, args)
- }
-
- fn flush(&mut self) -> Result<usize> {
- self.inner.flush()
- }
-}
diff --git a/core/src/layers/await_tree.rs b/core/src/layers/await_tree.rs
index b6ffa3198..8d6f66a80 100644
--- a/core/src/layers/await_tree.rs
+++ b/core/src/layers/await_tree.rs
@@ -71,13 +71,9 @@ pub struct AwaitTreeAccessor<A: Access> {
impl<A: Access> LayeredAccess for AwaitTreeAccessor<A> {
type Inner = A;
type Reader = AwaitTreeWrapper<A::Reader>;
- type BlockingReader = AwaitTreeWrapper<A::BlockingReader>;
type Writer = AwaitTreeWrapper<A::Writer>;
- type BlockingWriter = AwaitTreeWrapper<A::BlockingWriter>;
type Lister = AwaitTreeWrapper<A::Lister>;
- type BlockingLister = AwaitTreeWrapper<A::BlockingLister>;
type Deleter = AwaitTreeWrapper<A::Deleter>;
- type BlockingDeleter = AwaitTreeWrapper<A::BlockingDeleter>;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -142,30 +138,6 @@ impl<A: Access> LayeredAccess for AwaitTreeAccessor<A> {
.instrument_await(format!("opendal::{}", Operation::Presign))
.await
}
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.inner
- .blocking_read(path, args)
- .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.inner
- .blocking_write(path, args)
- .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.inner
- .blocking_list(path, args)
- .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner
- .blocking_delete()
- .map(|(rp, r)| (rp, AwaitTreeWrapper::new(r)))
- }
}
pub struct AwaitTreeWrapper<R> {
@@ -187,12 +159,6 @@ impl<R: oio::Read> oio::Read for AwaitTreeWrapper<R> {
}
}
-impl<R: oio::BlockingRead> oio::BlockingRead for AwaitTreeWrapper<R> {
- fn read(&mut self) -> Result<Buffer> {
- self.inner.read()
- }
-}
-
impl<R: oio::Write> oio::Write for AwaitTreeWrapper<R> {
fn write(&mut self, bs: Buffer) -> impl Future<Output = Result<()>> + MaybeSend {
self.inner
@@ -213,16 +179,6 @@ impl<R: oio::Write> oio::Write for AwaitTreeWrapper<R> {
}
}
-impl<R: oio::BlockingWrite> oio::BlockingWrite for AwaitTreeWrapper<R> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- self.inner.write(bs)
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.inner.close()
- }
-}
-
impl<R: oio::List> oio::List for AwaitTreeWrapper<R> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
self.inner
@@ -232,12 +188,6 @@ impl<R: oio::List> oio::List for AwaitTreeWrapper<R> {
}
}
-impl<R: oio::BlockingList> oio::BlockingList for AwaitTreeWrapper<R> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.inner.next()
- }
-}
-
impl<R: oio::Delete> oio::Delete for AwaitTreeWrapper<R> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
self.inner.delete(path, args)
@@ -250,13 +200,3 @@ impl<R: oio::Delete> oio::Delete for AwaitTreeWrapper<R> {
.await
}
}
-
-impl<R: oio::BlockingDelete> oio::BlockingDelete for AwaitTreeWrapper<R> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- self.inner.delete(path, args)
- }
-
- fn flush(&mut self) -> Result<usize> {
- self.inner.flush()
- }
-}
diff --git a/core/src/layers/blocking.rs b/core/src/layers/blocking.rs
deleted file mode 100644
index 63def0e3c..000000000
--- a/core/src/layers/blocking.rs
+++ /dev/null
@@ -1,362 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use tokio::runtime::Handle;
-
-use crate::raw::*;
-use crate::*;
-
-/// Add blocking API support for non-blocking services.
-///
-/// # Notes
-///
-/// - Please only enable this layer when the underlying service does not support blocking.
-///
-/// # Examples
-///
-/// ## In async context
-///
-/// BlockingLayer will use current async context's runtime to handle the async calls.
-///
-/// ```rust,no_run
-/// # use opendal::layers::BlockingLayer;
-/// # use opendal::services;
-/// # use opendal::BlockingOperator;
-/// # use opendal::Operator;
-/// # use opendal::Result;
-///
-/// #[tokio::main]
-/// async fn main() -> Result<()> {
-/// // Create fs backend builder.
-/// let mut builder = services::S3::default().bucket("test").region("us-east-1");
-///
-/// // Build an `BlockingOperator` with blocking layer to start operating the storage.
-/// let _: BlockingOperator = Operator::new(builder)?
-/// .layer(BlockingLayer::create()?)
-/// .finish()
-/// .blocking();
-///
-/// Ok(())
-/// }
-/// ```
-///
-/// ## In async context with blocking functions
-///
-/// If `BlockingLayer` is called in blocking function, please fetch a [`tokio::runtime::EnterGuard`]
-/// first. You can use [`Handle::try_current`] first to get the handle and then call [`Handle::enter`].
-/// This often happens in the case that async function calls blocking function.
-///
-/// ```rust,no_run
-/// # use opendal::layers::BlockingLayer;
-/// # use opendal::services;
-/// # use opendal::BlockingOperator;
-/// # use opendal::Operator;
-/// # use opendal::Result;
-///
-/// #[tokio::main]
-/// async fn main() -> Result<()> {
-/// let _ = blocking_fn()?;
-/// Ok(())
-/// }
-///
-/// fn blocking_fn() -> Result<BlockingOperator> {
-/// // Create fs backend builder.
-/// let mut builder = services::S3::default().bucket("test").region("us-east-1");
-///
-/// let handle = tokio::runtime::Handle::try_current().unwrap();
-/// let _guard = handle.enter();
-/// // Build an `BlockingOperator` with blocking layer to start operating the storage.
-/// let op: BlockingOperator = Operator::new(builder)?
-/// .layer(BlockingLayer::create()?)
-/// .finish()
-/// .blocking();
-/// Ok(op)
-/// }
-/// ```
-///
-/// ## In blocking context
-///
-/// In a pure blocking context, we can create a runtime and use it to create the `BlockingLayer`.
-///
-/// > The following code uses a global statically created runtime as an example, please manage the
-/// > runtime on demand.
-///
-/// ```rust,no_run
-/// # use std::sync::LazyLock;
-/// # use opendal::layers::BlockingLayer;
-/// # use opendal::services;
-/// # use opendal::BlockingOperator;
-/// # use opendal::Operator;
-/// # use opendal::Result;
-///
-/// static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
-/// tokio::runtime::Builder::new_multi_thread()
-/// .enable_all()
-/// .build()
-/// .unwrap()
-/// });
-///
-/// fn main() -> Result<()> {
-/// // Create fs backend builder.
-/// let mut builder = services::S3::default().bucket("test").region("us-east-1");
-///
-/// // Fetch the `EnterGuard` from global runtime.
-/// let _guard = RUNTIME.enter();
-/// // Build an `BlockingOperator` with blocking layer to start operating the storage.
-/// let _: BlockingOperator = Operator::new(builder)?
-/// .layer(BlockingLayer::create()?)
-/// .finish()
-/// .blocking();
-///
-/// Ok(())
-/// }
-/// ```
-#[derive(Debug, Clone)]
-pub struct BlockingLayer {
- handle: Handle,
-}
-
-impl BlockingLayer {
- /// Create a new `BlockingLayer` with the current runtime's handle
- pub fn create() -> Result<Self> {
- Ok(Self {
- handle: Handle::try_current()
- .map_err(|_| Error::new(ErrorKind::Unexpected, "failed to get current handle"))?,
- })
- }
-}
-
-impl<A: Access> Layer<A> for BlockingLayer {
- type LayeredAccess = BlockingAccessor<A>;
-
- fn layer(&self, inner: A) -> Self::LayeredAccess {
- let info = inner.info();
- info.update_full_capability(|mut cap| {
- cap.blocking = true;
- cap
- });
-
- BlockingAccessor {
- inner,
- handle: self.handle.clone(),
- }
- }
-}
-
-#[derive(Clone, Debug)]
-pub struct BlockingAccessor<A: Access> {
- inner: A,
-
- handle: Handle,
-}
-
-impl<A: Access> LayeredAccess for BlockingAccessor<A> {
- type Inner = A;
- type Reader = A::Reader;
- type BlockingReader = BlockingWrapper<A::Reader>;
- type Writer = A::Writer;
- type BlockingWriter = BlockingWrapper<A::Writer>;
- type Lister = A::Lister;
- type BlockingLister = BlockingWrapper<A::Lister>;
- type Deleter = A::Deleter;
- type BlockingDeleter = BlockingWrapper<A::Deleter>;
-
- fn inner(&self) -> &Self::Inner {
- &self.inner
- }
-
- async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.inner.create_dir(path, args).await
- }
-
- async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- self.inner.read(path, args).await
- }
-
- async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
- self.inner.write(path, args).await
- }
-
- async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- self.inner.copy(from, to, args).await
- }
-
- async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- self.inner.rename(from, to, args).await
- }
-
- async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.inner.stat(path, args).await
- }
-
- async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
- self.inner.delete().await
- }
-
- async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
- self.inner.list(path, args).await
- }
-
- async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
- self.inner.presign(path, args).await
- }
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.handle.block_on(self.inner.create_dir(path, args))
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.handle.block_on(async {
- let (rp, reader) = self.inner.read(path, args).await?;
- let blocking_reader = Self::BlockingReader::new(self.handle.clone(), reader);
-
- Ok((rp, blocking_reader))
- })
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.handle.block_on(async {
- let (rp, writer) = self.inner.write(path, args).await?;
- let blocking_writer = Self::BlockingWriter::new(self.handle.clone(), writer);
- Ok((rp, blocking_writer))
- })
- }
-
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- self.handle.block_on(self.inner.copy(from, to, args))
- }
-
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- self.handle.block_on(self.inner.rename(from, to, args))
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.handle.block_on(self.inner.stat(path, args))
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.handle.block_on(async {
- let (rp, writer) = self.inner.delete().await?;
- let blocking_deleter = Self::BlockingDeleter::new(self.handle.clone(), writer);
- Ok((rp, blocking_deleter))
- })
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.handle.block_on(async {
- let (rp, lister) = self.inner.list(path, args).await?;
- let blocking_lister = Self::BlockingLister::new(self.handle.clone(), lister);
- Ok((rp, blocking_lister))
- })
- }
-}
-
-pub struct BlockingWrapper<I> {
- handle: Handle,
- inner: Option<I>,
-}
-
-impl<I> BlockingWrapper<I> {
- fn new(handle: Handle, inner: I) -> Self {
- Self {
- handle,
- inner: Some(inner),
- }
- }
-}
-
-impl<I: oio::Read + 'static> oio::BlockingRead for BlockingWrapper<I> {
- fn read(&mut self) -> Result<Buffer> {
- self.handle.block_on(self.inner.as_mut().unwrap().read())
- }
-}
-
-impl<I: oio::Write + 'static> oio::BlockingWrite for BlockingWrapper<I> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- self.handle.block_on(self.inner.as_mut().unwrap().write(bs))
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.handle.block_on(self.inner.as_mut().unwrap().close())
- }
-}
-
-impl<I> Drop for BlockingWrapper<I> {
- fn drop(&mut self) {
- if let Some(inner) = self.inner.take() {
- self.handle.block_on(async move {
- drop(inner);
- });
- }
- }
-}
-
-impl<I: oio::List> oio::BlockingList for BlockingWrapper<I> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.handle.block_on(self.inner.as_mut().unwrap().next())
- }
-}
-
-impl<I: oio::Delete + 'static> oio::BlockingDelete for BlockingWrapper<I> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- self.inner.as_mut().unwrap().delete(path, args)
- }
-
- fn flush(&mut self) -> Result<usize> {
- self.handle.block_on(self.inner.as_mut().unwrap().flush())
- }
-}
-
-#[cfg(test)]
-mod tests {
- use std::sync::LazyLock;
-
- use super::*;
- use crate::types::Result;
-
- static RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
- tokio::runtime::Builder::new_multi_thread()
- .enable_all()
- .build()
- .unwrap()
- });
-
- fn create_blocking_layer() -> Result<BlockingLayer> {
- let _guard = RUNTIME.enter();
- BlockingLayer::create()
- }
-
- #[test]
- fn test_blocking_layer_in_blocking_context() {
- // create in a blocking context should fail
- let layer = BlockingLayer::create();
- assert!(layer.is_err());
-
- // create in an async context and drop in a blocking context
- let layer = create_blocking_layer();
- assert!(layer.is_ok())
- }
-
- #[test]
- fn test_blocking_layer_in_async_context() {
- // create and drop in an async context
- let _guard = RUNTIME.enter();
-
- let layer = BlockingLayer::create();
- assert!(layer.is_ok());
- }
-}
diff --git a/core/src/layers/capability_check.rs b/core/src/layers/capability_check.rs
index 2c9ce5849..6d5c3c9a6 100644
--- a/core/src/layers/capability_check.rs
+++ b/core/src/layers/capability_check.rs
@@ -84,10 +84,6 @@ impl<A: Access> LayeredAccess for CapabilityAccessor<A> {
type Writer = A::Writer;
type Lister = A::Lister;
type Deleter = A::Deleter;
- type BlockingReader = A::BlockingReader;
- type BlockingWriter = A::BlockingWriter;
- type BlockingLister = A::BlockingLister;
- type BlockingDeleter = A::BlockingDeleter;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -140,66 +136,6 @@ impl<A: Access> LayeredAccess for CapabilityAccessor<A> {
self.inner.list(path, args).await
}
-
- fn blocking_read(
- &self,
- path: &str,
- args: OpRead,
- ) -> crate::Result<(RpRead, Self::BlockingReader)> {
- self.inner().blocking_read(path, args)
- }
-
- fn blocking_write(
- &self,
- path: &str,
- args: OpWrite,
- ) -> crate::Result<(RpWrite, Self::BlockingWriter)> {
- let capability = self.info.full_capability();
- if !capability.write_with_content_type && args.content_type().is_some() {
- return Err(new_unsupported_error(
- self.info.as_ref(),
- Operation::Write,
- "content_type",
- ));
- }
- if !capability.write_with_cache_control && args.cache_control().is_some() {
- return Err(new_unsupported_error(
- self.info.as_ref(),
- Operation::Write,
- "cache_control",
- ));
- }
- if !capability.write_with_content_disposition && args.content_disposition().is_some() {
- return Err(new_unsupported_error(
- self.info.as_ref(),
- Operation::Write,
- "content_disposition",
- ));
- }
-
- self.inner.blocking_write(path, args)
- }
-
- fn blocking_delete(&self) -> crate::Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner.blocking_delete()
- }
-
- fn blocking_list(
- &self,
- path: &str,
- args: OpList,
- ) -> crate::Result<(RpList, Self::BlockingLister)> {
- let capability = self.info.full_capability();
- if !capability.list_with_versions && args.versions() {
- return Err(new_unsupported_error(
- self.info.as_ref(),
- Operation::List,
- "version",
- ));
- }
-
- self.inner.blocking_list(path, args)
- }
}
#[cfg(test)]
@@ -217,10 +153,6 @@ mod tests {
type Writer = oio::Writer;
type Lister = oio::Lister;
type Deleter = oio::Deleter;
- type BlockingReader = oio::BlockingReader;
- type BlockingWriter = oio::BlockingWriter;
- type BlockingLister = oio::BlockingLister;
- type BlockingDeleter = oio::BlockingDeleter;
fn info(&self) -> Arc<AccessorInfo> {
let info = AccessorInfo::default();
diff --git a/core/src/layers/chaos.rs b/core/src/layers/chaos.rs
index 342e56857..8a9424ec9 100644
--- a/core/src/layers/chaos.rs
+++ b/core/src/layers/chaos.rs
@@ -100,13 +100,9 @@ pub struct ChaosAccessor<A> {
impl<A: Access> LayeredAccess for ChaosAccessor<A> {
type Inner = A;
type Reader = ChaosReader<A::Reader>;
- type BlockingReader = ChaosReader<A::BlockingReader>;
type Writer = A::Writer;
- type BlockingWriter = A::BlockingWriter;
type Lister = A::Lister;
- type BlockingLister = A::BlockingLister;
type Deleter = A::Deleter;
- type BlockingDeleter = A::BlockingDeleter;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -119,35 +115,17 @@ impl<A: Access> LayeredAccess for ChaosAccessor<A> {
.map(|(rp, r)| (rp, ChaosReader::new(r, self.rng.clone(), self.error_ratio)))
}
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.inner
- .blocking_read(path, args)
- .map(|(rp, r)| (rp, ChaosReader::new(r, self.rng.clone(), self.error_ratio)))
- }
-
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
self.inner.write(path, args).await
}
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.inner.blocking_write(path, args)
- }
-
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
self.inner.list(path, args).await
}
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.inner.blocking_list(path, args)
- }
-
async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
self.inner.delete().await
}
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner.blocking_delete()
- }
}
/// ChaosReader will inject error into read operations.
@@ -190,13 +168,3 @@ impl<R: oio::Read> oio::Read for ChaosReader<R> {
}
}
}
-
-impl<R: oio::BlockingRead> oio::BlockingRead for ChaosReader<R> {
- fn read(&mut self) -> Result<Buffer> {
- if self.i_feel_lucky() {
- self.inner.read()
- } else {
- Err(Self::unexpected_eof())
- }
- }
-}
diff --git a/core/src/layers/complete.rs b/core/src/layers/complete.rs
index ffc91d80f..c2756146d 100644
--- a/core/src/layers/complete.rs
+++ b/core/src/layers/complete.rs
@@ -41,51 +41,6 @@ use std::sync::Arc;
///
/// Not all services support stat dir natively, but we can simulate it via list.
///
-/// ## Read Completion
-///
-/// OpenDAL requires all reader implements [`oio::Read`] and
-/// [`oio::BlockingRead`]. However, not all services have the
-/// capabilities. CompleteLayer will add those capabilities in
-/// a zero cost way.
-///
-/// Underlying services will return [`AccessorInfo`] to indicate the
-/// features that returning readers support.
-///
-/// - If both `seekable` and `streamable`, return directly.
-/// - If not `streamable`, with [`oio::into_read_from_stream`].
-/// - If not `seekable`, with [`oio::into_seekable_read_by_range`]
-/// - If neither not supported, wrap both by_range and into_streamable.
-///
-/// All implementations of Reader should be `zero cost`. In our cases,
-/// which means others must pay the same cost for the same feature provide
-/// by us.
-///
-/// For examples, call `read` without `seek` should always act the same as
-/// calling `read` on plain reader.
-///
-/// ### Read is Seekable
-///
-/// We use [`Capability`] to decide the most suitable implementations.
-///
-/// If [`Capability`] `read_can_seek` is true, we will open it with given args
-/// directly. Otherwise, we will pick a seekable reader implementation based
-/// on input range for it.
-///
-/// - `Some(offset), Some(size)` => `RangeReader`
-/// - `Some(offset), None` and `None, None` => `OffsetReader`
-/// - `None, Some(size)` => get the total size first to convert as `RangeReader`
-///
-/// No matter which reader we use, we will make sure the `read` operation
-/// is zero cost.
-///
-/// ### Read is Streamable
-///
-/// We use internal `AccessorHint::ReadStreamable` to decide the most
-/// suitable implementations.
-///
-/// If [`Capability`] `read_can_next` is true, we will use existing reader
-/// directly. Otherwise, we will use transform this reader as a stream.
-///
/// ## List Completion
///
/// There are two styles of list, but not all services support both of
@@ -146,21 +101,6 @@ impl<A: Access> CompleteAccessor<A> {
self.inner.create_dir(path, args).await
}
- fn complete_blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- let capability = self.info.native_capability();
- if capability.create_dir && capability.blocking {
- return self.inner().blocking_create_dir(path, args);
- }
-
- if capability.write_can_empty && capability.list && capability.blocking {
- let (_, mut w) = self.inner.blocking_write(path, OpWrite::default())?;
- oio::BlockingWrite::close(&mut w)?;
- return Ok(RpCreateDir::default());
- }
-
- self.inner.blocking_create_dir(path, args)
- }
-
async fn complete_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
let capability = self.info.native_capability();
@@ -203,47 +143,6 @@ impl<A: Access> CompleteAccessor<A> {
self.inner.stat(path, args).await
}
- fn complete_blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- let capability = self.info.native_capability();
-
- if path == "/" {
- return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
- }
-
- // Forward to inner if create dir is supported.
- if path.ends_with('/') && capability.create_dir {
- let meta = self.inner.blocking_stat(path, args)?.into_metadata();
-
- if meta.is_file() {
- return Err(Error::new(
- ErrorKind::NotFound,
- "stat expected a directory, but found a file",
- ));
- }
-
- return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
- }
-
- // Otherwise, we can simulate stat a dir path via `list`.
- if path.ends_with('/') && capability.list_with_recursive {
- let (_, mut l) = self
- .inner
- .blocking_list(path, OpList::default().with_recursive(true).with_limit(1))?;
-
- return if oio::BlockingList::next(&mut l)?.is_some() {
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
- } else {
- Err(Error::new(
- ErrorKind::NotFound,
- "the directory is not found",
- ))
- };
- }
-
- // Forward to underlying storage directly since we don't know how to handle stat dir.
- self.inner.blocking_stat(path, args)
- }
-
async fn complete_list(
&self,
path: &str,
@@ -288,63 +187,14 @@ impl<A: Access> CompleteAccessor<A> {
}
}
}
-
- fn complete_blocking_list(
- &self,
- path: &str,
- args: OpList,
- ) -> Result<(RpList, CompleteLister<A, A::BlockingLister>)> {
- let cap = self.info.native_capability();
-
- let recursive = args.recursive();
-
- match (recursive, cap.list_with_recursive) {
- // - If service can list_with_recursive, we can forward list to it directly.
- (_, true) => {
- let (rp, p) = self.inner.blocking_list(path, args)?;
- Ok((rp, CompleteLister::One(p)))
- }
- // If recursive is true but service can't list_with_recursive
- (true, false) => {
- // Forward path that ends with /
- if path.ends_with('/') {
- let p = FlatLister::new(self.inner.clone(), path);
- Ok((RpList::default(), CompleteLister::Two(p)))
- } else {
- let parent = get_parent(path);
- let p = FlatLister::new(self.inner.clone(), parent);
- let p = PrefixLister::new(p, path);
- Ok((RpList::default(), CompleteLister::Four(p)))
- }
- }
- // If recursive and service doesn't support list_with_recursive, we need to handle
- // list prefix by ourselves.
- (false, false) => {
- // Forward path that ends with /
- if path.ends_with('/') {
- let (rp, p) = self.inner.blocking_list(path, args)?;
- Ok((rp, CompleteLister::One(p)))
- } else {
- let parent = get_parent(path);
- let (rp, p) = self.inner.blocking_list(parent, args)?;
- let p = PrefixLister::new(p, path);
- Ok((rp, CompleteLister::Three(p)))
- }
- }
- }
- }
}
impl<A: Access> LayeredAccess for CompleteAccessor<A> {
type Inner = A;
type Reader = CompleteReader<A::Reader>;
- type BlockingReader = CompleteReader<A::BlockingReader>;
type Writer = CompleteWriter<A::Writer>;
- type BlockingWriter = CompleteWriter<A::BlockingWriter>;
type Lister = CompleteLister<A, A::Lister>;
- type BlockingLister = CompleteLister<A, A::BlockingLister>;
type Deleter = A::Deleter;
- type BlockingDeleter = A::BlockingDeleter;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -387,36 +237,6 @@ impl<A: Access> LayeredAccess for CompleteAccessor<A> {
async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
self.inner.presign(path, args).await
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.complete_blocking_create_dir(path, args)
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let size = args.range().size();
- self.inner
- .blocking_read(path, args)
- .map(|(rp, r)| (rp, CompleteReader::new(r, size)))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let append = args.append();
- self.inner
- .blocking_write(path, args)
- .map(|(rp, w)| (rp, CompleteWriter::new(w, append)))
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.complete_blocking_stat(path, args)
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner().blocking_delete()
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.complete_blocking_list(path, args)
- }
}
pub type CompleteLister<A, P> =
@@ -472,20 +292,6 @@ impl<R: oio::Read> oio::Read for CompleteReader<R> {
}
}
-impl<R: oio::BlockingRead> oio::BlockingRead for CompleteReader<R> {
- fn read(&mut self) -> Result<Buffer> {
- let buf = self.inner.read()?;
-
- if buf.is_empty() {
- self.check()?;
- } else {
- self.read += buf.len() as u64;
- }
-
- Ok(buf)
- }
-}
-
pub struct CompleteWriter<W> {
inner: Option<W>,
append: bool,
@@ -577,35 +383,3 @@ where
Ok(())
}
}
-
-impl<W> oio::BlockingWrite for CompleteWriter<W>
-where
- W: oio::BlockingWrite,
-{
- fn write(&mut self, bs: Buffer) -> Result<()> {
- let w = self.inner.as_mut().ok_or_else(|| {
- Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
- })?;
-
- let len = bs.len();
- w.write(bs)?;
- self.size += len as u64;
-
- Ok(())
- }
-
- fn close(&mut self) -> Result<Metadata> {
- let w = self.inner.as_mut().ok_or_else(|| {
- Error::new(ErrorKind::Unexpected, "writer has been closed or aborted")
- })?;
-
- let mut ret = w.close()?;
- self.check(ret.content_length())?;
- if ret.content_length() == 0 {
- ret = ret.with_content_length(self.size);
- }
- self.inner = None;
-
- Ok(ret)
- }
-}
diff --git a/core/src/layers/concurrent_limit.rs b/core/src/layers/concurrent_limit.rs
index 738a1e034..435149190 100644
--- a/core/src/layers/concurrent_limit.rs
+++ b/core/src/layers/concurrent_limit.rs
@@ -184,13 +184,9 @@ pub struct ConcurrentLimitAccessor<A: Access> {
impl<A: Access> LayeredAccess for ConcurrentLimitAccessor<A> {
type Inner = A;
type Reader = ConcurrentLimitWrapper<A::Reader>;
- type BlockingReader = ConcurrentLimitWrapper<A::BlockingReader>;
type Writer = ConcurrentLimitWrapper<A::Writer>;
- type BlockingWriter = ConcurrentLimitWrapper<A::BlockingWriter>;
type Lister = ConcurrentLimitWrapper<A::Lister>;
- type BlockingLister = ConcurrentLimitWrapper<A::BlockingLister>;
type Deleter = ConcurrentLimitWrapper<A::Deleter>;
- type BlockingDeleter = ConcurrentLimitWrapper<A::BlockingDeleter>;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -271,72 +267,6 @@ impl<A: Access> LayeredAccess for ConcurrentLimitAccessor<A> {
.await
.map(|(rp, s)| (rp, ConcurrentLimitWrapper::new(s, permit)))
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- let _permit = self
- .semaphore
- .try_acquire()
- .expect("semaphore must be valid");
-
- self.inner.blocking_create_dir(path, args)
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let permit = self
- .semaphore
- .clone()
- .try_acquire_owned()
- .expect("semaphore must be valid");
-
- self.inner
- .blocking_read(path, args)
- .map(|(rp, r)| (rp, ConcurrentLimitWrapper::new(r, permit)))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let permit = self
- .semaphore
- .clone()
- .try_acquire_owned()
- .expect("semaphore must be valid");
-
- self.inner
- .blocking_write(path, args)
- .map(|(rp, w)| (rp, ConcurrentLimitWrapper::new(w, permit)))
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- let _permit = self
- .semaphore
- .try_acquire()
- .expect("semaphore must be valid");
-
- self.inner.blocking_stat(path, args)
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- let permit = self
- .semaphore
- .clone()
- .try_acquire_owned()
- .expect("semaphore must be valid");
-
- self.inner
- .blocking_delete()
- .map(|(rp, w)| (rp, ConcurrentLimitWrapper::new(w, permit)))
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- let permit = self
- .semaphore
- .clone()
- .try_acquire_owned()
- .expect("semaphore must be valid");
-
- self.inner
- .blocking_list(path, args)
- .map(|(rp, it)| (rp, ConcurrentLimitWrapper::new(it, permit)))
- }
}
pub struct ConcurrentLimitWrapper<R> {
@@ -361,12 +291,6 @@ impl<R: oio::Read> oio::Read for ConcurrentLimitWrapper<R> {
}
}
-impl<R: oio::BlockingRead> oio::BlockingRead for ConcurrentLimitWrapper<R> {
- fn read(&mut self) -> Result<Buffer> {
- self.inner.read()
- }
-}
-
impl<R: oio::Write> oio::Write for ConcurrentLimitWrapper<R> {
async fn write(&mut self, bs: Buffer) -> Result<()> {
self.inner.write(bs).await
@@ -381,28 +305,12 @@ impl<R: oio::Write> oio::Write for ConcurrentLimitWrapper<R> {
}
}
-impl<R: oio::BlockingWrite> oio::BlockingWrite for ConcurrentLimitWrapper<R> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- self.inner.write(bs)
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.inner.close()
- }
-}
-
impl<R: oio::List> oio::List for ConcurrentLimitWrapper<R> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
self.inner.next().await
}
}
-impl<R: oio::BlockingList> oio::BlockingList for ConcurrentLimitWrapper<R> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.inner.next()
- }
-}
-
impl<R: oio::Delete> oio::Delete for ConcurrentLimitWrapper<R> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
self.inner.delete(path, args)
@@ -412,13 +320,3 @@ impl<R: oio::Delete> oio::Delete for ConcurrentLimitWrapper<R> {
self.inner.flush().await
}
}
-
-impl<R: oio::BlockingDelete> oio::BlockingDelete for ConcurrentLimitWrapper<R> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- self.inner.delete(path, args)
- }
-
- fn flush(&mut self) -> Result<usize> {
- self.inner.flush()
- }
-}
diff --git a/core/src/layers/correctness_check.rs b/core/src/layers/correctness_check.rs
index 8a7658630..47e8a0716 100644
--- a/core/src/layers/correctness_check.rs
+++ b/core/src/layers/correctness_check.rs
@@ -80,10 +80,6 @@ impl<A: Access> LayeredAccess for CorrectnessAccessor<A> {
type Writer = A::Writer;
type Lister = A::Lister;
type Deleter = CheckWrapper<A::Deleter>;
- type BlockingReader = A::BlockingReader;
- type BlockingWriter = A::BlockingWriter;
- type BlockingLister = A::BlockingLister;
- type BlockingDeleter = CheckWrapper<A::BlockingDeleter>;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -216,70 +212,6 @@ impl<A: Access> LayeredAccess for CorrectnessAccessor<A> {
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
self.inner.list(path, args).await
}
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let capability = self.info.full_capability();
- if !capability.read_with_version && args.version().is_some() {
- return Err(new_unsupported_error(
- self.info.as_ref(),
- Operation::Read,
- "version",
- ));
- }
-
- self.inner.blocking_read(path, args)
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let capability = self.info.full_capability();
- if args.append() && !capability.write_can_append {
- return Err(new_unsupported_error(
- &self.info,
- Operation::Write,
- "append",
- ));
- }
- if args.if_not_exists() && !capability.write_with_if_not_exists {
- return Err(new_unsupported_error(
- &self.info,
- Operation::Write,
- "if_not_exists",
- ));
- }
- if args.if_none_match().is_some() && !capability.write_with_if_none_match {
- return Err(new_unsupported_error(
- self.info.as_ref(),
- Operation::Write,
- "if_none_match",
- ));
- }
-
- self.inner.blocking_write(path, args)
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- let capability = self.info.full_capability();
- if !capability.stat_with_version && args.version().is_some() {
- return Err(new_unsupported_error(
- self.info.as_ref(),
- Operation::Stat,
- "version",
- ));
- }
-
- self.inner.blocking_stat(path, args)
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner.blocking_delete().map(|(rp, deleter)| {
- let deleter = CheckWrapper::new(deleter, self.info.clone());
- (rp, deleter)
- })
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.inner.blocking_list(path, args)
- }
}
pub struct CheckWrapper<T> {
@@ -316,17 +248,6 @@ impl<T: oio::Delete> oio::Delete for CheckWrapper<T> {
}
}
-impl<T: oio::BlockingDelete> oio::BlockingDelete for CheckWrapper<T> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- self.check_delete(&args)?;
- self.inner.delete(path, args)
- }
-
- fn flush(&mut self) -> Result<usize> {
- self.inner.flush()
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
@@ -343,10 +264,6 @@ mod tests {
type Writer = oio::Writer;
type Lister = oio::Lister;
type Deleter = oio::Deleter;
- type BlockingReader = oio::BlockingReader;
- type BlockingWriter = oio::BlockingWriter;
- type BlockingLister = oio::BlockingLister;
- type BlockingDeleter = oio::BlockingDeleter;
fn info(&self) -> Arc<AccessorInfo> {
let info = AccessorInfo::default();
diff --git a/core/src/layers/dtrace.rs b/core/src/layers/dtrace.rs
index e51e65f64..0f822530b 100644
--- a/core/src/layers/dtrace.rs
+++ b/core/src/layers/dtrace.rs
@@ -42,12 +42,7 @@ use crate::*;
/// 5. delete
/// 6. list
/// 7. presign
-/// 8. blocking_create_dir
-/// 9. blocking_read
-/// 10. blocking_write
-/// 11. blocking_stat
-/// 12. blocking_delete
-/// 13. blocking_list
+///
/// 2. ${operation}_end, arguments: path
/// 1. create_dir
/// 2. read
@@ -56,12 +51,6 @@ use crate::*;
/// 5. delete
/// 6. list
/// 7. presign
-/// 8. blocking_create_dir
-/// 9. blocking_read
-/// 10. blocking_write
-/// 11. blocking_stat
-/// 12. blocking_delete
-/// 13. blocking_list
///
/// ### For Reader
///
@@ -69,12 +58,6 @@ use crate::*;
/// 2. reader_read_ok, arguments: path, length
/// 3. reader_read_error, arguments: path
///
-/// ### For BlockingReader
-///
-/// 1. blocking_reader_read_start, arguments: path
-/// 2. blocking_reader_read_ok, arguments: path, length
-/// 3. blocking_reader_read_error, arguments: path
-///
/// ### For Writer
///
/// 1. writer_write_start, arguments: path
@@ -87,15 +70,6 @@ use crate::*;
/// 8. writer_close_ok, arguments: path
/// 9. writer_close_error, arguments: path
///
-/// ### For BlockingWriter
-///
-/// 1. blocking_writer_write_start, arguments: path
-/// 2. blocking_writer_write_ok, arguments: path, length
-/// 3. blocking_writer_write_error, arguments: path
-/// 4. blocking_writer_close_start, arguments: path
-/// 5. blocking_writer_close_ok, arguments: path
-/// 6. blocking_writer_close_error, arguments: path
-///
/// Example:
///
/// ```no_run
@@ -174,13 +148,9 @@ impl<A: Access> Debug for DTraceAccessor<A> {
impl<A: Access> LayeredAccess for DTraceAccessor<A> {
type Inner = A;
type Reader = DtraceLayerWrapper<A::Reader>;
- type BlockingReader = DtraceLayerWrapper<A::BlockingReader>;
type Writer = DtraceLayerWrapper<A::Writer>;
- type BlockingWriter = DtraceLayerWrapper<A::BlockingWriter>;
type Lister = A::Lister;
- type BlockingLister = A::BlockingLister;
type Deleter = A::Deleter;
- type BlockingDeleter = A::BlockingDeleter;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -246,56 +216,6 @@ impl<A: Access> LayeredAccess for DTraceAccessor<A> {
probe_lazy!(opendal, presign_end, c_path.as_ptr());
result
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- let c_path = CString::new(path).unwrap();
- probe_lazy!(opendal, blocking_create_dir_start, c_path.as_ptr());
- let result = self.inner.blocking_create_dir(path, args);
- probe_lazy!(opendal, blocking_create_dir_end, c_path.as_ptr());
- result
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let c_path = CString::new(path).unwrap();
- probe_lazy!(opendal, blocking_read_start, c_path.as_ptr());
- let result = self
- .inner
- .blocking_read(path, args)
- .map(|(rp, r)| (rp, DtraceLayerWrapper::new(r, &path.to_string())));
- probe_lazy!(opendal, blocking_read_end, c_path.as_ptr());
- result
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let c_path = CString::new(path).unwrap();
- probe_lazy!(opendal, blocking_write_start, c_path.as_ptr());
- let result = self
- .inner
- .blocking_write(path, args)
- .map(|(rp, r)| (rp, DtraceLayerWrapper::new(r, &path.to_string())));
- probe_lazy!(opendal, blocking_write_end, c_path.as_ptr());
- result
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- let c_path = CString::new(path).unwrap();
- probe_lazy!(opendal, blocking_stat_start, c_path.as_ptr());
- let result = self.inner.blocking_stat(path, args);
- probe_lazy!(opendal, blocking_stat_end, c_path.as_ptr());
- result
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner.blocking_delete()
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- let c_path = CString::new(path).unwrap();
- probe_lazy!(opendal, blocking_list_start, c_path.as_ptr());
- let result = self.inner.blocking_list(path, args);
- probe_lazy!(opendal, blocking_list_end, c_path.as_ptr());
- result
- }
}
pub struct DtraceLayerWrapper<R> {
@@ -329,26 +249,6 @@ impl<R: oio::Read> oio::Read for DtraceLayerWrapper<R> {
}
}
-impl<R: oio::BlockingRead> oio::BlockingRead for DtraceLayerWrapper<R> {
- fn read(&mut self) -> Result<Buffer> {
- let c_path = CString::new(self.path.clone()).unwrap();
- probe_lazy!(opendal, blocking_reader_read_start, c_path.as_ptr());
- self.inner
- .read()
- .inspect(|bs| {
- probe_lazy!(
- opendal,
- blocking_reader_read_ok,
- c_path.as_ptr(),
- bs.remaining()
- );
- })
- .inspect_err(|_| {
- probe_lazy!(opendal, blocking_reader_read_error, c_path.as_ptr());
- })
- }
-}
-
impl<R: oio::Write> oio::Write for DtraceLayerWrapper<R> {
async fn write(&mut self, bs: Buffer) -> Result<()> {
let c_path = CString::new(self.path.clone()).unwrap();
@@ -392,31 +292,3 @@ impl<R: oio::Write> oio::Write for DtraceLayerWrapper<R> {
})
}
}
-
-impl<R: oio::BlockingWrite> oio::BlockingWrite for DtraceLayerWrapper<R> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- let c_path = CString::new(self.path.clone()).unwrap();
- probe_lazy!(opendal, blocking_writer_write_start, c_path.as_ptr());
- self.inner
- .write(bs)
- .map(|_| {
- probe_lazy!(opendal, blocking_writer_write_ok, c_path.as_ptr());
- })
- .inspect_err(|_| {
- probe_lazy!(opendal, blocking_writer_write_error, c_path.as_ptr());
- })
- }
-
- fn close(&mut self) -> Result<Metadata> {
- let c_path = CString::new(self.path.clone()).unwrap();
- probe_lazy!(opendal, blocking_writer_close_start, c_path.as_ptr());
- self.inner
- .close()
- .inspect(|_| {
- probe_lazy!(opendal, blocking_writer_close_ok, c_path.as_ptr());
- })
- .inspect_err(|_| {
- probe_lazy!(opendal, blocking_writer_close_error, c_path.as_ptr());
- })
- }
-}
diff --git a/core/src/layers/error_context.rs b/core/src/layers/error_context.rs
index a820bd07a..f82a75152 100644
--- a/core/src/layers/error_context.rs
+++ b/core/src/layers/error_context.rs
@@ -66,13 +66,9 @@ impl<A: Access> Debug for ErrorContextAccessor<A> {
impl<A: Access> LayeredAccess for ErrorContextAccessor<A> {
type Inner = A;
type Reader = ErrorContextWrapper<A::Reader>;
- type BlockingReader = ErrorContextWrapper<A::BlockingReader>;
type Writer = ErrorContextWrapper<A::Writer>;
- type BlockingWriter = ErrorContextWrapper<A::BlockingWriter>;
type Lister = ErrorContextWrapper<A::Lister>;
- type BlockingLister = ErrorContextWrapper<A::BlockingLister>;
type Deleter = ErrorContextWrapper<A::Deleter>;
- type BlockingDeleter = ErrorContextWrapper<A::BlockingDeleter>;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -189,106 +185,6 @@ impl<A: Access> LayeredAccess for ErrorContextAccessor<A> {
.with_context("path", path)
})
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.inner.blocking_create_dir(path, args).map_err(|err| {
- err.with_operation(Operation::CreateDir)
- .with_context("service", self.info.scheme())
- .with_context("path", path)
- })
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let range = args.range();
- self.inner
- .blocking_read(path, args)
- .map(|(rp, os)| {
- (
- rp,
- ErrorContextWrapper::new(self.info.scheme(), path.to_string(), os)
- .with_range(range),
- )
- })
- .map_err(|err| {
- err.with_operation(Operation::Read)
- .with_context("service", self.info.scheme())
- .with_context("path", path)
- .with_context("range", range.to_string())
- })
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.inner
- .blocking_write(path, args)
- .map(|(rp, os)| {
- (
- rp,
- ErrorContextWrapper::new(self.info.scheme(), path.to_string(), os),
- )
- })
- .map_err(|err| {
- err.with_operation(Operation::Write)
- .with_context("service", self.info.scheme())
- .with_context("path", path)
- })
- }
-
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- self.inner.blocking_copy(from, to, args).map_err(|err| {
- err.with_operation(Operation::Copy)
- .with_context("service", self.info.scheme())
- .with_context("from", from)
- .with_context("to", to)
- })
- }
-
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- self.inner.blocking_rename(from, to, args).map_err(|err| {
- err.with_operation(Operation::Rename)
- .with_context("service", self.info.scheme())
- .with_context("from", from)
- .with_context("to", to)
- })
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.inner.blocking_stat(path, args).map_err(|err| {
- err.with_operation(Operation::Stat)
- .with_context("service", self.info.scheme())
- .with_context("path", path)
- })
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner
- .blocking_delete()
- .map(|(rp, w)| {
- (
- rp,
- ErrorContextWrapper::new(self.info.scheme(), "".to_string(), w),
- )
- })
- .map_err(|err| {
- err.with_operation(Operation::Delete)
- .with_context("service", self.info.scheme())
- })
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.inner
- .blocking_list(path, args)
- .map(|(rp, os)| {
- (
- rp,
- ErrorContextWrapper::new(self.info.scheme(), path.to_string(), os),
- )
- })
- .map_err(|err| {
- err.with_operation(Operation::List)
- .with_context("service", self.info.scheme())
- .with_context("path", path)
- })
- }
}
pub struct ErrorContextWrapper<T> {
@@ -334,23 +230,6 @@ impl<T: oio::Read> oio::Read for ErrorContextWrapper<T> {
}
}
-impl<T: oio::BlockingRead> oio::BlockingRead for ErrorContextWrapper<T> {
- fn read(&mut self) -> Result<Buffer> {
- self.inner
- .read()
- .inspect(|bs| {
- self.processed += bs.len() as u64;
- })
- .map_err(|err| {
- err.with_operation(Operation::Read)
- .with_context("service", self.scheme)
- .with_context("path", &self.path)
- .with_context("range", self.range.to_string())
- .with_context("read", self.processed.to_string())
- })
- }
-}
-
impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
async fn write(&mut self, bs: Buffer) -> Result<()> {
let size = bs.len();
@@ -388,33 +267,6 @@ impl<T: oio::Write> oio::Write for ErrorContextWrapper<T> {
}
}
-impl<T: oio::BlockingWrite> oio::BlockingWrite for ErrorContextWrapper<T> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- let size = bs.len();
- self.inner
- .write(bs)
- .map(|_| {
- self.processed += size as u64;
- })
- .map_err(|err| {
- err.with_operation(Operation::Write)
- .with_context("service", self.scheme)
- .with_context("path", &self.path)
- .with_context("size", size.to_string())
- .with_context("written", self.processed.to_string())
- })
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.inner.close().map_err(|err| {
- err.with_operation(Operation::Write)
- .with_context("service", self.scheme)
- .with_context("path", &self.path)
- .with_context("written", self.processed.to_string())
- })
- }
-}
-
impl<T: oio::List> oio::List for ErrorContextWrapper<T> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
self.inner
@@ -432,22 +284,6 @@ impl<T: oio::List> oio::List for ErrorContextWrapper<T> {
}
}
-impl<T: oio::BlockingList> oio::BlockingList for ErrorContextWrapper<T> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.inner
- .next()
- .inspect(|bs| {
- self.processed += bs.is_some() as u64;
- })
- .map_err(|err| {
- err.with_operation(Operation::List)
- .with_context("service", self.scheme)
- .with_context("path", &self.path)
- .with_context("listed", self.processed.to_string())
- })
- }
-}
-
impl<T: oio::Delete> oio::Delete for ErrorContextWrapper<T> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
self.inner.delete(path, args).map_err(|err| {
@@ -472,27 +308,3 @@ impl<T: oio::Delete> oio::Delete for ErrorContextWrapper<T> {
})
}
}
-
-impl<T: oio::BlockingDelete> oio::BlockingDelete for ErrorContextWrapper<T> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- self.inner.delete(path, args).map_err(|err| {
- err.with_operation(Operation::Delete)
- .with_context("service", self.scheme)
- .with_context("path", path)
- .with_context("deleted", self.processed.to_string())
- })
- }
-
- fn flush(&mut self) -> Result<usize> {
- self.inner
- .flush()
- .inspect(|&n| {
- self.processed += n as u64;
- })
- .map_err(|err| {
- err.with_operation(Operation::Delete)
- .with_context("service", self.scheme)
- .with_context("deleted", self.processed.to_string())
- })
- }
-}
diff --git a/core/src/layers/fastrace.rs b/core/src/layers/fastrace.rs
index f01f4d2bc..18dd50ca5 100644
--- a/core/src/layers/fastrace.rs
+++ b/core/src/layers/fastrace.rs
@@ -121,13 +121,9 @@ pub struct FastraceAccessor<A> {
impl<A: Access> LayeredAccess for FastraceAccessor<A> {
type Inner = A;
type Reader = FastraceWrapper<A::Reader>;
- type BlockingReader = FastraceWrapper<A::BlockingReader>;
type Writer = FastraceWrapper<A::Writer>;
- type BlockingWriter = FastraceWrapper<A::BlockingWriter>;
type Lister = FastraceWrapper<A::Lister>;
- type BlockingLister = FastraceWrapper<A::BlockingLister>;
type Deleter = FastraceWrapper<A::Deleter>;
- type BlockingDeleter = FastraceWrapper<A::BlockingDeleter>;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -214,78 +210,6 @@ impl<A: Access> LayeredAccess for FastraceAccessor<A> {
async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
self.inner.presign(path, args).await
}
-
- #[trace]
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.inner.blocking_create_dir(path, args)
- }
-
- #[trace]
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.inner.blocking_read(path, args).map(|(rp, r)| {
- (
- rp,
- FastraceWrapper::new(
- Span::enter_with_local_parent(Operation::Read.into_static()),
- r,
- ),
- )
- })
- }
-
- #[trace]
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.inner.blocking_write(path, args).map(|(rp, r)| {
- (
- rp,
- FastraceWrapper::new(
- Span::enter_with_local_parent(Operation::Write.into_static()),
- r,
- ),
- )
- })
- }
-
- #[trace]
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- self.inner().blocking_copy(from, to, args)
- }
-
- #[trace]
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- self.inner().blocking_rename(from, to, args)
- }
-
- #[trace]
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.inner.blocking_stat(path, args)
- }
-
- #[trace]
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner.blocking_delete().map(|(rp, r)| {
- (
- rp,
- FastraceWrapper::new(
- Span::enter_with_local_parent(Operation::Delete.into_static()),
- r,
- ),
- )
- })
- }
-
- #[trace]
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.inner.blocking_list(path, args).map(|(rp, it)| {
- (
- rp,
- FastraceWrapper::new(
- Span::enter_with_local_parent(Operation::List.into_static()),
- it,
- ),
- )
- })
- }
}
pub struct FastraceWrapper<R> {
@@ -306,14 +230,6 @@ impl<R: oio::Read> oio::Read for FastraceWrapper<R> {
}
}
-impl<R: oio::BlockingRead> oio::BlockingRead for FastraceWrapper<R> {
- fn read(&mut self) -> Result<Buffer> {
- let _g = self.span.set_local_parent();
- let _span = LocalSpan::enter_with_local_parent(Operation::Read.into_static());
- self.inner.read()
- }
-}
-
impl<R: oio::Write> oio::Write for FastraceWrapper<R> {
fn write(&mut self, bs: Buffer) -> impl Future<Output = Result<()>> + MaybeSend {
let _g = self.span.set_local_parent();
@@ -334,20 +250,6 @@ impl<R: oio::Write> oio::Write for FastraceWrapper<R> {
}
}
-impl<R: oio::BlockingWrite> oio::BlockingWrite for FastraceWrapper<R> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- let _g = self.span.set_local_parent();
- let _span = LocalSpan::enter_with_local_parent(Operation::Write.into_static());
- self.inner.write(bs)
- }
-
- fn close(&mut self) -> Result<Metadata> {
- let _g = self.span.set_local_parent();
- let _span = LocalSpan::enter_with_local_parent(Operation::Write.into_static());
- self.inner.close()
- }
-}
-
impl<R: oio::List> oio::List for FastraceWrapper<R> {
#[trace(enter_on_poll = true)]
async fn next(&mut self) -> Result<Option<oio::Entry>> {
@@ -355,14 +257,6 @@ impl<R: oio::List> oio::List for FastraceWrapper<R> {
}
}
-impl<R: oio::BlockingList> oio::BlockingList for FastraceWrapper<R> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- let _g = self.span.set_local_parent();
- let _span = LocalSpan::enter_with_local_parent(Operation::List.into_static());
- self.inner.next()
- }
-}
-
impl<R: oio::Delete> oio::Delete for FastraceWrapper<R> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
let _g = self.span.set_local_parent();
@@ -375,17 +269,3 @@ impl<R: oio::Delete> oio::Delete for FastraceWrapper<R> {
self.inner.flush().await
}
}
-
-impl<R: oio::BlockingDelete> oio::BlockingDelete for FastraceWrapper<R> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- let _g = self.span.set_local_parent();
- let _span = LocalSpan::enter_with_local_parent(Operation::Delete.into_static());
- self.inner.delete(path, args)
- }
-
- fn flush(&mut self) -> Result<usize> {
- let _g = self.span.set_local_parent();
- let _span = LocalSpan::enter_with_local_parent(Operation::Delete.into_static());
- self.inner.flush()
- }
-}
diff --git a/core/src/layers/immutable_index.rs b/core/src/layers/immutable_index.rs
index 44b6c96da..28f67ef81 100644
--- a/core/src/layers/immutable_index.rs
+++ b/core/src/layers/immutable_index.rs
@@ -147,10 +147,6 @@ impl<A: Access> LayeredAccess for ImmutableIndexAccessor<A> {
type Writer = A::Writer;
type Lister = ImmutableDir;
type Deleter = A::Deleter;
- type BlockingReader = A::BlockingReader;
- type BlockingWriter = A::BlockingWriter;
- type BlockingLister = ImmutableDir;
- type BlockingDeleter = A::BlockingDeleter;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -182,33 +178,6 @@ impl<A: Access> LayeredAccess for ImmutableIndexAccessor<A> {
async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
self.inner.delete().await
}
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.inner.blocking_read(path, args)
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.inner.blocking_write(path, args)
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- let mut path = path;
- if path == "/" {
- path = ""
- }
-
- let idx = if args.recursive() {
- self.children_flat(path)
- } else {
- self.children_hierarchy(path)
- };
-
- Ok((RpList::default(), ImmutableDir::new(idx)))
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner.blocking_delete()
- }
}
pub struct ImmutableDir {
@@ -241,12 +210,6 @@ impl oio::List for ImmutableDir {
}
}
-impl oio::BlockingList for ImmutableDir {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- Ok(self.inner_next())
- }
-}
-
#[cfg(test)]
#[cfg(feature = "services-http")]
mod tests {
diff --git a/core/src/layers/logging.rs b/core/src/layers/logging.rs
index c679b2e01..208d19ad2 100644
--- a/core/src/layers/logging.rs
+++ b/core/src/layers/logging.rs
@@ -245,13 +245,9 @@ static LOGGING_TARGET: &str = "opendal::services";
impl<A: Access, I: LoggingInterceptor> LayeredAccess for LoggingAccessor<A, I> {
type Inner = A;
type Reader = LoggingReader<A::Reader, I>;
- type BlockingReader = LoggingReader<A::BlockingReader, I>;
type Writer = LoggingWriter<A::Writer, I>;
- type BlockingWriter = LoggingWriter<A::BlockingWriter, I>;
type Lister = LoggingLister<A::Lister, I>;
- type BlockingLister = LoggingLister<A::BlockingLister, I>;
type Deleter = LoggingDeleter<A::Deleter, I>;
- type BlockingDeleter = LoggingDeleter<A::BlockingDeleter, I>;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -543,247 +539,6 @@ impl<A: Access, I: LoggingInterceptor> LayeredAccess for LoggingAccessor<A, I> {
);
})
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.logger.log(
- &self.info,
- Operation::CreateDir,
- &[("path", path)],
- "started",
- None,
- );
-
- self.inner
- .blocking_create_dir(path, args)
- .inspect(|_| {
- self.logger.log(
- &self.info,
- Operation::CreateDir,
- &[("path", path)],
- "finished",
- None,
- );
- })
- .inspect_err(|err| {
- self.logger.log(
- &self.info,
- Operation::CreateDir,
- &[("path", path)],
- "failed",
- Some(err),
- );
- })
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.logger.log(
- &self.info,
- Operation::Read,
- &[("path", path)],
- "started",
- None,
- );
-
- self.inner
- .blocking_read(path, args.clone())
- .map(|(rp, r)| {
- self.logger.log(
- &self.info,
- Operation::Read,
- &[("path", path)],
- "created reader",
- None,
- );
- let r = LoggingReader::new(self.info.clone(), self.logger.clone(), path, r);
- (rp, r)
- })
- .inspect_err(|err| {
- self.logger.log(
- &self.info,
- Operation::Read,
- &[("path", path)],
- "failed",
- Some(err),
- );
- })
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.logger.log(
- &self.info,
- Operation::Write,
- &[("path", path)],
- "started",
- None,
- );
-
- self.inner
- .blocking_write(path, args)
- .map(|(rp, w)| {
- self.logger.log(
- &self.info,
- Operation::Write,
- &[("path", path)],
- "created writer",
- None,
- );
- let w = LoggingWriter::new(self.info.clone(), self.logger.clone(), path, w);
- (rp, w)
- })
- .inspect_err(|err| {
- self.logger.log(
- &self.info,
- Operation::Write,
- &[("path", path)],
- "failed",
- Some(err),
- );
- })
- }
-
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- self.logger.log(
- &self.info,
- Operation::Copy,
- &[("from", from), ("to", to)],
- "started",
- None,
- );
-
- self.inner
- .blocking_copy(from, to, args)
- .inspect(|_| {
- self.logger.log(
- &self.info,
- Operation::Copy,
- &[("from", from), ("to", to)],
- "finished",
- None,
- );
- })
- .inspect_err(|err| {
- self.logger.log(
- &self.info,
- Operation::Copy,
- &[("from", from), ("to", to)],
- "",
- Some(err),
- );
- })
- }
-
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- self.logger.log(
- &self.info,
- Operation::Rename,
- &[("from", from), ("to", to)],
- "started",
- None,
- );
-
- self.inner
- .blocking_rename(from, to, args)
- .inspect(|_| {
- self.logger.log(
- &self.info,
- Operation::Rename,
- &[("from", from), ("to", to)],
- "finished",
- None,
- );
- })
- .inspect_err(|err| {
- self.logger.log(
- &self.info,
- Operation::Rename,
- &[("from", from), ("to", to)],
- "failed",
- Some(err),
- );
- })
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.logger.log(
- &self.info,
- Operation::Stat,
- &[("path", path)],
- "started",
- None,
- );
-
- self.inner
- .blocking_stat(path, args)
- .inspect(|_| {
- self.logger.log(
- &self.info,
- Operation::Stat,
- &[("path", path)],
- "finished",
- None,
- );
- })
- .inspect_err(|err| {
- self.logger.log(
- &self.info,
- Operation::Stat,
- &[("path", path)],
- "failed",
- Some(err),
- );
- })
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.logger
- .log(&self.info, Operation::Delete, &[], "started", None);
-
- self.inner
- .blocking_delete()
- .map(|(rp, d)| {
- self.logger
- .log(&self.info, Operation::Delete, &[], "finished", None);
- let d = LoggingDeleter::new(self.info.clone(), self.logger.clone(), d);
- (rp, d)
- })
- .inspect_err(|err| {
- self.logger
- .log(&self.info, Operation::Delete, &[], "failed", Some(err));
- })
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.logger.log(
- &self.info,
- Operation::List,
- &[("path", path)],
- "started",
- None,
- );
-
- self.inner
- .blocking_list(path, args)
- .map(|(rp, v)| {
- self.logger.log(
- &self.info,
- Operation::List,
- &[("path", path)],
- "created lister",
- None,
- );
- let li = LoggingLister::new(self.info.clone(), self.logger.clone(), path, v);
- (rp, li)
- })
- .inspect_err(|err| {
- self.logger.log(
- &self.info,
- Operation::List,
- &[("path", path)],
- "",
- Some(err),
- );
- })
- }
}
/// `LoggingReader` is a wrapper of `BytesReader`, with logging functionality.
@@ -853,50 +608,6 @@ impl<R: oio::Read, I: LoggingInterceptor> oio::Read for LoggingReader<R, I> {
}
}
-impl<R: oio::BlockingRead, I: LoggingInterceptor> oio::BlockingRead for LoggingReader<R, I> {
- fn read(&mut self) -> Result<Buffer> {
- self.logger.log(
- &self.info,
- Operation::Read,
- &[("path", &self.path), ("read", &self.read.to_string())],
- "started",
- None,
- );
-
- match self.inner.read() {
- Ok(bs) => {
- self.read += bs.len() as u64;
- self.logger.log(
- &self.info,
- Operation::Read,
- &[
- ("path", &self.path),
- ("read", &self.read.to_string()),
- ("size", &bs.len().to_string()),
- ],
- if bs.is_empty() {
- "finished"
- } else {
- "succeeded"
- },
- None,
- );
- Ok(bs)
- }
- Err(err) => {
- self.logger.log(
- &self.info,
- Operation::Read,
- &[("path", &self.path), ("read", &self.read.to_string())],
- "failed",
- Some(&err),
- );
- Err(err)
- }
- }
- }
-}
-
pub struct LoggingWriter<W, I> {
info: Arc<AccessorInfo>,
logger: I,
@@ -1035,88 +746,6 @@ impl<W: oio::Write, I: LoggingInterceptor> oio::Write for LoggingWriter<W, I> {
}
}
-impl<W: oio::BlockingWrite, I: LoggingInterceptor> oio::BlockingWrite for LoggingWriter<W, I> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- let size = bs.len();
-
- self.logger.log(
- &self.info,
- Operation::Write,
- &[
- ("path", &self.path),
- ("written", &self.written.to_string()),
- ("size", &size.to_string()),
- ],
- "started",
- None,
- );
-
- match self.inner.write(bs) {
- Ok(_) => {
- self.logger.log(
- &self.info,
- Operation::Write,
- &[
- ("path", &self.path),
- ("written", &self.written.to_string()),
- ("size", &size.to_string()),
- ],
- "succeeded",
- None,
- );
- Ok(())
- }
- Err(err) => {
- self.logger.log(
- &self.info,
- Operation::Write,
- &[
- ("path", &self.path),
- ("written", &self.written.to_string()),
- ("size", &size.to_string()),
- ],
- "failed",
- Some(&err),
- );
- Err(err)
- }
- }
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.logger.log(
- &self.info,
- Operation::Write,
- &[("path", &self.path), ("written", &self.written.to_string())],
- "started",
- None,
- );
-
- match self.inner.close() {
- Ok(meta) => {
- self.logger.log(
- &self.info,
- Operation::Write,
- &[("path", &self.path), ("written", &self.written.to_string())],
- "succeeded",
- None,
- );
- Ok(meta)
- }
- Err(err) => {
- self.logger.log(
- &self.info,
- Operation::Write,
- &[("path", &self.path), ("written", &self.written.to_string())],
- "failed",
- Some(&err),
- );
- Err(err)
- }
- }
- }
-}
-
pub struct LoggingLister<P, I: LoggingInterceptor> {
info: Arc<AccessorInfo>,
logger: I,
@@ -1190,56 +819,6 @@ impl<P: oio::List, I: LoggingInterceptor> oio::List for LoggingLister<P, I> {
}
}
-impl<P: oio::BlockingList, I: LoggingInterceptor> oio::BlockingList for LoggingLister<P, I> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.logger.log(
- &self.info,
- Operation::List,
- &[("path", &self.path), ("listed", &self.listed.to_string())],
- "started",
- None,
- );
-
- let res = self.inner.next();
- match &res {
- Ok(Some(de)) => {
- self.listed += 1;
- self.logger.log(
- &self.info,
- Operation::List,
- &[
- ("path", &self.path),
- ("listed", &self.listed.to_string()),
- ("entry", de.path()),
- ],
- "succeeded",
- None,
- );
- }
- Ok(None) => {
- self.logger.log(
- &self.info,
- Operation::List,
- &[("path", &self.path), ("listed", &self.listed.to_string())],
- "finished",
- None,
- );
- }
- Err(err) => {
- self.logger.log(
- &self.info,
- Operation::List,
- &[("path", &self.path), ("listed", &self.listed.to_string())],
- "failed",
- Some(err),
- );
- }
- };
-
- res
- }
-}
-
pub struct LoggingDeleter<D, I: LoggingInterceptor> {
info: Arc<AccessorInfo>,
logger: I,
@@ -1360,102 +939,3 @@ impl<D: oio::Delete, I: LoggingInterceptor> oio::Delete for LoggingDeleter<D, I>
res
}
}
-
-impl<D: oio::BlockingDelete, I: LoggingInterceptor> oio::BlockingDelete for LoggingDeleter<D, I> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- let version = args
- .version()
- .map(|v| v.to_string())
- .unwrap_or_else(|| "<latest>".to_string());
-
- self.logger.log(
- &self.info,
- Operation::Delete,
- &[("path", path), ("version", &version)],
- "started",
- None,
- );
-
- let res = self.inner.delete(path, args);
-
- match &res {
- Ok(_) => {
- self.queued += 1;
- self.logger.log(
- &self.info,
- Operation::Delete,
- &[
- ("path", path),
- ("version", &version),
- ("queued", &self.queued.to_string()),
- ("deleted", &self.deleted.to_string()),
- ],
- "succeeded",
- None,
- );
- }
- Err(err) => {
- self.logger.log(
- &self.info,
- Operation::Delete,
- &[
- ("path", path),
- ("version", &version),
- ("queued", &self.queued.to_string()),
- ("deleted", &self.deleted.to_string()),
- ],
- "failed",
- Some(err),
- );
- }
- };
-
- res
- }
-
- fn flush(&mut self) -> Result<usize> {
- self.logger.log(
- &self.info,
- Operation::Delete,
- &[
- ("queued", &self.queued.to_string()),
- ("deleted", &self.deleted.to_string()),
- ],
- "started",
- None,
- );
-
- let res = self.inner.flush();
-
- match &res {
- Ok(flushed) => {
- self.queued -= flushed;
- self.deleted += flushed;
- self.logger.log(
- &self.info,
- Operation::Delete,
- &[
- ("queued", &self.queued.to_string()),
- ("deleted", &self.deleted.to_string()),
- ],
- "succeeded",
- None,
- );
- }
- Err(err) => {
- self.logger.log(
- &self.info,
- Operation::Delete,
- &[
- ("queued", &self.queued.to_string()),
- ("deleted", &self.deleted.to_string()),
- ],
- "failed",
- Some(err),
- );
- }
- };
-
- res
- }
-}
diff --git a/core/src/layers/mime_guess.rs b/core/src/layers/mime_guess.rs
index 245aecf55..10f5762d5 100644
--- a/core/src/layers/mime_guess.rs
+++ b/core/src/layers/mime_guess.rs
@@ -109,10 +109,6 @@ impl<A: Access> LayeredAccess for MimeGuessAccessor<A> {
type Writer = A::Writer;
type Lister = A::Lister;
type Deleter = A::Deleter;
- type BlockingReader = A::BlockingReader;
- type BlockingWriter = A::BlockingWriter;
- type BlockingLister = A::BlockingLister;
- type BlockingDeleter = A::BlockingDeleter;
fn inner(&self) -> &Self::Inner {
&self.0
@@ -142,29 +138,6 @@ impl<A: Access> LayeredAccess for MimeGuessAccessor<A> {
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
self.inner().list(path, args).await
}
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.inner().blocking_read(path, args)
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.inner()
- .blocking_write(path, opwrite_with_mime(path, args))
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.inner()
- .blocking_stat(path, args)
- .map(|rp| rpstat_with_mime(path, rp))
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner().blocking_delete()
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.inner().blocking_list(path, args)
- }
}
#[cfg(test)]
@@ -222,38 +195,4 @@ mod tests {
assert_eq!(entries[1].content_type(), None);
assert_eq!(entries[2].content_type(), Some(CUSTOM));
}
-
- #[test]
- fn test_blocking() {
- let op = Operator::new(Memory::default())
- .unwrap()
- .layer(MimeGuessLayer::default())
- .finish()
- .blocking();
-
- op.write("test0.html", DATA).unwrap();
- assert_eq!(op.stat("test0.html").unwrap().content_type(), Some(HTML));
-
- op.write("test1.asdfghjkl", DATA).unwrap();
- assert_eq!(op.stat("test1.asdfghjkl").unwrap().content_type(), None);
-
- op.write_with("test2.html", DATA)
- .content_type(CUSTOM)
- .call()
- .unwrap();
- assert_eq!(op.stat("test2.html").unwrap().content_type(), Some(CUSTOM));
-
- let entries: Vec<Metadata> = op
- .lister_with("")
- .call()
- .unwrap()
- .map(|entry| {
- let op = op.clone();
- op.stat(entry.unwrap().path()).unwrap()
- })
- .collect();
- assert_eq!(entries[0].content_type(), Some(HTML));
- assert_eq!(entries[1].content_type(), None);
- assert_eq!(entries[2].content_type(), Some(CUSTOM));
- }
}
diff --git a/core/src/layers/mod.rs b/core/src/layers/mod.rs
index 0c7657c57..a4adf9161 100644
--- a/core/src/layers/mod.rs
+++ b/core/src/layers/mod.rs
@@ -39,11 +39,6 @@ pub use logging::LoggingLayer;
mod timeout;
pub use timeout::TimeoutLayer;
-#[cfg(feature = "layers-blocking")]
-mod blocking;
-#[cfg(feature = "layers-blocking")]
-pub use blocking::BlockingLayer;
-
#[cfg(feature = "layers-chaos")]
mod chaos;
#[cfg(feature = "layers-chaos")]
diff --git a/core/src/layers/observe/metrics.rs b/core/src/layers/observe/metrics.rs
index dc20b8f1d..20290338d 100644
--- a/core/src/layers/observe/metrics.rs
+++ b/core/src/layers/observe/metrics.rs
@@ -531,13 +531,9 @@ impl<A: Access, I: MetricsIntercept> Debug for MetricsAccessor<A, I> {
impl<A: Access, I: MetricsIntercept> LayeredAccess for MetricsAccessor<A, I> {
type Inner = A;
type Reader = MetricsWrapper<A::Reader, I>;
- type BlockingReader = MetricsWrapper<A::BlockingReader, I>;
type Writer = MetricsWrapper<A::Writer, I>;
- type BlockingWriter = MetricsWrapper<A::BlockingWriter, I>;
type Lister = MetricsWrapper<A::Lister, I>;
- type BlockingLister = MetricsWrapper<A::BlockingLister, I>;
type Deleter = MetricsWrapper<A::Deleter, I>;
- type BlockingDeleter = MetricsWrapper<A::BlockingDeleter, I>;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -786,215 +782,6 @@ impl<A: Access, I: MetricsIntercept> LayeredAccess for MetricsAccessor<A, I> {
.observe(labels, MetricValue::OperationExecuting(-1));
res
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- let labels = MetricLabels::new(self.info.clone(), Operation::CreateDir.into_static());
-
- let start = Instant::now();
-
- self.interceptor
- .observe(labels.clone(), MetricValue::OperationExecuting(1));
-
- let res = self
- .inner()
- .blocking_create_dir(path, args)
- .inspect(|_| {
- self.interceptor.observe(
- labels.clone(),
- MetricValue::OperationDurationSeconds(start.elapsed()),
- );
- })
- .inspect_err(|err| {
- self.interceptor.observe(
- labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- });
-
- self.interceptor
- .observe(labels, MetricValue::OperationExecuting(-1));
- res
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let labels = MetricLabels::new(self.info.clone(), Operation::Read.into_static());
-
- let start = Instant::now();
-
- self.interceptor
- .observe(labels.clone(), MetricValue::OperationExecuting(1));
-
- let (rp, reader) = self
- .inner
- .blocking_read(path, args)
- .inspect(|_| {
- self.interceptor.observe(
- labels.clone(),
- MetricValue::OperationTtfbSeconds(start.elapsed()),
- );
- })
- .inspect_err(|err| {
- self.interceptor.observe(
- labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- })?;
-
- Ok((
- rp,
- MetricsWrapper::new(reader, self.interceptor.clone(), labels, start),
- ))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let labels = MetricLabels::new(self.info.clone(), Operation::Write.into_static());
-
- let start = Instant::now();
-
- self.interceptor
- .observe(labels.clone(), MetricValue::OperationExecuting(1));
-
- let (rp, writer) = self.inner.blocking_write(path, args).inspect_err(|err| {
- self.interceptor.observe(
- labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- })?;
-
- Ok((
- rp,
- MetricsWrapper::new(writer, self.interceptor.clone(), labels, start),
- ))
- }
-
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- let labels = MetricLabels::new(self.info.clone(), Operation::Copy.into_static());
-
- let start = Instant::now();
-
- self.interceptor
- .observe(labels.clone(), MetricValue::OperationExecuting(1));
-
- let res = self
- .inner()
- .blocking_copy(from, to, args)
- .inspect(|_| {
- self.interceptor.observe(
- labels.clone(),
- MetricValue::OperationDurationSeconds(start.elapsed()),
- );
- })
- .inspect_err(|err| {
- self.interceptor.observe(
- labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- });
-
- self.interceptor
- .observe(labels, MetricValue::OperationExecuting(-1));
- res
- }
-
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- let labels = MetricLabels::new(self.info.clone(), Operation::Rename.into_static());
-
- let start = Instant::now();
-
- self.interceptor
- .observe(labels.clone(), MetricValue::OperationExecuting(1));
-
- let res = self
- .inner()
- .blocking_rename(from, to, args)
- .inspect(|_| {
- self.interceptor.observe(
- labels.clone(),
- MetricValue::OperationDurationSeconds(start.elapsed()),
- );
- })
- .inspect_err(|err| {
- self.interceptor.observe(
- labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- });
-
- self.interceptor
- .observe(labels, MetricValue::OperationExecuting(-1));
- res
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- let labels = MetricLabels::new(self.info.clone(), Operation::Stat.into_static());
-
- let start = Instant::now();
-
- self.interceptor
- .observe(labels.clone(), MetricValue::OperationExecuting(1));
-
- let res = self
- .inner()
- .blocking_stat(path, args)
- .inspect(|_| {
- self.interceptor.observe(
- labels.clone(),
- MetricValue::OperationDurationSeconds(start.elapsed()),
- );
- })
- .inspect_err(|err| {
- self.interceptor.observe(
- labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- });
-
- self.interceptor
- .observe(labels, MetricValue::OperationExecuting(-1));
- res
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- let labels = MetricLabels::new(self.info.clone(), Operation::Delete.into_static());
-
- let start = Instant::now();
-
- self.interceptor
- .observe(labels.clone(), MetricValue::OperationExecuting(1));
-
- let (rp, deleter) = self.inner.blocking_delete().inspect_err(|err| {
- self.interceptor.observe(
- labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- })?;
-
- Ok((
- rp,
- MetricsWrapper::new(deleter, self.interceptor.clone(), labels, start),
- ))
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- let labels = MetricLabels::new(self.info.clone(), Operation::List.into_static());
-
- let start = Instant::now();
-
- self.interceptor
- .observe(labels.clone(), MetricValue::OperationExecuting(1));
-
- let (rp, lister) = self.inner.blocking_list(path, args).inspect_err(|err| {
- self.interceptor.observe(
- labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- })?;
-
- Ok((
- rp,
- MetricsWrapper::new(lister, self.interceptor.clone(), labels, start),
- ))
- }
}
pub struct MetricsWrapper<R, I: MetricsIntercept> {
@@ -1069,22 +856,6 @@ impl<R: oio::Read, I: MetricsIntercept> oio::Read for MetricsWrapper<R, I> {
}
}
-impl<R: oio::BlockingRead, I: MetricsIntercept> oio::BlockingRead for MetricsWrapper<R, I> {
- fn read(&mut self) -> Result<Buffer> {
- self.inner
- .read()
- .inspect(|bs| {
- self.size += bs.len() as u64;
- })
- .inspect_err(|err| {
- self.interceptor.observe(
- self.labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- })
- }
-}
-
impl<R: oio::Write, I: MetricsIntercept> oio::Write for MetricsWrapper<R, I> {
async fn write(&mut self, bs: Buffer) -> Result<()> {
let size = bs.len();
@@ -1122,33 +893,6 @@ impl<R: oio::Write, I: MetricsIntercept> oio::Write for MetricsWrapper<R, I> {
}
}
-impl<R: oio::BlockingWrite, I: MetricsIntercept> oio::BlockingWrite for MetricsWrapper<R, I> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- let size = bs.len();
-
- self.inner
- .write(bs)
- .inspect(|_| {
- self.size += size as u64;
- })
- .inspect_err(|err| {
- self.interceptor.observe(
- self.labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- })
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.inner.close().inspect_err(|err| {
- self.interceptor.observe(
- self.labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- })
- }
-}
-
impl<R: oio::List, I: MetricsIntercept> oio::List for MetricsWrapper<R, I> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
self.inner
@@ -1166,22 +910,6 @@ impl<R: oio::List, I: MetricsIntercept> oio::List for MetricsWrapper<R, I> {
}
}
-impl<R: oio::BlockingList, I: MetricsIntercept> oio::BlockingList for MetricsWrapper<R, I> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.inner
- .next()
- .inspect(|_| {
- self.size += 1;
- })
- .inspect_err(|err| {
- self.interceptor.observe(
- self.labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- })
- }
-}
-
impl<R: oio::Delete, I: MetricsIntercept> oio::Delete for MetricsWrapper<R, I> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
self.inner
@@ -1206,28 +934,3 @@ impl<R: oio::Delete, I: MetricsIntercept> oio::Delete for MetricsWrapper<R, I> {
})
}
}
-
-impl<R: oio::BlockingDelete, I: MetricsIntercept> oio::BlockingDelete for MetricsWrapper<R, I> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- self.inner
- .delete(path, args)
- .inspect(|_| {
- self.size += 1;
- })
- .inspect_err(|err| {
- self.interceptor.observe(
- self.labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- })
- }
-
- fn flush(&mut self) -> Result<usize> {
- self.inner.flush().inspect_err(|err| {
- self.interceptor.observe(
- self.labels.clone().with_error(err.kind()),
- MetricValue::OperationErrorsTotal,
- );
- })
- }
-}
diff --git a/core/src/layers/oteltrace.rs b/core/src/layers/oteltrace.rs
index 1dd15c0ef..4b41ebece 100644
--- a/core/src/layers/oteltrace.rs
+++ b/core/src/layers/oteltrace.rs
@@ -67,13 +67,9 @@ pub struct OtelTraceAccessor<A> {
impl<A: Access> LayeredAccess for OtelTraceAccessor<A> {
type Inner = A;
type Reader = OtelTraceWrapper<A::Reader>;
- type BlockingReader = OtelTraceWrapper<A::BlockingReader>;
type Writer = OtelTraceWrapper<A::Writer>;
- type BlockingWriter = OtelTraceWrapper<A::BlockingWriter>;
type Lister = OtelTraceWrapper<A::Lister>;
- type BlockingLister = OtelTraceWrapper<A::BlockingLister>;
type Deleter = A::Deleter;
- type BlockingDeleter = A::BlockingDeleter;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -167,82 +163,6 @@ impl<A: Access> LayeredAccess for OtelTraceAccessor<A> {
let cx = TraceContext::current_with_span(span);
self.inner().presign(path, args).with_context(cx).await
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- let tracer = global::tracer("opendal");
- tracer.in_span("blocking_create_dir", |cx| {
- let span = cx.span(); // let mut span = cx.();
- span.set_attribute(KeyValue::new("path", path.to_string()));
- span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- self.inner().blocking_create_dir(path, args)
- })
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let tracer = global::tracer("opendal");
- let mut span = tracer.start("blocking_read");
- span.set_attribute(KeyValue::new("path", path.to_string()));
- span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- self.inner
- .blocking_read(path, args)
- .map(|(rp, r)| (rp, OtelTraceWrapper::new(span, r)))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let tracer = global::tracer("opendal");
- let mut span = tracer.start("blocking_write");
- span.set_attribute(KeyValue::new("path", path.to_string()));
- span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- self.inner
- .blocking_write(path, args)
- .map(|(rp, r)| (rp, OtelTraceWrapper::new(span, r)))
- }
-
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- let tracer = global::tracer("opendal");
- tracer.in_span("blocking_copy", |cx| {
- let span = cx.span();
- span.set_attribute(KeyValue::new("from", from.to_string()));
- span.set_attribute(KeyValue::new("to", to.to_string()));
- span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- self.inner().blocking_copy(from, to, args)
- })
- }
-
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- let tracer = global::tracer("opendal");
- tracer.in_span("blocking_rename", |cx| {
- let span = cx.span();
- span.set_attribute(KeyValue::new("from", from.to_string()));
- span.set_attribute(KeyValue::new("to", to.to_string()));
- span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- self.inner().blocking_rename(from, to, args)
- })
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- let tracer = global::tracer("opendal");
- tracer.in_span("blocking_stat", |cx| {
- let span = cx.span();
- span.set_attribute(KeyValue::new("path", path.to_string()));
- span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- self.inner().blocking_stat(path, args)
- })
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner().blocking_delete()
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- let tracer = global::tracer("opendal");
- let mut span = tracer.start("blocking_list");
- span.set_attribute(KeyValue::new("path", path.to_string()));
- span.set_attribute(KeyValue::new("args", format!("{:?}", args)));
- self.inner
- .blocking_list(path, args)
- .map(|(rp, it)| (rp, OtelTraceWrapper::new(span, it)))
- }
}
pub struct OtelTraceWrapper<R> {
@@ -262,12 +182,6 @@ impl<R: oio::Read> oio::Read for OtelTraceWrapper<R> {
}
}
-impl<R: oio::BlockingRead> oio::BlockingRead for OtelTraceWrapper<R> {
- fn read(&mut self) -> Result<Buffer> {
- self.inner.read()
- }
-}
-
impl<R: oio::Write> oio::Write for OtelTraceWrapper<R> {
fn write(&mut self, bs: Buffer) -> impl Future<Output = Result<()>> + MaybeSend {
self.inner.write(bs)
@@ -282,24 +196,8 @@ impl<R: oio::Write> oio::Write for OtelTraceWrapper<R> {
}
}
-impl<R: oio::BlockingWrite> oio::BlockingWrite for OtelTraceWrapper<R> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- self.inner.write(bs)
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.inner.close()
- }
-}
-
impl<R: oio::List> oio::List for OtelTraceWrapper<R> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
self.inner.next().await
}
}
-
-impl<R: oio::BlockingList> oio::BlockingList for OtelTraceWrapper<R> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- self.inner.next()
- }
-}
diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs
index b0ce9742e..fd64c4079 100644
--- a/core/src/layers/retry.rs
+++ b/core/src/layers/retry.rs
@@ -293,13 +293,9 @@ impl<A: Access, I: RetryInterceptor> Debug for RetryAccessor<A, I> {
impl<A: Access, I: RetryInterceptor> LayeredAccess for RetryAccessor<A, I> {
type Inner = A;
type Reader = RetryWrapper<RetryReader<A, A::Reader>, I>;
- type BlockingReader = RetryWrapper<RetryReader<A, A::BlockingReader>, I>;
type Writer = RetryWrapper<A::Writer, I>;
- type BlockingWriter = RetryWrapper<A::BlockingWriter, I>;
type Lister = RetryWrapper<A::Lister, I>;
- type BlockingLister = RetryWrapper<A::BlockingLister, I>;
type Deleter = RetryWrapper<A::Deleter, I>;
- type BlockingDeleter = RetryWrapper<A::BlockingDeleter, I>;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -384,89 +380,6 @@ impl<A: Access, I: RetryInterceptor> LayeredAccess for RetryAccessor<A, I> {
.map(|(rp, r)| (rp, RetryWrapper::new(r, self.notify.clone(), self.builder)))
.map_err(|e| e.set_persistent())
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- { || self.inner.blocking_create_dir(path, args.clone()) }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| self.notify.intercept(err, dur))
- .call()
- .map_err(|e| e.set_persistent())
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let (rp, reader) = { || self.inner.blocking_read(path, args.clone()) }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| self.notify.intercept(err, dur))
- .call()
- .map_err(|e| e.set_persistent())?;
-
- let retry_reader = RetryReader::new(self.inner.clone(), path.to_string(), args, reader);
- let retry_wrapper = RetryWrapper::new(retry_reader, self.notify.clone(), self.builder);
-
- Ok((rp, retry_wrapper))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- { || self.inner.blocking_write(path, args.clone()) }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| self.notify.intercept(err, dur))
- .call()
- .map(|(rp, r)| (rp, RetryWrapper::new(r, self.notify.clone(), self.builder)))
- .map_err(|e| e.set_persistent())
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- { || self.inner.blocking_stat(path, args.clone()) }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| self.notify.intercept(err, dur))
- .call()
- .map_err(|e| e.set_persistent())
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- { || self.inner.blocking_delete() }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| self.notify.intercept(err, dur))
- .call()
- .map(|(rp, r)| (rp, RetryWrapper::new(r, self.notify.clone(), self.builder)))
- .map_err(|e| e.set_persistent())
- }
-
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- { || self.inner.blocking_copy(from, to, args.clone()) }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| self.notify.intercept(err, dur))
- .call()
- .map_err(|e| e.set_persistent())
- }
-
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- { || self.inner.blocking_rename(from, to, args.clone()) }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| self.notify.intercept(err, dur))
- .call()
- .map_err(|e| e.set_persistent())
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- { || self.inner.blocking_list(path, args.clone()) }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| self.notify.intercept(err, dur))
- .call()
- .map(|(rp, p)| {
- let p = RetryWrapper::new(p, self.notify.clone(), self.builder);
- (rp, p)
- })
- .map_err(|e| e.set_persistent())
- }
}
pub struct RetryReader<A, R> {
@@ -509,26 +422,6 @@ impl<A: Access> oio::Read for RetryReader<A, A::Reader> {
}
}
-impl<A: Access> oio::BlockingRead for RetryReader<A, A::BlockingReader> {
- fn read(&mut self) -> Result<Buffer> {
- loop {
- match self.reader.take() {
- None => {
- let (_, r) = self.inner.blocking_read(&self.path, self.args.clone())?;
- self.reader = Some(r);
- continue;
- }
- Some(mut reader) => {
- let buf = reader.read()?;
- self.reader = Some(reader);
- self.args.range_mut().advance(buf.len() as u64);
- return Ok(buf);
- }
- }
- }
- }
-}
-
pub struct RetryWrapper<R, I> {
inner: Option<R>,
notify: Arc<I>,
@@ -579,30 +472,6 @@ impl<R: oio::Read, I: RetryInterceptor> oio::Read for RetryWrapper<R, I> {
}
}
-impl<R: oio::BlockingRead, I: RetryInterceptor> oio::BlockingRead for RetryWrapper<R, I> {
- fn read(&mut self) -> Result<Buffer> {
- use backon::BlockingRetryableWithContext;
-
- let inner = self.take_inner()?;
-
- let (inner, res) = {
- |mut r: R| {
- let res = r.read();
-
- (r, res)
- }
- }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .context(inner)
- .notify(|err, dur| self.notify.intercept(err, dur))
- .call();
-
- self.inner = Some(inner);
- res.map_err(|err| err.set_persistent())
- }
-}
-
impl<R: oio::Write, I: RetryInterceptor> oio::Write for RetryWrapper<R, I> {
async fn write(&mut self, bs: Buffer) -> Result<()> {
use backon::RetryableWithContext;
@@ -671,30 +540,6 @@ impl<R: oio::Write, I: RetryInterceptor> oio::Write for RetryWrapper<R, I> {
}
}
-impl<R: oio::BlockingWrite, I: RetryInterceptor> oio::BlockingWrite for RetryWrapper<R, I> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- { || self.inner.as_mut().unwrap().write(bs.clone()) }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| {
- self.notify.intercept(err, dur);
- })
- .call()
- .map_err(|e| e.set_persistent())
- }
-
- fn close(&mut self) -> Result<Metadata> {
- { || self.inner.as_mut().unwrap().close() }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| {
- self.notify.intercept(err, dur);
- })
- .call()
- .map_err(|e| e.set_persistent())
- }
-}
-
impl<P: oio::List, I: RetryInterceptor> oio::List for RetryWrapper<P, I> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
use backon::RetryableWithContext;
@@ -719,19 +564,6 @@ impl<P: oio::List, I: RetryInterceptor> oio::List for RetryWrapper<P, I> {
}
}
-impl<P: oio::BlockingList, I: RetryInterceptor> oio::BlockingList for RetryWrapper<P, I> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- { || self.inner.as_mut().unwrap().next() }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| {
- self.notify.intercept(err, dur);
- })
- .call()
- .map_err(|e| e.set_persistent())
- }
-}
-
impl<P: oio::Delete, I: RetryInterceptor> oio::Delete for RetryWrapper<P, I> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
{ || self.inner.as_mut().unwrap().delete(path, args.clone()) }
@@ -767,30 +599,6 @@ impl<P: oio::Delete, I: RetryInterceptor> oio::Delete for RetryWrapper<P, I> {
}
}
-impl<P: oio::BlockingDelete, I: RetryInterceptor> oio::BlockingDelete for RetryWrapper<P, I> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- { || self.inner.as_mut().unwrap().delete(path, args.clone()) }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| {
- self.notify.intercept(err, dur);
- })
- .call()
- .map_err(|e| e.set_persistent())
- }
-
- fn flush(&mut self) -> Result<usize> {
- { || self.inner.as_mut().unwrap().flush() }
- .retry(self.builder)
- .when(|e| e.is_temporary())
- .notify(|err, dur| {
- self.notify.intercept(err, dur);
- })
- .call()
- .map_err(|e| e.set_persistent())
- }
-}
-
#[cfg(test)]
mod tests {
use std::mem;
@@ -830,10 +638,6 @@ mod tests {
type Writer = MockWriter;
type Lister = MockLister;
type Deleter = MockDeleter;
- type BlockingReader = ();
- type BlockingWriter = ();
- type BlockingLister = ();
- type BlockingDeleter = ();
fn info(&self) -> Arc<AccessorInfo> {
let am = AccessorInfo::default();
diff --git a/core/src/layers/throttle.rs b/core/src/layers/throttle.rs
index 4a5e674e1..7d90a2778 100644
--- a/core/src/layers/throttle.rs
+++ b/core/src/layers/throttle.rs
@@ -17,7 +17,6 @@
use std::num::NonZeroU32;
use std::sync::Arc;
-use std::thread;
use governor::clock::Clock;
use governor::clock::DefaultClock;
@@ -117,10 +116,6 @@ impl<A: Access> LayeredAccess for ThrottleAccessor<A> {
type Writer = ThrottleWrapper<A::Writer>;
type Lister = A::Lister;
type Deleter = A::Deleter;
- type BlockingReader = ThrottleWrapper<A::BlockingReader>;
- type BlockingWriter = ThrottleWrapper<A::BlockingWriter>;
- type BlockingLister = A::BlockingLister;
- type BlockingDeleter = A::BlockingDeleter;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -151,30 +146,6 @@ impl<A: Access> LayeredAccess for ThrottleAccessor<A> {
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
self.inner.list(path, args).await
}
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let limiter = self.rate_limiter.clone();
-
- self.inner
- .blocking_read(path, args)
- .map(|(rp, r)| (rp, ThrottleWrapper::new(r, limiter)))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let limiter = self.rate_limiter.clone();
-
- self.inner
- .blocking_write(path, args)
- .map(|(rp, w)| (rp, ThrottleWrapper::new(w, limiter)))
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner.blocking_delete()
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.inner.blocking_list(path, args)
- }
}
pub struct ThrottleWrapper<R> {
@@ -197,12 +168,6 @@ impl<R: oio::Read> oio::Read for ThrottleWrapper<R> {
}
}
-impl<R: oio::BlockingRead> oio::BlockingRead for ThrottleWrapper<R> {
- fn read(&mut self) -> Result<Buffer> {
- self.inner.read()
- }
-}
-
impl<R: oio::Write> oio::Write for ThrottleWrapper<R> {
async fn write(&mut self, bs: Buffer) -> Result<()> {
let buf_length = NonZeroU32::new(bs.len() as u32).unwrap();
@@ -237,31 +202,3 @@ impl<R: oio::Write> oio::Write for ThrottleWrapper<R> {
self.inner.close().await
}
}
-
-impl<R: oio::BlockingWrite> oio::BlockingWrite for ThrottleWrapper<R> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- let buf_length = NonZeroU32::new(bs.len() as u32).unwrap();
-
- loop {
- match self.limiter.check_n(buf_length) {
- Ok(res) => match res {
- Ok(_) => return self.inner.write(bs),
- // the query is valid but the Decider can not accommodate them.
- Err(not_until) => {
- let wait_time = not_until.wait_time_from(DefaultClock::default().now());
- thread::sleep(wait_time);
- }
- },
- // the query was invalid as the rate limit parameters can "never" accommodate the number of cells queried for.
- Err(_) => return Err(Error::new(
- ErrorKind::RateLimited,
- "InsufficientCapacity due to burst size being smaller than the request size",
- )),
- }
- }
- }
-
- fn close(&mut self) -> Result<Metadata> {
- self.inner.close()
- }
-}
diff --git a/core/src/layers/timeout.rs b/core/src/layers/timeout.rs
index b89e13912..932794a94 100644
--- a/core/src/layers/timeout.rs
+++ b/core/src/layers/timeout.rs
@@ -220,13 +220,9 @@ impl<A: Access> TimeoutAccessor<A> {
impl<A: Access> LayeredAccess for TimeoutAccessor<A> {
type Inner = A;
type Reader = TimeoutWrapper<A::Reader>;
- type BlockingReader = A::BlockingReader;
type Writer = TimeoutWrapper<A::Writer>;
- type BlockingWriter = A::BlockingWriter;
type Lister = TimeoutWrapper<A::Lister>;
- type BlockingLister = A::BlockingLister;
type Deleter = TimeoutWrapper<A::Deleter>;
- type BlockingDeleter = A::BlockingDeleter;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -280,22 +276,6 @@ impl<A: Access> LayeredAccess for TimeoutAccessor<A> {
self.timeout(Operation::Presign, self.inner.presign(path, args))
.await
}
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.inner.blocking_read(path, args)
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.inner.blocking_write(path, args)
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.inner.blocking_list(path, args)
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner.blocking_delete()
- }
}
pub struct TimeoutExecutor {
@@ -410,11 +390,7 @@ mod tests {
type Reader = MockReader;
type Writer = ();
type Lister = MockLister;
- type BlockingReader = ();
- type BlockingWriter = ();
- type BlockingLister = ();
type Deleter = ();
- type BlockingDeleter = ();
fn info(&self) -> Arc<AccessorInfo> {
let am = AccessorInfo::default();
diff --git a/core/src/layers/tracing.rs b/core/src/layers/tracing.rs
index 5084a09dc..a432a1ab4 100644
--- a/core/src/layers/tracing.rs
+++ b/core/src/layers/tracing.rs
@@ -203,10 +203,6 @@ impl<A: Access> LayeredAccess for TracingAccessor<A> {
type Writer = TracingWrapper<A::Writer>;
type Lister = TracingWrapper<A::Lister>;
type Deleter = TracingWrapper<A::Deleter>;
- type BlockingReader = TracingWrapper<A::BlockingReader>;
- type BlockingWriter = TracingWrapper<A::BlockingWriter>;
- type BlockingLister = TracingWrapper<A::BlockingLister>;
- type BlockingDeleter = TracingWrapper<A::BlockingDeleter>;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -280,70 +276,6 @@ impl<A: Access> LayeredAccess for TracingAccessor<A> {
async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
self.inner.presign(path, args).await
}
-
- #[tracing::instrument(level = "debug", skip(self))]
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.inner.blocking_create_dir(path, args)
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let span = span!(Level::DEBUG, "read", path, ?args);
-
- let (rp, r) = {
- let _enter = span.enter();
- self.inner.blocking_read(path, args)?
- };
-
- Ok((rp, TracingWrapper::new(span, r)))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let span = span!(Level::DEBUG, "write", path, ?args);
-
- let (rp, r) = {
- let _enter = span.enter();
- self.inner.blocking_write(path, args)?
- };
-
- Ok((rp, TracingWrapper::new(span, r)))
- }
-
- #[tracing::instrument(level = "debug", skip(self))]
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- self.inner().blocking_copy(from, to, args)
- }
-
- #[tracing::instrument(level = "debug", skip(self))]
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- self.inner().blocking_rename(from, to, args)
- }
-
- #[tracing::instrument(level = "debug", skip(self))]
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.inner.blocking_stat(path, args)
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- let span = span!(Level::DEBUG, "delete");
-
- let (rp, r) = {
- let _enter = span.enter();
- self.inner.blocking_delete()?
- };
-
- Ok((rp, TracingWrapper::new(span, r)))
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- let span = span!(Level::DEBUG, "list", path, ?args);
-
- let (rp, r) = {
- let _enter = span.enter();
- self.inner.blocking_list(path, args)?
- };
-
- Ok((rp, TracingWrapper::new(span, r)))
- }
}
pub struct TracingWrapper<R> {
@@ -365,14 +297,6 @@ impl<R: oio::Read> oio::Read for TracingWrapper<R> {
}
}
-impl<R: oio::BlockingRead> oio::BlockingRead for TracingWrapper<R> {
- fn read(&mut self) -> Result<Buffer> {
- let _enter = self.span.enter();
-
- self.inner.read()
- }
-}
-
impl<R: oio::Write> oio::Write for TracingWrapper<R> {
async fn write(&mut self, bs: Buffer) -> Result<()> {
let _enter = self.span.enter();
@@ -393,20 +317,6 @@ impl<R: oio::Write> oio::Write for TracingWrapper<R> {
}
}
-impl<R: oio::BlockingWrite> oio::BlockingWrite for TracingWrapper<R> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- let _enter = self.span.enter();
-
- self.inner.write(bs)
- }
-
- fn close(&mut self) -> Result<Metadata> {
- let _enter = self.span.enter();
-
- self.inner.close()
- }
-}
-
impl<R: oio::List> oio::List for TracingWrapper<R> {
async fn next(&mut self) -> Result<Option<oio::Entry>> {
let _enter = self.span.enter();
@@ -415,14 +325,6 @@ impl<R: oio::List> oio::List for TracingWrapper<R> {
}
}
-impl<R: oio::BlockingList> oio::BlockingList for TracingWrapper<R> {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- let _enter = self.span.enter();
-
- self.inner.next()
- }
-}
-
impl<R: oio::Delete> oio::Delete for TracingWrapper<R> {
fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
let _enter = self.span.enter();
@@ -436,17 +338,3 @@ impl<R: oio::Delete> oio::Delete for TracingWrapper<R> {
self.inner.flush().await
}
}
-
-impl<R: oio::BlockingDelete> oio::BlockingDelete for TracingWrapper<R> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- let _enter = self.span.enter();
-
- self.inner.delete(path, args)
- }
-
- fn flush(&mut self) -> Result<usize> {
- let _enter = self.span.enter();
-
- self.inner.flush()
- }
-}
diff --git a/core/src/layers/type_eraser.rs b/core/src/layers/type_eraser.rs
index cfe161aae..4fe7547ff 100644
--- a/core/src/layers/type_eraser.rs
+++ b/core/src/layers/type_eraser.rs
@@ -55,10 +55,6 @@ impl<A: Access> LayeredAccess for TypeEraseAccessor<A> {
type Writer = oio::Writer;
type Lister = oio::Lister;
type Deleter = oio::Deleter;
- type BlockingReader = oio::BlockingReader;
- type BlockingWriter = oio::BlockingWriter;
- type BlockingLister = oio::BlockingLister;
- type BlockingDeleter = oio::BlockingDeleter;
fn inner(&self) -> &Self::Inner {
&self.inner
@@ -91,28 +87,4 @@ impl<A: Access> LayeredAccess for TypeEraseAccessor<A> {
.await
.map(|(rp, p)| (rp, Box::new(p) as oio::Lister))
}
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.inner
- .blocking_read(path, args)
- .map(|(rp, r)| (rp, Box::new(r) as oio::BlockingReader))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.inner
- .blocking_write(path, args)
- .map(|(rp, w)| (rp, Box::new(w) as oio::BlockingWriter))
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.inner
- .blocking_delete()
- .map(|(rp, p)| (rp, Box::new(p) as oio::BlockingDeleter))
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.inner
- .blocking_list(path, args)
- .map(|(rp, p)| (rp, Box::new(p) as oio::BlockingLister))
- }
}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 62bf1529b..e3901e2a5 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -26,7 +26,7 @@
//!
//! # Quick Start
//!
-//! OpenDAL's API entry points are [`Operator`] and [`BlockingOperator`]. All
+//! OpenDAL's API entry points are [`Operator`] and [`blocking::Operator`]. All
//! public APIs are accessible through the operator. To utilize OpenDAL, you
//! need to:
//!
@@ -88,7 +88,7 @@
//! ## Use operator
//!
//! The final step is to use the operator. OpenDAL supports both async [`Operator`]
-//! and blocking [`BlockingOperator`]. Please pick the one that fits your use case.
+//! and blocking [`blocking::Operator`]. Please pick the one that fits your use case.
//!
//! Every Operator API follows the same pattern, take `read` as an example:
//!
@@ -139,6 +139,8 @@ mod types;
pub use types::*;
// Public modules, they will be accessed like `opendal::layers::Xxxx`
+#[cfg(feature = "blocking")]
+pub mod blocking;
#[cfg(docsrs)]
pub mod docs;
pub mod layers;
@@ -172,10 +174,6 @@ mod tests {
impl AssertSendSync for Writer {}
impl AssertSendSync for Lister {}
impl AssertSendSync for Operator {}
- impl AssertSendSync for BlockingReader {}
- impl AssertSendSync for BlockingWriter {}
- impl AssertSendSync for BlockingLister {}
- impl AssertSendSync for BlockingOperator {}
/// This is used to make sure our public API implement Send + Sync
#[test]
diff --git a/core/src/raw/accessor.rs b/core/src/raw/accessor.rs
index 25eca2a7a..03928ac9d 100644
--- a/core/src/raw/accessor.rs
+++ b/core/src/raw/accessor.rs
@@ -67,15 +67,6 @@ pub trait Access: Send + Sync + Debug + Unpin + 'static {
/// Deleter is the associated deleter returned in `delete` operation.
type Deleter: oio::Delete;
- /// BlockingReader is the associated reader returned `blocking_read` operation.
- type BlockingReader: oio::BlockingRead;
- /// BlockingWriter is the associated writer returned `blocking_write` operation.
- type BlockingWriter: oio::BlockingWrite;
- /// BlockingLister is the associated lister returned `blocking_list` operation.
- type BlockingLister: oio::BlockingList;
- /// BlockingDeleter is the associated deleter returned `blocking_delete` operation.
- type BlockingDeleter: oio::BlockingDelete;
-
/// Invoke the `info` operation to get metadata of accessor.
///
/// # Notes
@@ -266,120 +257,6 @@ pub trait Access: Send + Sync + Debug + Unpin + 'static {
"operation is not supported",
)))
}
-
- /// Invoke the `blocking_create` operation on the specified path.
- ///
- /// This operation is the blocking version of [`Accessor::create_dir`]
- ///
- /// Require [`Capability::create_dir`] and [`Capability::blocking`]
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- let (_, _) = (path, args);
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "operation is not supported",
- ))
- }
-
- /// Invoke the `blocking_stat` operation on the specified path.
- ///
- /// This operation is the blocking version of [`Accessor::stat`]
- ///
- /// Require [`Capability::stat`] and [`Capability::blocking`]
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- let (_, _) = (path, args);
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "operation is not supported",
- ))
- }
-
- /// Invoke the `blocking_read` operation on the specified path.
- ///
- /// This operation is the blocking version of [`Accessor::read`]
- ///
- /// Require [`Capability::read`] and [`Capability::blocking`]
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let (_, _) = (path, args);
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "operation is not supported",
- ))
- }
-
- /// Invoke the `blocking_write` operation on the specified path.
- ///
- /// This operation is the blocking version of [`Accessor::write`]
- ///
- /// Require [`Capability::write`] and [`Capability::blocking`]
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let (_, _) = (path, args);
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "operation is not supported",
- ))
- }
-
- /// Invoke the `blocking_delete` operation on the specified path.
- ///
- /// This operation is the blocking version of [`Accessor::delete`]
- ///
- /// Require [`Capability::write`] and [`Capability::blocking`]
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- Err(Error::new(
- ErrorKind::Unsupported,
- "operation is not supported",
- ))
- }
-
- /// Invoke the `blocking_list` operation on the specified path.
- ///
- /// This operation is the blocking version of [`Accessor::list`]
- ///
- /// Require [`Capability::list`] and [`Capability::blocking`]
- ///
- /// # Behavior
- ///
- /// - List non-exist dir should return Empty.
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- let (_, _) = (path, args);
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "operation is not supported",
- ))
- }
-
- /// Invoke the `blocking_copy` operation on the specified `from` path and `to` path.
- ///
- /// This operation is the blocking version of [`Accessor::copy`]
- ///
- /// Require [`Capability::copy`] and [`Capability::blocking`]
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- let (_, _, _) = (from, to, args);
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "operation is not supported",
- ))
- }
-
- /// Invoke the `blocking_rename` operation on the specified `from` path and `to` path.
- ///
- /// This operation is the blocking version of [`Accessor::rename`]
- ///
- /// Require [`Capability::rename`] and [`Capability::blocking`]
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- let (_, _, _) = (from, to, args);
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "operation is not supported",
- ))
- }
}
/// `AccessDyn` is the dyn version of [`Access`] make it possible to use as
@@ -435,39 +312,15 @@ pub trait AccessDyn: Send + Sync + Debug + Unpin {
path: &'a str,
args: OpPresign,
) -> BoxedFuture<'a, Result<RpPresign>>;
- /// Dyn version of [`Accessor::blocking_create_dir`]
- fn blocking_create_dir_dyn(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir>;
- /// Dyn version of [`Accessor::blocking_stat`]
- fn blocking_stat_dyn(&self, path: &str, args: OpStat) -> Result<RpStat>;
- /// Dyn version of [`Accessor::blocking_read`]
- fn blocking_read_dyn(&self, path: &str, args: OpRead) -> Result<(RpRead, oio::BlockingReader)>;
- /// Dyn version of [`Accessor::blocking_write`]
- fn blocking_write_dyn(
- &self,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, oio::BlockingWriter)>;
- /// Dyn version of [`Accessor::blocking_delete`]
- fn blocking_delete_dyn(&self) -> Result<(RpDelete, oio::BlockingDeleter)>;
- /// Dyn version of [`Accessor::blocking_list`]
- fn blocking_list_dyn(&self, path: &str, args: OpList) -> Result<(RpList, oio::BlockingLister)>;
- /// Dyn version of [`Accessor::blocking_copy`]
- fn blocking_copy_dyn(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy>;
- /// Dyn version of [`Accessor::blocking_rename`]
- fn blocking_rename_dyn(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename>;
}
impl<A: ?Sized> AccessDyn for A
where
A: Access<
Reader = oio::Reader,
- BlockingReader = oio::BlockingReader,
Writer = oio::Writer,
- BlockingWriter = oio::BlockingWriter,
Lister = oio::Lister,
- BlockingLister = oio::BlockingLister,
Deleter = oio::Deleter,
- BlockingDeleter = oio::BlockingDeleter,
>,
{
fn info_dyn(&self) -> Arc<AccessorInfo> {
@@ -539,53 +392,13 @@ where
) -> BoxedFuture<'a, Result<RpPresign>> {
Box::pin(self.presign(path, args))
}
-
- fn blocking_create_dir_dyn(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.blocking_create_dir(path, args)
- }
-
- fn blocking_stat_dyn(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.blocking_stat(path, args)
- }
-
- fn blocking_read_dyn(&self, path: &str, args: OpRead) -> Result<(RpRead, oio::BlockingReader)> {
- self.blocking_read(path, args)
- }
-
- fn blocking_write_dyn(
- &self,
- path: &str,
- args: OpWrite,
- ) -> Result<(RpWrite, oio::BlockingWriter)> {
- self.blocking_write(path, args)
- }
-
- fn blocking_delete_dyn(&self) -> Result<(RpDelete, oio::BlockingDeleter)> {
- self.blocking_delete()
- }
-
- fn blocking_list_dyn(&self, path: &str, args: OpList) -> Result<(RpList, oio::BlockingLister)> {
- self.blocking_list(path, args)
- }
-
- fn blocking_copy_dyn(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- self.blocking_copy(from, to, args)
- }
-
- fn blocking_rename_dyn(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- self.blocking_rename(from, to, args)
- }
}
impl Access for dyn AccessDyn {
type Reader = oio::Reader;
- type BlockingReader = oio::BlockingReader;
type Writer = oio::Writer;
type Deleter = oio::Deleter;
- type BlockingWriter = oio::BlockingWriter;
type Lister = oio::Lister;
- type BlockingLister = oio::BlockingLister;
- type BlockingDeleter = oio::BlockingDeleter;
fn info(&self) -> Arc<AccessorInfo> {
self.info_dyn()
@@ -626,38 +439,6 @@ impl Access for dyn AccessDyn {
async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
self.presign_dyn(path, args).await
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.blocking_create_dir_dyn(path, args)
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.blocking_read_dyn(path, args)
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.blocking_stat_dyn(path, args)
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.blocking_write_dyn(path, args)
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.blocking_delete_dyn()
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.blocking_list_dyn(path, args)
- }
-
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- self.blocking_copy_dyn(from, to, args)
- }
-
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- self.blocking_rename_dyn(from, to, args)
- }
}
/// Dummy implementation of accessor.
@@ -666,10 +447,6 @@ impl Access for () {
type Writer = ();
type Lister = ();
type Deleter = ();
- type BlockingReader = ();
- type BlockingWriter = ();
- type BlockingLister = ();
- type BlockingDeleter = ();
fn info(&self) -> Arc<AccessorInfo> {
let ai = AccessorInfo::default();
@@ -691,10 +468,6 @@ impl<T: Access + ?Sized> Access for Arc<T> {
type Writer = T::Writer;
type Lister = T::Lister;
type Deleter = T::Deleter;
- type BlockingReader = T::BlockingReader;
- type BlockingWriter = T::BlockingWriter;
- type BlockingLister = T::BlockingLister;
- type BlockingDeleter = T::BlockingDeleter;
fn info(&self) -> Arc<AccessorInfo> {
self.as_ref().info()
@@ -765,38 +538,6 @@ impl<T: Access + ?Sized> Access for Arc<T> {
) -> impl Future<Output = Result<RpPresign>> + MaybeSend {
async move { self.as_ref().presign(path, args).await }
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.as_ref().blocking_create_dir(path, args)
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.as_ref().blocking_stat(path, args)
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- self.as_ref().blocking_read(path, args)
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- self.as_ref().blocking_write(path, args)
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- self.as_ref().blocking_delete()
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- self.as_ref().blocking_list(path, args)
- }
-
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- self.as_ref().blocking_copy(from, to, args)
- }
-
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- self.as_ref().blocking_rename(from, to, args)
- }
}
/// Accessor is the type erased accessor with `Arc<dyn Accessor>`.
diff --git a/core/src/raw/adapters/kv/api.rs b/core/src/raw/adapters/kv/api.rs
index 5cea5ebf4..282093ca0 100644
--- a/core/src/raw/adapters/kv/api.rs
+++ b/core/src/raw/adapters/kv/api.rs
@@ -104,49 +104,14 @@ pub trait Adapter: Send + Sync + Debug + Unpin + 'static {
/// - return `Ok(None)` if this key is not exist.
fn get(&self, path: &str) -> impl Future<Output = Result<Option<Buffer>>> + MaybeSend;
- /// The blocking version of get.
- fn blocking_get(&self, path: &str) -> Result<Option<Buffer>> {
- let _ = path;
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "kv adapter doesn't support this operation",
- )
- .with_operation("kv::Adapter::blocking_get"))
- }
-
/// Set a key into service.
fn set(&self, path: &str, value: Buffer) -> impl Future<Output = Result<()>> + MaybeSend;
- /// The blocking version of set.
- fn blocking_set(&self, path: &str, value: Buffer) -> Result<()> {
- let _ = (path, value);
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "kv adapter doesn't support this operation",
- )
- .with_operation("kv::Adapter::blocking_set"))
- }
-
/// Delete a key from service.
///
/// - return `Ok(())` even if this key is not exist.
fn delete(&self, path: &str) -> impl Future<Output = Result<()>> + MaybeSend;
- /// Delete a key from service in blocking way.
- ///
- /// - return `Ok(())` even if this key is not exist.
- fn blocking_delete(&self, path: &str) -> Result<()> {
- let _ = path;
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "kv adapter doesn't support this operation",
- )
- .with_operation("kv::Adapter::blocking_delete"))
- }
-
/// Scan a key prefix to get all keys that start with this key.
fn scan(&self, path: &str) -> impl Future<Output = Result<Self::Scanner>> + MaybeSend {
let _ = path;
@@ -158,18 +123,6 @@ pub trait Adapter: Send + Sync + Debug + Unpin + 'static {
.with_operation("kv::Adapter::scan")))
}
- /// Scan a key prefix to get all keys that start with this key
- /// in blocking way.
- fn blocking_scan(&self, path: &str) -> Result<Vec<String>> {
- let _ = path;
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "kv adapter doesn't support this operation",
- )
- .with_operation("kv::Adapter::blocking_scan"))
- }
-
/// Append a key into service
fn append(&self, path: &str, value: &[u8]) -> impl Future<Output = Result<()>> + MaybeSend {
let _ = path;
@@ -181,19 +134,6 @@ pub trait Adapter: Send + Sync + Debug + Unpin + 'static {
)
.with_operation("kv::Adapter::append")))
}
-
- /// Append a key into service
- /// in blocking way.
- fn blocking_append(&self, path: &str, value: &[u8]) -> Result<()> {
- let _ = path;
- let _ = value;
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "kv adapter doesn't support this operation",
- )
- .with_operation("kv::Adapter::blocking_append"))
- }
}
/// Info for this key value accessor.
diff --git a/core/src/raw/adapters/kv/backend.rs b/core/src/raw/adapters/kv/backend.rs
index 873f118fc..3f5380ac6 100644
--- a/core/src/raw/adapters/kv/backend.rs
+++ b/core/src/raw/adapters/kv/backend.rs
@@ -16,7 +16,6 @@
// under the License.
use std::sync::Arc;
-use std::vec::IntoIter;
use super::{Adapter, Scan};
use crate::raw::oio::HierarchyLister;
@@ -96,10 +95,6 @@ impl<S: Adapter> Access for Backend<S> {
type Writer = KvWriter<S>;
type Lister = HierarchyLister<KvLister<S::Scanner>>;
type Deleter = oio::OneShotDeleter<KvDeleter<S>>;
- type BlockingReader = Buffer;
- type BlockingWriter = KvWriter<S>;
- type BlockingLister = HierarchyLister<BlockingKvLister>;
- type BlockingDeleter = oio::OneShotDeleter<KvDeleter<S>>;
fn info(&self) -> Arc<AccessorInfo> {
self.info.clone()
@@ -151,53 +146,6 @@ impl<S: Adapter> Access for Backend<S> {
Ok((RpList::default(), lister))
}
-
- fn blocking_stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
- let p = build_abs_path(&self.root, path);
-
- if p == build_abs_path(&self.root, "") {
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
- } else {
- let bs = self.kv.blocking_get(&p)?;
- match bs {
- Some(bs) => Ok(RpStat::new(
- Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64),
- )),
- None => Err(Error::new(ErrorKind::NotFound, "kv doesn't have this path")),
- }
- }
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let p = build_abs_path(&self.root, path);
- let bs = match self.kv.blocking_get(&p)? {
- Some(bs) => bs,
- None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't have this path")),
- };
- Ok((RpRead::new(), bs.slice(args.range().to_range_as_usize())))
- }
-
- fn blocking_write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let p = build_abs_path(&self.root, path);
-
- Ok((RpWrite::new(), KvWriter::new(self.kv.clone(), p)))
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(KvDeleter::new(self.kv.clone(), self.root.clone())),
- ))
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- let p = build_abs_path(&self.root, path);
- let res = self.kv.blocking_scan(&p)?;
- let lister = BlockingKvLister::new(&self.root, res);
- let lister = HierarchyLister::new(lister, path, args.recursive());
-
- Ok((RpList::default(), lister))
- }
}
pub struct KvLister<Iter> {
@@ -241,41 +189,6 @@ where
}
}
-pub struct BlockingKvLister {
- root: String,
- inner: IntoIter<String>,
-}
-
-impl BlockingKvLister {
- fn new(root: &str, inner: Vec<String>) -> Self {
- Self {
- root: root.to_string(),
- inner: inner.into_iter(),
- }
- }
-
- fn inner_next(&mut self) -> Option<oio::Entry> {
- self.inner.next().map(|v| {
- let mode = if v.ends_with('/') {
- EntryMode::DIR
- } else {
- EntryMode::FILE
- };
- let mut path = build_rel_path(&self.root, &v);
- if path.is_empty() {
- path = "/".to_string();
- }
- oio::Entry::new(&path, Metadata::new(mode))
- })
- }
-}
-
-impl oio::BlockingList for BlockingKvLister {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- Ok(self.inner_next())
- }
-}
-
pub struct KvWriter<S> {
kv: Arc<S>,
path: String,
@@ -318,22 +231,6 @@ impl<S: Adapter> oio::Write for KvWriter<S> {
}
}
-impl<S: Adapter> oio::BlockingWrite for KvWriter<S> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- self.buffer.push(bs);
- Ok(())
- }
-
- fn close(&mut self) -> Result<Metadata> {
- let buf = self.buffer.clone().collect();
- let length = buf.len() as u64;
- self.kv.blocking_set(&self.path, buf)?;
-
- let meta = Metadata::new(EntryMode::from_path(&self.path)).with_content_length(length);
- Ok(meta)
- }
-}
-
pub struct KvDeleter<S> {
kv: Arc<S>,
root: String,
@@ -353,12 +250,3 @@ impl<S: Adapter> oio::OneShotDelete for KvDeleter<S> {
Ok(())
}
}
-
-impl<S: Adapter> oio::BlockingOneShotDelete for KvDeleter<S> {
- fn blocking_delete_once(&self, path: String, _: OpDelete) -> Result<()> {
- let p = build_abs_path(&self.root, &path);
-
- self.kv.blocking_delete(&p)?;
- Ok(())
- }
-}
diff --git a/core/src/raw/adapters/typed_kv/api.rs b/core/src/raw/adapters/typed_kv/api.rs
index a3d049084..d0f8b84cc 100644
--- a/core/src/raw/adapters/typed_kv/api.rs
+++ b/core/src/raw/adapters/typed_kv/api.rs
@@ -51,21 +51,12 @@ pub trait Adapter: Send + Sync + Debug + Unpin + 'static {
/// Get a value from adapter.
fn get(&self, path: &str) -> impl Future<Output = Result<Option<Value>>> + MaybeSend;
- /// Get a value from adapter.
- fn blocking_get(&self, path: &str) -> Result<Option<Value>>;
-
/// Set a value into adapter.
fn set(&self, path: &str, value: Value) -> impl Future<Output = Result<()>> + MaybeSend;
- /// Set a value into adapter.
- fn blocking_set(&self, path: &str, value: Value) -> Result<()>;
-
/// Delete a value from adapter.
fn delete(&self, path: &str) -> impl Future<Output = Result<()>> + MaybeSend;
- /// Delete a value from adapter.
- fn blocking_delete(&self, path: &str) -> Result<()>;
-
/// Scan a key prefix to get all keys that start with this key.
fn scan(&self, path: &str) -> impl Future<Output = Result<Vec<String>>> + MaybeSend {
let _ = path;
@@ -76,18 +67,6 @@ pub trait Adapter: Send + Sync + Debug + Unpin + 'static {
)
.with_operation("typed_kv::Adapter::scan")))
}
-
- /// Scan a key prefix to get all keys that start with this key
- /// in blocking way.
- fn blocking_scan(&self, path: &str) -> Result<Vec<String>> {
- let _ = path;
-
- Err(Error::new(
- ErrorKind::Unsupported,
- "typed_kv adapter doesn't support this operation",
- )
- .with_operation("typed_kv::Adapter::blocking_scan"))
- }
}
/// Value is the typed value stored in adapter.
diff --git a/core/src/raw/adapters/typed_kv/backend.rs b/core/src/raw/adapters/typed_kv/backend.rs
index 95238dedf..8e08438b0 100644
--- a/core/src/raw/adapters/typed_kv/backend.rs
+++ b/core/src/raw/adapters/typed_kv/backend.rs
@@ -74,8 +74,6 @@ where
cap.shared = true;
}
- cap.blocking = true;
-
am.set_native_capability(cap);
am.into()
@@ -97,10 +95,6 @@ impl<S: Adapter> Access for Backend<S> {
type Writer = KvWriter<S>;
type Lister = HierarchyLister<KvLister>;
type Deleter = oio::OneShotDeleter<KvDeleter<S>>;
- type BlockingReader = Buffer;
- type BlockingWriter = KvWriter<S>;
- type BlockingLister = HierarchyLister<KvLister>;
- type BlockingDeleter = oio::OneShotDeleter<KvDeleter<S>>;
fn info(&self) -> Arc<AccessorInfo> {
self.info.clone()
@@ -153,54 +147,6 @@ impl<S: Adapter> Access for Backend<S> {
Ok((RpList::default(), lister))
}
-
- fn blocking_stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
- let p = build_abs_path(&self.root, path);
-
- if p == build_abs_path(&self.root, "") {
- Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
- } else {
- let bs = self.kv.blocking_get(&p)?;
- match bs {
- Some(bs) => Ok(RpStat::new(bs.metadata)),
- None => Err(Error::new(ErrorKind::NotFound, "kv doesn't have this path")),
- }
- }
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- let p = build_abs_path(&self.root, path);
-
- let bs = match self.kv.blocking_get(&p)? {
- // TODO: we can reuse the metadata in value to build content range.
- Some(bs) => bs.value,
- None => return Err(Error::new(ErrorKind::NotFound, "kv doesn't have this path")),
- };
-
- Ok((RpRead::new(), bs.slice(args.range().to_range_as_usize())))
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- let p = build_abs_path(&self.root, path);
-
- Ok((RpWrite::new(), KvWriter::new(self.kv.clone(), p, args)))
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- Ok((
- RpDelete::default(),
- oio::OneShotDeleter::new(KvDeleter::new(self.kv.clone(), self.root.clone())),
- ))
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- let p = build_abs_path(&self.root, path);
- let res = self.kv.blocking_scan(&p)?;
- let lister = KvLister::new(&self.root, res);
- let lister = HierarchyLister::new(lister, path, args.recursive());
-
- Ok((RpList::default(), lister))
- }
}
pub struct KvLister {
@@ -238,12 +184,6 @@ impl oio::List for KvLister {
}
}
-impl oio::BlockingList for KvLister {
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- Ok(self.inner_next())
- }
-}
-
pub struct KvWriter<S> {
kv: Arc<S>,
path: String,
@@ -318,31 +258,6 @@ impl<S: Adapter> oio::Write for KvWriter<S> {
}
}
-impl<S: Adapter> oio::BlockingWrite for KvWriter<S> {
- fn write(&mut self, bs: Buffer) -> Result<()> {
- let mut buf = self.buf.take().unwrap_or_default();
- buf.push(bs);
- self.buf = Some(buf);
- Ok(())
- }
-
- fn close(&mut self) -> Result<Metadata> {
- let kv = self.kv.clone();
- let value = match &self.value {
- Some(value) => value.clone(),
- None => {
- let value = self.build();
- self.value = Some(value.clone());
- value
- }
- };
-
- let meta = value.metadata.clone();
- kv.blocking_set(&self.path, value)?;
- Ok(meta)
- }
-}
-
pub struct KvDeleter<S> {
kv: Arc<S>,
root: String,
@@ -362,12 +277,3 @@ impl<S: Adapter> oio::OneShotDelete for KvDeleter<S> {
Ok(())
}
}
-
-impl<S: Adapter> oio::BlockingOneShotDelete for KvDeleter<S> {
- fn blocking_delete_once(&self, path: String, _: OpDelete) -> Result<()> {
- let p = build_abs_path(&self.root, &path);
-
- self.kv.blocking_delete(&p)?;
- Ok(())
- }
-}
diff --git a/core/src/raw/enum_utils.rs b/core/src/raw/enum_utils.rs
index 9a539943b..93822aa44 100644
--- a/core/src/raw/enum_utils.rs
+++ b/core/src/raw/enum_utils.rs
@@ -60,15 +60,6 @@ impl<ONE: oio::Read, TWO: oio::Read> oio::Read for TwoWays<ONE, TWO> {
}
}
-impl<ONE: oio::BlockingRead, TWO: oio::BlockingRead> oio::BlockingRead for TwoWays<ONE, TWO> {
- fn read(&mut self) -> Result<Buffer> {
- match self {
- Self::One(v) => v.read(),
- Self::Two(v) => v.read(),
- }
- }
-}
-
impl<ONE: oio::Write, TWO: oio::Write> oio::Write for TwoWays<ONE, TWO> {
async fn write(&mut self, bs: Buffer) -> Result<()> {
match self {
@@ -123,18 +114,6 @@ impl<ONE: oio::Read, TWO: oio::Read, THREE: oio::Read> oio::Read for ThreeWays<O
}
}
-impl<ONE: oio::BlockingRead, TWO: oio::BlockingRead, THREE: oio::BlockingRead> oio::BlockingRead
- for ThreeWays<ONE, TWO, THREE>
-{
- fn read(&mut self) -> Result<Buffer> {
- match self {
- Self::One(v) => v.read(),
- Self::Two(v) => v.read(),
- Self::Three(v) => v.read(),
- }
- }
-}
-
impl<ONE: oio::Write, TWO: oio::Write, THREE: oio::Write> oio::Write
for ThreeWays<ONE, TWO, THREE>
{
@@ -204,23 +183,6 @@ where
}
}
-impl<ONE, TWO, THREE, FOUR> oio::BlockingRead for FourWays<ONE, TWO, THREE, FOUR>
-where
- ONE: oio::BlockingRead,
- TWO: oio::BlockingRead,
- THREE: oio::BlockingRead,
- FOUR: oio::BlockingRead,
-{
- fn read(&mut self) -> Result<Buffer> {
- match self {
- Self::One(v) => v.read(),
- Self::Two(v) => v.read(),
- Self::Three(v) => v.read(),
- Self::Four(v) => v.read(),
- }
- }
-}
-
impl<ONE, TWO, THREE, FOUR> oio::List for FourWays<ONE, TWO, THREE, FOUR>
where
ONE: oio::List,
@@ -237,20 +199,3 @@ where
}
}
}
-
-impl<ONE, TWO, THREE, FOUR> oio::BlockingList for FourWays<ONE, TWO, THREE, FOUR>
-where
- ONE: oio::BlockingList,
- TWO: oio::BlockingList,
- THREE: oio::BlockingList,
- FOUR: oio::BlockingList,
-{
- fn next(&mut self) -> Result<Option<oio::Entry>> {
- match self {
- Self::One(v) => v.next(),
- Self::Two(v) => v.next(),
- Self::Three(v) => v.next(),
- Self::Four(v) => v.next(),
- }
- }
-}
diff --git a/core/src/raw/layer.rs b/core/src/raw/layer.rs
index a3af764e1..87d99f81e 100644
--- a/core/src/raw/layer.rs
+++ b/core/src/raw/layer.rs
@@ -57,13 +57,9 @@ use crate::*;
/// impl<A: Access> LayeredAccess for TraceAccessor<A> {
/// type Inner = A;
/// type Reader = A::Reader;
-/// type BlockingReader = A::BlockingReader;
/// type Writer = A::Writer;
-/// type BlockingWriter = A::BlockingWriter;
/// type Lister = A::Lister;
-/// type BlockingLister = A::BlockingLister;
/// type Deleter = A::Deleter;
-/// type BlockingDeleter = A::BlockingDeleter;
///
/// fn inner(&self) -> &Self::Inner {
/// &self.inner
@@ -73,45 +69,17 @@ use crate::*;
/// self.inner.read(path, args).await
/// }
///
-/// fn blocking_read(
-/// &self,
-/// path: &str,
-/// args: OpRead,
-/// ) -> Result<(RpRead, Self::BlockingReader)> {
-/// self.inner.blocking_read(path, args)
-/// }
-///
/// async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
/// self.inner.write(path, args).await
/// }
///
-/// fn blocking_write(
-/// &self,
-/// path: &str,
-/// args: OpWrite,
-/// ) -> Result<(RpWrite, Self::BlockingWriter)> {
-/// self.inner.blocking_write(path, args)
-/// }
-///
/// async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
/// self.inner.list(path, args).await
/// }
///
-/// fn blocking_list(
-/// &self,
-/// path: &str,
-/// args: OpList,
-/// ) -> Result<(RpList, Self::BlockingLister)> {
-/// self.inner.blocking_list(path, args)
-/// }
-///
/// async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
/// self.inner.delete().await
/// }
-///
-/// fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
-/// self.inner.blocking_delete()
-/// }
/// }
///
/// /// The public struct that exposed to users.
@@ -145,10 +113,6 @@ pub trait LayeredAccess: Send + Sync + Debug + Unpin + 'static {
type Writer: oio::Write;
type Lister: oio::List;
type Deleter: oio::Delete;
- type BlockingReader: oio::BlockingRead;
- type BlockingWriter: oio::BlockingWrite;
- type BlockingLister: oio::BlockingList;
- type BlockingDeleter: oio::BlockingDelete;
fn inner(&self) -> &Self::Inner;
@@ -213,30 +177,6 @@ pub trait LayeredAccess: Send + Sync + Debug + Unpin + 'static {
) -> impl Future<Output = Result<RpPresign>> + MaybeSend {
self.inner().presign(path, args)
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- self.inner().blocking_create_dir(path, args)
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)>;
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)>;
-
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- self.inner().blocking_copy(from, to, args)
- }
-
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- self.inner().blocking_rename(from, to, args)
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- self.inner().blocking_stat(path, args)
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)>;
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)>;
}
impl<L: LayeredAccess> Access for L {
@@ -245,11 +185,6 @@ impl<L: LayeredAccess> Access for L {
type Lister = L::Lister;
type Deleter = L::Deleter;
- type BlockingReader = L::BlockingReader;
- type BlockingWriter = L::BlockingWriter;
- type BlockingLister = L::BlockingLister;
- type BlockingDeleter = L::BlockingDeleter;
-
fn info(&self) -> Arc<AccessorInfo> {
LayeredAccess::info(self)
}
@@ -289,38 +224,6 @@ impl<L: LayeredAccess> Access for L {
async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
LayeredAccess::presign(self, path, args).await
}
-
- fn blocking_create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
- LayeredAccess::blocking_create_dir(self, path, args)
- }
-
- fn blocking_read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::BlockingReader)> {
- LayeredAccess::blocking_read(self, path, args)
- }
-
- fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
- LayeredAccess::blocking_write(self, path, args)
- }
-
- fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
- LayeredAccess::blocking_copy(self, from, to, args)
- }
-
- fn blocking_rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
- LayeredAccess::blocking_rename(self, from, to, args)
- }
-
- fn blocking_stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
- LayeredAccess::blocking_stat(self, path, args)
- }
-
- fn blocking_delete(&self) -> Result<(RpDelete, Self::BlockingDeleter)> {
- LayeredAccess::blocking_delete(self)
- }
-
- fn blocking_list(&self, path: &str, args: OpList) -> Result<(RpList, Self::BlockingLister)> {
- LayeredAccess::blocking_list(self, path, args)
- }
}
#[cfg(test)]
@@ -352,13 +255,9 @@ mod tests {
impl<A: Access> Access for Test<A> {
type Reader = ();
- type BlockingReader = ();
type Writer = ();
- type BlockingWriter = ();
type Lister = ();
- type BlockingLister = ();
type Deleter = ();
- type BlockingDeleter = ();
fn info(&self) -> Arc<AccessorInfo> {
let am = AccessorInfo::default();
diff --git a/core/src/raw/oio/delete/api.rs b/core/src/raw/oio/delete/api.rs
index e0db7733b..bf605334f 100644
--- a/core/src/raw/oio/delete/api.rs
+++ b/core/src/raw/oio/delete/api.rs
@@ -97,55 +97,3 @@ impl<T: DeleteDyn + ?Sized> Delete for Box<T> {
self.deref_mut().flush_dyn().await
}
}
-
-/// BlockingDeleter is a type erased [`BlockingDelete`]
-pub type BlockingDeleter = Box<dyn BlockingDelete>;
-
-/// BlockingDelete is the trait to perform delete operations.
-pub trait BlockingDelete: Send + Sync + 'static {
- /// Delete given path with optional arguments.
- ///
- /// # Behavior
- ///
- /// - `Ok(())` means the path has been queued for deletion.
- /// - `Err(err)` means error happens and no deletion has been queued.
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()>;
-
- /// Flushes the deletion queue to ensure queued deletions are executed
- ///
- /// # Returns
- /// - `Ok(0)`: All queued deletions have been processed or the queue is empty.
- /// - `Ok(count)`: The number of resources successfully deleted. Implementations should
- /// return an error if the queue is non-empty but no resources were deleted
- /// - `Err(err)`: An error occurred while performing the deletions
- fn flush(&mut self) -> Result<usize>;
-}
-
-impl BlockingDelete for () {
- fn delete(&mut self, _: &str, _: OpDelete) -> Result<()> {
- Err(Error::new(
- ErrorKind::Unsupported,
- "output deleter doesn't support delete",
- ))
- }
-
- fn flush(&mut self) -> Result<usize> {
- Err(Error::new(
- ErrorKind::Unsupported,
- "output deleter doesn't support flush",
- ))
- }
-}
-
-/// `Box<dyn BlockingDelete>` won't implement `BlockingDelete` automatically.
-///
-/// To make BlockingWriter work as expected, we must add this impl.
-impl<T: BlockingDelete + ?Sized> BlockingDelete for Box<T> {
- fn delete(&mut self, path: &str, args: OpDelete) -> Result<()> {
- (**self).delete(path, args)
- }
-
- fn flush(&mut self) -> Result<usize> {
- (**self).flush()
- }
-}
... 6629 lines suppressed ...