You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/09/08 17:53:59 UTC
(opendal) branch main updated: refactor: unify async synchronization with asyncband (#8261)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 37fe179d04 refactor: unify async synchronization with asyncband (#8261)
37fe179d04 is described below
commit 37fe179d04897c210011d915a2d2eabcad645b78
Author: tison <wa...@gmail.com>
AuthorDate: Wed Sep 9 01:53:53 2026 +0800
refactor: unify async synchronization with asyncband (#8261)
---
bindings/dotnet/Cargo.toml | 1 +
bindings/dotnet/src/operator.rs | 9 +++++----
core/Cargo.lock | 2 ++
core/services/gcs/Cargo.toml | 3 ++-
core/services/gcs/src/composer.rs | 32 +++++++++++---------------------
core/services/goosefs/Cargo.toml | 2 +-
core/services/goosefs/src/core.rs | 2 +-
7 files changed, 23 insertions(+), 28 deletions(-)
diff --git a/bindings/dotnet/Cargo.toml b/bindings/dotnet/Cargo.toml
index 275349dbca..267a83ddb8 100644
--- a/bindings/dotnet/Cargo.toml
+++ b/bindings/dotnet/Cargo.toml
@@ -32,6 +32,7 @@ doc = false
[dependencies]
# this crate won't be published, we always use the local version
+asyncband = { version = "0.7.1", features = ["mutex"] }
bytes = "1.12.1"
futures = "0.3"
opendal = { version = ">=0", path = "../../core", features = [
diff --git a/bindings/dotnet/src/operator.rs b/bindings/dotnet/src/operator.rs
index 71dd514d4e..d5fc2868d0 100644
--- a/bindings/dotnet/src/operator.rs
+++ b/bindings/dotnet/src/operator.rs
@@ -45,6 +45,7 @@ use std::os::raw::c_char;
use std::sync::Arc;
use std::time::Duration;
+use asyncband::mutex::Mutex;
use futures::StreamExt;
use crate::executor::Executor;
@@ -1261,7 +1262,7 @@ pub extern "C" fn operator_input_stream_create(
/// reads — and the `Arc` lets in-flight tasks outlive an early free safely.
struct InputStream {
executor: Arc<Executor>,
- inner: Arc<tokio::sync::Mutex<opendal::FuturesBytesStream>>,
+ inner: Arc<Mutex<opendal::FuturesBytesStream>>,
}
/// Convert one polled chunk into an FFI read payload.
@@ -1324,7 +1325,7 @@ fn operator_input_stream_create_inner(
Ok(Box::into_raw(Box::new(InputStream {
executor,
- inner: Arc::new(tokio::sync::Mutex::new(stream)),
+ inner: Arc::new(Mutex::new(stream)),
})) as *mut c_void)
}
@@ -1453,7 +1454,7 @@ pub extern "C" fn operator_output_stream_create(
/// against an early free.
struct OutputStream {
executor: Arc<Executor>,
- inner: Arc<tokio::sync::Mutex<opendal::Writer>>,
+ inner: Arc<Mutex<opendal::Writer>>,
}
fn operator_output_stream_create_inner(
@@ -1477,7 +1478,7 @@ fn operator_output_stream_create_inner(
Ok(Box::into_raw(Box::new(OutputStream {
executor,
- inner: Arc::new(tokio::sync::Mutex::new(writer)),
+ inner: Arc::new(Mutex::new(writer)),
})) as *mut c_void)
}
diff --git a/core/Cargo.lock b/core/Cargo.lock
index f94ebcda56..4870c4c6a5 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -6960,6 +6960,7 @@ name = "opendal-service-gcs"
version = "0.59.1"
dependencies = [
"async-trait",
+ "asyncband",
"bytes",
"futures",
"http 1.4.2",
@@ -7048,6 +7049,7 @@ dependencies = [
name = "opendal-service-goosefs"
version = "0.59.1"
dependencies = [
+ "asyncband",
"bytes",
"goosefs-sdk",
"log",
diff --git a/core/services/gcs/Cargo.toml b/core/services/gcs/Cargo.toml
index 9e6dd3f3dd..d643de0227 100644
--- a/core/services/gcs/Cargo.toml
+++ b/core/services/gcs/Cargo.toml
@@ -50,6 +50,7 @@ tokio = { workspace = true, features = ["rt"] }
uuid = { workspace = true, features = ["v4"] }
[dev-dependencies]
+asyncband = { workspace = true, features = ["latch", "mutex"] }
futures = { workspace = true }
serde_json = { workspace = true }
-tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync"] }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/core/services/gcs/src/composer.rs b/core/services/gcs/src/composer.rs
index ece0a457cb..db8c40f0c6 100644
--- a/core/services/gcs/src/composer.rs
+++ b/core/services/gcs/src/composer.rs
@@ -365,11 +365,12 @@ fn new_tasks(
#[cfg(test)]
mod tests {
- use std::sync::Mutex;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::time::Duration;
+ use asyncband::latch::Latch;
+ use asyncband::mutex::Mutex;
use futures::stream;
use http::Method;
use http::Response;
@@ -378,28 +379,25 @@ mod tests {
use reqsign_core::Signer;
use reqsign_google::RequestSigner;
use reqsign_google::TokenCredentialProvider;
- use tokio::sync::Notify;
use super::*;
use opendal_core::raw::oio::Compose;
struct ComposeTransport {
- started: AtomicUsize,
+ started: Latch,
active: AtomicUsize,
max_active: AtomicUsize,
generation: AtomicUsize,
- notify: Notify,
requests: Mutex<Vec<serde_json::Value>>,
}
impl ComposeTransport {
fn new() -> Self {
Self {
- started: AtomicUsize::new(0),
+ started: Latch::new(2),
active: AtomicUsize::new(0),
max_active: AtomicUsize::new(0),
generation: AtomicUsize::new(1),
- notify: Notify::new(),
requests: Mutex::new(Vec::new()),
}
}
@@ -432,21 +430,13 @@ mod tests {
let value: serde_json::Value = serde_json::from_slice(&req.body().to_bytes())
.expect("compose request body must be valid JSON");
- self.0.requests.lock().unwrap().push(value.clone());
+ self.0.requests.lock().await.push(value.clone());
if req.uri().path().contains("__opendal%2Fcompose%2F") {
let active = self.0.active.fetch_add(1, Ordering::SeqCst) + 1;
self.0.max_active.fetch_max(active, Ordering::SeqCst);
- self.0.started.fetch_add(1, Ordering::SeqCst);
- self.0.notify.notify_waiters();
-
- loop {
- let notified = self.0.notify.notified();
- if self.0.started.load(Ordering::SeqCst) >= 2 {
- break;
- }
- notified.await;
- }
+ self.0.started.count_down();
+ self.0.started.wait().await;
self.0.active.fetch_sub(1, Ordering::SeqCst);
}
@@ -485,7 +475,7 @@ mod tests {
if req.method() == Method::GET {
self.0.metadata_reads.fetch_add(1, Ordering::SeqCst);
- let token = self.0.token.lock().unwrap().clone().unwrap();
+ let token = self.0.token.lock().await.clone().unwrap();
let body = Buffer::from(
serde_json::to_vec(&serde_json::json!({
"size": "32",
@@ -504,7 +494,7 @@ mod tests {
.as_str()
.unwrap()
.to_string();
- *self.0.token.lock().unwrap() = Some(token);
+ *self.0.token.lock().await = Some(token);
if self.0.intermediate_posts.fetch_add(1, Ordering::SeqCst) == 0 {
return Err(Error::new(
@@ -586,7 +576,7 @@ mod tests {
.expect("composition must succeed");
assert_eq!(transport.max_active.load(Ordering::SeqCst), 2);
- let requests = transport.requests.lock().unwrap();
+ let requests = transport.requests.lock().await;
assert_eq!(requests.len(), 3);
let source_lists: Vec<&Vec<serde_json::Value>> = requests
@@ -672,7 +662,7 @@ mod tests {
}
composer.close().await.expect("composition must succeed");
- let requests = transport.requests.lock().unwrap();
+ let requests = transport.requests.lock().await;
assert_eq!(requests.len(), 34);
let source_lists: Vec<&Vec<serde_json::Value>> = requests
.iter()
diff --git a/core/services/goosefs/Cargo.toml b/core/services/goosefs/Cargo.toml
index 7d75cd7732..63c60fd6a5 100644
--- a/core/services/goosefs/Cargo.toml
+++ b/core/services/goosefs/Cargo.toml
@@ -32,12 +32,12 @@ version = { workspace = true }
all-features = true
[dependencies]
+asyncband = { workspace = true, features = ["rwlock"] }
bytes = { workspace = true }
goosefs-sdk = "0.1.9"
log = { workspace = true }
opendal-core = { path = "../../core", version = "0.59.1", default-features = false }
serde = { workspace = true, features = ["derive"] }
-tokio = { workspace = true }
[dev-dependencies]
opendal = { path = "../..", version = "0.59.1", features = [
diff --git a/core/services/goosefs/src/core.rs b/core/services/goosefs/src/core.rs
index 281c00afa7..a6174cc30e 100644
--- a/core/services/goosefs/src/core.rs
+++ b/core/services/goosefs/src/core.rs
@@ -19,12 +19,12 @@ use std::fmt::Debug;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
+use asyncband::rwlock::RwLock;
use goosefs_sdk::client::MasterClient;
use goosefs_sdk::config::GoosefsConfig as ClientConfig;
use goosefs_sdk::context::FileSystemContext;
use goosefs_sdk::io::{GoosefsFileReader, GoosefsFileWriter};
use goosefs_sdk::proto::grpc::file::FileInfo;
-use tokio::sync::RwLock;
use opendal_core::raw::*;
use opendal_core::*;