You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/10/26 07:40:45 UTC
(opendal-yinyang) 01/02: fix(mount): release the namespace lock across node materialization
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/linux-fuse-writable
in repository https://gitbox.apache.org/repos/asf/opendal-yinyang.git
commit b2840620dd1b1cbb81afb4f182fba811a1a86562
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Oct 26 15:29:19 2026 +0800
fix(mount): release the namespace lock across node materialization
First opens now serialize per NodeId through a short-lived gate while the instance node map is held only for lookup and accounting, so opening one node cannot block an independent node's remote materialization. NodeGuard keeps refresh, fsync and status from racing reclaim. Deterministic tests pause one observe and prove another node opens independently, concurrent first opens share one handle, and cancellation releases the gate.
Co-authored-by: Chojan Shang <ps...@apache.org>
---
specs/mount-runtime.md | 9 ++-
src/mount.rs | 132 ++++++++++++++++++++++++++++---------
tests/mount.rs | 172 ++++++++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 279 insertions(+), 34 deletions(-)
diff --git a/specs/mount-runtime.md b/specs/mount-runtime.md
index 70ac3a9..105e8fb 100644
--- a/specs/mount-runtime.md
+++ b/specs/mount-runtime.md
@@ -11,8 +11,13 @@ Do not mix private library handles into this staging directory.
writes from other references immediately after local acceptance. Writes,
truncate, append and fsync are serialized per node; append chooses its offset
under the same lock. Different nodes do not share an I/O lock. Initial
-materialization, refresh and reclaim serialize through the instance node map.
-Both read-only and writable references currently use full-file staging.
+materialization is serialized per NodeId by a short-lived open gate, and the
+instance node map is not held during a remote observation or staging read.
+Different nodes can therefore complete their first opens independently. Refresh
+and reclaim use the map for lookup and reference accounting; data-plane calls
+keep per-node locks. Cancelling a first open releases its gate; an incomplete
+local open is never exposed and is discarded when staging is reopened. Both
+read-only and writable references currently use full-file staging.
Dropping a reference performs no I/O and never publishes or deletes data.
`reclaim` rejects referenced, pending, frozen or failed nodes. Clean unreferenced
diff --git a/src/mount.rs b/src/mount.rs
index 58baa9f..1aabd54 100644
--- a/src/mount.rs
+++ b/src/mount.rs
@@ -19,19 +19,47 @@
use crate::runtime::{Error, ErrorKind, FileHandle, HandleStatus, Result, Runtime, RuntimeStatus};
use std::collections::BTreeMap;
use std::path::Path;
-use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
-use tokio::sync::Mutex;
+use std::sync::{Arc, Mutex as StdMutex};
+use tokio::sync::{Mutex, Semaphore};
use yinyang_core::{Authority, NodeId, Receipt};
struct Node {
handle: Mutex<FileHandle>,
references: AtomicUsize,
}
+struct OpenGate {
+ inner: Arc<Inner>,
+ id: NodeId,
+ ready: Arc<Semaphore>,
+}
+impl Drop for OpenGate {
+ fn drop(&mut self) {
+ if let Ok(mut opening) = self.inner.opening.lock() {
+ opening.remove(&self.id);
+ }
+ self.ready.add_permits(1);
+ }
+}
+struct NodeGuard {
+ node: Arc<Node>,
+}
+impl NodeGuard {
+ fn new(node: Arc<Node>) -> Self {
+ node.references.fetch_add(1, Ordering::Relaxed);
+ Self { node }
+ }
+}
+impl Drop for NodeGuard {
+ fn drop(&mut self) {
+ self.node.references.fetch_sub(1, Ordering::Relaxed);
+ }
+}
struct Inner {
runtime: Runtime,
read_only: bool,
nodes: Mutex<BTreeMap<NodeId, Arc<Node>>>,
+ opening: StdMutex<BTreeMap<NodeId, Arc<Semaphore>>>,
}
/// A mount instance owns one recoverable staged version per node, not per open.
/// It does not install a mount or manage kernel caches.
@@ -76,6 +104,7 @@ impl Mount {
runtime,
read_only,
nodes: Mutex::new(nodes),
+ opening: StdMutex::new(BTreeMap::new()),
}),
})
}
@@ -88,14 +117,14 @@ impl Mount {
/// Observe staged metadata for one shared node without opening a reference.
/// `None` means this mount has no shared handle for the node.
pub async fn node_status(&self, id: NodeId) -> Result<Option<HandleStatus>> {
- let node = {
+ let guard = {
let nodes = self.inner.nodes.lock().await;
- nodes.get(&id).cloned()
+ nodes.get(&id).cloned().map(NodeGuard::new)
};
- let Some(node) = node else {
+ let Some(guard) = guard else {
return Ok(None);
};
- node.handle.lock().await.status().await.map(Some)
+ guard.node.handle.lock().await.status().await.map(Some)
}
/// Namespace operations use the same conditional runtime contract. Do not
/// open private FileHandles through this runtime alongside Mount references.
@@ -106,37 +135,75 @@ impl Mount {
if writable && self.inner.read_only {
return Err(Error::State(ErrorKind::ReadOnly, "mount is read-only"));
}
- // Serialize first materialization with opens and reclaim. File I/O and
- // publication use per-node locks, never this namespace lock.
- let mut nodes = self.inner.nodes.lock().await;
- let node = match nodes.get(&id) {
- Some(node) => node.clone(),
- None => {
- let handle = self
- .inner
- .runtime
- .open_node(id, !self.inner.read_only)
- .await?;
- let node = Arc::new(Node {
- handle: Mutex::new(handle),
- references: AtomicUsize::new(0),
- });
+ loop {
+ let existing = {
+ let nodes = self.inner.nodes.lock().await;
+ nodes.get(&id).map(|node| {
+ node.references.fetch_add(1, Ordering::Relaxed);
+ node.clone()
+ })
+ };
+ if let Some(node) = existing {
+ return Ok(MountFile { node, writable });
+ }
+
+ let (waiting, ready) = {
+ let mut opening = self.inner.opening.lock().map_err(Error::local)?;
+ match opening.get(&id) {
+ Some(ready) => (true, ready.clone()),
+ None => {
+ let ready = Arc::new(Semaphore::new(0));
+ opening.insert(id, ready.clone());
+ (false, ready)
+ }
+ }
+ };
+ if waiting {
+ ready
+ .acquire()
+ .await
+ .map_err(|_| Error::local("mount open coordination closed"))?
+ .forget();
+ continue;
+ }
+
+ // Only one task materializes a NodeId. The namespace map is not
+ // held across the runtime's remote observation and staging read.
+ let gate = OpenGate {
+ inner: self.inner.clone(),
+ id,
+ ready,
+ };
+ let handle = self
+ .inner
+ .runtime
+ .open_node(id, !self.inner.read_only)
+ .await?;
+ let node = Arc::new(Node {
+ handle: Mutex::new(handle),
+ references: AtomicUsize::new(0),
+ });
+ {
+ let mut nodes = self.inner.nodes.lock().await;
nodes.insert(id, node.clone());
- node
+ node.references.fetch_add(1, Ordering::Relaxed);
}
- };
- node.references.fetch_add(1, Ordering::Relaxed);
- Ok(MountFile { node, writable })
+ drop(gate);
+ return Ok(MountFile { node, writable });
+ }
}
/// Replace a clean local version with a fresh remote observation. Pending,
/// frozen or failed versions are never rebased. Adapters must coordinate any
/// OS cache invalidation outside this call; no OS callbacks run under locks.
pub async fn refresh(&self, id: NodeId) -> Result<()> {
- let nodes = self.inner.nodes.lock().await;
- let Some(node) = nodes.get(&id) else {
+ let guard = {
+ let nodes = self.inner.nodes.lock().await;
+ nodes.get(&id).cloned().map(NodeGuard::new)
+ };
+ let Some(guard) = guard else {
return Ok(());
};
- let mut handle = node.handle.lock().await;
+ let mut handle = guard.node.handle.lock().await;
let state = handle.status().await?;
if state.pending || state.frozen || state.error.is_some() {
return Err(Error::State(
@@ -157,10 +224,13 @@ impl Mount {
/// Persist every currently staged node remotely. Continue after one failure
/// so independent nodes can finish; the original failures remain in status.
pub async fn fsync(&self) -> Result<()> {
- let nodes: Vec<_> = self.inner.nodes.lock().await.values().cloned().collect();
+ let nodes: Vec<_> = {
+ let nodes = self.inner.nodes.lock().await;
+ nodes.values().cloned().map(NodeGuard::new).collect()
+ };
let mut failure = None;
- for node in nodes {
- if let Err(error) = node.handle.lock().await.fsync().await {
+ for node in &nodes {
+ if let Err(error) = node.node.handle.lock().await.fsync().await {
failure.get_or_insert(error);
}
}
diff --git a/tests/mount.rs b/tests/mount.rs
index 338a786..7b05420 100644
--- a/tests/mount.rs
+++ b/tests/mount.rs
@@ -18,12 +18,79 @@
#[path = "../crates/core/tests/support/mod.rs"]
mod support;
use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::Duration;
+use std::{future::Future, pin::Pin};
use support::TestBackend;
+use tokio::sync::Semaphore;
use yinyang::core::service::{MetadataService, ServiceClient, serve};
-use yinyang::core::{Authority, BackendProfile, Fs};
+use yinyang::core::{
+ Authority, BackendProfile, CommitOutcome, DataStore, Fs, NodeId, Revision, Snapshot,
+ Transaction,
+};
use yinyang::mount::Mount;
use yinyang::runtime::{ErrorKind, Runtime};
+struct PausingObserveAuthority {
+ inner: Arc<dyn Authority>,
+ entered: Semaphore,
+ release: Semaphore,
+ pause_next_observe: AtomicBool,
+}
+impl PausingObserveAuthority {
+ fn new(inner: Arc<dyn Authority>) -> Self {
+ Self {
+ inner,
+ entered: Semaphore::new(0),
+ release: Semaphore::new(0),
+ pause_next_observe: AtomicBool::new(false),
+ }
+ }
+ fn pause_next_observe(&self) {
+ self.pause_next_observe.store(true, Ordering::SeqCst);
+ }
+}
+impl std::fmt::Debug for PausingObserveAuthority {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("PausingObserveAuthority")
+ .finish_non_exhaustive()
+ }
+}
+impl Authority for PausingObserveAuthority {
+ fn data(&self) -> &DataStore {
+ self.inner.data()
+ }
+ fn filesystem(&self) -> NodeId {
+ self.inner.filesystem()
+ }
+ fn root(&self) -> NodeId {
+ self.inner.root()
+ }
+ fn observe_latest(
+ &self,
+ ) -> Pin<Box<dyn Future<Output = yinyang::core::Result<Snapshot>> + Send + '_>> {
+ Box::pin(async move {
+ if self.pause_next_observe.swap(false, Ordering::SeqCst) {
+ self.entered.add_permits(1);
+ self.release.acquire().await.unwrap().forget();
+ }
+ self.inner.observe_latest().await
+ })
+ }
+ fn observe_revision(
+ &self,
+ revision: Revision,
+ ) -> Pin<Box<dyn Future<Output = yinyang::core::Result<Snapshot>> + Send + '_>> {
+ self.inner.observe_revision(revision)
+ }
+ fn commit_batch<'a>(
+ &'a self,
+ requests: &'a [Transaction],
+ ) -> Pin<Box<dyn Future<Output = yinyang::core::Result<Vec<CommitOutcome>>> + Send + 'a>> {
+ self.inner.commit_batch(requests)
+ }
+}
+
async fn exercise(authority: Arc<dyn Authority>) {
let temp = tempfile::tempdir().unwrap();
let mount = Mount::open(authority.clone(), temp.path(), false)
@@ -238,3 +305,106 @@ async fn flush_publishes_but_reference_release_does_not() {
mount.reclaim(pending).await.unwrap();
assert!(mount.node_status(pending).await.unwrap().is_none());
}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn independent_node_opens_do_not_share_the_namespace_lock() {
+ let backend = TestBackend::default();
+ let inner: Arc<dyn Authority> = Arc::new(
+ Fs::create(backend.operator(), BackendProfile::Minio)
+ .await
+ .unwrap(),
+ );
+ let setup_temp = tempfile::tempdir().unwrap();
+ let setup = Runtime::open(inner.clone(), setup_temp.path())
+ .await
+ .unwrap();
+ let (a, _) = setup
+ .create_file_with_node(inner.root(), "a", false)
+ .await
+ .unwrap();
+ let (b, _) = setup
+ .create_file_with_node(inner.root(), "b", false)
+ .await
+ .unwrap();
+ drop(setup);
+
+ let pausing = Arc::new(PausingObserveAuthority::new(inner));
+ let authority: Arc<dyn Authority> = pausing.clone();
+ let mount_temp = tempfile::tempdir().unwrap();
+ let mount = Mount::open(authority, mount_temp.path(), false)
+ .await
+ .unwrap();
+ pausing.pause_next_observe();
+ let a_task = {
+ let mount = mount.clone();
+ tokio::spawn(async move { mount.open_node(a, false).await })
+ };
+ pausing.entered.acquire().await.unwrap().forget();
+ let mut b_task = {
+ let mount = mount.clone();
+ tokio::spawn(async move { mount.open_node(b, false).await })
+ };
+ let b_file = tokio::time::timeout(Duration::from_secs(5), &mut b_task)
+ .await
+ .expect("opening an independent node waited for another node's remote observation")
+ .unwrap()
+ .unwrap();
+ drop(b_file);
+ let a_second = {
+ let mount = mount.clone();
+ tokio::spawn(async move { mount.open_node(a, false).await })
+ };
+
+ pausing.release.add_permits(1);
+ let a_file = a_task.await.unwrap().unwrap();
+ let a_second = a_second.await.unwrap().unwrap();
+ assert_eq!(
+ a_file.status().await.unwrap().id,
+ a_second.status().await.unwrap().id
+ );
+ drop(a_second);
+ drop(a_file);
+ assert!(mount.node_status(a).await.unwrap().is_some());
+ assert!(mount.node_status(b).await.unwrap().is_some());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn cancelled_first_open_releases_the_node_gate() {
+ let backend = TestBackend::default();
+ let inner: Arc<dyn Authority> = Arc::new(
+ Fs::create(backend.operator(), BackendProfile::Minio)
+ .await
+ .unwrap(),
+ );
+ let setup_temp = tempfile::tempdir().unwrap();
+ let setup = Runtime::open(inner.clone(), setup_temp.path())
+ .await
+ .unwrap();
+ let (a, _) = setup
+ .create_file_with_node(inner.root(), "a", false)
+ .await
+ .unwrap();
+ drop(setup);
+
+ let pausing = Arc::new(PausingObserveAuthority::new(inner));
+ let authority: Arc<dyn Authority> = pausing.clone();
+ let mount_temp = tempfile::tempdir().unwrap();
+ let mount = Mount::open(authority, mount_temp.path(), false)
+ .await
+ .unwrap();
+ pausing.pause_next_observe();
+ let cancelled = {
+ let mount = mount.clone();
+ tokio::spawn(async move { mount.open_node(a, false).await })
+ };
+ pausing.entered.acquire().await.unwrap().forget();
+ cancelled.abort();
+ assert!(cancelled.await.unwrap_err().is_cancelled());
+
+ let file = tokio::time::timeout(Duration::from_secs(5), mount.open_node(a, false))
+ .await
+ .expect("cancelled open left the per-node gate claimed")
+ .unwrap();
+ assert!(mount.node_status(a).await.unwrap().is_some());
+ drop(file);
+}