You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/10/26 07:30:07 UTC
(opendal-yinyang) branch xuanwo/linux-fuse-acceptance updated: test: harden Linux FUSE acceptance semantics
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/linux-fuse-acceptance
in repository https://gitbox.apache.org/repos/asf/opendal-yinyang.git
The following commit(s) were added to refs/heads/xuanwo/linux-fuse-acceptance by this push:
new e38aae4 test: harden Linux FUSE acceptance semantics
e38aae4 is described below
commit e38aae4475254093005541a4bbc99f6af290ebc2
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Oct 26 15:30:01 2026 +0800
test: harden Linux FUSE acceptance semantics
Keep external-change descriptors pinned, coordinate concurrent mounts, interrupt real in-flight fsync, preserve stale service addresses across restart, and require clean unmount and SIGTERM lease release.
Co-authored-by: Chojan Shang <ps...@apache.org>
---
docs/linux-fuse-acceptance.md | 21 +++--
tests/linux_fuse/acceptance.rs | 192 +++++++++++++++++++++++++++++++++++++----
tests/linux_fuse/harness.rs | 142 ++++++++++++++++++++++--------
3 files changed, 296 insertions(+), 59 deletions(-)
diff --git a/docs/linux-fuse-acceptance.md b/docs/linux-fuse-acceptance.md
index f86b66a..8befa12 100644
--- a/docs/linux-fuse-acceptance.md
+++ b/docs/linux-fuse-acceptance.md
@@ -50,18 +50,27 @@ cargo x test-linux-fuse --read-only
after `unlink`.
- Unsupported operations: symlink and hard link return an explicit OS error
instead of advertising POSIX links.
-- External changes and dirty refresh: independent external growth/shrink is
- authoritative on reopen, while a dirty descriptor keeps its staged bytes and
- `fsync` reports failure instead of silently rebasing.
+- External changes and dirty refresh: a descriptor opened before an external
+ grow/shrink keeps its pinned size, EOF and pread results, a fresh reopen
+ reflects the authoritative size, and a dirty descriptor keeps its staged
+ bytes while `fsync` reports failure instead of silently rebasing.
- Concurrent mounts: two live `yy mount` processes on separate staging
- directories publish and read distinct files concurrently.
+ directories publish distinct files concurrently under a barrier, then read
+ across each other's mounts.
- Process kill recovery: `SIGKILL` of the mount process retains staged bytes
- through persistent staging; a reopened mount can read and publish them.
+ through persistent staging while the descriptor is still open; a reopened
+ mount can read and publish them.
- Fsync/network failure retention: stopping the metadata service makes
`fsync` fail within a bound; staged bytes survive, the service restarts, and
- a reopened mount publishes them.
+ a reopened mount publishes them through the unchanged service address.
+- Interrupted call retention: a `fsync` left in flight against a paused
+ metadata service is interrupted by killing the mount daemon, the authority is
+ untouched, and a reopened mount publishes the staged bytes.
- Busy unmount: `fusermount3 -u` fails while a descriptor is open and succeeds
after the descriptor is released.
+- Unmount lifecycle: after a successful external `fusermount3 -u`, `yy` exits
+ on its own, removes the mount entry, and releases the staging lease. A
+ separate SIGTERM scenario verifies the same process and lease lifecycle.
The assertions follow the core specs. They do not add POSIX guarantees that the
transactional filesystem does not provide; for example symlinks are rejected,
diff --git a/tests/linux_fuse/acceptance.rs b/tests/linux_fuse/acceptance.rs
index 55a59dc..28cec80 100644
--- a/tests/linux_fuse/acceptance.rs
+++ b/tests/linux_fuse/acceptance.rs
@@ -17,9 +17,12 @@
//! Real /dev/fuse acceptance scenarios for the YinYang mount adapter.
-use crate::harness::{Fixture, Frontend, Publication, assert_errno, sync_all_bounded};
+use crate::harness::{Fixture, Frontend, Publication, assert_errno, sync_all_with_deadline};
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
+use std::os::unix::fs::FileExt;
+use std::sync::{Arc, Barrier, mpsc};
+use std::thread;
use std::time::Duration;
const EACCES: i32 = 13;
@@ -294,34 +297,57 @@ fn external_changes(fixture: &Fixture) {
fixture.volume_config(&format!("lib-external-{suffix}"), Frontend::Library, false);
fixture.library_seed(&library, "external.txt", b"AAAA");
fixture.library_seed(&library, "shrink.txt", b"0123456789");
+ fixture.library_seed(&library, "dirty.txt", b"BBBBBBBB");
let config = fixture.volume_config(&format!("stage-external-{suffix}"), Frontend::Fuse, false);
let mountpoint = fixture.temp.path().join(format!("mnt-external-{suffix}"));
fs::create_dir(&mountpoint).expect("create mountpoint");
let mut mounted = fixture.mount(&config, &mountpoint);
+ let mut old = File::open(mounted.mountpoint().join("external.txt"))
+ .expect("open descriptor before external growth");
+ let mut initial = Vec::new();
+ old.read_to_end(&mut initial).expect("read original bytes");
+ assert_eq!(initial, b"AAAA");
+ assert_eq!(old.metadata().expect("stat original").len(), 4);
fixture.remote_write(&library, "external.txt", b"BBBBBBBB");
assert_eq!(
fixture.remote_read(&library, "external.txt", 16),
b"BBBBBBBB",
"independent external growth must be authoritative"
);
+ assert_eq!(
+ old.metadata().expect("stat old descriptor").len(),
+ 4,
+ "external growth must not resize a descriptor that was opened before the change"
+ );
+ let mut beyond_old_eof = [0u8; 4];
+ let count = old
+ .read_at(&mut beyond_old_eof, 4)
+ .expect("pread beyond the old EOF");
+ assert_eq!(
+ count, 0,
+ "pread past the old EOF must not leak bytes from the externally grown file"
+ );
+ drop(old);
+ let dirty_path = mounted.mountpoint().join("dirty.txt");
let mut dirty = OpenOptions::new()
+ .read(true)
.write(true)
- .open(mounted.mountpoint().join("external.txt"))
+ .open(&dirty_path)
.expect("open file for dirty write");
dirty.write_all(b"LOCAL").expect("stage dirty write");
- fixture.remote_write(&library, "external.txt", b"ZZZZZZZZZZ");
+ fixture.remote_write(&library, "dirty.txt", b"ZZZZZZZZZZ");
dirty.seek(SeekFrom::Start(0)).expect("rewind dirty file");
let mut local = Vec::new();
dirty
.read_to_end(&mut local)
.expect("read dirty staged bytes");
assert_eq!(
- local, b"LOCAL",
+ local, b"LOCALBBB",
"an external change must not silently rebase a dirty descriptor"
);
- let conflict = sync_all_bounded(dirty.path(), Duration::from_secs(30));
+ let conflict = dirty.sync_all();
assert!(
conflict.is_err(),
"fsync of a stale dirty base must report failure instead of claiming remote durability"
@@ -336,11 +362,31 @@ fn external_changes(fixture: &Fixture) {
.join(format!("mnt-external-reopen-{suffix}"));
fs::create_dir(&mountpoint).expect("create reopen mountpoint");
let mut reopen = fixture.mount(&config, &mountpoint);
+ let mut old_shrink = File::open(reopen.mountpoint().join("shrink.txt"))
+ .expect("open descriptor before external shrink");
+ let mut before = Vec::new();
+ old_shrink
+ .read_to_end(&mut before)
+ .expect("read original shrink file");
+ assert_eq!(before, b"0123456789");
+ assert_eq!(old_shrink.metadata().expect("stat original").len(), 10);
+ fixture.remote_truncate(&library, "shrink.txt", 4);
assert_eq!(
- fs::read(reopen.mountpoint().join("shrink.txt")).expect("read shrunk file"),
- b"0123456789"
+ old_shrink.metadata().expect("stat old descriptor").len(),
+ 10,
+ "external shrink must not resize a descriptor that was opened before the change"
+ );
+ let mut tail = [0u8; 2];
+ let count = old_shrink
+ .read_at(&mut tail, 8)
+ .expect("pread within the old EOF");
+ assert_eq!(
+ count,
+ tail.len(),
+ "pread at the old EOF must still return pinned bytes"
);
- fixture.remote_truncate(&library, "shrink.txt", 4);
+ assert_eq!(&tail, b"89");
+ drop(old_shrink);
reopen.unmount();
let mountpoint = fixture
@@ -349,6 +395,15 @@ fn external_changes(fixture: &Fixture) {
.join(format!("mnt-external-final-{suffix}"));
fs::create_dir(&mountpoint).expect("create final mountpoint");
let mut final_mount = fixture.mount(&config, &mountpoint);
+ assert_eq!(
+ final_mount
+ .mountpoint()
+ .join("shrink.txt")
+ .metadata()
+ .expect("stat externally shrunk file")
+ .len(),
+ 4
+ );
assert_eq!(
fs::read(final_mount.mountpoint().join("shrink.txt")).expect("read externally shrunk file"),
b"0123"
@@ -370,14 +425,32 @@ fn fuse_concurrent_mounts_object() {
let mut a = fixture.mount(&config_a, &mount_a);
let mut b = fixture.mount(&config_b, &mount_b);
- let mut file_a = File::create(a.mountpoint().join("from-a.txt")).expect("create on mount A");
- file_a.write_all(b"a").expect("write via mount A");
- file_a.sync_all().expect("fsync via mount A");
- drop(file_a);
- let mut file_b = File::create(b.mountpoint().join("from-b.txt")).expect("create on mount B");
- file_b.write_all(b"b").expect("write via mount B");
- file_b.sync_all().expect("fsync via mount B");
- drop(file_b);
+ let start = Arc::new(Barrier::new(3));
+ let publish = Arc::new(Barrier::new(3));
+ let path_a = a.mountpoint().join("from-a.txt");
+ let path_b = b.mountpoint().join("from-b.txt");
+ let start_a = start.clone();
+ let start_b = start.clone();
+ let publish_a = publish.clone();
+ let publish_b = publish.clone();
+ let writer_a = thread::spawn(move || {
+ start_a.wait();
+ let mut file = File::create(&path_a).expect("create on mount A");
+ file.write_all(b"a").expect("write via mount A");
+ publish_a.wait();
+ file.sync_all().expect("fsync via mount A");
+ });
+ let writer_b = thread::spawn(move || {
+ start_b.wait();
+ let mut file = File::create(&path_b).expect("create on mount B");
+ file.write_all(b"b").expect("write via mount B");
+ publish_b.wait();
+ file.sync_all().expect("fsync via mount B");
+ });
+ start.wait();
+ publish.wait();
+ writer_a.join().expect("mount A writer panicked");
+ writer_b.join().expect("mount B writer panicked");
assert_eq!(
fs::read(a.mountpoint().join("from-b.txt")).expect("cross read"),
@@ -409,8 +482,13 @@ fn fuse_process_kill_recovery_object() {
.open(mounted.mountpoint().join("recover.txt"))
.expect("open file for staged write");
dirty.write_all(b"dirty-local").expect("stage dirty write");
- drop(dirty);
+ assert_eq!(
+ fixture.remote_read(&library, "recover.txt", 64),
+ b"seed",
+ "an un-fsynced mounted write must not publish before the daemon is interrupted"
+ );
mounted.kill();
+ drop(dirty);
let mountpoint = fixture.temp.path().join("mnt-kill-reopen");
fs::create_dir(&mountpoint).expect("create reopen mountpoint");
@@ -446,12 +524,13 @@ fn fuse_fsync_failure_retention_service() {
let mut mounted = fixture.mount(&config, &mountpoint);
let mut dirty = OpenOptions::new()
+ .read(true)
.write(true)
.open(mounted.mountpoint().join("network.txt"))
.expect("open file for staged write");
dirty.write_all(b"survives").expect("stage dirty write");
fixture.stop_service();
- let failure = sync_all_bounded(dirty.path(), Duration::from_secs(20));
+ let failure = sync_all_with_deadline(&dirty, Duration::from_secs(20));
assert!(
failure.is_err(),
"fsync must fail while the metadata service is unavailable"
@@ -469,7 +548,6 @@ fn fuse_fsync_failure_retention_service() {
mounted.kill();
fixture.restart_service();
- let config = fixture.volume_config("stage-network", Frontend::Fuse, false);
let mountpoint = fixture.temp.path().join("mnt-network-reopen");
fs::create_dir(&mountpoint).expect("create reopen mountpoint");
let mut reopened = fixture.mount(&config, &mountpoint);
@@ -490,6 +568,71 @@ fn fuse_fsync_failure_retention_service() {
reopened.unmount();
}
+#[test]
+#[ignore = "requires /dev/fuse and isolated S3; run with cargo x test-linux-fuse"]
+fn fuse_interrupted_call_retention_service() {
+ let fixture = Fixture::new(Publication::Service);
+ let library = fixture.volume_config("lib-interrupted", Frontend::Library, false);
+ fixture.library_seed(&library, "interrupted.txt", b"seed");
+ let config = fixture.volume_config("stage-interrupted", Frontend::Fuse, false);
+ let mountpoint = fixture.temp.path().join("mnt-interrupted");
+ fs::create_dir(&mountpoint).expect("create mountpoint");
+ let mut mounted = fixture.mount(&config, &mountpoint);
+
+ let mut dirty = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .open(mounted.mountpoint().join("interrupted.txt"))
+ .expect("open file for staged write");
+ dirty
+ .write_all(b"interrupted-local")
+ .expect("stage dirty write");
+ let in_flight = dirty.try_clone().expect("duplicate descriptor");
+ fixture.pause_service();
+ let (sender, receiver) = mpsc::channel();
+ let caller = thread::spawn(move || {
+ let _ = sender.send(in_flight.sync_all());
+ });
+ thread::sleep(Duration::from_millis(200));
+ mounted.kill();
+ let outcome = receiver
+ .recv_timeout(Duration::from_secs(20))
+ .unwrap_or_else(|error| {
+ panic!("fsync did not return after the mount daemon was killed: {error}")
+ });
+ caller.join().expect("fsync caller panicked");
+ assert!(
+ outcome.is_err(),
+ "an fsync interrupted by killing the mount daemon must fail, not report remote durability"
+ );
+ drop(dirty);
+
+ fixture.resume_service();
+ assert_eq!(
+ fixture.remote_read(&library, "interrupted.txt", 64),
+ b"seed",
+ "the interrupted publication must not have reached the authority"
+ );
+ let mountpoint = fixture.temp.path().join("mnt-interrupted-reopen");
+ fs::create_dir(&mountpoint).expect("create reopen mountpoint");
+ let mut reopened = fixture.mount(&config, &mountpoint);
+ assert_eq!(
+ fs::read(reopened.mountpoint().join("interrupted.txt")).expect("read staged recovery"),
+ b"interrupted-local"
+ );
+ let mut publish = OpenOptions::new()
+ .write(true)
+ .open(reopened.mountpoint().join("interrupted.txt"))
+ .expect("reopen recovered file");
+ publish.sync_all().expect("recovered handle publishes");
+ drop(publish);
+ assert_eq!(
+ fixture.remote_read(&library, "interrupted.txt", 64),
+ b"interrupted-local"
+ );
+ reopened.unmount();
+}
+
#[test]
#[ignore = "requires /dev/fuse and isolated S3; run with cargo x test-linux-fuse"]
fn fuse_busy_unmount_object() {
@@ -515,3 +658,14 @@ fn fuse_busy_unmount_object() {
drop(file);
mounted.unmount();
}
+
+#[test]
+#[ignore = "requires /dev/fuse and isolated S3; run with cargo x test-linux-fuse"]
+fn fuse_sigterm_lifecycle_object() {
+ let fixture = Fixture::new(Publication::Object);
+ let config = fixture.volume_config("stage-sigterm", Frontend::Fuse, false);
+ let mountpoint = fixture.temp.path().join("mnt-sigterm");
+ fs::create_dir(&mountpoint).expect("create mountpoint");
+ let mut mounted = fixture.mount(&config, &mountpoint);
+ mounted.terminate();
+}
diff --git a/tests/linux_fuse/harness.rs b/tests/linux_fuse/harness.rs
index ddec6b3..7445f25 100644
--- a/tests/linux_fuse/harness.rs
+++ b/tests/linux_fuse/harness.rs
@@ -230,10 +230,14 @@ impl Fixture {
}
fn spawn_service(&mut self) {
+ self.spawn_service_on("127.0.0.1:0".parse().expect("loopback wildcard"));
+ }
+
+ fn spawn_service_on(&mut self, listen: SocketAddr) {
let database = self.temp.path().join("metadata.db");
let database = database.to_str().expect("UTF-8 database path");
- let mut cmd =
- self.yy_command(&["serve", "--database", database, "--listen", "127.0.0.1:0"]);
+ let listen = listen.to_string();
+ let mut cmd = self.yy_command(&["serve", "--database", database, "--listen", &listen]);
cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
let mut child = cmd
.spawn()
@@ -290,8 +294,9 @@ impl Fixture {
}
pub fn restart_service(&mut self) {
+ let address = self.service_address();
self.stop_service();
- self.spawn_service();
+ self.spawn_service_on(address);
}
pub fn service_address(&self) -> SocketAddr {
@@ -301,8 +306,37 @@ impl Fixture {
.address
}
+ pub fn pause_service(&self) {
+ let pid = self
+ .service
+ .as_ref()
+ .expect("service publication requires a running metadata service")
+ .child
+ .id();
+ let output = run_timeout(
+ Command::new("kill").args(["-STOP", &pid.to_string()]),
+ Duration::from_secs(30),
+ );
+ output.success_or("pause metadata service");
+ }
+
+ pub fn resume_service(&self) {
+ let pid = self
+ .service
+ .as_ref()
+ .expect("service publication requires a running metadata service")
+ .child
+ .id();
+ let output = run_timeout(
+ Command::new("kill").args(["-CONT", &pid.to_string()]),
+ Duration::from_secs(30),
+ );
+ output.success_or("resume metadata service");
+ }
+
/// Writes a strict volume config and returns its path. Rewriting the same
- /// path is intentional: a restarted service gets a fresh loopback address.
+ /// path is intentional: a restarted service keeps its original loopback
+ /// address, so an unchanged config keeps working.
pub fn volume_config(&self, name: &str, frontend: Frontend, read_only: bool) -> PathBuf {
let staging = self.temp.path().join(name);
let mut json = format!(
@@ -382,6 +416,10 @@ impl Fixture {
}
pub fn mount(&self, config: &Path, mountpoint: &Path) -> Mounted {
+ let staging = self
+ .temp
+ .path()
+ .join(config.file_stem().expect("volume config has a file stem"));
let mountpoint = mountpoint.to_str().expect("UTF-8 mountpoint");
let config = config.to_str().expect("UTF-8 config path");
let mut cmd = self.yy_command(&["--volume", config, "mount", mountpoint]);
@@ -398,6 +436,7 @@ impl Fixture {
stdout: Some(stdout_thread),
stderr: Some(stderr_thread),
mountpoint: PathBuf::from(mountpoint),
+ staging,
};
mounted.wait_mounted();
mounted
@@ -431,6 +470,7 @@ pub struct Mounted {
stdout: Option<JoinHandle<Vec<u8>>>,
stderr: Option<JoinHandle<Vec<u8>>>,
mountpoint: PathBuf,
+ staging: PathBuf,
}
impl Mounted {
@@ -478,8 +518,29 @@ impl Mounted {
pub fn kill(&mut self) {
let _ = self.child.kill();
- self.wait_exit(Duration::from_secs(30));
+ let _ = self.wait_exit(Duration::from_secs(30));
self.unmount_force();
+ self.assert_staging_lease_released();
+ }
+
+ pub fn terminate(&mut self) {
+ let pid = self.child.id().to_string();
+ let output = run_timeout(
+ Command::new("kill").args(["-TERM", &pid]),
+ Duration::from_secs(30),
+ );
+ output.success_or("send SIGTERM to yy mount");
+ let status = self.wait_exit(Duration::from_secs(30));
+ assert!(
+ status.success(),
+ "yy mount did not exit cleanly after SIGTERM: {status}"
+ );
+ assert!(
+ mount_fs_type(&self.mountpoint).is_none(),
+ "yy did not remove {} after a graceful SIGTERM",
+ self.mountpoint.display()
+ );
+ self.assert_staging_lease_released();
}
pub fn unmount(&mut self) {
@@ -489,7 +550,17 @@ impl Mounted {
Duration::from_secs(30),
);
output.success_or(&format!("fusermount3 -u {mountpoint}"));
- self.wait_exit(Duration::from_secs(30));
+ let status = self.wait_exit(Duration::from_secs(30));
+ assert!(
+ status.success(),
+ "yy mount did not exit cleanly after external unmount: {status}"
+ );
+ assert!(
+ mount_fs_type(&self.mountpoint).is_none(),
+ "external unmount left {} in /proc/mounts",
+ self.mountpoint.display()
+ );
+ self.assert_staging_lease_released();
}
fn unmount_force(&mut self) {
@@ -503,13 +574,13 @@ impl Mounted {
);
}
- fn wait_exit(&mut self, timeout: Duration) {
+ fn wait_exit(&mut self, timeout: Duration) -> ExitStatus {
let deadline = Instant::now() + timeout;
loop {
- if self.child.try_wait().expect("wait for yy mount").is_some() {
+ if let Some(status) = self.child.try_wait().expect("wait for yy mount") {
let _ = self.stdout.take().map(JoinHandle::join);
let _ = self.stderr.take().map(JoinHandle::join);
- return;
+ return status;
}
if Instant::now() >= deadline {
let _ = self.child.kill();
@@ -531,6 +602,23 @@ impl Mounted {
.and_then(|thread| thread.join().ok())
.unwrap_or_default()
}
+
+ fn assert_staging_lease_released(&self) {
+ let path = self.staging.join("runtime.lock");
+ let lock = fs::OpenOptions::new()
+ .create(true)
+ .truncate(false)
+ .read(true)
+ .write(true)
+ .open(&path)
+ .unwrap_or_else(|error| panic!("open staging lock {}: {error}", path.display()));
+ lock.try_lock().unwrap_or_else(|error| {
+ panic!(
+ "yy mount still holds the staging lease {} after process exit: {error}",
+ path.display()
+ )
+ });
+ }
}
impl Drop for Mounted {
@@ -586,29 +674,15 @@ pub fn assert_errno(error: std::io::Error, allowed: &[i32], operation: &str) {
);
}
-pub fn sync_all_bounded(path: &Path, timeout: Duration) -> std::io::Result<()> {
- // Run fsync in a fresh subprocess so a hung FUSE request cannot wedge the
- // test process; timeout(1) gives a bounded, explicit failure.
- let script = "import os, sys\nfd = os.open(sys.argv[1], os.O_RDWR)\nos.fsync(fd)\nos.close(fd)";
- let output = run_timeout(
- Command::new("timeout")
- .arg(timeout.as_secs().to_string())
- .arg("python3")
- .arg("-c")
- .arg(script)
- .arg(path),
- timeout + Duration::from_secs(5),
- );
- if output.status.success() {
- Ok(())
- } else {
- Err(std::io::Error::new(
- std::io::ErrorKind::Other,
- format!(
- "fsync helper for {} failed with {}",
- path.display(),
- output.status
- ),
- ))
- }
+pub fn sync_all_with_deadline(file: &fs::File, timeout: Duration) -> std::io::Result<()> {
+ let clone = file.try_clone()?;
+ let (sender, receiver) = mpsc::channel();
+ let worker = thread::spawn(move || {
+ let _ = sender.send(clone.sync_all());
+ });
+ let result = receiver.recv_timeout(timeout).unwrap_or_else(|error| {
+ panic!("fsync did not complete within {timeout:?}; a hang is not acceptance: {error}")
+ });
+ worker.join().expect("fsync worker panicked");
+ result
}