You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/10/26 07:30:07 UTC

(opendal-yinyang) branch xuanwo/linux-fuse-acceptance updated: test: harden Linux FUSE acceptance semantics

This is an automated email from the ASF dual-hosted git repository.

Xuanwo pushed a commit to branch xuanwo/linux-fuse-acceptance
in repository https://gitbox.apache.org/repos/asf/opendal-yinyang.git


The following commit(s) were added to refs/heads/xuanwo/linux-fuse-acceptance by this push:
     new e38aae4  test: harden Linux FUSE acceptance semantics
e38aae4 is described below

commit e38aae4475254093005541a4bbc99f6af290ebc2
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Oct 26 15:30:01 2026 +0800

    test: harden Linux FUSE acceptance semantics
    
    Keep external-change descriptors pinned, coordinate concurrent mounts, interrupt real in-flight fsync, preserve stale service addresses across restart, and require clean unmount and SIGTERM lease release.
    
    Co-authored-by: Chojan Shang <ps...@apache.org>
---
 docs/linux-fuse-acceptance.md  |  21 +++--
 tests/linux_fuse/acceptance.rs | 192 +++++++++++++++++++++++++++++++++++++----
 tests/linux_fuse/harness.rs    | 142 ++++++++++++++++++++++--------
 3 files changed, 296 insertions(+), 59 deletions(-)

diff --git a/docs/linux-fuse-acceptance.md b/docs/linux-fuse-acceptance.md
index f86b66a..8befa12 100644
--- a/docs/linux-fuse-acceptance.md
+++ b/docs/linux-fuse-acceptance.md
@@ -50,18 +50,27 @@ cargo x test-linux-fuse --read-only
   after `unlink`.
 - Unsupported operations: symlink and hard link return an explicit OS error
   instead of advertising POSIX links.
-- External changes and dirty refresh: independent external growth/shrink is
-  authoritative on reopen, while a dirty descriptor keeps its staged bytes and
-  `fsync` reports failure instead of silently rebasing.
+- External changes and dirty refresh: a descriptor opened before an external
+  grow/shrink keeps its pinned size, EOF and pread results, a fresh reopen
+  reflects the authoritative size, and a dirty descriptor keeps its staged
+  bytes while `fsync` reports failure instead of silently rebasing.
 - Concurrent mounts: two live `yy mount` processes on separate staging
-  directories publish and read distinct files concurrently.
+  directories publish distinct files concurrently under a barrier, then read
+  across each other's mounts.
 - Process kill recovery: `SIGKILL` of the mount process retains staged bytes
-  through persistent staging; a reopened mount can read and publish them.
+  through persistent staging while the descriptor is still open; a reopened
+  mount can read and publish them.
 - Fsync/network failure retention: stopping the metadata service makes
   `fsync` fail within a bound; staged bytes survive, the service restarts, and
-  a reopened mount publishes them.
+  a reopened mount publishes them through the unchanged service address.
+- Interrupted call retention: a `fsync` left in flight against a paused
+  metadata service is interrupted by killing the mount daemon, the authority is
+  untouched, and a reopened mount publishes the staged bytes.
 - Busy unmount: `fusermount3 -u` fails while a descriptor is open and succeeds
   after the descriptor is released.
+- Unmount lifecycle: after a successful external `fusermount3 -u`, `yy` exits
+  on its own, removes the mount entry, and releases the staging lease. A
+  separate SIGTERM scenario verifies the same process and lease lifecycle.
 
 The assertions follow the core specs. They do not add POSIX guarantees that the
 transactional filesystem does not provide; for example symlinks are rejected,
diff --git a/tests/linux_fuse/acceptance.rs b/tests/linux_fuse/acceptance.rs
index 55a59dc..28cec80 100644
--- a/tests/linux_fuse/acceptance.rs
+++ b/tests/linux_fuse/acceptance.rs
@@ -17,9 +17,12 @@
 
 //! Real /dev/fuse acceptance scenarios for the YinYang mount adapter.
 
-use crate::harness::{Fixture, Frontend, Publication, assert_errno, sync_all_bounded};
+use crate::harness::{Fixture, Frontend, Publication, assert_errno, sync_all_with_deadline};
 use std::fs::{self, File, OpenOptions};
 use std::io::{Read, Seek, SeekFrom, Write};
+use std::os::unix::fs::FileExt;
+use std::sync::{Arc, Barrier, mpsc};
+use std::thread;
 use std::time::Duration;
 
 const EACCES: i32 = 13;
@@ -294,34 +297,57 @@ fn external_changes(fixture: &Fixture) {
         fixture.volume_config(&format!("lib-external-{suffix}"), Frontend::Library, false);
     fixture.library_seed(&library, "external.txt", b"AAAA");
     fixture.library_seed(&library, "shrink.txt", b"0123456789");
+    fixture.library_seed(&library, "dirty.txt", b"BBBBBBBB");
     let config = fixture.volume_config(&format!("stage-external-{suffix}"), Frontend::Fuse, false);
     let mountpoint = fixture.temp.path().join(format!("mnt-external-{suffix}"));
     fs::create_dir(&mountpoint).expect("create mountpoint");
     let mut mounted = fixture.mount(&config, &mountpoint);
 
+    let mut old = File::open(mounted.mountpoint().join("external.txt"))
+        .expect("open descriptor before external growth");
+    let mut initial = Vec::new();
+    old.read_to_end(&mut initial).expect("read original bytes");
+    assert_eq!(initial, b"AAAA");
+    assert_eq!(old.metadata().expect("stat original").len(), 4);
     fixture.remote_write(&library, "external.txt", b"BBBBBBBB");
     assert_eq!(
         fixture.remote_read(&library, "external.txt", 16),
         b"BBBBBBBB",
         "independent external growth must be authoritative"
     );
+    assert_eq!(
+        old.metadata().expect("stat old descriptor").len(),
+        4,
+        "external growth must not resize a descriptor that was opened before the change"
+    );
+    let mut beyond_old_eof = [0u8; 4];
+    let count = old
+        .read_at(&mut beyond_old_eof, 4)
+        .expect("pread beyond the old EOF");
+    assert_eq!(
+        count, 0,
+        "pread past the old EOF must not leak bytes from the externally grown file"
+    );
+    drop(old);
 
+    let dirty_path = mounted.mountpoint().join("dirty.txt");
     let mut dirty = OpenOptions::new()
+        .read(true)
         .write(true)
-        .open(mounted.mountpoint().join("external.txt"))
+        .open(&dirty_path)
         .expect("open file for dirty write");
     dirty.write_all(b"LOCAL").expect("stage dirty write");
-    fixture.remote_write(&library, "external.txt", b"ZZZZZZZZZZ");
+    fixture.remote_write(&library, "dirty.txt", b"ZZZZZZZZZZ");
     dirty.seek(SeekFrom::Start(0)).expect("rewind dirty file");
     let mut local = Vec::new();
     dirty
         .read_to_end(&mut local)
         .expect("read dirty staged bytes");
     assert_eq!(
-        local, b"LOCAL",
+        local, b"LOCALBBB",
         "an external change must not silently rebase a dirty descriptor"
     );
-    let conflict = sync_all_bounded(dirty.path(), Duration::from_secs(30));
+    let conflict = dirty.sync_all();
     assert!(
         conflict.is_err(),
         "fsync of a stale dirty base must report failure instead of claiming remote durability"
@@ -336,11 +362,31 @@ fn external_changes(fixture: &Fixture) {
         .join(format!("mnt-external-reopen-{suffix}"));
     fs::create_dir(&mountpoint).expect("create reopen mountpoint");
     let mut reopen = fixture.mount(&config, &mountpoint);
+    let mut old_shrink = File::open(reopen.mountpoint().join("shrink.txt"))
+        .expect("open descriptor before external shrink");
+    let mut before = Vec::new();
+    old_shrink
+        .read_to_end(&mut before)
+        .expect("read original shrink file");
+    assert_eq!(before, b"0123456789");
+    assert_eq!(old_shrink.metadata().expect("stat original").len(), 10);
+    fixture.remote_truncate(&library, "shrink.txt", 4);
     assert_eq!(
-        fs::read(reopen.mountpoint().join("shrink.txt")).expect("read shrunk file"),
-        b"0123456789"
+        old_shrink.metadata().expect("stat old descriptor").len(),
+        10,
+        "external shrink must not resize a descriptor that was opened before the change"
+    );
+    let mut tail = [0u8; 2];
+    let count = old_shrink
+        .read_at(&mut tail, 8)
+        .expect("pread within the old EOF");
+    assert_eq!(
+        count,
+        tail.len(),
+        "pread at the old EOF must still return pinned bytes"
     );
-    fixture.remote_truncate(&library, "shrink.txt", 4);
+    assert_eq!(&tail, b"89");
+    drop(old_shrink);
     reopen.unmount();
 
     let mountpoint = fixture
@@ -349,6 +395,15 @@ fn external_changes(fixture: &Fixture) {
         .join(format!("mnt-external-final-{suffix}"));
     fs::create_dir(&mountpoint).expect("create final mountpoint");
     let mut final_mount = fixture.mount(&config, &mountpoint);
+    assert_eq!(
+        final_mount
+            .mountpoint()
+            .join("shrink.txt")
+            .metadata()
+            .expect("stat externally shrunk file")
+            .len(),
+        4
+    );
     assert_eq!(
         fs::read(final_mount.mountpoint().join("shrink.txt")).expect("read externally shrunk file"),
         b"0123"
@@ -370,14 +425,32 @@ fn fuse_concurrent_mounts_object() {
     let mut a = fixture.mount(&config_a, &mount_a);
     let mut b = fixture.mount(&config_b, &mount_b);
 
-    let mut file_a = File::create(a.mountpoint().join("from-a.txt")).expect("create on mount A");
-    file_a.write_all(b"a").expect("write via mount A");
-    file_a.sync_all().expect("fsync via mount A");
-    drop(file_a);
-    let mut file_b = File::create(b.mountpoint().join("from-b.txt")).expect("create on mount B");
-    file_b.write_all(b"b").expect("write via mount B");
-    file_b.sync_all().expect("fsync via mount B");
-    drop(file_b);
+    let start = Arc::new(Barrier::new(3));
+    let publish = Arc::new(Barrier::new(3));
+    let path_a = a.mountpoint().join("from-a.txt");
+    let path_b = b.mountpoint().join("from-b.txt");
+    let start_a = start.clone();
+    let start_b = start.clone();
+    let publish_a = publish.clone();
+    let publish_b = publish.clone();
+    let writer_a = thread::spawn(move || {
+        start_a.wait();
+        let mut file = File::create(&path_a).expect("create on mount A");
+        file.write_all(b"a").expect("write via mount A");
+        publish_a.wait();
+        file.sync_all().expect("fsync via mount A");
+    });
+    let writer_b = thread::spawn(move || {
+        start_b.wait();
+        let mut file = File::create(&path_b).expect("create on mount B");
+        file.write_all(b"b").expect("write via mount B");
+        publish_b.wait();
+        file.sync_all().expect("fsync via mount B");
+    });
+    start.wait();
+    publish.wait();
+    writer_a.join().expect("mount A writer panicked");
+    writer_b.join().expect("mount B writer panicked");
 
     assert_eq!(
         fs::read(a.mountpoint().join("from-b.txt")).expect("cross read"),
@@ -409,8 +482,13 @@ fn fuse_process_kill_recovery_object() {
         .open(mounted.mountpoint().join("recover.txt"))
         .expect("open file for staged write");
     dirty.write_all(b"dirty-local").expect("stage dirty write");
-    drop(dirty);
+    assert_eq!(
+        fixture.remote_read(&library, "recover.txt", 64),
+        b"seed",
+        "an un-fsynced mounted write must not publish before the daemon is interrupted"
+    );
     mounted.kill();
+    drop(dirty);
 
     let mountpoint = fixture.temp.path().join("mnt-kill-reopen");
     fs::create_dir(&mountpoint).expect("create reopen mountpoint");
@@ -446,12 +524,13 @@ fn fuse_fsync_failure_retention_service() {
     let mut mounted = fixture.mount(&config, &mountpoint);
 
     let mut dirty = OpenOptions::new()
+        .read(true)
         .write(true)
         .open(mounted.mountpoint().join("network.txt"))
         .expect("open file for staged write");
     dirty.write_all(b"survives").expect("stage dirty write");
     fixture.stop_service();
-    let failure = sync_all_bounded(dirty.path(), Duration::from_secs(20));
+    let failure = sync_all_with_deadline(&dirty, Duration::from_secs(20));
     assert!(
         failure.is_err(),
         "fsync must fail while the metadata service is unavailable"
@@ -469,7 +548,6 @@ fn fuse_fsync_failure_retention_service() {
     mounted.kill();
 
     fixture.restart_service();
-    let config = fixture.volume_config("stage-network", Frontend::Fuse, false);
     let mountpoint = fixture.temp.path().join("mnt-network-reopen");
     fs::create_dir(&mountpoint).expect("create reopen mountpoint");
     let mut reopened = fixture.mount(&config, &mountpoint);
@@ -490,6 +568,71 @@ fn fuse_fsync_failure_retention_service() {
     reopened.unmount();
 }
 
+#[test]
+#[ignore = "requires /dev/fuse and isolated S3; run with cargo x test-linux-fuse"]
+fn fuse_interrupted_call_retention_service() {
+    let fixture = Fixture::new(Publication::Service);
+    let library = fixture.volume_config("lib-interrupted", Frontend::Library, false);
+    fixture.library_seed(&library, "interrupted.txt", b"seed");
+    let config = fixture.volume_config("stage-interrupted", Frontend::Fuse, false);
+    let mountpoint = fixture.temp.path().join("mnt-interrupted");
+    fs::create_dir(&mountpoint).expect("create mountpoint");
+    let mut mounted = fixture.mount(&config, &mountpoint);
+
+    let mut dirty = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(mounted.mountpoint().join("interrupted.txt"))
+        .expect("open file for staged write");
+    dirty
+        .write_all(b"interrupted-local")
+        .expect("stage dirty write");
+    let in_flight = dirty.try_clone().expect("duplicate descriptor");
+    fixture.pause_service();
+    let (sender, receiver) = mpsc::channel();
+    let caller = thread::spawn(move || {
+        let _ = sender.send(in_flight.sync_all());
+    });
+    thread::sleep(Duration::from_millis(200));
+    mounted.kill();
+    let outcome = receiver
+        .recv_timeout(Duration::from_secs(20))
+        .unwrap_or_else(|error| {
+            panic!("fsync did not return after the mount daemon was killed: {error}")
+        });
+    caller.join().expect("fsync caller panicked");
+    assert!(
+        outcome.is_err(),
+        "an fsync interrupted by killing the mount daemon must fail, not report remote durability"
+    );
+    drop(dirty);
+
+    fixture.resume_service();
+    assert_eq!(
+        fixture.remote_read(&library, "interrupted.txt", 64),
+        b"seed",
+        "the interrupted publication must not have reached the authority"
+    );
+    let mountpoint = fixture.temp.path().join("mnt-interrupted-reopen");
+    fs::create_dir(&mountpoint).expect("create reopen mountpoint");
+    let mut reopened = fixture.mount(&config, &mountpoint);
+    assert_eq!(
+        fs::read(reopened.mountpoint().join("interrupted.txt")).expect("read staged recovery"),
+        b"interrupted-local"
+    );
+    let mut publish = OpenOptions::new()
+        .write(true)
+        .open(reopened.mountpoint().join("interrupted.txt"))
+        .expect("reopen recovered file");
+    publish.sync_all().expect("recovered handle publishes");
+    drop(publish);
+    assert_eq!(
+        fixture.remote_read(&library, "interrupted.txt", 64),
+        b"interrupted-local"
+    );
+    reopened.unmount();
+}
+
 #[test]
 #[ignore = "requires /dev/fuse and isolated S3; run with cargo x test-linux-fuse"]
 fn fuse_busy_unmount_object() {
@@ -515,3 +658,14 @@ fn fuse_busy_unmount_object() {
     drop(file);
     mounted.unmount();
 }
+
+#[test]
+#[ignore = "requires /dev/fuse and isolated S3; run with cargo x test-linux-fuse"]
+fn fuse_sigterm_lifecycle_object() {
+    let fixture = Fixture::new(Publication::Object);
+    let config = fixture.volume_config("stage-sigterm", Frontend::Fuse, false);
+    let mountpoint = fixture.temp.path().join("mnt-sigterm");
+    fs::create_dir(&mountpoint).expect("create mountpoint");
+    let mut mounted = fixture.mount(&config, &mountpoint);
+    mounted.terminate();
+}
diff --git a/tests/linux_fuse/harness.rs b/tests/linux_fuse/harness.rs
index ddec6b3..7445f25 100644
--- a/tests/linux_fuse/harness.rs
+++ b/tests/linux_fuse/harness.rs
@@ -230,10 +230,14 @@ impl Fixture {
     }
 
     fn spawn_service(&mut self) {
+        self.spawn_service_on("127.0.0.1:0".parse().expect("loopback wildcard"));
+    }
+
+    fn spawn_service_on(&mut self, listen: SocketAddr) {
         let database = self.temp.path().join("metadata.db");
         let database = database.to_str().expect("UTF-8 database path");
-        let mut cmd =
-            self.yy_command(&["serve", "--database", database, "--listen", "127.0.0.1:0"]);
+        let listen = listen.to_string();
+        let mut cmd = self.yy_command(&["serve", "--database", database, "--listen", &listen]);
         cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
         let mut child = cmd
             .spawn()
@@ -290,8 +294,9 @@ impl Fixture {
     }
 
     pub fn restart_service(&mut self) {
+        let address = self.service_address();
         self.stop_service();
-        self.spawn_service();
+        self.spawn_service_on(address);
     }
 
     pub fn service_address(&self) -> SocketAddr {
@@ -301,8 +306,37 @@ impl Fixture {
             .address
     }
 
+    pub fn pause_service(&self) {
+        let pid = self
+            .service
+            .as_ref()
+            .expect("service publication requires a running metadata service")
+            .child
+            .id();
+        let output = run_timeout(
+            Command::new("kill").args(["-STOP", &pid.to_string()]),
+            Duration::from_secs(30),
+        );
+        output.success_or("pause metadata service");
+    }
+
+    pub fn resume_service(&self) {
+        let pid = self
+            .service
+            .as_ref()
+            .expect("service publication requires a running metadata service")
+            .child
+            .id();
+        let output = run_timeout(
+            Command::new("kill").args(["-CONT", &pid.to_string()]),
+            Duration::from_secs(30),
+        );
+        output.success_or("resume metadata service");
+    }
+
     /// Writes a strict volume config and returns its path. Rewriting the same
-    /// path is intentional: a restarted service gets a fresh loopback address.
+    /// path is intentional: a restarted service keeps its original loopback
+    /// address, so an unchanged config keeps working.
     pub fn volume_config(&self, name: &str, frontend: Frontend, read_only: bool) -> PathBuf {
         let staging = self.temp.path().join(name);
         let mut json = format!(
@@ -382,6 +416,10 @@ impl Fixture {
     }
 
     pub fn mount(&self, config: &Path, mountpoint: &Path) -> Mounted {
+        let staging = self
+            .temp
+            .path()
+            .join(config.file_stem().expect("volume config has a file stem"));
         let mountpoint = mountpoint.to_str().expect("UTF-8 mountpoint");
         let config = config.to_str().expect("UTF-8 config path");
         let mut cmd = self.yy_command(&["--volume", config, "mount", mountpoint]);
@@ -398,6 +436,7 @@ impl Fixture {
             stdout: Some(stdout_thread),
             stderr: Some(stderr_thread),
             mountpoint: PathBuf::from(mountpoint),
+            staging,
         };
         mounted.wait_mounted();
         mounted
@@ -431,6 +470,7 @@ pub struct Mounted {
     stdout: Option<JoinHandle<Vec<u8>>>,
     stderr: Option<JoinHandle<Vec<u8>>>,
     mountpoint: PathBuf,
+    staging: PathBuf,
 }
 
 impl Mounted {
@@ -478,8 +518,29 @@ impl Mounted {
 
     pub fn kill(&mut self) {
         let _ = self.child.kill();
-        self.wait_exit(Duration::from_secs(30));
+        let _ = self.wait_exit(Duration::from_secs(30));
         self.unmount_force();
+        self.assert_staging_lease_released();
+    }
+
+    pub fn terminate(&mut self) {
+        let pid = self.child.id().to_string();
+        let output = run_timeout(
+            Command::new("kill").args(["-TERM", &pid]),
+            Duration::from_secs(30),
+        );
+        output.success_or("send SIGTERM to yy mount");
+        let status = self.wait_exit(Duration::from_secs(30));
+        assert!(
+            status.success(),
+            "yy mount did not exit cleanly after SIGTERM: {status}"
+        );
+        assert!(
+            mount_fs_type(&self.mountpoint).is_none(),
+            "yy did not remove {} after a graceful SIGTERM",
+            self.mountpoint.display()
+        );
+        self.assert_staging_lease_released();
     }
 
     pub fn unmount(&mut self) {
@@ -489,7 +550,17 @@ impl Mounted {
             Duration::from_secs(30),
         );
         output.success_or(&format!("fusermount3 -u {mountpoint}"));
-        self.wait_exit(Duration::from_secs(30));
+        let status = self.wait_exit(Duration::from_secs(30));
+        assert!(
+            status.success(),
+            "yy mount did not exit cleanly after external unmount: {status}"
+        );
+        assert!(
+            mount_fs_type(&self.mountpoint).is_none(),
+            "external unmount left {} in /proc/mounts",
+            self.mountpoint.display()
+        );
+        self.assert_staging_lease_released();
     }
 
     fn unmount_force(&mut self) {
@@ -503,13 +574,13 @@ impl Mounted {
         );
     }
 
-    fn wait_exit(&mut self, timeout: Duration) {
+    fn wait_exit(&mut self, timeout: Duration) -> ExitStatus {
         let deadline = Instant::now() + timeout;
         loop {
-            if self.child.try_wait().expect("wait for yy mount").is_some() {
+            if let Some(status) = self.child.try_wait().expect("wait for yy mount") {
                 let _ = self.stdout.take().map(JoinHandle::join);
                 let _ = self.stderr.take().map(JoinHandle::join);
-                return;
+                return status;
             }
             if Instant::now() >= deadline {
                 let _ = self.child.kill();
@@ -531,6 +602,23 @@ impl Mounted {
             .and_then(|thread| thread.join().ok())
             .unwrap_or_default()
     }
+
+    fn assert_staging_lease_released(&self) {
+        let path = self.staging.join("runtime.lock");
+        let lock = fs::OpenOptions::new()
+            .create(true)
+            .truncate(false)
+            .read(true)
+            .write(true)
+            .open(&path)
+            .unwrap_or_else(|error| panic!("open staging lock {}: {error}", path.display()));
+        lock.try_lock().unwrap_or_else(|error| {
+            panic!(
+                "yy mount still holds the staging lease {} after process exit: {error}",
+                path.display()
+            )
+        });
+    }
 }
 
 impl Drop for Mounted {
@@ -586,29 +674,15 @@ pub fn assert_errno(error: std::io::Error, allowed: &[i32], operation: &str) {
     );
 }
 
-pub fn sync_all_bounded(path: &Path, timeout: Duration) -> std::io::Result<()> {
-    // Run fsync in a fresh subprocess so a hung FUSE request cannot wedge the
-    // test process; timeout(1) gives a bounded, explicit failure.
-    let script = "import os, sys\nfd = os.open(sys.argv[1], os.O_RDWR)\nos.fsync(fd)\nos.close(fd)";
-    let output = run_timeout(
-        Command::new("timeout")
-            .arg(timeout.as_secs().to_string())
-            .arg("python3")
-            .arg("-c")
-            .arg(script)
-            .arg(path),
-        timeout + Duration::from_secs(5),
-    );
-    if output.status.success() {
-        Ok(())
-    } else {
-        Err(std::io::Error::new(
-            std::io::ErrorKind::Other,
-            format!(
-                "fsync helper for {} failed with {}",
-                path.display(),
-                output.status
-            ),
-        ))
-    }
+pub fn sync_all_with_deadline(file: &fs::File, timeout: Duration) -> std::io::Result<()> {
+    let clone = file.try_clone()?;
+    let (sender, receiver) = mpsc::channel();
+    let worker = thread::spawn(move || {
+        let _ = sender.send(clone.sync_all());
+    });
+    let result = receiver.recv_timeout(timeout).unwrap_or_else(|error| {
+        panic!("fsync did not complete within {timeout:?}; a hang is not acceptance: {error}")
+    });
+    worker.join().expect("fsync worker panicked");
+    result
 }