You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/06/03 03:02:22 UTC

(opendal) 01/01: fix(ofs): Close file during flush

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch try-fix-ofs
in repository https://gitbox.apache.org/repos/asf/opendal.git

commit ac1f0a6e48b27a17144eb5ed88be54a22dfde5dc
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Jun 3 11:01:50 2024 +0800

    fix(ofs): Close file during flush
    
    We don't want this, but we have to
    
    Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
 bin/ofs/Cargo.lock          |  9 ++++++++
 bin/ofs/Cargo.toml          |  1 +
 bin/ofs/src/fuse/adapter.rs | 50 ++++++++++++++++++++++++++++++---------------
 bin/ofs/tests/common/mod.rs | 35 ++++++-------------------------
 4 files changed, 49 insertions(+), 46 deletions(-)

diff --git a/bin/ofs/Cargo.lock b/bin/ofs/Cargo.lock
index 3cbe072632..93a78540e0 100644
--- a/bin/ofs/Cargo.lock
+++ b/bin/ofs/Cargo.lock
@@ -391,6 +391,12 @@ dependencies = [
  "const-random",
 ]
 
+[[package]]
+name = "dotenvy"
+version = "0.15.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
+
 [[package]]
 name = "either"
 version = "1.9.0"
@@ -964,6 +970,7 @@ dependencies = [
  "bytes",
  "chrono",
  "crc32c",
+ "dotenvy",
  "flagset",
  "futures",
  "getrandom",
@@ -973,10 +980,12 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "quick-xml",
+ "rand",
  "reqsign",
  "reqwest",
  "serde",
  "serde_json",
+ "sha2",
  "tokio",
  "uuid",
 ]
diff --git a/bin/ofs/Cargo.toml b/bin/ofs/Cargo.toml
index 42fb3d88cc..76bcb4c28d 100644
--- a/bin/ofs/Cargo.toml
+++ b/bin/ofs/Cargo.toml
@@ -62,3 +62,4 @@ test-context = "0.3.0"
 urlencoding = "2.1.3"
 uuid = "1.7.0"
 walkdir = "2.5.0"
+opendal = { version = "0.46.0", path = "../../core", features = ["tests"] }
\ No newline at end of file
diff --git a/bin/ofs/src/fuse/adapter.rs b/bin/ofs/src/fuse/adapter.rs
index 45ebf79b31..5c14c767ea 100644
--- a/bin/ofs/src/fuse/adapter.rs
+++ b/bin/ofs/src/fuse/adapter.rs
@@ -387,15 +387,12 @@ impl PathFilesystem for FuseAdapter {
         let (is_read, is_trunc, is_append) = self.check_flags(flags | libc::O_CREAT as u32)?;
 
         let path = PathBuf::from(parent).join(name);
-        self.op
-            .write(&path.to_string_lossy(), Bytes::new())
-            .await
-            .map_err(opendal_error2errno)?;
 
         let inner_writer = if is_trunc || is_append {
             let writer = self
                 .op
                 .writer_with(&path.to_string_lossy())
+                .chunk(4 * 1024 * 1024)
                 .append(is_append)
                 .await
                 .map_err(opendal_error2errno)?;
@@ -425,22 +422,22 @@ impl PathFilesystem for FuseAdapter {
         })
     }
 
-    async fn release(
+    /// In design, flush could be called multiple times for a single open. But there is the only
+    /// place that we can handle the write operations.
+    ///
+    /// So we only support the use case that flush only be called once.
+    async fn flush(
         &self,
         _req: Request,
         path: Option<&OsStr>,
         fh: u64,
-        flags: u32,
         lock_owner: u64,
-        flush: bool,
     ) -> Result<()> {
         log::debug!(
-            "release(path={:?}, fh={}, flags=0x{:x}, lock_owner={}, flush={})",
+            "flush(path={:?}, fh={}, lock_owner={})",
             path,
             fh,
-            flags,
             lock_owner,
-            flush
         );
 
         let file = self
@@ -449,13 +446,9 @@ impl PathFilesystem for FuseAdapter {
             .ok_or(Errno::from(libc::EBADF))?;
 
         if let Some(inner_writer) = file.inner_writer {
-            inner_writer
-                .lock_owned()
-                .await
-                .writer
-                .close()
-                .await
-                .map_err(opendal_error2errno)?;
+            let mut lock = inner_writer.lock().await;
+            let res = lock.writer.close().await.map_err(opendal_error2errno);
+            return res;
         }
 
         if matches!(path, Some(ref p) if p != &file.path) {
@@ -465,6 +458,29 @@ impl PathFilesystem for FuseAdapter {
         Ok(())
     }
 
+    async fn release(
+        &self,
+        _req: Request,
+        path: Option<&OsStr>,
+        fh: u64,
+        flags: u32,
+        lock_owner: u64,
+        flush: bool,
+    ) -> Result<()> {
+        log::debug!(
+            "release(path={:?}, fh={}, flags=0x{:x}, lock_owner={}, flush={})",
+            path,
+            fh,
+            flags,
+            lock_owner,
+            flush
+        );
+
+        // Just take and forget it.
+        let _ = self.opened_files.take(FileKey::try_from(fh)?.0);
+        Ok(())
+    }
+
     async fn open(&self, _req: Request, path: &OsStr, flags: u32) -> Result<ReplyOpen> {
         log::debug!("open(path={:?}, flags=0x{:x})", path, flags);
 
diff --git a/bin/ofs/tests/common/mod.rs b/bin/ofs/tests/common/mod.rs
index f2df4a1293..2b99a70716 100644
--- a/bin/ofs/tests/common/mod.rs
+++ b/bin/ofs/tests/common/mod.rs
@@ -15,9 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::{collections::HashMap, env, sync::OnceLock};
+use std::sync::OnceLock;
 
-use opendal::{Capability, Operator};
+use opendal::raw::tests;
+use opendal::Capability;
 use tempfile::TempDir;
 use test_context::TestContext;
 use tokio::runtime::{self, Runtime};
@@ -33,7 +34,9 @@ pub struct OfsTestContext {
 
 impl TestContext for OfsTestContext {
     fn setup() -> Self {
-        let backend = backend();
+        let backend = tests::init_test_service()
+            .expect("init test services failed")
+            .expect("no test services has been configured");
         let capability = backend.info().full_capability();
 
         INIT_LOGGER.get_or_init(env_logger::init);
@@ -70,29 +73,3 @@ impl TestContext for OfsTestContext {
         self.mount_point.close().unwrap();
     }
 }
-
-fn backend() -> Operator {
-    let scheme = env::var("OPENDAL_TEST").unwrap().parse().unwrap();
-    let prefix = format!("opendal_{scheme}_");
-
-    let mut cfg = env::vars()
-        .filter_map(|(k, v)| {
-            k.to_lowercase()
-                .strip_prefix(&prefix)
-                .map(|k| (k.to_string(), v))
-        })
-        .collect::<HashMap<String, String>>();
-
-    // Use random root unless OPENDAL_DISABLE_RANDOM_ROOT is set to true.
-    let disable_random_root = env::var("OPENDAL_DISABLE_RANDOM_ROOT").unwrap_or_default() == "true";
-    if !disable_random_root {
-        let root = format!(
-            "{}{}/",
-            cfg.get("root").cloned().unwrap_or_else(|| "/".to_string()),
-            uuid::Uuid::new_v4()
-        );
-        cfg.insert("root".to_string(), root);
-    }
-
-    Operator::via_map(scheme, cfg).unwrap()
-}