You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/10/26 03:30:43 UTC

[incubator-opendal] branch main updated: feat: Add write_total_max_size in Capability (#3309)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 3aad395fc feat: Add write_total_max_size in Capability (#3309)
3aad395fc is described below

commit 3aad395fc8d8c894a28d05d8e6bf904d1eb38d08
Author: taobo <mt...@gmail.com>
AuthorDate: Thu Oct 26 11:30:39 2023 +0800

    feat: Add write_total_max_size in Capability (#3309)
    
    * feat: behavior test support config max_size
    
    Signed-off-by: taobo <mt...@gmail.com>
    
    * feat: update gen_bytes func
    
    * fix: make clippy happy
    
    ---------
    
    Signed-off-by: taobo <mt...@gmail.com>
---
 core/Cargo.toml                        |  6 ++--
 core/src/services/d1/backend.rs        |  3 ++
 core/src/types/capability.rs           |  4 +++
 core/src/types/operator/builder.rs     |  2 ++
 core/tests/behavior/append.rs          | 12 +++----
 core/tests/behavior/blocking_append.rs | 12 +++----
 core/tests/behavior/blocking_copy.rs   | 12 +++----
 core/tests/behavior/blocking_list.rs   |  6 ++--
 core/tests/behavior/blocking_rename.rs | 12 +++----
 core/tests/behavior/blocking_write.rs  | 26 +++++++-------
 core/tests/behavior/copy.rs            | 14 ++++----
 core/tests/behavior/list.rs            |  6 ++--
 core/tests/behavior/presign.rs         |  6 ++--
 core/tests/behavior/rename.rs          | 12 +++----
 core/tests/behavior/utils.rs           |  5 +--
 core/tests/behavior/write.rs           | 62 +++++++++++++++++-----------------
 16 files changed, 105 insertions(+), 95 deletions(-)

diff --git a/core/Cargo.toml b/core/Cargo.toml
index 3679f3955..9ddae4539 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -115,6 +115,7 @@ services-azdls = [
   "reqsign?/services-azblob",
   "reqsign?/reqwest_request",
 ]
+services-azfile = []
 services-cacache = ["dep:cacache"]
 services-cloudflare-kv = []
 services-cos = [
@@ -145,6 +146,7 @@ services-memcached = ["dep:bb8"]
 services-memory = []
 services-mini-moka = ["dep:mini-moka"]
 services-moka = ["dep:moka"]
+services-mongodb = ["dep:mongodb"]
 services-mysql = ["dep:mysql_async"]
 services-obs = [
   "dep:reqsign",
@@ -182,8 +184,6 @@ services-wasabi = [
 ]
 services-webdav = []
 services-webhdfs = []
-services-azfile = []
-services-mongodb = ["dep:mongodb"]
 
 [lib]
 bench = false
@@ -243,6 +243,7 @@ metrics = { version = "0.20", optional = true }
 mini-moka = { version = "0.10", optional = true }
 minitrace = { version = "0.5", optional = true }
 moka = { version = "0.10", optional = true, features = ["future"] }
+mongodb = { version = "2.7.0", optional = true, features = ["tokio-runtime"] }
 mysql_async = { version = "0.32.2", optional = true }
 once_cell = "1"
 openssh = { version = "0.9.9", optional = true }
@@ -286,7 +287,6 @@ tokio = "1.27"
 tokio-postgres = { version = "0.7.8", optional = true }
 tracing = { version = "0.1", optional = true }
 uuid = { version = "1", features = ["serde", "v4"] }
-mongodb = { version = "2.7.0", optional = true, features = ["tokio-runtime"] }
 
 [dev-dependencies]
 criterion = { version = "0.4", features = ["async", "async_tokio"] }
diff --git a/core/src/services/d1/backend.rs b/core/src/services/d1/backend.rs
index faf0c472e..15e8d2854 100644
--- a/core/src/services/d1/backend.rs
+++ b/core/src/services/d1/backend.rs
@@ -271,6 +271,9 @@ impl kv::Adapter for Adapter {
             Capability {
                 read: true,
                 write: true,
+                // Cloudflare D1 supports 1MB as max in write_total.
+                // refer to https://developers.cloudflare.com/d1/platform/limits/
+                write_total_max_size: Some(1000 * 1000),
                 ..Default::default()
             },
         )
diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs
index 2767c29e4..437a3e9ba 100644
--- a/core/src/types/capability.rs
+++ b/core/src/types/capability.rs
@@ -98,6 +98,10 @@ pub struct Capability {
     ///
     /// For example, Google GCS requires align size to 256KiB in write_multi.
     pub write_multi_align_size: Option<usize>,
+    /// write_total_max_size is the max size that services support in write_total.
+    ///
+    /// For example, Cloudflare D1 supports 1MB as max in write_total.
+    pub write_total_max_size: Option<usize>,
 
     /// If operator supports create dir.
     pub create_dir: bool,
diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs
index 1f7a4a5aa..c7976e2c7 100644
--- a/core/src/types/operator/builder.rs
+++ b/core/src/types/operator/builder.rs
@@ -163,6 +163,8 @@ impl Operator {
             Scheme::Cacache => Self::from_map::<services::Cacache>(map)?.finish(),
             #[cfg(feature = "services-cos")]
             Scheme::Cos => Self::from_map::<services::Cos>(map)?.finish(),
+            #[cfg(feature = "services-d1")]
+            Scheme::D1 => Self::from_map::<services::D1>(map)?.finish(),
             #[cfg(feature = "services-dashmap")]
             Scheme::Dashmap => Self::from_map::<services::Dashmap>(map)?.finish(),
             #[cfg(feature = "services-dropbox")]
diff --git a/core/tests/behavior/append.rs b/core/tests/behavior/append.rs
index 533ca15b7..91ed3bee3 100644
--- a/core/tests/behavior/append.rs
+++ b/core/tests/behavior/append.rs
@@ -47,8 +47,8 @@ pub fn behavior_append_tests(op: &Operator) -> Vec<Trial> {
 /// Test append to a file must success.
 pub async fn test_append_create_append(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
-    let (content_one, size_one) = gen_bytes();
-    let (content_two, size_two) = gen_bytes();
+    let (content_one, size_one) = gen_bytes(op.info().full_capability());
+    let (content_two, size_two) = gen_bytes(op.info().full_capability());
 
     op.write_with(&path, content_one.clone())
         .append(true)
@@ -74,7 +74,7 @@ pub async fn test_append_create_append(op: Operator) -> Result<()> {
 /// Test append to a directory path must fail.
 pub async fn test_append_with_dir_path(op: Operator) -> Result<()> {
     let path = format!("{}/", uuid::Uuid::new_v4());
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let res = op.write_with(&path, content).append(true).await;
     assert!(res.is_err());
@@ -90,7 +90,7 @@ pub async fn test_append_with_cache_control(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let target_cache_control = "no-cache, no-store, max-age=300";
     op.write_with(&path, content)
@@ -117,7 +117,7 @@ pub async fn test_append_with_content_type(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let target_content_type = "application/json";
     op.write_with(&path, content)
@@ -145,7 +145,7 @@ pub async fn test_append_with_content_disposition(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let target_content_disposition = "attachment; filename=\"filename.jpg\"";
     op.write_with(&path, content)
diff --git a/core/tests/behavior/blocking_append.rs b/core/tests/behavior/blocking_append.rs
index d402d97e5..8161f825a 100644
--- a/core/tests/behavior/blocking_append.rs
+++ b/core/tests/behavior/blocking_append.rs
@@ -47,8 +47,8 @@ pub fn behavior_blocking_append_tests(op: &Operator) -> Vec<Trial> {
 /// Test append to a file must success.
 pub fn test_blocking_append_create_append(op: BlockingOperator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
-    let (content_one, size_one) = gen_bytes();
-    let (content_two, size_two) = gen_bytes();
+    let (content_one, size_one) = gen_bytes(op.info().full_capability());
+    let (content_two, size_two) = gen_bytes(op.info().full_capability());
 
     op.write_with(&path, content_one.clone())
         .append(true)
@@ -74,7 +74,7 @@ pub fn test_blocking_append_create_append(op: BlockingOperator) -> Result<()> {
 /// Test append to a directory path must fail.
 pub fn test_blocking_append_with_dir_path(op: BlockingOperator) -> Result<()> {
     let path = format!("{}/", uuid::Uuid::new_v4());
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let res = op.write_with(&path, content).append(true).call();
     assert!(res.is_err());
@@ -90,7 +90,7 @@ pub fn test_blocking_append_with_cache_control(op: BlockingOperator) -> Result<(
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let target_cache_control = "no-cache, no-store, max-age=300";
     op.write_with(&path, content)
@@ -117,7 +117,7 @@ pub fn test_blocking_append_with_content_type(op: BlockingOperator) -> Result<()
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let target_content_type = "application/json";
     op.write_with(&path, content)
@@ -145,7 +145,7 @@ pub fn test_blocking_append_with_content_disposition(op: BlockingOperator) -> Re
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let target_content_disposition = "attachment; filename=\"filename.jpg\"";
     op.write_with(&path, content)
diff --git a/core/tests/behavior/blocking_copy.rs b/core/tests/behavior/blocking_copy.rs
index 822369a22..163c55990 100644
--- a/core/tests/behavior/blocking_copy.rs
+++ b/core/tests/behavior/blocking_copy.rs
@@ -41,7 +41,7 @@ pub fn behavior_blocking_copy_tests(op: &Operator) -> Vec<Trial> {
 /// Copy a file and test with stat.
 pub fn test_blocking_copy_file(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
@@ -86,7 +86,7 @@ pub fn test_blocking_copy_source_dir(op: BlockingOperator) -> Result<()> {
 /// Copy to a dir should return an error.
 pub fn test_blocking_copy_target_dir(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content)?;
 
@@ -107,7 +107,7 @@ pub fn test_blocking_copy_target_dir(op: BlockingOperator) -> Result<()> {
 /// Copy a file to self should return an error.
 pub fn test_blocking_copy_self(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _size) = gen_bytes();
+    let (source_content, _size) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content)?;
 
@@ -123,7 +123,7 @@ pub fn test_blocking_copy_self(op: BlockingOperator) -> Result<()> {
 /// Copy to a nested path, parent path should be created successfully.
 pub fn test_blocking_copy_nested(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
@@ -147,12 +147,12 @@ pub fn test_blocking_copy_nested(op: BlockingOperator) -> Result<()> {
 /// Copy to a exist path should overwrite successfully.
 pub fn test_blocking_copy_overwrite(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
     let target_path = uuid::Uuid::new_v4().to_string();
-    let (target_content, _) = gen_bytes();
+    let (target_content, _) = gen_bytes(op.info().full_capability());
     assert_ne!(source_content, target_content);
 
     op.write(&target_path, target_content)?;
diff --git a/core/tests/behavior/blocking_list.rs b/core/tests/behavior/blocking_list.rs
index c58e4feea..e53b8a1de 100644
--- a/core/tests/behavior/blocking_list.rs
+++ b/core/tests/behavior/blocking_list.rs
@@ -46,7 +46,7 @@ pub fn test_blocking_list_dir(op: BlockingOperator) -> Result<()> {
     let parent = uuid::Uuid::new_v4().to_string();
     let path = format!("{parent}/{}", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).expect("write must succeed");
 
@@ -74,7 +74,7 @@ pub fn test_blocking_list_dir_with_metakey(op: BlockingOperator) -> Result<()> {
     let parent = uuid::Uuid::new_v4().to_string();
     let path = format!("{parent}/{}", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).expect("write must succeed");
 
@@ -124,7 +124,7 @@ pub fn test_blocking_list_dir_with_metakey_complete(op: BlockingOperator) -> Res
     let parent = uuid::Uuid::new_v4().to_string();
     let path = format!("{parent}/{}", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).expect("write must succeed");
 
diff --git a/core/tests/behavior/blocking_rename.rs b/core/tests/behavior/blocking_rename.rs
index 398a3f4da..602d59399 100644
--- a/core/tests/behavior/blocking_rename.rs
+++ b/core/tests/behavior/blocking_rename.rs
@@ -41,7 +41,7 @@ pub fn behavior_blocking_rename_tests(op: &Operator) -> Vec<Trial> {
 /// Rename a file and test with stat.
 pub fn test_blocking_rename_file(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
@@ -89,7 +89,7 @@ pub fn test_blocking_rename_source_dir(op: BlockingOperator) -> Result<()> {
 /// Rename to a dir should return an error.
 pub fn test_blocking_rename_target_dir(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content)?;
 
@@ -110,7 +110,7 @@ pub fn test_blocking_rename_target_dir(op: BlockingOperator) -> Result<()> {
 /// Rename a file to self should return an error.
 pub fn test_blocking_rename_self(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _size) = gen_bytes();
+    let (source_content, _size) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content)?;
 
@@ -126,7 +126,7 @@ pub fn test_blocking_rename_self(op: BlockingOperator) -> Result<()> {
 /// Rename to a nested path, parent path should be created successfully.
 pub fn test_blocking_rename_nested(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
@@ -153,12 +153,12 @@ pub fn test_blocking_rename_nested(op: BlockingOperator) -> Result<()> {
 /// Rename to a exist path should overwrite successfully.
 pub fn test_blocking_rename_overwrite(op: BlockingOperator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone())?;
 
     let target_path = uuid::Uuid::new_v4().to_string();
-    let (target_content, _) = gen_bytes();
+    let (target_content, _) = gen_bytes(op.info().full_capability());
     assert_ne!(source_content, target_content);
 
     op.write(&target_path, target_content)?;
diff --git a/core/tests/behavior/blocking_write.rs b/core/tests/behavior/blocking_write.rs
index 1c305c5c6..25442a62f 100644
--- a/core/tests/behavior/blocking_write.rs
+++ b/core/tests/behavior/blocking_write.rs
@@ -88,7 +88,7 @@ pub fn test_blocking_create_dir_existing(op: BlockingOperator) -> Result<()> {
 pub fn test_blocking_write_file(op: BlockingOperator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content)?;
 
@@ -102,7 +102,7 @@ pub fn test_blocking_write_file(op: BlockingOperator) -> Result<()> {
 /// Write file with dir path should return an error
 pub fn test_blocking_write_with_dir_path(op: BlockingOperator) -> Result<()> {
     let path = format!("{}/", uuid::Uuid::new_v4());
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let result = op.write(&path, content);
     assert!(result.is_err());
@@ -126,7 +126,7 @@ pub fn test_blocking_write_with_special_chars(op: BlockingOperator) -> Result<()
 
     let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content)?;
 
@@ -141,7 +141,7 @@ pub fn test_blocking_write_with_special_chars(op: BlockingOperator) -> Result<()
 pub fn test_blocking_stat_file(op: BlockingOperator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).expect("write must succeed");
 
@@ -181,7 +181,7 @@ pub fn test_blocking_stat_with_special_chars(op: BlockingOperator) -> Result<()>
 
     let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).expect("write must succeed");
 
@@ -208,7 +208,7 @@ pub fn test_blocking_stat_not_exist(op: BlockingOperator) -> Result<()> {
 pub fn test_blocking_read_full(op: BlockingOperator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .expect("write must succeed");
@@ -233,7 +233,7 @@ pub fn test_blocking_read_range(op: BlockingOperator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
     let (offset, length) = gen_offset_length(size);
 
     op.write(&path, content.clone())
@@ -262,7 +262,7 @@ pub fn test_blocking_read_large_range(op: BlockingOperator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
     let (offset, _) = gen_offset_length(size);
 
     op.write(&path, content.clone())
@@ -302,7 +302,7 @@ pub fn test_blocking_fuzz_range_reader(op: BlockingOperator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .expect("write must succeed");
@@ -342,7 +342,7 @@ pub fn test_blocking_fuzz_offset_reader(op: BlockingOperator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .expect("write must succeed");
@@ -379,7 +379,7 @@ pub fn test_blocking_fuzz_part_reader(op: BlockingOperator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
     let (offset, length) = gen_offset_length(size);
 
     op.write(&path, content.clone())
@@ -417,7 +417,7 @@ pub fn test_blocking_fuzz_part_reader(op: BlockingOperator) -> Result<()> {
 pub fn test_blocking_delete_file(op: BlockingOperator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).expect("write must succeed");
 
@@ -432,7 +432,7 @@ pub fn test_blocking_delete_file(op: BlockingOperator) -> Result<()> {
 /// Remove one file
 pub fn test_blocking_remove_one_file(op: BlockingOperator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).expect("write must succeed");
 
diff --git a/core/tests/behavior/copy.rs b/core/tests/behavior/copy.rs
index b8a4573ba..bd19a9c38 100644
--- a/core/tests/behavior/copy.rs
+++ b/core/tests/behavior/copy.rs
@@ -42,7 +42,7 @@ pub fn behavior_copy_tests(op: &Operator) -> Vec<Trial> {
 /// Copy a file with ascii name and test contents.
 pub async fn test_copy_file_with_ascii_name(op: Operator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone()).await?;
 
@@ -62,7 +62,7 @@ pub async fn test_copy_file_with_ascii_name(op: Operator) -> Result<()> {
 pub async fn test_copy_file_with_non_ascii_name(op: Operator) -> Result<()> {
     let source_path = "🐂🍺中文.docx";
     let target_path = "😈🐅Français.docx";
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(source_path, source_content.clone()).await?;
     op.copy(source_path, target_path).await?;
@@ -106,7 +106,7 @@ pub async fn test_copy_source_dir(op: Operator) -> Result<()> {
 /// Copy to a dir should return an error.
 pub async fn test_copy_target_dir(op: Operator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, content).await?;
 
@@ -128,7 +128,7 @@ pub async fn test_copy_target_dir(op: Operator) -> Result<()> {
 /// Copy a file to self should return an error.
 pub async fn test_copy_self(op: Operator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, content).await?;
 
@@ -145,7 +145,7 @@ pub async fn test_copy_self(op: Operator) -> Result<()> {
 /// Copy to a nested path, parent path should be created successfully.
 pub async fn test_copy_nested(op: Operator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone()).await?;
 
@@ -169,12 +169,12 @@ pub async fn test_copy_nested(op: Operator) -> Result<()> {
 /// Copy to a exist path should overwrite successfully.
 pub async fn test_copy_overwrite(op: Operator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone()).await?;
 
     let target_path = uuid::Uuid::new_v4().to_string();
-    let (target_content, _) = gen_bytes();
+    let (target_content, _) = gen_bytes(op.info().full_capability());
     assert_ne!(source_content, target_content);
 
     op.write(&target_path, target_content).await?;
diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/list.rs
index 65ac1b013..a65fb126b 100644
--- a/core/tests/behavior/list.rs
+++ b/core/tests/behavior/list.rs
@@ -64,7 +64,7 @@ pub async fn test_list_dir(op: Operator) -> Result<()> {
     let parent = uuid::Uuid::new_v4().to_string();
     let path = format!("{parent}/{}", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await.expect("write must succeed");
 
@@ -91,7 +91,7 @@ pub async fn test_list_dir_with_metakey(op: Operator) -> Result<()> {
     let parent = uuid::Uuid::new_v4().to_string();
     let path = format!("{parent}/{}", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await.expect("write must succeed");
 
@@ -141,7 +141,7 @@ pub async fn test_list_dir_with_metakey_complete(op: Operator) -> Result<()> {
     let parent = uuid::Uuid::new_v4().to_string();
     let path = format!("{parent}/{}", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await.expect("write must succeed");
 
diff --git a/core/tests/behavior/presign.rs b/core/tests/behavior/presign.rs
index 7ddc9082b..87bc2365b 100644
--- a/core/tests/behavior/presign.rs
+++ b/core/tests/behavior/presign.rs
@@ -42,7 +42,7 @@ pub fn behavior_presign_tests(op: &Operator) -> Vec<Trial> {
 pub async fn test_presign_write(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let signed_req = op.presign_write(&path, Duration::from_secs(3600)).await?;
     debug!("Generated request: {signed_req:?}");
@@ -74,7 +74,7 @@ pub async fn test_presign_write(op: Operator) -> Result<()> {
 pub async fn test_presign_stat(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
     op.write(&path, content.clone())
         .await
         .expect("write must succeed");
@@ -104,7 +104,7 @@ pub async fn test_presign_stat(op: Operator) -> Result<()> {
 pub async fn test_presign_read(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
diff --git a/core/tests/behavior/rename.rs b/core/tests/behavior/rename.rs
index ac4974c8f..2dc3c1df0 100644
--- a/core/tests/behavior/rename.rs
+++ b/core/tests/behavior/rename.rs
@@ -41,7 +41,7 @@ pub fn behavior_rename_tests(op: &Operator) -> Vec<Trial> {
 /// Rename a file and test with stat.
 pub async fn test_rename_file(op: Operator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone()).await?;
 
@@ -91,7 +91,7 @@ pub async fn test_rename_source_dir(op: Operator) -> Result<()> {
 /// Rename to a dir should return an error.
 pub async fn test_rename_target_dir(op: Operator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, content).await?;
 
@@ -113,7 +113,7 @@ pub async fn test_rename_target_dir(op: Operator) -> Result<()> {
 /// Rename a file to self should return an error.
 pub async fn test_rename_self(op: Operator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, content).await?;
 
@@ -130,7 +130,7 @@ pub async fn test_rename_self(op: Operator) -> Result<()> {
 /// Rename to a nested path, parent path should be created successfully.
 pub async fn test_rename_nested(op: Operator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone()).await?;
 
@@ -157,12 +157,12 @@ pub async fn test_rename_nested(op: Operator) -> Result<()> {
 /// Rename to a exist path should overwrite successfully.
 pub async fn test_rename_overwrite(op: Operator) -> Result<()> {
     let source_path = uuid::Uuid::new_v4().to_string();
-    let (source_content, _) = gen_bytes();
+    let (source_content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&source_path, source_content.clone()).await?;
 
     let target_path = uuid::Uuid::new_v4().to_string();
-    let (target_content, _) = gen_bytes();
+    let (target_content, _) = gen_bytes(op.info().full_capability());
     assert_ne!(source_content, target_content);
 
     op.write(&target_path, target_content).await?;
diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs
index d1baf9ac5..6b91e8d2b 100644
--- a/core/tests/behavior/utils.rs
+++ b/core/tests/behavior/utils.rs
@@ -109,8 +109,9 @@ pub fn gen_bytes_with_range(range: impl SampleRange<usize>) -> (Vec<u8>, usize)
     (content, size)
 }
 
-pub fn gen_bytes() -> (Vec<u8>, usize) {
-    gen_bytes_with_range(1..4 * 1024 * 1024)
+pub fn gen_bytes(cap: Capability) -> (Vec<u8>, usize) {
+    let max_size = cap.write_total_max_size.unwrap_or(4 * 1024 * 1024);
+    gen_bytes_with_range(1..max_size)
 }
 
 pub fn gen_fixed_bytes(size: usize) -> Vec<u8> {
diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs
index 675d0c7a3..a2685605b 100644
--- a/core/tests/behavior/write.rs
+++ b/core/tests/behavior/write.rs
@@ -126,7 +126,7 @@ pub async fn test_create_dir_existing(op: Operator) -> Result<()> {
 /// Write a single file and test with stat.
 pub async fn test_write_only(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await?;
 
@@ -157,7 +157,7 @@ pub async fn test_write_with_empty_content(op: Operator) -> Result<()> {
 /// Write file with dir path should return an error
 pub async fn test_write_with_dir_path(op: Operator) -> Result<()> {
     let path = format!("{}/", uuid::Uuid::new_v4());
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let result = op.write(&path, content).await;
     assert!(result.is_err());
@@ -180,7 +180,7 @@ pub async fn test_write_with_special_chars(op: Operator) -> Result<()> {
     }
 
     let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await?;
 
@@ -198,7 +198,7 @@ pub async fn test_write_with_cache_control(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let target_cache_control = "no-cache, no-store, max-age=300";
     op.write_with(&path, content)
@@ -224,7 +224,7 @@ pub async fn test_write_with_content_type(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let target_content_type = "application/json";
     op.write_with(&path, content)
@@ -251,7 +251,7 @@ pub async fn test_write_with_content_disposition(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     let target_content_disposition = "attachment; filename=\"filename.jpg\"";
     op.write_with(&path, content)
@@ -274,7 +274,7 @@ pub async fn test_write_with_content_disposition(op: Operator) -> Result<()> {
 /// Stat existing file should return metadata
 pub async fn test_stat_file(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await.expect("write must succeed");
 
@@ -313,7 +313,7 @@ pub async fn test_stat_with_special_chars(op: Operator) -> Result<()> {
     }
 
     let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await.expect("write must succeed");
 
@@ -329,7 +329,7 @@ pub async fn test_stat_with_special_chars(op: Operator) -> Result<()> {
 pub async fn test_stat_not_cleaned_path(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await.expect("write must succeed");
 
@@ -360,7 +360,7 @@ pub async fn test_stat_with_if_match(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -392,7 +392,7 @@ pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -435,7 +435,7 @@ pub async fn test_stat_root(op: Operator) -> Result<()> {
 pub async fn test_read_full(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -461,7 +461,7 @@ pub async fn test_read_range(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
     let (offset, length) = gen_offset_length(size);
 
     op.write(&path, content.clone())
@@ -491,7 +491,7 @@ pub async fn test_read_large_range(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
     let (offset, _) = gen_offset_length(size);
 
     op.write(&path, content.clone())
@@ -522,7 +522,7 @@ pub async fn test_reader_range(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
     let (offset, length) = gen_offset_length(size);
 
     op.write(&path, content.clone())
@@ -555,7 +555,7 @@ pub async fn test_reader_from(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
     let (offset, _) = gen_offset_length(size);
 
     op.write(&path, content.clone())
@@ -586,7 +586,7 @@ pub async fn test_reader_tail(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
     let (_, length) = gen_offset_length(size);
 
     op.write(&path, content.clone())
@@ -636,7 +636,7 @@ pub async fn test_read_with_if_match(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -667,7 +667,7 @@ pub async fn test_read_with_if_none_match(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -700,7 +700,7 @@ pub async fn test_fuzz_reader_with_range(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -741,7 +741,7 @@ pub async fn test_fuzz_offset_reader(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -782,7 +782,7 @@ pub async fn test_fuzz_part_reader(op: Operator) -> Result<()> {
 
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
     let (offset, length) = gen_offset_length(size);
 
     op.write(&path, content.clone())
@@ -847,7 +847,7 @@ pub async fn test_read_with_special_chars(op: Operator) -> Result<()> {
 
     let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, size) = gen_bytes();
+    let (content, size) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -874,7 +874,7 @@ pub async fn test_read_with_override_cache_control(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -924,7 +924,7 @@ pub async fn test_read_with_override_content_disposition(op: Operator) -> Result
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -974,7 +974,7 @@ pub async fn test_read_with_override_content_type(op: Operator) -> Result<()> {
     }
 
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await
@@ -1018,7 +1018,7 @@ pub async fn test_read_with_override_content_type(op: Operator) -> Result<()> {
 /// Delete existing file should succeed.
 pub async fn test_writer_abort(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     let mut writer = match op.writer(&path).await {
         Ok(writer) => writer,
@@ -1046,7 +1046,7 @@ pub async fn test_writer_abort(op: Operator) -> Result<()> {
 /// Delete existing file should succeed.
 pub async fn test_delete_file(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await.expect("write must succeed");
 
@@ -1084,7 +1084,7 @@ pub async fn test_delete_with_special_chars(op: Operator) -> Result<()> {
 
     let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
     debug!("Generate a random file: {}", &path);
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await.expect("write must succeed");
 
@@ -1108,7 +1108,7 @@ pub async fn test_delete_not_existing(op: Operator) -> Result<()> {
 /// Remove one file
 pub async fn test_remove_one_file(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content).await.expect("write must succeed");
 
@@ -1325,7 +1325,7 @@ pub async fn test_fuzz_unsized_writer(op: Operator) -> Result<()> {
 pub async fn test_invalid_reader_seek(op: Operator) -> Result<()> {
     let path = uuid::Uuid::new_v4().to_string();
     debug!("Generate a random file: {}", &path);
-    let (content, _) = gen_bytes();
+    let (content, _) = gen_bytes(op.info().full_capability());
 
     op.write(&path, content.clone())
         .await