You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/10/26 03:30:43 UTC
[incubator-opendal] branch main updated: feat: Add write_total_max_size in Capability (#3309)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 3aad395fc feat: Add write_total_max_size in Capability (#3309)
3aad395fc is described below
commit 3aad395fc8d8c894a28d05d8e6bf904d1eb38d08
Author: taobo <mt...@gmail.com>
AuthorDate: Thu Oct 26 11:30:39 2023 +0800
feat: Add write_total_max_size in Capability (#3309)
* feat: behavior test support config max_size
Signed-off-by: taobo <mt...@gmail.com>
* feat: update gen_bytes func
* fix: make clippy happy
---------
Signed-off-by: taobo <mt...@gmail.com>
---
core/Cargo.toml | 6 ++--
core/src/services/d1/backend.rs | 3 ++
core/src/types/capability.rs | 4 +++
core/src/types/operator/builder.rs | 2 ++
core/tests/behavior/append.rs | 12 +++----
core/tests/behavior/blocking_append.rs | 12 +++----
core/tests/behavior/blocking_copy.rs | 12 +++----
core/tests/behavior/blocking_list.rs | 6 ++--
core/tests/behavior/blocking_rename.rs | 12 +++----
core/tests/behavior/blocking_write.rs | 26 +++++++-------
core/tests/behavior/copy.rs | 14 ++++----
core/tests/behavior/list.rs | 6 ++--
core/tests/behavior/presign.rs | 6 ++--
core/tests/behavior/rename.rs | 12 +++----
core/tests/behavior/utils.rs | 5 +--
core/tests/behavior/write.rs | 62 +++++++++++++++++-----------------
16 files changed, 105 insertions(+), 95 deletions(-)
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 3679f3955..9ddae4539 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -115,6 +115,7 @@ services-azdls = [
"reqsign?/services-azblob",
"reqsign?/reqwest_request",
]
+services-azfile = []
services-cacache = ["dep:cacache"]
services-cloudflare-kv = []
services-cos = [
@@ -145,6 +146,7 @@ services-memcached = ["dep:bb8"]
services-memory = []
services-mini-moka = ["dep:mini-moka"]
services-moka = ["dep:moka"]
+services-mongodb = ["dep:mongodb"]
services-mysql = ["dep:mysql_async"]
services-obs = [
"dep:reqsign",
@@ -182,8 +184,6 @@ services-wasabi = [
]
services-webdav = []
services-webhdfs = []
-services-azfile = []
-services-mongodb = ["dep:mongodb"]
[lib]
bench = false
@@ -243,6 +243,7 @@ metrics = { version = "0.20", optional = true }
mini-moka = { version = "0.10", optional = true }
minitrace = { version = "0.5", optional = true }
moka = { version = "0.10", optional = true, features = ["future"] }
+mongodb = { version = "2.7.0", optional = true, features = ["tokio-runtime"] }
mysql_async = { version = "0.32.2", optional = true }
once_cell = "1"
openssh = { version = "0.9.9", optional = true }
@@ -286,7 +287,6 @@ tokio = "1.27"
tokio-postgres = { version = "0.7.8", optional = true }
tracing = { version = "0.1", optional = true }
uuid = { version = "1", features = ["serde", "v4"] }
-mongodb = { version = "2.7.0", optional = true, features = ["tokio-runtime"] }
[dev-dependencies]
criterion = { version = "0.4", features = ["async", "async_tokio"] }
diff --git a/core/src/services/d1/backend.rs b/core/src/services/d1/backend.rs
index faf0c472e..15e8d2854 100644
--- a/core/src/services/d1/backend.rs
+++ b/core/src/services/d1/backend.rs
@@ -271,6 +271,9 @@ impl kv::Adapter for Adapter {
Capability {
read: true,
write: true,
+ // Cloudflare D1 supports 1MB as max in write_total.
+ // refer to https://developers.cloudflare.com/d1/platform/limits/
+ write_total_max_size: Some(1000 * 1000),
..Default::default()
},
)
diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs
index 2767c29e4..437a3e9ba 100644
--- a/core/src/types/capability.rs
+++ b/core/src/types/capability.rs
@@ -98,6 +98,10 @@ pub struct Capability {
///
/// For example, Google GCS requires align size to 256KiB in write_multi.
pub write_multi_align_size: Option<usize>,
+ /// write_total_max_size is the max size that services support in write_total.
+ ///
+ /// For example, Cloudflare D1 supports 1MB as max in write_total.
+ pub write_total_max_size: Option<usize>,
/// If operator supports create dir.
pub create_dir: bool,
diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs
index 1f7a4a5aa..c7976e2c7 100644
--- a/core/src/types/operator/builder.rs
+++ b/core/src/types/operator/builder.rs
@@ -163,6 +163,8 @@ impl Operator {
Scheme::Cacache => Self::from_map::<services::Cacache>(map)?.finish(),
#[cfg(feature = "services-cos")]
Scheme::Cos => Self::from_map::<services::Cos>(map)?.finish(),
+ #[cfg(feature = "services-d1")]
+ Scheme::D1 => Self::from_map::<services::D1>(map)?.finish(),
#[cfg(feature = "services-dashmap")]
Scheme::Dashmap => Self::from_map::<services::Dashmap>(map)?.finish(),
#[cfg(feature = "services-dropbox")]
diff --git a/core/tests/behavior/append.rs b/core/tests/behavior/append.rs
index 533ca15b7..91ed3bee3 100644
--- a/core/tests/behavior/append.rs
+++ b/core/tests/behavior/append.rs
@@ -47,8 +47,8 @@ pub fn behavior_append_tests(op: &Operator) -> Vec<Trial> {
/// Test append to a file must success.
pub async fn test_append_create_append(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- let (content_one, size_one) = gen_bytes();
- let (content_two, size_two) = gen_bytes();
+ let (content_one, size_one) = gen_bytes(op.info().full_capability());
+ let (content_two, size_two) = gen_bytes(op.info().full_capability());
op.write_with(&path, content_one.clone())
.append(true)
@@ -74,7 +74,7 @@ pub async fn test_append_create_append(op: Operator) -> Result<()> {
/// Test append to a directory path must fail.
pub async fn test_append_with_dir_path(op: Operator) -> Result<()> {
let path = format!("{}/", uuid::Uuid::new_v4());
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
let res = op.write_with(&path, content).append(true).await;
assert!(res.is_err());
@@ -90,7 +90,7 @@ pub async fn test_append_with_cache_control(op: Operator) -> Result<()> {
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
let target_cache_control = "no-cache, no-store, max-age=300";
op.write_with(&path, content)
@@ -117,7 +117,7 @@ pub async fn test_append_with_content_type(op: Operator) -> Result<()> {
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let target_content_type = "application/json";
op.write_with(&path, content)
@@ -145,7 +145,7 @@ pub async fn test_append_with_content_disposition(op: Operator) -> Result<()> {
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let target_content_disposition = "attachment; filename=\"filename.jpg\"";
op.write_with(&path, content)
diff --git a/core/tests/behavior/blocking_append.rs b/core/tests/behavior/blocking_append.rs
index d402d97e5..8161f825a 100644
--- a/core/tests/behavior/blocking_append.rs
+++ b/core/tests/behavior/blocking_append.rs
@@ -47,8 +47,8 @@ pub fn behavior_blocking_append_tests(op: &Operator) -> Vec<Trial> {
/// Test append to a file must success.
pub fn test_blocking_append_create_append(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- let (content_one, size_one) = gen_bytes();
- let (content_two, size_two) = gen_bytes();
+ let (content_one, size_one) = gen_bytes(op.info().full_capability());
+ let (content_two, size_two) = gen_bytes(op.info().full_capability());
op.write_with(&path, content_one.clone())
.append(true)
@@ -74,7 +74,7 @@ pub fn test_blocking_append_create_append(op: BlockingOperator) -> Result<()> {
/// Test append to a directory path must fail.
pub fn test_blocking_append_with_dir_path(op: BlockingOperator) -> Result<()> {
let path = format!("{}/", uuid::Uuid::new_v4());
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
let res = op.write_with(&path, content).append(true).call();
assert!(res.is_err());
@@ -90,7 +90,7 @@ pub fn test_blocking_append_with_cache_control(op: BlockingOperator) -> Result<(
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
let target_cache_control = "no-cache, no-store, max-age=300";
op.write_with(&path, content)
@@ -117,7 +117,7 @@ pub fn test_blocking_append_with_content_type(op: BlockingOperator) -> Result<()
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let target_content_type = "application/json";
op.write_with(&path, content)
@@ -145,7 +145,7 @@ pub fn test_blocking_append_with_content_disposition(op: BlockingOperator) -> Re
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let target_content_disposition = "attachment; filename=\"filename.jpg\"";
op.write_with(&path, content)
diff --git a/core/tests/behavior/blocking_copy.rs b/core/tests/behavior/blocking_copy.rs
index 822369a22..163c55990 100644
--- a/core/tests/behavior/blocking_copy.rs
+++ b/core/tests/behavior/blocking_copy.rs
@@ -41,7 +41,7 @@ pub fn behavior_blocking_copy_tests(op: &Operator) -> Vec<Trial> {
/// Copy a file and test with stat.
pub fn test_blocking_copy_file(op: BlockingOperator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone())?;
@@ -86,7 +86,7 @@ pub fn test_blocking_copy_source_dir(op: BlockingOperator) -> Result<()> {
/// Copy to a dir should return an error.
pub fn test_blocking_copy_target_dir(op: BlockingOperator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content)?;
@@ -107,7 +107,7 @@ pub fn test_blocking_copy_target_dir(op: BlockingOperator) -> Result<()> {
/// Copy a file to self should return an error.
pub fn test_blocking_copy_self(op: BlockingOperator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _size) = gen_bytes();
+ let (source_content, _size) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content)?;
@@ -123,7 +123,7 @@ pub fn test_blocking_copy_self(op: BlockingOperator) -> Result<()> {
/// Copy to a nested path, parent path should be created successfully.
pub fn test_blocking_copy_nested(op: BlockingOperator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone())?;
@@ -147,12 +147,12 @@ pub fn test_blocking_copy_nested(op: BlockingOperator) -> Result<()> {
/// Copy to a exist path should overwrite successfully.
pub fn test_blocking_copy_overwrite(op: BlockingOperator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone())?;
let target_path = uuid::Uuid::new_v4().to_string();
- let (target_content, _) = gen_bytes();
+ let (target_content, _) = gen_bytes(op.info().full_capability());
assert_ne!(source_content, target_content);
op.write(&target_path, target_content)?;
diff --git a/core/tests/behavior/blocking_list.rs b/core/tests/behavior/blocking_list.rs
index c58e4feea..e53b8a1de 100644
--- a/core/tests/behavior/blocking_list.rs
+++ b/core/tests/behavior/blocking_list.rs
@@ -46,7 +46,7 @@ pub fn test_blocking_list_dir(op: BlockingOperator) -> Result<()> {
let parent = uuid::Uuid::new_v4().to_string();
let path = format!("{parent}/{}", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).expect("write must succeed");
@@ -74,7 +74,7 @@ pub fn test_blocking_list_dir_with_metakey(op: BlockingOperator) -> Result<()> {
let parent = uuid::Uuid::new_v4().to_string();
let path = format!("{parent}/{}", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).expect("write must succeed");
@@ -124,7 +124,7 @@ pub fn test_blocking_list_dir_with_metakey_complete(op: BlockingOperator) -> Res
let parent = uuid::Uuid::new_v4().to_string();
let path = format!("{parent}/{}", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).expect("write must succeed");
diff --git a/core/tests/behavior/blocking_rename.rs b/core/tests/behavior/blocking_rename.rs
index 398a3f4da..602d59399 100644
--- a/core/tests/behavior/blocking_rename.rs
+++ b/core/tests/behavior/blocking_rename.rs
@@ -41,7 +41,7 @@ pub fn behavior_blocking_rename_tests(op: &Operator) -> Vec<Trial> {
/// Rename a file and test with stat.
pub fn test_blocking_rename_file(op: BlockingOperator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone())?;
@@ -89,7 +89,7 @@ pub fn test_blocking_rename_source_dir(op: BlockingOperator) -> Result<()> {
/// Rename to a dir should return an error.
pub fn test_blocking_rename_target_dir(op: BlockingOperator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content)?;
@@ -110,7 +110,7 @@ pub fn test_blocking_rename_target_dir(op: BlockingOperator) -> Result<()> {
/// Rename a file to self should return an error.
pub fn test_blocking_rename_self(op: BlockingOperator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _size) = gen_bytes();
+ let (source_content, _size) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content)?;
@@ -126,7 +126,7 @@ pub fn test_blocking_rename_self(op: BlockingOperator) -> Result<()> {
/// Rename to a nested path, parent path should be created successfully.
pub fn test_blocking_rename_nested(op: BlockingOperator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone())?;
@@ -153,12 +153,12 @@ pub fn test_blocking_rename_nested(op: BlockingOperator) -> Result<()> {
/// Rename to a exist path should overwrite successfully.
pub fn test_blocking_rename_overwrite(op: BlockingOperator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone())?;
let target_path = uuid::Uuid::new_v4().to_string();
- let (target_content, _) = gen_bytes();
+ let (target_content, _) = gen_bytes(op.info().full_capability());
assert_ne!(source_content, target_content);
op.write(&target_path, target_content)?;
diff --git a/core/tests/behavior/blocking_write.rs b/core/tests/behavior/blocking_write.rs
index 1c305c5c6..25442a62f 100644
--- a/core/tests/behavior/blocking_write.rs
+++ b/core/tests/behavior/blocking_write.rs
@@ -88,7 +88,7 @@ pub fn test_blocking_create_dir_existing(op: BlockingOperator) -> Result<()> {
pub fn test_blocking_write_file(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content)?;
@@ -102,7 +102,7 @@ pub fn test_blocking_write_file(op: BlockingOperator) -> Result<()> {
/// Write file with dir path should return an error
pub fn test_blocking_write_with_dir_path(op: BlockingOperator) -> Result<()> {
let path = format!("{}/", uuid::Uuid::new_v4());
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
let result = op.write(&path, content);
assert!(result.is_err());
@@ -126,7 +126,7 @@ pub fn test_blocking_write_with_special_chars(op: BlockingOperator) -> Result<()
let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content)?;
@@ -141,7 +141,7 @@ pub fn test_blocking_write_with_special_chars(op: BlockingOperator) -> Result<()
pub fn test_blocking_stat_file(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).expect("write must succeed");
@@ -181,7 +181,7 @@ pub fn test_blocking_stat_with_special_chars(op: BlockingOperator) -> Result<()>
let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).expect("write must succeed");
@@ -208,7 +208,7 @@ pub fn test_blocking_stat_not_exist(op: BlockingOperator) -> Result<()> {
pub fn test_blocking_read_full(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.expect("write must succeed");
@@ -233,7 +233,7 @@ pub fn test_blocking_read_range(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let (offset, length) = gen_offset_length(size);
op.write(&path, content.clone())
@@ -262,7 +262,7 @@ pub fn test_blocking_read_large_range(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let (offset, _) = gen_offset_length(size);
op.write(&path, content.clone())
@@ -302,7 +302,7 @@ pub fn test_blocking_fuzz_range_reader(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.expect("write must succeed");
@@ -342,7 +342,7 @@ pub fn test_blocking_fuzz_offset_reader(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.expect("write must succeed");
@@ -379,7 +379,7 @@ pub fn test_blocking_fuzz_part_reader(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let (offset, length) = gen_offset_length(size);
op.write(&path, content.clone())
@@ -417,7 +417,7 @@ pub fn test_blocking_fuzz_part_reader(op: BlockingOperator) -> Result<()> {
pub fn test_blocking_delete_file(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content).expect("write must succeed");
@@ -432,7 +432,7 @@ pub fn test_blocking_delete_file(op: BlockingOperator) -> Result<()> {
/// Remove one file
pub fn test_blocking_remove_one_file(op: BlockingOperator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content).expect("write must succeed");
diff --git a/core/tests/behavior/copy.rs b/core/tests/behavior/copy.rs
index b8a4573ba..bd19a9c38 100644
--- a/core/tests/behavior/copy.rs
+++ b/core/tests/behavior/copy.rs
@@ -42,7 +42,7 @@ pub fn behavior_copy_tests(op: &Operator) -> Vec<Trial> {
/// Copy a file with ascii name and test contents.
pub async fn test_copy_file_with_ascii_name(op: Operator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone()).await?;
@@ -62,7 +62,7 @@ pub async fn test_copy_file_with_ascii_name(op: Operator) -> Result<()> {
pub async fn test_copy_file_with_non_ascii_name(op: Operator) -> Result<()> {
let source_path = "🐂🍺中文.docx";
let target_path = "😈🐅Français.docx";
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(source_path, source_content.clone()).await?;
op.copy(source_path, target_path).await?;
@@ -106,7 +106,7 @@ pub async fn test_copy_source_dir(op: Operator) -> Result<()> {
/// Copy to a dir should return an error.
pub async fn test_copy_target_dir(op: Operator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, content).await?;
@@ -128,7 +128,7 @@ pub async fn test_copy_target_dir(op: Operator) -> Result<()> {
/// Copy a file to self should return an error.
pub async fn test_copy_self(op: Operator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, content).await?;
@@ -145,7 +145,7 @@ pub async fn test_copy_self(op: Operator) -> Result<()> {
/// Copy to a nested path, parent path should be created successfully.
pub async fn test_copy_nested(op: Operator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone()).await?;
@@ -169,12 +169,12 @@ pub async fn test_copy_nested(op: Operator) -> Result<()> {
/// Copy to a exist path should overwrite successfully.
pub async fn test_copy_overwrite(op: Operator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone()).await?;
let target_path = uuid::Uuid::new_v4().to_string();
- let (target_content, _) = gen_bytes();
+ let (target_content, _) = gen_bytes(op.info().full_capability());
assert_ne!(source_content, target_content);
op.write(&target_path, target_content).await?;
diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/list.rs
index 65ac1b013..a65fb126b 100644
--- a/core/tests/behavior/list.rs
+++ b/core/tests/behavior/list.rs
@@ -64,7 +64,7 @@ pub async fn test_list_dir(op: Operator) -> Result<()> {
let parent = uuid::Uuid::new_v4().to_string();
let path = format!("{parent}/{}", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).await.expect("write must succeed");
@@ -91,7 +91,7 @@ pub async fn test_list_dir_with_metakey(op: Operator) -> Result<()> {
let parent = uuid::Uuid::new_v4().to_string();
let path = format!("{parent}/{}", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).await.expect("write must succeed");
@@ -141,7 +141,7 @@ pub async fn test_list_dir_with_metakey_complete(op: Operator) -> Result<()> {
let parent = uuid::Uuid::new_v4().to_string();
let path = format!("{parent}/{}", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).await.expect("write must succeed");
diff --git a/core/tests/behavior/presign.rs b/core/tests/behavior/presign.rs
index 7ddc9082b..87bc2365b 100644
--- a/core/tests/behavior/presign.rs
+++ b/core/tests/behavior/presign.rs
@@ -42,7 +42,7 @@ pub fn behavior_presign_tests(op: &Operator) -> Vec<Trial> {
pub async fn test_presign_write(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let signed_req = op.presign_write(&path, Duration::from_secs(3600)).await?;
debug!("Generated request: {signed_req:?}");
@@ -74,7 +74,7 @@ pub async fn test_presign_write(op: Operator) -> Result<()> {
pub async fn test_presign_stat(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
.expect("write must succeed");
@@ -104,7 +104,7 @@ pub async fn test_presign_stat(op: Operator) -> Result<()> {
pub async fn test_presign_read(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
diff --git a/core/tests/behavior/rename.rs b/core/tests/behavior/rename.rs
index ac4974c8f..2dc3c1df0 100644
--- a/core/tests/behavior/rename.rs
+++ b/core/tests/behavior/rename.rs
@@ -41,7 +41,7 @@ pub fn behavior_rename_tests(op: &Operator) -> Vec<Trial> {
/// Rename a file and test with stat.
pub async fn test_rename_file(op: Operator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone()).await?;
@@ -91,7 +91,7 @@ pub async fn test_rename_source_dir(op: Operator) -> Result<()> {
/// Rename to a dir should return an error.
pub async fn test_rename_target_dir(op: Operator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, content).await?;
@@ -113,7 +113,7 @@ pub async fn test_rename_target_dir(op: Operator) -> Result<()> {
/// Rename a file to self should return an error.
pub async fn test_rename_self(op: Operator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, content).await?;
@@ -130,7 +130,7 @@ pub async fn test_rename_self(op: Operator) -> Result<()> {
/// Rename to a nested path, parent path should be created successfully.
pub async fn test_rename_nested(op: Operator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone()).await?;
@@ -157,12 +157,12 @@ pub async fn test_rename_nested(op: Operator) -> Result<()> {
/// Rename to a exist path should overwrite successfully.
pub async fn test_rename_overwrite(op: Operator) -> Result<()> {
let source_path = uuid::Uuid::new_v4().to_string();
- let (source_content, _) = gen_bytes();
+ let (source_content, _) = gen_bytes(op.info().full_capability());
op.write(&source_path, source_content.clone()).await?;
let target_path = uuid::Uuid::new_v4().to_string();
- let (target_content, _) = gen_bytes();
+ let (target_content, _) = gen_bytes(op.info().full_capability());
assert_ne!(source_content, target_content);
op.write(&target_path, target_content).await?;
diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs
index d1baf9ac5..6b91e8d2b 100644
--- a/core/tests/behavior/utils.rs
+++ b/core/tests/behavior/utils.rs
@@ -109,8 +109,9 @@ pub fn gen_bytes_with_range(range: impl SampleRange<usize>) -> (Vec<u8>, usize)
(content, size)
}
-pub fn gen_bytes() -> (Vec<u8>, usize) {
- gen_bytes_with_range(1..4 * 1024 * 1024)
+pub fn gen_bytes(cap: Capability) -> (Vec<u8>, usize) {
+ let max_size = cap.write_total_max_size.unwrap_or(4 * 1024 * 1024);
+ gen_bytes_with_range(1..max_size)
}
pub fn gen_fixed_bytes(size: usize) -> Vec<u8> {
diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs
index 675d0c7a3..a2685605b 100644
--- a/core/tests/behavior/write.rs
+++ b/core/tests/behavior/write.rs
@@ -126,7 +126,7 @@ pub async fn test_create_dir_existing(op: Operator) -> Result<()> {
/// Write a single file and test with stat.
pub async fn test_write_only(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).await?;
@@ -157,7 +157,7 @@ pub async fn test_write_with_empty_content(op: Operator) -> Result<()> {
/// Write file with dir path should return an error
pub async fn test_write_with_dir_path(op: Operator) -> Result<()> {
let path = format!("{}/", uuid::Uuid::new_v4());
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
let result = op.write(&path, content).await;
assert!(result.is_err());
@@ -180,7 +180,7 @@ pub async fn test_write_with_special_chars(op: Operator) -> Result<()> {
}
let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).await?;
@@ -198,7 +198,7 @@ pub async fn test_write_with_cache_control(op: Operator) -> Result<()> {
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
let target_cache_control = "no-cache, no-store, max-age=300";
op.write_with(&path, content)
@@ -224,7 +224,7 @@ pub async fn test_write_with_content_type(op: Operator) -> Result<()> {
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let target_content_type = "application/json";
op.write_with(&path, content)
@@ -251,7 +251,7 @@ pub async fn test_write_with_content_disposition(op: Operator) -> Result<()> {
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let target_content_disposition = "attachment; filename=\"filename.jpg\"";
op.write_with(&path, content)
@@ -274,7 +274,7 @@ pub async fn test_write_with_content_disposition(op: Operator) -> Result<()> {
/// Stat existing file should return metadata
pub async fn test_stat_file(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).await.expect("write must succeed");
@@ -313,7 +313,7 @@ pub async fn test_stat_with_special_chars(op: Operator) -> Result<()> {
}
let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).await.expect("write must succeed");
@@ -329,7 +329,7 @@ pub async fn test_stat_with_special_chars(op: Operator) -> Result<()> {
pub async fn test_stat_not_cleaned_path(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content).await.expect("write must succeed");
@@ -360,7 +360,7 @@ pub async fn test_stat_with_if_match(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -392,7 +392,7 @@ pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -435,7 +435,7 @@ pub async fn test_stat_root(op: Operator) -> Result<()> {
pub async fn test_read_full(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -461,7 +461,7 @@ pub async fn test_read_range(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let (offset, length) = gen_offset_length(size);
op.write(&path, content.clone())
@@ -491,7 +491,7 @@ pub async fn test_read_large_range(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let (offset, _) = gen_offset_length(size);
op.write(&path, content.clone())
@@ -522,7 +522,7 @@ pub async fn test_reader_range(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let (offset, length) = gen_offset_length(size);
op.write(&path, content.clone())
@@ -555,7 +555,7 @@ pub async fn test_reader_from(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let (offset, _) = gen_offset_length(size);
op.write(&path, content.clone())
@@ -586,7 +586,7 @@ pub async fn test_reader_tail(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let (_, length) = gen_offset_length(size);
op.write(&path, content.clone())
@@ -636,7 +636,7 @@ pub async fn test_read_with_if_match(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -667,7 +667,7 @@ pub async fn test_read_with_if_none_match(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -700,7 +700,7 @@ pub async fn test_fuzz_reader_with_range(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -741,7 +741,7 @@ pub async fn test_fuzz_offset_reader(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -782,7 +782,7 @@ pub async fn test_fuzz_part_reader(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
let (offset, length) = gen_offset_length(size);
op.write(&path, content.clone())
@@ -847,7 +847,7 @@ pub async fn test_read_with_special_chars(op: Operator) -> Result<()> {
let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
- let (content, size) = gen_bytes();
+ let (content, size) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -874,7 +874,7 @@ pub async fn test_read_with_override_cache_control(op: Operator) -> Result<()> {
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -924,7 +924,7 @@ pub async fn test_read_with_override_content_disposition(op: Operator) -> Result
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -974,7 +974,7 @@ pub async fn test_read_with_override_content_type(op: Operator) -> Result<()> {
}
let path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await
@@ -1018,7 +1018,7 @@ pub async fn test_read_with_override_content_type(op: Operator) -> Result<()> {
/// Delete existing file should succeed.
pub async fn test_writer_abort(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
let mut writer = match op.writer(&path).await {
Ok(writer) => writer,
@@ -1046,7 +1046,7 @@ pub async fn test_writer_abort(op: Operator) -> Result<()> {
/// Delete existing file should succeed.
pub async fn test_delete_file(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content).await.expect("write must succeed");
@@ -1084,7 +1084,7 @@ pub async fn test_delete_with_special_chars(op: Operator) -> Result<()> {
let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
debug!("Generate a random file: {}", &path);
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content).await.expect("write must succeed");
@@ -1108,7 +1108,7 @@ pub async fn test_delete_not_existing(op: Operator) -> Result<()> {
/// Remove one file
pub async fn test_remove_one_file(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content).await.expect("write must succeed");
@@ -1325,7 +1325,7 @@ pub async fn test_fuzz_unsized_writer(op: Operator) -> Result<()> {
pub async fn test_invalid_reader_seek(op: Operator) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
debug!("Generate a random file: {}", &path);
- let (content, _) = gen_bytes();
+ let (content, _) = gen_bytes(op.info().full_capability());
op.write(&path, content.clone())
.await