You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/04/24 10:03:14 UTC
[incubator-opendal] 01/01: feat(tests): Add fuzz test for writer
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch add-fuzz-writer-test
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 6ec01f7213d50cd650d57bf60eca26a5138ad4f1
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Apr 24 18:02:51 2023 +0800
feat(tests): Add fuzz test for writer
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/fs/backend.rs | 1 +
core/src/services/gcs/backend.rs | 1 +
core/src/services/oss/backend.rs | 1 +
core/src/services/s3/backend.rs | 2 +
core/tests/behavior/utils.rs | 96 ++++++++++++++++++++++++++++++++++++++++
core/tests/behavior/write.rs | 28 ++++++++++++
6 files changed, 129 insertions(+)
diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs
index 4e78321c..5509663b 100644
--- a/core/src/services/fs/backend.rs
+++ b/core/src/services/fs/backend.rs
@@ -304,6 +304,7 @@ impl Accessor for FsBackend {
read: true,
read_can_seek: true,
write: true,
+ write_without_content_length: true,
create_dir: true,
list: true,
copy: true,
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index c2388dae..778c7781 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -377,6 +377,7 @@ impl Accessor for GcsBackend {
read: true,
read_can_next: true,
write: true,
+ write_without_content_length: true,
list: true,
scan: true,
copy: true,
diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs
index 276fc32e..8748c7dd 100644
--- a/core/src/services/oss/backend.rs
+++ b/core/src/services/oss/backend.rs
@@ -367,6 +367,7 @@ impl Accessor for OssBackend {
read: true,
read_can_next: true,
write: true,
+ write_without_content_length: true,
list: true,
scan: true,
copy: true,
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index eed0335e..0adf125a 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -924,6 +924,8 @@ impl Accessor for S3Backend {
read_with_override_content_disposition: true,
write: true,
+ write_without_content_length: true,
+
list: true,
scan: true,
copy: true,
diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs
index 5592ecfe..fdf877d6 100644
--- a/core/tests/behavior/utils.rs
+++ b/core/tests/behavior/utils.rs
@@ -17,6 +17,9 @@
use std::collections::HashMap;
use std::env;
+use std::fmt;
+use std::fmt::Debug;
+use std::fmt::Formatter;
use std::io::SeekFrom;
use std::usize;
@@ -279,3 +282,96 @@ impl ObjectReaderFuzzer {
}
}
}
+
+/// ObjectWriterFuzzer is the fuzzer for object writer.
+///
+/// We will generate random write operations to operate on object
+/// write to check if the output is expected.
+///
+/// # TODO
+///
+/// This fuzzer only generate valid operations.
+///
+/// In the future, we need to generate invalid operations to check if we
+/// handled correctly.
+pub struct ObjectWriterFuzzer {
+ name: String,
+ bs: Vec<u8>,
+
+ size: Option<usize>,
+ cur: usize,
+ rng: ThreadRng,
+ actions: Vec<ObjectWriterAction>,
+}
+
+#[derive(Clone)]
+pub enum ObjectWriterAction {
+ Write(Bytes),
+}
+
+impl Debug for ObjectWriterAction {
+ fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+ match self {
+ ObjectWriterAction::Write(bs) => write!(f, "Write({})", bs.len()),
+ }
+ }
+}
+
+impl ObjectWriterFuzzer {
+ /// Create a new fuzzer.
+ pub fn new(name: &str, size: Option<usize>) -> Self {
+ Self {
+ name: name.to_string(),
+ bs: Vec::new(),
+
+ size,
+ cur: 0,
+
+ rng: thread_rng(),
+ actions: vec![],
+ }
+ }
+
+ /// Generate a new action.
+ pub fn fuzz(&mut self) -> ObjectWriterAction {
+ let max = if let Some(size) = self.size {
+ size - self.cur
+ } else {
+ // Set max to 1MiB
+ 1024 * 1024
+ };
+
+ let size = self.rng.gen_range(0..max);
+
+ let mut bs = vec![0; size];
+ self.rng.fill_bytes(&mut bs);
+
+ let bs = Bytes::from(bs);
+ self.bs.extend_from_slice(&bs);
+ self.cur += bs.len();
+
+ let action = ObjectWriterAction::Write(bs);
+ debug!("{} perform fuzz action: {:?}", self.name, action);
+
+ self.actions.push(action.clone());
+
+ action
+ }
+
+ /// Check if read operation is expected.
+ pub fn check(&mut self, actual_bs: &[u8]) {
+ assert_eq!(
+ self.bs.len(),
+ actual_bs.len(),
+ "check failed: expected len is different with actual len, actions: {:?}",
+ self.actions
+ );
+
+ assert_eq!(
+ format!("{:x}", Sha256::digest(&self.bs)),
+ format!("{:x}", Sha256::digest(actual_bs)),
+ "check failed: expected bs is different with actual bs, actions: {:?}",
+ self.actions,
+ );
+ }
+}
diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs
index 4b76128d..39a6d98d 100644
--- a/core/tests/behavior/write.rs
+++ b/core/tests/behavior/write.rs
@@ -104,6 +104,7 @@ macro_rules! behavior_write_tests {
test_writer_write,
test_writer_abort,
test_writer_futures_copy,
+ test_fuzz_unsized_writer,
);
)*
};
@@ -885,3 +886,30 @@ pub async fn test_writer_futures_copy(op: Operator) -> Result<()> {
op.delete(&path).await.expect("delete must succeed");
Ok(())
}
+
+/// Add test for unsized writer
+pub async fn test_fuzz_unsized_writer(op: Operator) -> Result<()> {
+ if !op.info().capability().write_without_content_length {
+ warn!("{op:?} doesn't support write without content length, test skip");
+ return Ok(());
+ }
+
+ let path = uuid::Uuid::new_v4().to_string();
+
+ let mut fuzzer = ObjectWriterFuzzer::new(&path, None);
+
+ let mut w = op.writer(&path).await?;
+
+ for _ in 0..100 {
+ match fuzzer.fuzz() {
+ ObjectWriterAction::Write(bs) => w.write(bs).await?,
+ }
+ }
+ w.close().await?;
+
+ let content = op.read(&path).await?;
+ fuzzer.check(&content);
+
+ op.delete(&path).await.expect("delete must succeed");
+ Ok(())
+}