You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/10/30 03:53:10 UTC
(incubator-opendal) branch main updated: refactor(core/fuzz): Fix some bugs inside fuzzer (#3418)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 80f954de3 refactor(core/fuzz): Fix some bugs inside fuzzer (#3418)
80f954de3 is described below
commit 80f954de384e2a45ef515e6a10bc59b4cae17da6
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Oct 30 11:53:05 2023 +0800
refactor(core/fuzz): Fix some bugs inside fuzzer (#3418)
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
Cargo.lock | 1 +
core/fuzz/Cargo.toml | 4 +++
core/fuzz/fuzz_reader.rs | 69 +++++++++++++++++++++++++++++++++++++-----------
core/fuzz/utils.rs | 8 +++++-
4 files changed, 66 insertions(+), 16 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 719810b15..e5fa7d138 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4279,6 +4279,7 @@ dependencies = [
"rand 0.8.5",
"sha2",
"tokio",
+ "tracing-subscriber",
"uuid",
]
diff --git a/core/fuzz/Cargo.toml b/core/fuzz/Cargo.toml
index 8020dfaf4..036433934 100644
--- a/core/fuzz/Cargo.toml
+++ b/core/fuzz/Cargo.toml
@@ -35,6 +35,10 @@ rand = "0.8"
sha2 = { version = "0.10.6" }
tokio = { version = "1", features = ["full"] }
uuid = { version = "1", features = ["v4"] }
+tracing-subscriber = { version = "0.3", features = [
+ "env-filter",
+ "tracing-log",
+] }
[[bin]]
name = "fuzz_reader"
diff --git a/core/fuzz/fuzz_reader.rs b/core/fuzz/fuzz_reader.rs
index d6eb19c1c..799d6f52b 100644
--- a/core/fuzz/fuzz_reader.rs
+++ b/core/fuzz/fuzz_reader.rs
@@ -17,6 +17,7 @@
#![no_main]
+use std::fmt::{Debug, Formatter};
use std::io::SeekFrom;
use bytes::Bytes;
@@ -35,20 +36,37 @@ mod utils;
const MAX_DATA_SIZE: usize = 16 * 1024 * 1024;
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Eq, PartialEq)]
enum ReadAction {
Read { size: usize },
Seek(SeekFrom),
Next,
}
-#[derive(Debug, Clone)]
+#[derive(Clone)]
struct FuzzInput {
+ path: String,
size: usize,
range: BytesRange,
actions: Vec<ReadAction>,
}
+impl Debug for FuzzInput {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut actions = self.actions.clone();
+ // Remove all Read(0) entry.
+ let empty = ReadAction::Read { size: 0 };
+ actions.retain(|e| e != &empty);
+
+ f.debug_struct("FuzzInput")
+ .field("path", &self.path)
+ .field("size", &self.size)
+ .field("range", &self.range.to_string())
+ .field("actions", &actions)
+ .finish()
+ }
+}
+
impl Arbitrary<'_> for FuzzInput {
fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result<Self> {
let total_size = u.int_in_range(1..=MAX_DATA_SIZE)?;
@@ -109,6 +127,7 @@ impl Arbitrary<'_> for FuzzInput {
}
Ok(FuzzInput {
+ path: uuid::Uuid::new_v4().to_string(),
size: total_size,
range,
actions,
@@ -142,17 +161,30 @@ impl ReadChecker {
}
}
- fn check_read(&mut self, n: usize, output: &[u8]) {
- if n == 0 {
+ fn check_read(&mut self, buf_size: usize, output: &[u8]) {
+ if buf_size == 0 {
assert_eq!(
output.len(),
0,
- "check read failed: output bs is not empty when read size is 0"
+ "check read failed: output must be empty if buf_size is 0"
+ );
+ return;
+ }
+
+ if buf_size > 0 && output.is_empty() {
+ assert!(
+ self.cur >= self.ranged_data.len(),
+ "check read failed: no data read means cur must outsides of ranged_data",
);
return;
}
- let expected = &self.ranged_data[self.cur..self.cur + n];
+ assert!(
+ self.cur + output.len() <= self.ranged_data.len(),
+ "check read failed: cur + output length must be less than ranged_data length, cur: {}, output: {}, ranged_data: {}", self.cur, output.len(), self.ranged_data.len(),
+ );
+
+ let expected = &self.ranged_data[self.cur..self.cur + output.len()];
// Check the read result
assert_eq!(
@@ -162,7 +194,7 @@ impl ReadChecker {
);
// Update the current position
- self.cur += n;
+ self.cur += output.len();
}
fn check_seek(&mut self, seek_from: SeekFrom, output: Result<u64>) {
@@ -206,7 +238,8 @@ impl ReadChecker {
"{:x}",
Sha256::digest(&self.ranged_data[self.cur..self.cur + output.len()])
),
- "check next failed: output bs is different with expected bs",
+ "check next failed: output bs is different with expected bs, current: {}, output length: {}",
+ self.cur, output.len(),
);
// update the current position
@@ -221,19 +254,20 @@ impl ReadChecker {
}
async fn fuzz_reader(op: Operator, input: FuzzInput) -> Result<()> {
- let path = uuid::Uuid::new_v4().to_string();
-
let mut checker = ReadChecker::new(input.size, input.range);
- op.write(&path, checker.raw_data.clone()).await?;
+ op.write(&input.path, checker.raw_data.clone()).await?;
- let mut o = op.reader_with(&path).range(input.range.to_range()).await?;
+ let mut o = op
+ .reader_with(&input.path)
+ .range(input.range.to_range())
+ .await?;
for action in input.actions {
match action {
ReadAction::Read { size } => {
let mut buf = vec![0; size];
let n = o.read(&mut buf).await?;
- checker.check_read(n, &buf[..n]);
+ checker.check_read(size, &buf[..n]);
}
ReadAction::Seek(seek_from) => {
@@ -248,12 +282,17 @@ async fn fuzz_reader(op: Operator, input: FuzzInput) -> Result<()> {
}
}
- op.delete(&path).await?;
+ op.delete(&input.path).await?;
Ok(())
}
fuzz_target!(|input: FuzzInput| {
let _ = dotenvy::dotenv();
+ let _ = tracing_subscriber::fmt()
+ .pretty()
+ .with_test_writer()
+ .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+ .try_init();
let runtime = tokio::runtime::Runtime::new().expect("init runtime must succeed");
@@ -261,7 +300,7 @@ fuzz_target!(|input: FuzzInput| {
runtime.block_on(async {
fuzz_reader(op, input.clone())
.await
- .unwrap_or_else(|_| panic!("fuzz reader must succeed"));
+ .unwrap_or_else(|err| panic!("fuzz reader must succeed: {err:?}"));
})
}
});
diff --git a/core/fuzz/utils.rs b/core/fuzz/utils.rs
index 37598fdb3..e83e4c184 100644
--- a/core/fuzz/utils.rs
+++ b/core/fuzz/utils.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use opendal::layers::{LoggingLayer, RetryLayer};
use std::env;
use std::str::FromStr;
@@ -44,5 +45,10 @@ pub fn init_service() -> Option<Operator> {
})
.collect();
- Some(Operator::via_map(scheme, envs).unwrap_or_else(|_| panic!("init {} must succeed", scheme)))
+ Some(
+ Operator::via_map(scheme, envs)
+ .unwrap_or_else(|_| panic!("init {} must succeed", scheme))
+ .layer(LoggingLayer::default())
+ .layer(RetryLayer::default()),
+ )
}