You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/12/30 17:22:33 UTC
(incubator-opendal) branch main updated: feat: add fuzz reader with buffer tests (#3866)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 86e7852d6 feat: add fuzz reader with buffer tests (#3866)
86e7852d6 is described below
commit 86e7852d68a762879c90e1d157c92b0fb741a662
Author: Weny Xu <we...@gmail.com>
AuthorDate: Sun Dec 31 02:22:29 2023 +0900
feat: add fuzz reader with buffer tests (#3866)
---
.github/workflows/fuzz_test.yml | 7 +-
core/fuzz/Cargo.toml | 4 +
core/fuzz/fuzz_reader_with_buffer.rs | 164 +++++++++++++++++++++++++++++++++++
3 files changed, 172 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/fuzz_test.yml b/.github/workflows/fuzz_test.yml
index 3c1abc92d..8d6dee79e 100644
--- a/.github/workflows/fuzz_test.yml
+++ b/.github/workflows/fuzz_test.yml
@@ -65,6 +65,7 @@ jobs:
name: fuzz_targets
path: |
./target/x86_64-unknown-linux-gnu/release/fuzz_reader
+ ./target/x86_64-unknown-linux-gnu/release/fuzz_reader_with_buffer
./target/x86_64-unknown-linux-gnu/release/fuzz_writer
fuzz-test-s3:
@@ -81,7 +82,7 @@ jobs:
strategy:
fail-fast: true
matrix:
- fuzz-targets: [fuzz_reader, fuzz_writer]
+ fuzz-targets: [fuzz_reader, fuzz_reader_with_buffer, fuzz_writer]
steps:
- name: Install libfuzzer
run: sudo apt-get install -y libfuzzer-14-dev
@@ -116,7 +117,7 @@ jobs:
strategy:
fail-fast: true
matrix:
- fuzz-targets: [fuzz_reader, fuzz_writer]
+ fuzz-targets: [fuzz_reader, fuzz_reader_with_buffer, fuzz_writer]
steps:
- name: Install libfuzzer
run: sudo apt-get install -y libfuzzer-14-dev
@@ -141,7 +142,7 @@ jobs:
strategy:
fail-fast: true
matrix:
- fuzz-targets: [fuzz_reader, fuzz_writer]
+ fuzz-targets: [fuzz_reader, fuzz_reader_with_buffer, fuzz_writer]
steps:
- name: Install libfuzzer
run: sudo apt-get install -y libfuzzer-14-dev
diff --git a/core/fuzz/Cargo.toml b/core/fuzz/Cargo.toml
index e12454af2..e6b34e879 100644
--- a/core/fuzz/Cargo.toml
+++ b/core/fuzz/Cargo.toml
@@ -40,6 +40,10 @@ tracing-subscriber = { version = "0.3", features = [
] }
uuid = { version = "1", features = ["v4"] }
+[[bin]]
+name = "fuzz_reader_with_buffer"
+path = "fuzz_reader_with_buffer.rs"
+
[[bin]]
name = "fuzz_reader"
path = "fuzz_reader.rs"
diff --git a/core/fuzz/fuzz_reader_with_buffer.rs b/core/fuzz/fuzz_reader_with_buffer.rs
new file mode 100644
index 000000000..aa1baf6f8
--- /dev/null
+++ b/core/fuzz/fuzz_reader_with_buffer.rs
@@ -0,0 +1,164 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#![no_main]
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::io::SeekFrom;
+
+use libfuzzer_sys::arbitrary::Arbitrary;
+use libfuzzer_sys::arbitrary::Unstructured;
+use libfuzzer_sys::fuzz_target;
+use opendal::raw::tests::init_test_service;
+use opendal::raw::tests::ReadAction;
+use opendal::raw::tests::ReadChecker;
+use opendal::raw::tests::TEST_RUNTIME;
+use opendal::raw::BytesRange;
+use opendal::Operator;
+use opendal::Result;
+
+const MAX_DATA_SIZE: usize = 16 * 1024 * 1024;
+
+#[derive(Clone)]
+struct FuzzInput {
+ path: String,
+ size: usize,
+ range: BytesRange,
+ actions: Vec<ReadAction>,
+ buffer: usize,
+}
+
+impl Debug for FuzzInput {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ let mut actions = self.actions.clone();
+ // Remove all Read(0) entry.
+ let empty = ReadAction::Read(0);
+ actions.retain(|e| e != &empty);
+
+ f.debug_struct("FuzzInput")
+ .field("path", &self.path)
+ .field("size", &self.size)
+ .field("buffer", &self.buffer)
+ .field("range", &self.range.to_string())
+ .field("actions", &actions)
+ .finish()
+ }
+}
+
+impl Arbitrary<'_> for FuzzInput {
+ fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result<Self> {
+ let total_size = u.int_in_range(1..=MAX_DATA_SIZE)?;
+ let buffer = u.int_in_range(1..=MAX_DATA_SIZE + 1024)?;
+
+ // TODO: it's valid that size is larger than total_size.
+ let (offset, size) = match u.int_in_range(0..=3)? {
+ // Full range
+ 0 => (None, None),
+ 1 => {
+ let offset = u.int_in_range(0..=total_size as u64 - 1)?;
+ (Some(offset), None)
+ }
+ 2 => {
+ let size = u.int_in_range(1..=total_size as u64)?;
+ (None, Some(size))
+ }
+ 3 => {
+ let offset = u.int_in_range(0..=total_size as u64 - 1)?;
+ let size = u.int_in_range(1..=total_size as u64 - offset)?;
+ (Some(offset), Some(size))
+ }
+ _ => unreachable!("invalid int generated by arbitrary"),
+ };
+ let range = BytesRange::new(offset, size);
+
+ let count = u.int_in_range(1..=1024)?;
+ let mut actions = vec![];
+
+ for _ in 0..count {
+ let action = match u.int_in_range(0..=4)? {
+ // Read
+ 0 => {
+ let size = u.int_in_range(0..=total_size * 2)?;
+ ReadAction::Read(size)
+ }
+ // Next
+ 1 => ReadAction::Next,
+ // Seek Start
+ 2 => {
+ // NOTE: seek out of the end of file is valid.
+ let offset = u.int_in_range(0..=total_size * 2)?;
+ ReadAction::Seek(SeekFrom::Start(offset as u64))
+ }
+ // Seek Current
+ 3 => {
+ let offset = u.int_in_range(-(total_size as i64)..=(total_size as i64))?;
+ ReadAction::Seek(SeekFrom::Current(offset))
+ }
+ // Seek End
+ 4 => {
+ let offset = u.int_in_range(-(total_size as i64)..=(total_size as i64))?;
+ ReadAction::Seek(SeekFrom::End(offset))
+ }
+ _ => unreachable!("invalid int generated by arbitrary"),
+ };
+
+ actions.push(action);
+ }
+
+ Ok(FuzzInput {
+ path: uuid::Uuid::new_v4().to_string(),
+ size: total_size,
+ range,
+ actions,
+ buffer,
+ })
+ }
+}
+
+async fn fuzz_reader_with_buffer(op: Operator, input: FuzzInput) -> Result<()> {
+ let mut checker = ReadChecker::new(input.size, input.range);
+ op.write(&input.path, checker.data()).await?;
+
+ let r = op
+ .reader_with(&input.path)
+ .range(input.range.to_range())
+ .buffer(input.buffer)
+ .await?;
+
+ checker.check(r, &input.actions).await;
+
+ op.delete(&input.path).await?;
+ Ok(())
+}
+
+fuzz_target!(|input: FuzzInput| {
+ let _ = tracing_subscriber::fmt()
+ .pretty()
+ .with_test_writer()
+ .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+ .try_init();
+
+ let op = init_test_service().expect("operator init must succeed");
+ if let Some(op) = op {
+ TEST_RUNTIME.block_on(async {
+ fuzz_reader_with_buffer(op, input.clone())
+ .await
+ .unwrap_or_else(|err| panic!("fuzz reader must succeed: {err:?}"));
+ })
+ }
+});