You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2023/10/30 06:19:32 UTC
(incubator-opendal) branch fuzzer-checker created (now e7b63a92c)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a change to branch fuzzer-checker
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
at e7b63a92c refactor(tests): Extract tests related logic into raw::tests for reuse
This branch includes the following new commits:
new 185943daf Remove unused files
new e7b63a92c refactor(tests): Extract tests related logic into raw::tests for reuse
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-opendal) 01/02: Remove unused files
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch fuzzer-checker
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 185943daf9515522ea2a0c359c8b61e579659681
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Oct 30 12:32:35 2023 +0800
Remove unused files
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/tests/behavior/file_write_full_disk.rs | 25 -------------------------
core/tests/behavior/main.rs | 2 --
2 files changed, 27 deletions(-)
diff --git a/core/tests/behavior/file_write_full_disk.rs b/core/tests/behavior/file_write_full_disk.rs
deleted file mode 100644
index 0198af25a..000000000
--- a/core/tests/behavior/file_write_full_disk.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[cfg(test)]
-mod tests {
- #[tokio::test]
- async fn test_file_write_on_full_disk() {
- let data = [96u8; 1024 * 512]; // The writable was smaller than 512KB
- op.write("/test", data.to_vec()).await.unwrap();
- }
-}
diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs
index ab4e900e1..45d1ba589 100644
--- a/core/tests/behavior/main.rs
+++ b/core/tests/behavior/main.rs
@@ -48,8 +48,6 @@ mod blocking_list;
mod blocking_read_only;
mod blocking_rename;
mod blocking_write;
-mod file_write_full_disk;
-
use blocking_append::behavior_blocking_append_tests;
use blocking_copy::behavior_blocking_copy_tests;
use blocking_list::behavior_blocking_list_tests;
(incubator-opendal) 02/02: refactor(tests): Extract tests related logic into raw::tests for reuse
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch fuzzer-checker
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit e7b63a92cea3c2ff3ecf34e285985908b1e384dd
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Oct 30 14:18:55 2023 +0800
refactor(tests): Extract tests related logic into raw::tests for reuse
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
Cargo.lock | 2 -
core/Cargo.toml | 10 ++
core/benches/ops/read.rs | 10 +-
core/benches/ops/utils.rs | 32 +-----
core/benches/ops/write.rs | 6 +-
core/fuzz/Cargo.toml | 4 +-
core/fuzz/fuzz_reader.rs | 172 +++----------------------------
core/fuzz/fuzz_writer.rs | 78 ++++----------
core/fuzz/utils.rs | 54 ----------
core/src/raw/mod.rs | 2 +
core/src/raw/tests/mod.rs | 30 ++++++
core/src/raw/tests/read.rs | 236 +++++++++++++++++++++++++++++++++++++++++++
core/src/raw/tests/utils.rs | 85 ++++++++++++++++
core/src/raw/tests/write.rs | 76 ++++++++++++++
core/tests/behavior/fuzz.rs | 54 +++++-----
core/tests/behavior/main.rs | 11 +-
core/tests/behavior/utils.rs | 69 +------------
17 files changed, 517 insertions(+), 414 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index e5fa7d138..76d235e32 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4276,8 +4276,6 @@ dependencies = [
"dotenvy",
"libfuzzer-sys",
"opendal",
- "rand 0.8.5",
- "sha2",
"tokio",
"tracing-subscriber",
"uuid",
diff --git a/core/Cargo.toml b/core/Cargo.toml
index e736896c8..2bf456813 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -59,6 +59,14 @@ default = [
# And doesn't have any other effects.
docs = []
+# Build test utils or not.
+#
+# This features is used to control whether or not to build opendal's test utils.
+# And doesn't have any other effects.
+#
+# You should never enable this feature unless you are developing opendal.
+tests = ["dep:rand", "dep:sha2", "dep:dotenvy"]
+
# Enable trust-dns for pure rust dns cache.
trust-dns = ["reqwest/trust-dns"]
@@ -201,6 +209,7 @@ name = "oio"
harness = false
name = "behavior"
path = "tests/behavior/main.rs"
+required-features = ["tests"]
[dependencies]
anyhow = { version = "1.0.30", features = ["std"] }
@@ -286,6 +295,7 @@ tokio = "1.27"
tokio-postgres = { version = "0.7.8", optional = true }
tracing = { version = "0.1", optional = true }
uuid = { version = "1", features = ["serde", "v4"] }
+dotenvy = { version = "0.15", optional = true }
[dev-dependencies]
criterion = { version = "0.4", features = ["async", "async_tokio"] }
diff --git a/core/benches/ops/read.rs b/core/benches/ops/read.rs
index 3be87ce38..8171daebd 100644
--- a/core/benches/ops/read.rs
+++ b/core/benches/ops/read.rs
@@ -18,6 +18,8 @@
use criterion::Criterion;
use futures::io;
use futures::AsyncReadExt;
+use opendal::raw::tests::init_test_service;
+use opendal::raw::tests::TEST_RUNTIME;
use opendal::Operator;
use rand::prelude::*;
use size::Size;
@@ -25,7 +27,7 @@ use size::Size;
use super::utils::*;
pub fn bench(c: &mut Criterion) {
- if let Some(op) = init_service() {
+ if let Some(op) = init_test_service().unwrap() {
bench_read_full(c, op.info().scheme().into_static(), op.clone());
bench_read_part(c, op.info().scheme().into_static(), op.clone());
bench_read_parallel(c, op.info().scheme().into_static(), op.clone());
@@ -49,7 +51,7 @@ fn bench_read_full(c: &mut Criterion, name: &str, op: Operator) {
group.throughput(criterion::Throughput::Bytes(size.bytes() as u64));
group.bench_with_input(size.to_string(), &(op.clone(), &path), |b, (op, path)| {
- b.to_async(&*TOKIO).iter(|| async {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
let r = op
.reader_with(path)
.range(0..=size.bytes() as u64)
@@ -84,7 +86,7 @@ fn bench_read_part(c: &mut Criterion, name: &str, op: Operator) {
group.throughput(criterion::Throughput::Bytes(size.bytes() as u64));
group.bench_with_input(size.to_string(), &(op.clone(), &path), |b, (op, path)| {
- b.to_async(&*TOKIO).iter(|| async {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
let r = op.reader_with(path).range(offset..).await.unwrap();
io::copy(r, &mut io::sink()).await.unwrap();
})
@@ -119,7 +121,7 @@ fn bench_read_parallel(c: &mut Criterion, name: &str, op: Operator) {
format!("{}x{}", parallel, size.to_string()),
&(op.clone(), &path, buf.clone()),
|b, (op, path, buf)| {
- b.to_async(&*TOKIO).iter(|| async {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
let futures = (0..parallel)
.map(|_| async {
let mut buf = buf.clone();
diff --git a/core/benches/ops/utils.rs b/core/benches/ops/utils.rs
index 4c6aa866a..39947835a 100644
--- a/core/benches/ops/utils.rs
+++ b/core/benches/ops/utils.rs
@@ -15,37 +15,11 @@
// specific language governing permissions and limitations
// under the License.
-use std::env;
-use std::str::FromStr;
-
use bytes::Bytes;
-use once_cell::sync::Lazy;
+use opendal::raw::tests::TEST_RUNTIME;
use opendal::*;
use rand::prelude::*;
-pub static TOKIO: Lazy<tokio::runtime::Runtime> =
- Lazy::new(|| tokio::runtime::Runtime::new().expect("build tokio runtime"));
-
-pub fn init_service() -> Option<Operator> {
- let scheme = if let Ok(v) = env::var("OPENDAL_TEST") {
- v
- } else {
- return None;
- };
- let scheme = Scheme::from_str(&scheme).unwrap();
-
- let prefix = format!("opendal_{scheme}_");
- let envs = env::vars()
- .filter_map(move |(k, v)| {
- k.to_lowercase()
- .strip_prefix(&prefix)
- .map(|k| (k.to_string(), v))
- })
- .collect();
-
- Some(Operator::via_map(scheme, envs).unwrap_or_else(|_| panic!("init {scheme} must succeed")))
-}
-
pub fn gen_bytes(rng: &mut ThreadRng, size: usize) -> Bytes {
let mut content = vec![0; size];
rng.fill_bytes(&mut content);
@@ -67,7 +41,7 @@ impl TempData {
}
pub fn generate(op: Operator, path: &str, content: Bytes) -> Self {
- TOKIO.block_on(async { op.write(path, content).await.expect("create test data") });
+ TEST_RUNTIME.block_on(async { op.write(path, content).await.expect("create test data") });
Self {
op,
@@ -78,7 +52,7 @@ impl TempData {
impl Drop for TempData {
fn drop(&mut self) {
- TOKIO.block_on(async {
+ TEST_RUNTIME.block_on(async {
self.op.delete(&self.path).await.expect("cleanup test data");
})
}
diff --git a/core/benches/ops/write.rs b/core/benches/ops/write.rs
index 2d9d98fd6..aefce8288 100644
--- a/core/benches/ops/write.rs
+++ b/core/benches/ops/write.rs
@@ -16,6 +16,8 @@
// under the License.
use criterion::Criterion;
+use opendal::raw::tests::init_test_service;
+use opendal::raw::tests::TEST_RUNTIME;
use opendal::Operator;
use rand::prelude::*;
use size::Size;
@@ -23,7 +25,7 @@ use size::Size;
use super::utils::*;
pub fn bench(c: &mut Criterion) {
- if let Some(op) = init_service() {
+ if let Some(op) = init_test_service().unwrap() {
bench_write_once(c, op.info().scheme().into_static(), op.clone());
}
}
@@ -48,7 +50,7 @@ fn bench_write_once(c: &mut Criterion, name: &str, op: Operator) {
size.to_string(),
&(op.clone(), &path, content.clone()),
|b, (op, path, content)| {
- b.to_async(&*TOKIO).iter(|| async {
+ b.to_async(&*TEST_RUNTIME).iter(|| async {
op.write(path, content.clone()).await.unwrap();
})
},
diff --git a/core/fuzz/Cargo.toml b/core/fuzz/Cargo.toml
index 036433934..f0a3ad726 100644
--- a/core/fuzz/Cargo.toml
+++ b/core/fuzz/Cargo.toml
@@ -30,9 +30,7 @@ arbitrary = { version = "1.3.0", features = ["derive"] }
bytes = "1.2"
dotenvy = "0.15.6"
libfuzzer-sys = "0.4"
-opendal = { path = ".." }
-rand = "0.8"
-sha2 = { version = "0.10.6" }
+opendal = { path = "..", features = ["tests"] }
tokio = { version = "1", features = ["full"] }
uuid = { version = "1", features = ["v4"] }
tracing-subscriber = { version = "0.3", features = [
diff --git a/core/fuzz/fuzz_reader.rs b/core/fuzz/fuzz_reader.rs
index 799d6f52b..bda286a18 100644
--- a/core/fuzz/fuzz_reader.rs
+++ b/core/fuzz/fuzz_reader.rs
@@ -20,29 +20,19 @@
use std::fmt::{Debug, Formatter};
use std::io::SeekFrom;
-use bytes::Bytes;
use libfuzzer_sys::arbitrary::Arbitrary;
use libfuzzer_sys::arbitrary::Unstructured;
use libfuzzer_sys::fuzz_target;
-use opendal::raw::oio::ReadExt;
+use opendal::raw::tests::init_test_service;
+use opendal::raw::tests::ReadAction;
+use opendal::raw::tests::ReadChecker;
+use opendal::raw::tests::TEST_RUNTIME;
use opendal::raw::BytesRange;
use opendal::Operator;
use opendal::Result;
-use rand::prelude::*;
-use sha2::Digest;
-use sha2::Sha256;
-
-mod utils;
const MAX_DATA_SIZE: usize = 16 * 1024 * 1024;
-#[derive(Debug, Clone, Eq, PartialEq)]
-enum ReadAction {
- Read { size: usize },
- Seek(SeekFrom),
- Next,
-}
-
#[derive(Clone)]
struct FuzzInput {
path: String,
@@ -55,7 +45,7 @@ impl Debug for FuzzInput {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut actions = self.actions.clone();
// Remove all Read(0) entry.
- let empty = ReadAction::Read { size: 0 };
+ let empty = ReadAction::Read(0);
actions.retain(|e| e != &empty);
f.debug_struct("FuzzInput")
@@ -100,7 +90,7 @@ impl Arbitrary<'_> for FuzzInput {
// Read
0 => {
let size = u.int_in_range(0..=total_size * 2)?;
- ReadAction::Read { size }
+ ReadAction::Read(size)
}
// Next
1 => ReadAction::Next,
@@ -135,169 +125,31 @@ impl Arbitrary<'_> for FuzzInput {
}
}
-struct ReadChecker {
- /// Raw Data is the data we write to the storage.
- raw_data: Bytes,
- /// Ranged Data is the data that we read from the storage.
- ranged_data: Bytes,
-
- cur: usize,
-}
-
-impl ReadChecker {
- fn new(size: usize, range: BytesRange) -> Self {
- let mut rng = thread_rng();
- let mut data = vec![0; size];
- rng.fill_bytes(&mut data);
-
- let raw_data = Bytes::from(data);
- let ranged_data = range.apply_on_bytes(raw_data.clone());
-
- Self {
- raw_data,
- ranged_data,
-
- cur: 0,
- }
- }
-
- fn check_read(&mut self, buf_size: usize, output: &[u8]) {
- if buf_size == 0 {
- assert_eq!(
- output.len(),
- 0,
- "check read failed: output must be empty if buf_size is 0"
- );
- return;
- }
-
- if buf_size > 0 && output.is_empty() {
- assert!(
- self.cur >= self.ranged_data.len(),
- "check read failed: no data read means cur must outsides of ranged_data",
- );
- return;
- }
-
- assert!(
- self.cur + output.len() <= self.ranged_data.len(),
- "check read failed: cur + output length must be less than ranged_data length, cur: {}, output: {}, ranged_data: {}", self.cur, output.len(), self.ranged_data.len(),
- );
-
- let expected = &self.ranged_data[self.cur..self.cur + output.len()];
-
- // Check the read result
- assert_eq!(
- format!("{:x}", Sha256::digest(output)),
- format!("{:x}", Sha256::digest(expected)),
- "check read failed: output bs is different with expected bs",
- );
-
- // Update the current position
- self.cur += output.len();
- }
-
- fn check_seek(&mut self, seek_from: SeekFrom, output: Result<u64>) {
- let expected = match seek_from {
- SeekFrom::Start(offset) => offset as i64,
- SeekFrom::End(offset) => self.ranged_data.len() as i64 + offset,
- SeekFrom::Current(offset) => self.cur as i64 + offset,
- };
-
- if expected < 0 {
- assert!(output.is_err(), "check seek failed: seek should fail");
- assert_eq!(
- output.unwrap_err().kind(),
- opendal::ErrorKind::InvalidInput,
- "check seek failed: seek result is different with expected result"
- );
-
- return;
- }
-
- assert_eq!(
- output.unwrap(),
- expected as u64,
- "check seek failed: seek result is different with expected result",
- );
-
- // only update the current position when seek succeed
- self.cur = expected as usize;
- }
-
- fn check_next(&mut self, output: Option<Bytes>) {
- if let Some(output) = output {
- assert!(
- self.cur + output.len() <= self.ranged_data.len(),
- "check next failed: output bs is larger than remaining bs",
- );
-
- assert_eq!(
- format!("{:x}", Sha256::digest(&output)),
- format!(
- "{:x}",
- Sha256::digest(&self.ranged_data[self.cur..self.cur + output.len()])
- ),
- "check next failed: output bs is different with expected bs, current: {}, output length: {}",
- self.cur, output.len(),
- );
-
- // update the current position
- self.cur += output.len();
- } else {
- assert!(
- self.cur >= self.ranged_data.len(),
- "check next failed: output bs is None, we still have bytes to read",
- )
- }
- }
-}
-
async fn fuzz_reader(op: Operator, input: FuzzInput) -> Result<()> {
let mut checker = ReadChecker::new(input.size, input.range);
- op.write(&input.path, checker.raw_data.clone()).await?;
+ op.write(&input.path, checker.data()).await?;
- let mut o = op
+ let r = op
.reader_with(&input.path)
.range(input.range.to_range())
.await?;
- for action in input.actions {
- match action {
- ReadAction::Read { size } => {
- let mut buf = vec![0; size];
- let n = o.read(&mut buf).await?;
- checker.check_read(size, &buf[..n]);
- }
-
- ReadAction::Seek(seek_from) => {
- let res = o.seek(seek_from).await;
- checker.check_seek(seek_from, res);
- }
-
- ReadAction::Next => {
- let res = o.next().await.transpose()?;
- checker.check_next(res);
- }
- }
- }
+ checker.check(r, &input.actions).await;
op.delete(&input.path).await?;
Ok(())
}
fuzz_target!(|input: FuzzInput| {
- let _ = dotenvy::dotenv();
let _ = tracing_subscriber::fmt()
.pretty()
.with_test_writer()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.try_init();
- let runtime = tokio::runtime::Runtime::new().expect("init runtime must succeed");
-
- if let Some(op) = utils::init_service() {
- runtime.block_on(async {
+ let op = init_test_service().expect("operator init must succeed");
+ if let Some(op) = op {
+ TEST_RUNTIME.block_on(async {
fuzz_reader(op, input.clone())
.await
.unwrap_or_else(|err| panic!("fuzz reader must succeed: {err:?}"));
diff --git a/core/fuzz/fuzz_writer.rs b/core/fuzz/fuzz_writer.rs
index edd25ac26..866dfb718 100644
--- a/core/fuzz/fuzz_writer.rs
+++ b/core/fuzz/fuzz_writer.rs
@@ -17,29 +17,21 @@
#![no_main]
-use bytes::Bytes;
-use bytes::BytesMut;
use libfuzzer_sys::arbitrary::Arbitrary;
use libfuzzer_sys::arbitrary::Unstructured;
use libfuzzer_sys::fuzz_target;
+use opendal::raw::tests::init_test_service;
+use opendal::raw::tests::WriteAction;
+use opendal::raw::tests::WriteChecker;
+use opendal::raw::tests::TEST_RUNTIME;
use opendal::Operator;
use opendal::Result;
-use rand::prelude::*;
-use sha2::Digest;
-use sha2::Sha256;
-
-mod utils;
const MAX_DATA_SIZE: usize = 16 * 1024 * 1024;
-#[derive(Debug, Clone)]
-enum WriterAction {
- Write { size: usize },
-}
-
#[derive(Debug, Clone)]
struct FuzzInput {
- actions: Vec<WriterAction>,
+ actions: Vec<WriteAction>,
}
impl Arbitrary<'_> for FuzzInput {
@@ -50,50 +42,13 @@ impl Arbitrary<'_> for FuzzInput {
for _ in 0..count {
let size = u.int_in_range(1..=MAX_DATA_SIZE)?;
- actions.push(WriterAction::Write { size });
+ actions.push(WriteAction::Write(size));
}
Ok(FuzzInput { actions })
}
}
-struct WriteChecker {
- chunks: Vec<Bytes>,
- data: Bytes,
-}
-
-impl WriteChecker {
- fn new(size: Vec<usize>) -> Self {
- let mut rng = thread_rng();
-
- let mut chunks = Vec::with_capacity(size.len());
-
- for i in size {
- let mut bs = vec![0u8; i];
- rng.fill_bytes(&mut bs);
- chunks.push(Bytes::from(bs));
- }
-
- let data = chunks.iter().fold(BytesMut::new(), |mut acc, x| {
- acc.extend_from_slice(x);
- acc
- });
-
- WriteChecker {
- chunks,
- data: data.freeze(),
- }
- }
-
- fn check(&self, actual: &[u8]) {
- assert_eq!(
- format!("{:x}", Sha256::digest(actual)),
- format!("{:x}", Sha256::digest(&self.data)),
- "check failed: result is not expected"
- )
- }
-}
-
async fn fuzz_writer(op: Operator, input: FuzzInput) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
@@ -101,7 +56,7 @@ async fn fuzz_writer(op: Operator, input: FuzzInput) -> Result<()> {
.actions
.iter()
.map(|a| match a {
- WriterAction::Write { size } => *size,
+ WriteAction::Write(size) => *size,
})
.collect();
@@ -109,7 +64,7 @@ async fn fuzz_writer(op: Operator, input: FuzzInput) -> Result<()> {
let mut writer = op.writer_with(&path).buffer(8 * 1024 * 1024).await?;
- for chunk in &checker.chunks {
+ for chunk in checker.chunks() {
writer.write(chunk.clone()).await?;
}
@@ -124,15 +79,18 @@ async fn fuzz_writer(op: Operator, input: FuzzInput) -> Result<()> {
}
fuzz_target!(|input: FuzzInput| {
- let _ = dotenvy::dotenv();
-
- let runtime = tokio::runtime::Runtime::new().expect("init runtime must succeed");
-
- if let Some(op) = utils::init_service() {
- runtime.block_on(async {
+ let _ = tracing_subscriber::fmt()
+ .pretty()
+ .with_test_writer()
+ .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+ .try_init();
+
+ let op = init_test_service().expect("operator init must succeed");
+ if let Some(op) = op {
+ TEST_RUNTIME.block_on(async {
fuzz_writer(op, input.clone())
.await
- .unwrap_or_else(|_| panic!("fuzz writer must succeed"));
+ .unwrap_or_else(|err| panic!("fuzz reader must succeed: {err:?}"));
})
}
});
diff --git a/core/fuzz/utils.rs b/core/fuzz/utils.rs
deleted file mode 100644
index e83e4c184..000000000
--- a/core/fuzz/utils.rs
+++ /dev/null
@@ -1,54 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use opendal::layers::{LoggingLayer, RetryLayer};
-use std::env;
-use std::str::FromStr;
-
-use opendal::Operator;
-use opendal::Scheme;
-
-pub fn init_service() -> Option<Operator> {
- let scheme = if let Ok(v) = env::var("OPENDAL_TEST") {
- v
- } else {
- return None;
- };
- let scheme = Scheme::from_str(&scheme).unwrap();
-
- let args: Vec<String> = env::args().collect();
- if args[0].ends_with(&scheme.to_string()) {
- // if not exist, fallback to .env
- let _ = dotenvy::from_filename_override(format!(".{scheme}.env"));
- }
-
- let prefix = format!("opendal_{}_", scheme);
- let envs = env::vars()
- .filter_map(move |(k, v)| {
- k.to_lowercase()
- .strip_prefix(&prefix)
- .map(|k| (k.to_string(), v))
- })
- .collect();
-
- Some(
- Operator::via_map(scheme, envs)
- .unwrap_or_else(|_| panic!("init {} must succeed", scheme))
- .layer(LoggingLayer::default())
- .layer(RetryLayer::default()),
- )
-}
diff --git a/core/src/raw/mod.rs b/core/src/raw/mod.rs
index 313c5d04f..12fe91305 100644
--- a/core/src/raw/mod.rs
+++ b/core/src/raw/mod.rs
@@ -62,3 +62,5 @@ pub use tokio_util::*;
// Expose as a pub mod to avoid confusing.
pub mod adapters;
pub mod oio;
+#[cfg(feature = "tests")]
+pub mod tests;
diff --git a/core/src/raw/tests/mod.rs b/core/src/raw/tests/mod.rs
new file mode 100644
index 000000000..7a3619cab
--- /dev/null
+++ b/core/src/raw/tests/mod.rs
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Utilities for opendal testing.
+
+mod read;
+pub use read::ReadAction;
+pub use read::ReadChecker;
+
+mod write;
+pub use write::WriteAction;
+pub use write::WriteChecker;
+
+mod utils;
+pub use utils::init_test_service;
+pub use utils::TEST_RUNTIME;
diff --git a/core/src/raw/tests/read.rs b/core/src/raw/tests/read.rs
new file mode 100644
index 000000000..526af48e5
--- /dev/null
+++ b/core/src/raw/tests/read.rs
@@ -0,0 +1,236 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::*;
+use crate::*;
+use bytes::Bytes;
+use rand::{thread_rng, RngCore};
+use sha2::{Digest, Sha256};
+use std::io::SeekFrom;
+
+/// ReadAction represents a read action.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum ReadAction {
+ /// Read represents a read action with given input buf size.
+ ///
+ /// # NOTE
+ ///
+ /// The size is the input buf size, it's possible that the actual read size is smaller.
+ Read(usize),
+ /// Seek represents a seek action with given seek position.
+ ///
+ /// # NOTE
+ ///
+ /// It's valid that seek outside of the file's end.
+ Seek(SeekFrom),
+ /// Next represents a next action.
+ Next,
+}
+
+/// ReadChecker is used to check the correctness of the read process.
+pub struct ReadChecker {
+ /// Raw Data is the data we write to the storage.
+ raw_data: Bytes,
+ /// Ranged Data is the data that we read from the storage.
+ ranged_data: Bytes,
+ /// Cur is the current position of the read process.
+ cur: usize,
+}
+
+impl ReadChecker {
+ /// Create a new read checker by given size and range.
+ ///
+ /// It's by design that we use a random generator to generate the raw data. The content of data
+ /// is not important, we only care about the correctness of the read process.
+ pub fn new(size: usize, range: impl Into<BytesRange>) -> Self {
+ let mut rng = thread_rng();
+ let mut data = vec![0; size];
+ rng.fill_bytes(&mut data);
+
+ let raw_data = Bytes::from(data);
+ let ranged_data = range.into().apply_on_bytes(raw_data.clone());
+
+ Self {
+ raw_data,
+ ranged_data,
+
+ cur: 0,
+ }
+ }
+
+ /// Return the raw data of this read checker.
+ pub fn data(&self) -> Bytes {
+ self.raw_data.clone()
+ }
+
+ /// check_read checks the correctness of the read process after a read action.
+ ///
+ /// - buf_size is the read action's buf size.
+ /// - output is the output of this read action.
+ fn check_read(&mut self, input: usize, output: &[u8]) {
+ if input == 0 {
+ assert_eq!(
+ output.len(),
+ 0,
+ "check read failed: output must be empty if buf_size is 0"
+ );
+ return;
+ }
+
+ if input > 0 && output.is_empty() {
+ assert!(
+ self.cur >= self.ranged_data.len(),
+ "check read failed: no data read means cur must outsides of ranged_data",
+ );
+ return;
+ }
+
+ assert!(
+ self.cur + output.len() <= self.ranged_data.len(),
+ "check read failed: cur + output length must be less than ranged_data length, cur: {}, output: {}, ranged_data: {}", self.cur, output.len(), self.ranged_data.len(),
+ );
+
+ let expected = &self.ranged_data[self.cur..self.cur + output.len()];
+
+ // Check the read result
+ assert_eq!(
+ format!("{:x}", Sha256::digest(output)),
+ format!("{:x}", Sha256::digest(expected)),
+ "check read failed: output bs is different with expected bs",
+ );
+
+ // Update the current position
+ self.cur += output.len();
+ }
+
+ /// check_seek checks the correctness of the read process after a seek action.
+ ///
+ /// - input is the `SeekFrom` passed by SeekAction.
+ /// - output ts the result after the seek operation.
+ fn check_seek(&mut self, input: SeekFrom, output: Result<u64>) {
+ let expected = match input {
+ SeekFrom::Start(offset) => offset as i64,
+ SeekFrom::End(offset) => self.ranged_data.len() as i64 + offset,
+ SeekFrom::Current(offset) => self.cur as i64 + offset,
+ };
+
+ if expected < 0 {
+ let Err(err) = output else {
+ panic!("check seek failed: seek should fail with negative offset");
+ };
+
+ assert_eq!(
+ err.kind(),
+ ErrorKind::InvalidInput,
+ "check seek failed: seek should fail with error InvalidInput with negative offset"
+ );
+ return;
+ }
+
+ assert_eq!(
+ output.unwrap(),
+ expected as u64,
+ "check seek failed: seek result is different with expected result",
+ );
+
+ // only update the current position when seek succeed
+ self.cur = expected as usize;
+ }
+
+ /// check_next checks the correctness of the read process after a next action.
+ fn check_next(&mut self, output: Option<Bytes>) {
+ if self.cur < self.ranged_data.len() {
+ let Some(bs) = output else {
+ panic!("check next failed: there are remaining bytes, but next output is None",);
+ };
+
+ assert_eq!(
+ format!("{:x}", Sha256::digest(&bs)),
+ format!(
+ "{:x}",
+ Sha256::digest(&self.ranged_data[self.cur..self.cur + bs.len()])
+ ),
+ "check next failed: output bs is different with expected bs, current: {}, output length: {}",
+ self.cur, bs.len(),
+ );
+
+ // update the current position
+ self.cur += bs.len();
+ } else {
+ assert!(
+ output.is_none(),
+ "check next failed: there are no remaining bytes, the next output is Some",
+ );
+ }
+ }
+
+ /// Check will check the correctness of the read process via given actions.
+ ///
+ /// Check will panic if any check failed.
+ pub async fn check(&mut self, mut r: Reader, actions: &[ReadAction]) {
+ use oio::ReadExt;
+
+ for action in actions {
+ match action {
+ ReadAction::Read(size) => {
+ let mut buf = vec![0; *size];
+ let n = r.read(&mut buf).await.expect("read must success");
+ self.check_read(*size, &buf[..n]);
+ }
+
+ ReadAction::Seek(pos) => {
+ let res = r.seek(*pos).await;
+ self.check_seek(*pos, res);
+ }
+
+ ReadAction::Next => {
+ let res = r.next().await.transpose().expect("next must success");
+ self.check_next(res);
+ }
+ }
+ }
+ }
+
+ /// Check will check the correctness of the read process via given actions.
+ ///
+ /// Check will panic if any check failed.
+ pub fn blocking_check(&mut self, mut r: BlockingReader, actions: &[ReadAction]) {
+ for action in actions {
+ match action {
+ ReadAction::Read(size) => {
+ use oio::BlockingRead;
+
+ let mut buf = vec![0; *size];
+ let n = r.read(&mut buf).expect("read must success");
+ self.check_read(*size, &buf[..n]);
+ }
+
+ ReadAction::Seek(pos) => {
+ use oio::BlockingRead;
+
+ let res = r.seek(*pos);
+ self.check_seek(*pos, res);
+ }
+
+ ReadAction::Next => {
+ let res = r.next().transpose().expect("next must success");
+ self.check_next(res);
+ }
+ }
+ }
+ }
+}
diff --git a/core/src/raw/tests/utils.rs b/core/src/raw/tests/utils.rs
new file mode 100644
index 000000000..2ecda5b79
--- /dev/null
+++ b/core/src/raw/tests/utils.rs
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::*;
+use once_cell::sync::Lazy;
+use std::collections::HashMap;
+use std::env;
+use std::str::FromStr;
+
+/// TEST_RUNTIME is the runtime used for running tests.
+pub static TEST_RUNTIME: Lazy<tokio::runtime::Runtime> = Lazy::new(|| {
+ tokio::runtime::Builder::new_multi_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+});
+
+/// Init a service with given scheme.
+///
+/// - Load scheme from `OPENDAL_TEST`
+/// - Construct a new Operator with given root.
+/// - Else, returns a `None` to represent no valid config for operator.
+pub fn init_test_service() -> Result<Option<Operator>> {
+ let _ = dotenvy::dotenv();
+
+ let scheme = if let Ok(v) = env::var("OPENDAL_TEST") {
+ v
+ } else {
+ return Ok(None);
+ };
+ let scheme = Scheme::from_str(&scheme).unwrap();
+
+ let prefix = format!("opendal_{scheme}_");
+
+ let mut cfg = env::vars()
+ .filter_map(|(k, v)| {
+ k.to_lowercase()
+ .strip_prefix(&prefix)
+ .map(|k| (k.to_string(), v))
+ })
+ .collect::<HashMap<String, String>>();
+
+ // Use random root unless OPENDAL_DISABLE_RANDOM_ROOT is set to true.
+ let disable_random_root = env::var("OPENDAL_DISABLE_RANDOM_ROOT").unwrap_or_default() == "true";
+ if !disable_random_root {
+ let root = format!(
+ "{}{}/",
+ cfg.get("root").cloned().unwrap_or_else(|| "/".to_string()),
+ uuid::Uuid::new_v4()
+ );
+ cfg.insert("root".to_string(), root);
+ }
+
+ let op = Operator::via_map(scheme, cfg).expect("must succeed");
+
+ #[cfg(feature = "layers-chaos")]
+ let op = { op.layer(layers::ChaosLayer::new(0.1)) };
+
+ let mut op = op
+ .layer(layers::LoggingLayer::default().with_backtrace_output(true))
+ .layer(layers::TimeoutLayer::new())
+ .layer(layers::RetryLayer::new().with_max_times(4));
+
+ // Enable blocking layer if needed.
+ if !op.info().full_capability().blocking {
+ let _guard = TEST_RUNTIME.enter();
+ op = op.layer(layers::BlockingLayer::create().expect("blocking layer must be created"))
+ }
+
+ Ok(Some(op))
+}
diff --git a/core/src/raw/tests/write.rs b/core/src/raw/tests/write.rs
new file mode 100644
index 000000000..511d5f143
--- /dev/null
+++ b/core/src/raw/tests/write.rs
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::{Bytes, BytesMut};
+use rand::{thread_rng, RngCore};
+use sha2::{Digest, Sha256};
+
+/// WriteAction represents a read action.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum WriteAction {
+ /// Write represents a write action with given input buf size.
+ ///
+ /// # NOTE
+ ///
+ /// The size is the input buf size, it's possible that the actual write size is smaller.
+ Write(usize),
+}
+
+/// WriteAction is used to check the correctness of the write process.
+pub struct WriteChecker {
+ chunks: Vec<Bytes>,
+ data: Bytes,
+}
+
+impl WriteChecker {
+ /// Create a new WriteChecker with given size.
+ pub fn new(size: Vec<usize>) -> Self {
+ let mut rng = thread_rng();
+
+ let mut chunks = Vec::with_capacity(size.len());
+
+ for i in size {
+ let mut bs = vec![0u8; i];
+ rng.fill_bytes(&mut bs);
+ chunks.push(Bytes::from(bs));
+ }
+
+ let data = chunks.iter().fold(BytesMut::new(), |mut acc, x| {
+ acc.extend_from_slice(x);
+ acc
+ });
+
+ WriteChecker {
+ chunks,
+ data: data.freeze(),
+ }
+ }
+
+ /// Get the check's chunks.
+ pub fn chunks(&self) -> &[Bytes] {
+ &self.chunks
+ }
+
+ /// Check the correctness of the write process.
+ pub fn check(&self, actual: &[u8]) {
+ assert_eq!(
+ format!("{:x}", Sha256::digest(actual)),
+ format!("{:x}", Sha256::digest(&self.data)),
+ "check failed: result is not expected"
+ )
+ }
+}
diff --git a/core/tests/behavior/fuzz.rs b/core/tests/behavior/fuzz.rs
index b8f47fdfb..6ad261795 100644
--- a/core/tests/behavior/fuzz.rs
+++ b/core/tests/behavior/fuzz.rs
@@ -15,13 +15,12 @@
// specific language governing permissions and limitations
// under the License.
-use std::io;
use std::io::SeekFrom;
use std::vec;
use anyhow::Result;
-use futures::AsyncSeekExt;
-use log::debug;
+use opendal::raw::tests::{ReadAction, ReadChecker};
+use opendal::raw::BytesRange;
use crate::*;
@@ -29,6 +28,32 @@ pub fn behavior_fuzz_tests(op: &Operator) -> Vec<Trial> {
async_trials!(op, test_fuzz_issue_2717)
}
+async fn test_fuzz_read(
+ op: Operator,
+ size: usize,
+ range: impl Into<BytesRange>,
+ actions: &[ReadAction],
+) -> Result<()> {
+ let cap = op.info().full_capability();
+
+ if !(cap.read && cap.write & cap.read_with_range) {
+ return Ok(());
+ }
+
+ let path = uuid::Uuid::new_v4().to_string();
+
+ let mut checker = ReadChecker::new(size, range.into());
+
+ op.write(&path, checker.data())
+ .await
+ .expect("write must succeed");
+
+ let r = op.reader(&path).await.expect("reader must be created");
+
+ checker.check(r, actions).await;
+ Ok(())
+}
+
/// This fuzz test is to reproduce <https://github.com/apache/incubator-opendal/issues/2717>.
///
/// The simplified cases could be seen as:
@@ -67,26 +92,7 @@ pub fn behavior_fuzz_tests(op: &Operator) -> Vec<Trial> {
/// is invalid for given range `1..2`. However, the actual behavior is we seek to `0`
/// and results in a panic.
pub async fn test_fuzz_issue_2717(op: Operator) -> Result<()> {
- let cap = op.info().full_capability();
-
- if !(cap.read && cap.write & cap.read_with_range) {
- return Ok(());
- }
-
- let path = uuid::Uuid::new_v4().to_string();
- debug!("Generate a random file: {}", &path);
- let content = gen_fixed_bytes(2);
-
- op.write(&path, content.clone())
- .await
- .expect("write must succeed");
+ let actions = [ReadAction::Seek(SeekFrom::End(-2))];
- let mut r = op.reader_with(&path).range(1..2).await?;
-
- // Perform a seek
- let result = r.seek(SeekFrom::End(-2)).await;
- assert!(result.is_err());
- assert_eq!(result.unwrap_err().kind(), io::ErrorKind::InvalidInput);
-
- Ok(())
+ test_fuzz_read(op, 2, .., &actions).await
}
diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs
index 45d1ba589..6f21a13ba 100644
--- a/core/tests/behavior/main.rs
+++ b/core/tests/behavior/main.rs
@@ -56,20 +56,13 @@ use blocking_rename::behavior_blocking_rename_tests;
use blocking_write::behavior_blocking_write_tests;
// External dependences
use libtest_mimic::{Arguments, Trial};
-use once_cell::sync::Lazy;
+use opendal::raw::tests::init_test_service;
use opendal::*;
-static RUNTIME: Lazy<tokio::runtime::Runtime> = Lazy::new(|| {
- tokio::runtime::Builder::new_multi_thread()
- .enable_all()
- .build()
- .unwrap()
-});
-
fn main() -> anyhow::Result<()> {
let args = Arguments::from_args();
- let op = if let Some(op) = init_service()? {
+ let op = if let Some(op) = init_test_service()? {
op
} else {
return Ok(());
diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs
index 6b91e8d2b..60302a91a 100644
--- a/core/tests/behavior/utils.rs
+++ b/core/tests/behavior/utils.rs
@@ -15,13 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use std::collections::HashMap;
-use std::env;
use std::fmt;
use std::fmt::Debug;
use std::fmt::Formatter;
use std::io::SeekFrom;
-use std::str::FromStr;
use std::usize;
use bytes::Bytes;
@@ -29,75 +26,13 @@ use futures::Future;
use libtest_mimic::Failed;
use libtest_mimic::Trial;
use log::debug;
-use opendal::layers::BlockingLayer;
-use opendal::layers::LoggingLayer;
-use opendal::layers::RetryLayer;
-use opendal::layers::TimeoutLayer;
use opendal::*;
use rand::distributions::uniform::SampleRange;
use rand::prelude::*;
use sha2::Digest;
use sha2::Sha256;
-use crate::RUNTIME;
-
-/// Init a service with given scheme.
-///
-/// - Load scheme from `OPENDAL_TEST`
-/// - Construct a new Operator with given root.
-/// - Else, returns a `None` to represent no valid config for operator.
-pub fn init_service() -> anyhow::Result<Option<Operator>> {
- let _ = dotenvy::dotenv();
-
- let scheme = if let Ok(v) = env::var("OPENDAL_TEST") {
- v
- } else {
- return Ok(None);
- };
- let scheme = Scheme::from_str(&scheme).unwrap();
-
- let prefix = format!("opendal_{scheme}_");
-
- let mut cfg = env::vars()
- .filter_map(|(k, v)| {
- k.to_lowercase()
- .strip_prefix(&prefix)
- .map(|k| (k.to_string(), v))
- })
- .collect::<HashMap<String, String>>();
-
- // Use random root unless OPENDAL_DISABLE_RANDOM_ROOT is set to true.
- let disable_random_root = env::var("OPENDAL_DISABLE_RANDOM_ROOT").unwrap_or_default() == "true";
- if !disable_random_root {
- let root = format!(
- "{}{}/",
- cfg.get("root").cloned().unwrap_or_else(|| "/".to_string()),
- uuid::Uuid::new_v4()
- );
- cfg.insert("root".to_string(), root);
- }
-
- let op = Operator::via_map(scheme, cfg).expect("must succeed");
-
- #[cfg(feature = "layers-chaos")]
- let op = {
- use opendal::layers::ChaosLayer;
- op.layer(ChaosLayer::new(0.1))
- };
-
- let mut op = op
- .layer(LoggingLayer::default().with_backtrace_output(true))
- .layer(TimeoutLayer::new())
- .layer(RetryLayer::new().with_max_times(4));
-
- // Enable blocking layer if needed.
- if !op.info().full_capability().blocking {
- let _guard = RUNTIME.enter();
- op = op.layer(BlockingLayer::create().expect("blocking layer must be created"))
- }
-
- Ok(Some(op))
-}
+use opendal::raw::tests::TEST_RUNTIME;
pub fn gen_bytes_with_range(range: impl SampleRange<usize>) -> (Vec<u8>, usize) {
let mut rng = thread_rng();
@@ -136,7 +71,7 @@ where
F: FnOnce(Operator) -> Fut + Send + 'static,
Fut: Future<Output = anyhow::Result<()>>,
{
- let handle = RUNTIME.handle().clone();
+ let handle = TEST_RUNTIME.handle().clone();
let op = op.clone();
Trial::test(format!("behavior::{name}"), move || {