You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/07/12 07:49:28 UTC
[avro] branch branch-1.11 updated: AVRO-3575: Rust: add fuzz harness (#1761)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch branch-1.11
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/branch-1.11 by this push:
new b8463f21d AVRO-3575: Rust: add fuzz harness (#1761)
b8463f21d is described below
commit b8463f21db3088690e6b1ce267966d8c57f1b706
Author: Evan Richter <ev...@gmail.com>
AuthorDate: Tue Jul 12 02:48:49 2022 -0500
AVRO-3575: Rust: add fuzz harness (#1761)
* AVRO-3575: Rust: cargo fuzz harness
* AVRO-3575: Rust: Fuzzing fixes
* AVRO-3575: Add ASLv2 licences and mark other test related modules as publish = false
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
Co-authored-by: Martin Tzvetanov Grigorov <mg...@apache.org>
(cherry picked from commit eb2477555d32d25ec1e8f5469d52ff76beaf33b6)
---
lang/rust/Cargo.toml | 3 +
lang/rust/avro/src/decode.rs | 2 +-
lang/rust/avro/src/reader.rs | 6 +-
lang/rust/avro/src/util.rs | 5 +-
lang/rust/avro_test_helper/Cargo.toml | 1 +
lang/rust/fuzz/.gitignore | 4 +
lang/rust/{ => fuzz}/Cargo.toml | 36 +++++--
lang/rust/fuzz/corpus/roundtrip/infinite_iteration | Bin 0 -> 132 bytes
.../fuzz/corpus/roundtrip/negate_with_overflow | Bin 0 -> 17 bytes
lang/rust/fuzz/corpus/roundtrip/unchecked_resize | Bin 0 -> 58 bytes
lang/rust/fuzz/fuzz_targets/roundtrip.rs | 117 +++++++++++++++++++++
lang/rust/wasm-demo/Cargo.toml | 1 +
12 files changed, 165 insertions(+), 10 deletions(-)
diff --git a/lang/rust/Cargo.toml b/lang/rust/Cargo.toml
index 689a83e51..8a6dfaa03 100644
--- a/lang/rust/Cargo.toml
+++ b/lang/rust/Cargo.toml
@@ -22,3 +22,6 @@ members = [
"avro",
"wasm-demo"
]
+exclude = [
+ "fuzz"
+]
diff --git a/lang/rust/avro/src/decode.rs b/lang/rust/avro/src/decode.rs
index 6017d9165..cb32ebdb7 100644
--- a/lang/rust/avro/src/decode.rs
+++ b/lang/rust/avro/src/decode.rs
@@ -59,7 +59,7 @@ fn decode_seq_len<R: Read>(reader: &mut R) -> AvroResult<usize> {
std::cmp::Ordering::Equal => return Ok(0),
std::cmp::Ordering::Less => {
let _size = zag_i64(reader)?;
- -raw_len
+ raw_len.checked_neg().ok_or(Error::IntegerOverflow)?
}
std::cmp::Ordering::Greater => raw_len,
})
diff --git a/lang/rust/avro/src/reader.rs b/lang/rust/avro/src/reader.rs
index 193449e12..de68c18da 100644
--- a/lang/rust/avro/src/reader.rs
+++ b/lang/rust/avro/src/reader.rs
@@ -114,7 +114,7 @@ impl<R: Read> Block<R> {
// We need to resize to ensure that the buffer len is safe to read `n` elements.
//
// TODO: Figure out a way to avoid having to truncate for the second case.
- self.buf.resize(n, 0);
+ self.buf.resize(util::safe_len(n)?, 0);
self.reader
.read_exact(&mut self.buf)
.map_err(Error::ReadIntoBuf)?;
@@ -179,6 +179,10 @@ impl<R: Read> Block<R> {
let mut block_bytes = &self.buf[self.buf_idx..];
let b_original = block_bytes.len();
let item = from_avro_datum(&self.writer_schema, &mut block_bytes, read_schema)?;
+ if b_original == block_bytes.len() {
+ // from_avro_datum did not consume any bytes, so return an error to avoid an infinite loop
+ return Err(Error::ReadBlock);
+ }
self.buf_idx += b_original - block_bytes.len();
self.message_count -= 1;
Ok(Some(item))
diff --git a/lang/rust/avro/src/util.rs b/lang/rust/avro/src/util.rs
index 5929c06dd..e18b5641f 100644
--- a/lang/rust/avro/src/util.rs
+++ b/lang/rust/avro/src/util.rs
@@ -23,7 +23,8 @@ use std::{convert::TryFrom, i64, io::Read, sync::Once};
/// Avro-encoded values. This is a protection against ill-formed
/// data, whose length field might be interpreted as enormous.
/// See max_allocation_bytes to change this limit.
-pub static mut MAX_ALLOCATION_BYTES: usize = 512 * 1024 * 1024;
+pub const DEFAULT_MAX_ALLOCATION_BYTES: usize = 512 * 1024 * 1024;
+static mut MAX_ALLOCATION_BYTES: usize = DEFAULT_MAX_ALLOCATION_BYTES;
static MAX_ALLOCATION_BYTES_ONCE: Once = Once::new();
pub trait MapHelper {
@@ -140,7 +141,7 @@ pub fn max_allocation_bytes(num_bytes: usize) -> usize {
}
pub fn safe_len(len: usize) -> AvroResult<usize> {
- let max_bytes = max_allocation_bytes(512 * 1024 * 1024);
+ let max_bytes = max_allocation_bytes(DEFAULT_MAX_ALLOCATION_BYTES);
if len <= max_bytes {
Ok(len)
diff --git a/lang/rust/avro_test_helper/Cargo.toml b/lang/rust/avro_test_helper/Cargo.toml
index 221f3cdde..790a72627 100644
--- a/lang/rust/avro_test_helper/Cargo.toml
+++ b/lang/rust/avro_test_helper/Cargo.toml
@@ -19,6 +19,7 @@
name = "apache-avro-test-helper"
version = "0.1.0"
edition = "2018"
+publish = false
description = "Avro test helper. This crate is not supposed to be published!"
[dependencies]
diff --git a/lang/rust/fuzz/.gitignore b/lang/rust/fuzz/.gitignore
new file mode 100644
index 000000000..1a45eee77
--- /dev/null
+++ b/lang/rust/fuzz/.gitignore
@@ -0,0 +1,4 @@
+target
+corpus
+artifacts
+coverage
diff --git a/lang/rust/Cargo.toml b/lang/rust/fuzz/Cargo.toml
similarity index 64%
copy from lang/rust/Cargo.toml
copy to lang/rust/fuzz/Cargo.toml
index 689a83e51..b2e38b482 100644
--- a/lang/rust/Cargo.toml
+++ b/lang/rust/fuzz/Cargo.toml
@@ -15,10 +15,34 @@
# specific language governing permissions and limitations
# under the License.
+[package]
+name = "apache-avro-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2018"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+serde = { version = "1", features = ["derive"] }
+
+[dependencies.apache-avro]
+path = "../avro"
+features = ["derive"]
+
[workspace]
-members = [
- "avro_test_helper",
- "avro_derive",
- "avro",
- "wasm-demo"
-]
+members = ["."]
+
+[features]
+debug = []
+
+[[bin]]
+name = "roundtrip"
+path = "fuzz_targets/roundtrip.rs"
+test = false
+doc = false
+
+[profile.release]
+debug = true
diff --git a/lang/rust/fuzz/corpus/roundtrip/infinite_iteration b/lang/rust/fuzz/corpus/roundtrip/infinite_iteration
new file mode 100644
index 000000000..d6606806d
Binary files /dev/null and b/lang/rust/fuzz/corpus/roundtrip/infinite_iteration differ
diff --git a/lang/rust/fuzz/corpus/roundtrip/negate_with_overflow b/lang/rust/fuzz/corpus/roundtrip/negate_with_overflow
new file mode 100644
index 000000000..698f2b56b
Binary files /dev/null and b/lang/rust/fuzz/corpus/roundtrip/negate_with_overflow differ
diff --git a/lang/rust/fuzz/corpus/roundtrip/unchecked_resize b/lang/rust/fuzz/corpus/roundtrip/unchecked_resize
new file mode 100644
index 000000000..82a5f947b
Binary files /dev/null and b/lang/rust/fuzz/corpus/roundtrip/unchecked_resize differ
diff --git a/lang/rust/fuzz/fuzz_targets/roundtrip.rs b/lang/rust/fuzz/fuzz_targets/roundtrip.rs
new file mode 100644
index 000000000..14bc59c8c
--- /dev/null
+++ b/lang/rust/fuzz/fuzz_targets/roundtrip.rs
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+enum PlainEnum {
+ A,
+ B,
+ C,
+ D,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+enum Enum {
+ A(u8),
+ B(()),
+ C(Vec<PlainEnum>),
+ D(i128),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+enum FloatEnum {
+ A(Enum),
+ E(Option<f32>),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+struct Struct {
+ _a: (),
+ _b: u8,
+ _c: Vec<Enum>,
+ _d: (u128, i8, (), PlainEnum, String),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+struct FloatStruct {
+ _a: Struct,
+ _b: f64,
+}
+
+macro_rules! round_trip {
+ ($ty:ty, $data:ident, $equality:expr) => {{
+ #[cfg(feature = "debug")]
+ println!("roundtripping {}", stringify!($ty));
+
+ use ::apache_avro::{from_value, Reader};
+
+ let reader = match Reader::new($data) {
+ Ok(r) => r,
+ _ => return,
+ };
+
+ for value in reader {
+ let value = match value {
+ Ok(v) => v,
+ _ => continue,
+ };
+ #[cfg(feature = "debug")]
+ println!("value {:?}", &value);
+
+ let _: Result<$ty, _> = from_value(&value);
+ }
+ }};
+}
+
+macro_rules! from_bytes {
+ ($ty:ty, $data:ident, $equality:expr) => {{
+ round_trip!($ty, $data, $equality);
+ round_trip!(Vec<$ty>, $data, $equality);
+ round_trip!(Option<$ty>, $data, $equality);
+ }};
+}
+
+fuzz_target!(|data: &[u8]| {
+ // limit avro memory usage
+ apache_avro::max_allocation_bytes(2 * 1024 * 1024); // 2 MB
+
+ from_bytes!(bool, data, true);
+ from_bytes!(i8, data, true);
+ from_bytes!(i16, data, true);
+ from_bytes!(i32, data, true);
+ from_bytes!(i64, data, true);
+ from_bytes!(i128, data, true);
+ from_bytes!(u8, data, true);
+ from_bytes!(u16, data, true);
+ from_bytes!(u32, data, true);
+ from_bytes!(u64, data, true);
+ from_bytes!(u128, data, true);
+ from_bytes!(f32, data, false);
+ from_bytes!(f64, data, false);
+ from_bytes!(char, data, true);
+ from_bytes!(&str, data, true);
+ from_bytes!((), data, true);
+ from_bytes!(PlainEnum, data, true);
+ from_bytes!(Enum, data, true);
+ from_bytes!(FloatEnum, data, false);
+ from_bytes!(Struct, data, true);
+ from_bytes!(FloatStruct, data, false);
+});
diff --git a/lang/rust/wasm-demo/Cargo.toml b/lang/rust/wasm-demo/Cargo.toml
index 9fa891300..ca4e0b554 100644
--- a/lang/rust/wasm-demo/Cargo.toml
+++ b/lang/rust/wasm-demo/Cargo.toml
@@ -27,6 +27,7 @@ edition = "2018"
keywords = ["avro", "data", "serialization", "wasm", "web assembly"]
categories = ["encoding"]
documentation = "https://docs.rs/apache-avro"
+publish = false
[lib]