You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/02/21 14:15:54 UTC
[avro] branch branch-1.11 updated: Use BufReader/BufWriter in the examples
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch branch-1.11
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/branch-1.11 by this push:
new d69b8fc Use BufReader/BufWriter in the examples
d69b8fc is described below
commit d69b8fcc13beb861773eb5a94ce1d46bb34f5530
Author: Martin Tzvetanov Grigorov <mg...@apache.org>
AuthorDate: Mon Feb 21 16:11:24 2022 +0200
Use BufReader/BufWriter in the examples
Buffering readers/writers is considered a best practice.
It is better to use them in the example apps, so users do it too in
their apps.
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
(cherry picked from commit 52b6987b7281587bab6f7a307333b631ed9fd39c)
---
lang/rust/examples/benchmark.rs | 58 +++++++++++++++++++----------
lang/rust/examples/generate_interop_data.rs | 8 ++--
lang/rust/examples/test_interop_data.rs | 9 +++--
3 files changed, 48 insertions(+), 27 deletions(-)
diff --git a/lang/rust/examples/benchmark.rs b/lang/rust/examples/benchmark.rs
index 9728ead..7e3a0c4 100644
--- a/lang/rust/examples/benchmark.rs
+++ b/lang/rust/examples/benchmark.rs
@@ -20,7 +20,10 @@ use apache_avro::{
types::{Record, Value},
Reader, Writer,
};
-use std::time::{Duration, Instant};
+use std::{
+ io::{BufReader, BufWriter},
+ time::{Duration, Instant},
+};
fn nanos(duration: Duration) -> u64 {
duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64
@@ -36,7 +39,13 @@ fn duration(nanos: u64) -> Duration {
}
*/
-fn benchmark(schema: &Schema, record: &Value, s: &str, count: usize, runs: usize) {
+fn benchmark(
+ schema: &Schema,
+ record: &Value,
+ big_or_small: &str,
+ count: usize,
+ runs: usize,
+) -> anyhow::Result<()> {
let mut records = Vec::new();
for __ in 0..count {
records.push(record.clone());
@@ -49,13 +58,13 @@ fn benchmark(schema: &Schema, record: &Value, s: &str, count: usize, runs: usize
let records = records.clone();
let start = Instant::now();
- let mut writer = Writer::new(schema, Vec::new());
- writer.extend(records.into_iter()).unwrap();
+ let mut writer = Writer::new(schema, BufWriter::new(Vec::new()));
+ writer.extend(records.into_iter())?;
let duration = Instant::now().duration_since(start);
durations.push(duration);
- bytes = Some(writer.into_inner().unwrap());
+ bytes = Some(writer.into_inner()?.into_inner()?);
}
let total_duration_write = durations.into_iter().fold(0u64, |a, b| a + nanos(b));
@@ -68,7 +77,7 @@ fn benchmark(schema: &Schema, record: &Value, s: &str, count: usize, runs: usize
for _ in 0..runs {
let start = Instant::now();
- let reader = Reader::with_schema(schema, &bytes[..]).unwrap();
+ let reader = Reader::with_schema(schema, BufReader::new(&bytes[..]))?;
let mut read_records = Vec::with_capacity(count);
for record in reader {
@@ -84,12 +93,17 @@ fn benchmark(schema: &Schema, record: &Value, s: &str, count: usize, runs: usize
let total_duration_read = durations.into_iter().fold(0u64, |a, b| a + nanos(b));
// println!("Read: {} {} {:?}", count, runs, seconds(total_duration));
- let (s_w, s_r) = (seconds(total_duration_write), seconds(total_duration_read));
-
- println!("{},{},{},{},{}", count, runs, s, s_w, s_r);
+ let (total_write_secs, total_read_secs) =
+ (seconds(total_duration_write), seconds(total_duration_read));
+
+ println!(
+ "{},{},{},{},{}",
+ count, runs, big_or_small, total_write_secs, total_read_secs
+ );
+ Ok(())
}
-fn main() {
+fn main() -> anyhow::Result<()> {
let raw_small_schema = r#"
{"namespace": "test", "type": "record", "name": "Test", "fields": [{"type": {"type": "string"}, "name": "field"}]}
"#;
@@ -98,8 +112,8 @@ fn main() {
{"namespace": "my.example", "type": "record", "name": "userInfo", "fields": [{"default": "NONE", "type": "string", "name": "username"}, {"default": -1, "type": "int", "name": "age"}, {"default": "NONE", "type": "string", "name": "phone"}, {"default": "NONE", "type": "string", "name": "housenum"}, {"default": {}, "type": {"fields": [{"default": "NONE", "type": "string", "name": "street"}, {"default": "NONE", "type": "string", "name": "city"}, {"default": "NONE", "type": "string", [...]
"#;
- let small_schema = Schema::parse_str(raw_small_schema).unwrap();
- let big_schema = Schema::parse_str(raw_big_schema).unwrap();
+ let small_schema = Schema::parse_str(raw_small_schema)?;
+ let big_schema = Schema::parse_str(raw_big_schema)?;
println!("{:?}", small_schema);
println!("{:?}", big_schema);
@@ -125,14 +139,18 @@ fn main() {
big_record.put("address", address);
let big_record = big_record.into();
- benchmark(&small_schema, &small_record, "S", 10_000, 1);
- benchmark(&big_schema, &big_record, "B", 10_000, 1);
+ println!();
+
+ benchmark(&small_schema, &small_record, "Small", 10_000, 1)?;
+ benchmark(&big_schema, &big_record, "Big", 10_000, 1)?;
+
+ benchmark(&small_schema, &small_record, "Small", 1, 100_000)?;
+ benchmark(&small_schema, &small_record, "Small", 100, 1000)?;
+ benchmark(&small_schema, &small_record, "Small", 10_000, 10)?;
- benchmark(&small_schema, &small_record, "S", 1, 100_000);
- benchmark(&small_schema, &small_record, "S", 100, 1000);
- benchmark(&small_schema, &small_record, "S", 10_000, 10);
+ benchmark(&big_schema, &big_record, "Big", 1, 100_000)?;
+ benchmark(&big_schema, &big_record, "Big", 100, 1000)?;
+ benchmark(&big_schema, &big_record, "Big", 10_000, 10)?;
- benchmark(&big_schema, &big_record, "B", 1, 100_000);
- benchmark(&big_schema, &big_record, "B", 100, 1000);
- benchmark(&big_schema, &big_record, "B", 10_000, 10);
+ Ok(())
}
diff --git a/lang/rust/examples/generate_interop_data.rs b/lang/rust/examples/generate_interop_data.rs
index 3bbbb3a..7aa0358 100644
--- a/lang/rust/examples/generate_interop_data.rs
+++ b/lang/rust/examples/generate_interop_data.rs
@@ -20,7 +20,7 @@ use apache_avro::{
types::{Record, Value},
Codec, Writer,
};
-use std::collections::HashMap;
+use std::{collections::HashMap, io::BufWriter};
use strum::IntoEnumIterator;
fn create_datum(schema: &Schema) -> Record {
@@ -77,7 +77,7 @@ fn main() -> anyhow::Result<()> {
let schema = Schema::parse_str(schema_str.as_str())?;
for codec in Codec::iter() {
- let mut writer = Writer::with_codec(&schema, Vec::new(), codec);
+ let mut writer = Writer::with_codec(&schema, BufWriter::new(Vec::new()), codec);
write_user_metadata(&mut writer)?;
let datum = create_datum(&schema);
@@ -93,14 +93,14 @@ fn main() -> anyhow::Result<()> {
std::fs::write(
format!("../../build/interop/data/rust{}.avro", suffix),
- bytes,
+ bytes.into_inner()?,
)?;
}
Ok(())
}
-fn write_user_metadata(writer: &mut Writer<Vec<u8>>) -> anyhow::Result<()> {
+fn write_user_metadata(writer: &mut Writer<BufWriter<Vec<u8>>>) -> anyhow::Result<()> {
writer.add_user_metadata("user_metadata".to_string(), b"someByteArray")?;
Ok(())
diff --git a/lang/rust/examples/test_interop_data.rs b/lang/rust/examples/test_interop_data.rs
index 82d2aa1..08934dc 100644
--- a/lang/rust/examples/test_interop_data.rs
+++ b/lang/rust/examples/test_interop_data.rs
@@ -16,7 +16,7 @@
// under the License.
use apache_avro::Reader;
-use std::{collections::HashMap, ffi::OsStr, fs::File};
+use std::{collections::HashMap, ffi::OsStr, fs::File, io::BufReader};
fn main() -> anyhow::Result<()> {
let mut expected_user_metadata: HashMap<String, Vec<u8>> = HashMap::new();
@@ -38,7 +38,7 @@ fn main() -> anyhow::Result<()> {
if ext == "avro" {
println!("Checking {:?}", &path);
let content = std::fs::File::open(&path)?;
- let reader = Reader::new(&content)?;
+ let reader = Reader::new(BufReader::new(&content))?;
test_user_metadata(&reader, &expected_user_metadata);
@@ -64,7 +64,10 @@ fn main() -> anyhow::Result<()> {
}
}
-fn test_user_metadata(reader: &Reader<&File>, expected_user_metadata: &HashMap<String, Vec<u8>>) {
+fn test_user_metadata(
+ reader: &Reader<BufReader<&File>>,
+ expected_user_metadata: &HashMap<String, Vec<u8>>,
+) {
let user_metadata = reader.user_metadata();
if !user_metadata.is_empty() {
assert_eq!(user_metadata, expected_user_metadata);