You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/06/02 13:25:29 UTC
[avro] branch rust-improve-cpu-and-memory-usage created (now 015184f9d)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a change to branch rust-improve-cpu-and-memory-usage
in repository https://gitbox.apache.org/repos/asf/avro.git
at 015184f9d Implement PartialEq for Schema that does not use canonical parsing form
This branch includes the following new commits:
new 1a5dec97c Print the schemas with pretty-print
new 98ca9adc3 Add a new build.sh task for memory profiling the example apps
new 1ed1d784c Use a block size that is pow2
new 015184f9d Implement PartialEq for Schema that does not use canonical parsing form
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[avro] 04/04: Implement PartialEq for Schema that does not use canonical parsing form
Posted by mg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch rust-improve-cpu-and-memory-usage
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 015184f9debc0f5ec21c0cf8556622a8269664d2
Author: Martin Tzvetanov Grigorov <mg...@apache.org>
AuthorDate: Thu Jun 2 16:24:13 2022 +0300
Implement PartialEq for Schema that does not use canonical parsing form
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
---
lang/rust/avro/src/schema.rs | 87 +++++++++++++++++++++++++++++++++++++++++---
1 file changed, 81 insertions(+), 6 deletions(-)
diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs
index dfb9b146e..d7f21307f 100644
--- a/lang/rust/avro/src/schema.rs
+++ b/lang/rust/avro/src/schema.rs
@@ -149,10 +149,8 @@ pub enum Schema {
TimestampMicros,
/// An amount of time defined by a number of months, days and milliseconds.
Duration,
- // A reference to another schema.
- Ref {
- name: Name,
- },
+ /// A reference to another schema.
+ Ref { name: Name },
}
impl PartialEq for Schema {
@@ -161,7 +159,78 @@ impl PartialEq for Schema {
/// [Parsing Canonical Form]:
/// https://avro.apache.org/docs/1.8.2/spec.html#Parsing+Canonical+Form+for+Schemas
fn eq(&self, other: &Self) -> bool {
- self.canonical_form() == other.canonical_form()
+ let maybe_eq = match (self, other) {
+ (Schema::Null, Schema::Null) => true,
+ (Schema::Boolean, Schema::Boolean) => true,
+ (Schema::Int, Schema::Int) => true,
+ (Schema::Long, Schema::Long) => true,
+ (Schema::Float, Schema::Float) => true,
+ (Schema::Double, Schema::Double) => true,
+ (Schema::Bytes, Schema::Bytes) => true,
+ (Schema::String, Schema::String) => true,
+ (Schema::Array(a), Schema::Array(b)) => a == b,
+ (Schema::Map(a), Schema::Map(b)) => a == b,
+ (Schema::Union(_), Schema::Union(_)) => true,
+ (
+ Schema::Record {
+ name: name_a,
+ fields: fields_a,
+ ..
+ },
+ Schema::Record {
+ name: name_b,
+ fields: fields_b,
+ ..
+ },
+ ) => name_a == name_b && fields_a == fields_b,
+ (
+ Schema::Enum {
+ name: name_a,
+ symbols: symbols_a,
+ ..
+ },
+ Schema::Enum {
+ name: name_b,
+ symbols: symbols_b,
+ ..
+ },
+ ) => name_a == name_b && symbols_a == symbols_b,
+ (
+ Schema::Fixed {
+ name: name_a,
+ size: size_a,
+ ..
+ },
+ Schema::Fixed {
+ name: name_b,
+ size: size_b,
+ ..
+ },
+ ) => name_a == name_b && size_a == size_b,
+ (
+ Schema::Decimal {
+ precision: precision_a,
+ scale: scale_a,
+ ..
+ },
+ Schema::Decimal {
+ precision: precision_b,
+ scale: scale_b,
+ ..
+ },
+ ) => precision_a == precision_b && scale_a == scale_b,
+ (Schema::Uuid, Schema::Uuid) => true,
+ (Schema::Date, Schema::Date) => true,
+ (Schema::TimeMillis, Schema::TimeMillis) => true,
+ (Schema::TimeMicros, Schema::TimeMicros) => true,
+ (Schema::TimestampMillis, Schema::TimestampMillis) => true,
+ (Schema::TimestampMicros, Schema::TimestampMicros) => true,
+ (Schema::Duration, Schema::Duration) => true,
+ (Schema::Ref { .. }, Schema::Ref { .. }) => true,
+ _ => false,
+ };
+
+ maybe_eq/* && self.canonical_form() == other.canonical_form()*/
}
}
@@ -557,7 +626,7 @@ impl ResolvedOwnedSchema {
}
/// Represents a `field` in a `record` Avro schema.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
pub struct RecordField {
/// Name of the field.
pub name: String,
@@ -577,6 +646,12 @@ pub struct RecordField {
pub position: usize,
}
+impl PartialEq for RecordField {
+ fn eq(&self, other: &Self) -> bool {
+ self.name == other.name && self.schema == other.schema
+ }
+}
+
/// Represents any valid order for a `field` in a `record` Avro schema.
#[derive(Clone, Debug, PartialEq, EnumString)]
#[strum(serialize_all = "kebab_case")]
[avro] 03/04: Use a block size that is pow2
Posted by mg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch rust-improve-cpu-and-memory-usage
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 1ed1d784c5c5b0068cf4f1324e58f654765e6572
Author: Martin Tzvetanov Grigorov <mg...@apache.org>
AuthorDate: Thu Jun 2 16:23:39 2022 +0300
Use a block size that is pow2
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
---
lang/rust/avro/src/writer.rs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lang/rust/avro/src/writer.rs b/lang/rust/avro/src/writer.rs
index 18eb833c0..6e370cf41 100644
--- a/lang/rust/avro/src/writer.rs
+++ b/lang/rust/avro/src/writer.rs
@@ -28,7 +28,7 @@ use rand::random;
use serde::Serialize;
use std::{collections::HashMap, convert::TryFrom, io::Write, marker::PhantomData};
-const DEFAULT_BLOCK_SIZE: usize = 16000;
+const DEFAULT_BLOCK_SIZE: usize = 16 * 1024;
const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01";
/// Main interface for writing Avro formatted values.
[avro] 01/04: Print the schemas with pretty-print
Posted by mg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch rust-improve-cpu-and-memory-usage
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 1a5dec97c50a38335f538413472dcb58cd5dc6e5
Author: Martin Tzvetanov Grigorov <mg...@apache.org>
AuthorDate: Thu Jun 2 16:22:37 2022 +0300
Print the schemas with pretty-print
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
---
lang/rust/avro/examples/benchmark.rs | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/lang/rust/avro/examples/benchmark.rs b/lang/rust/avro/examples/benchmark.rs
index 1829d2171..4ba49d068 100644
--- a/lang/rust/avro/examples/benchmark.rs
+++ b/lang/rust/avro/examples/benchmark.rs
@@ -115,8 +115,8 @@ fn main() -> anyhow::Result<()> {
let small_schema = Schema::parse_str(raw_small_schema)?;
let big_schema = Schema::parse_str(raw_big_schema)?;
- println!("{:?}", small_schema);
- println!("{:?}", big_schema);
+ println!("{:#?}", small_schema);
+ println!("{:#?}", big_schema);
let mut small_record = Record::new(&small_schema).unwrap();
small_record.put("field", "foo");
@@ -139,8 +139,7 @@ fn main() -> anyhow::Result<()> {
big_record.put("address", address);
let big_record = big_record.into();
- println!();
- println!("Count\t\tRuns\t\tBig/Small\tTotal write secs\tTotal read secs");
+ println!("\nCount\t\tRuns\t\tBig/Small\tTotal write secs\tTotal read secs");
benchmark(&small_schema, &small_record, "Small", 10_000, 1)?;
benchmark(&big_schema, &big_record, "Big", 10_000, 1)?;
[avro] 02/04: Add a new build.sh task for memory profiling the example apps
Posted by mg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch rust-improve-cpu-and-memory-usage
in repository https://gitbox.apache.org/repos/asf/avro.git
commit 98ca9adc39abf61a3b00bdf086e605da1a87b886
Author: Martin Tzvetanov Grigorov <mg...@apache.org>
AuthorDate: Thu Jun 2 16:22:59 2022 +0300
Add a new build.sh task for memory profiling the example apps
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
---
lang/rust/build.sh | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/lang/rust/build.sh b/lang/rust/build.sh
index e8f9bf69f..3a20e07ac 100755
--- a/lang/rust/build.sh
+++ b/lang/rust/build.sh
@@ -68,8 +68,19 @@ do
echo -e "\nRunning single object encoding interop data tests"
cargo run --example test_interop_single_object_encoding
;;
+ profile-heap)
+ echo "Heap memory profiling using KDE heaptrack tool"
+ EXAMPLE_APP=$2
+ EXAMPLE_APP=${EXAMPLE_APP:-"benchmark"}
+ echo $EXAMPLE_APP
+ RUSTFLAGS=-g cargo build --release --example $EXAMPLE_APP
+ rm -rf heaptrack.${EXAMPLE_APP}.*
+ heaptrack ./target/release/examples/$EXAMPLE_APP
+ heaptrack --analyze heaptrack.${EXAMPLE_APP}.*
+ exit
+ ;;
*)
- echo "Usage: $0 {lint|test|dist|clean|interop-data-generate|interop-data-test}" >&2
+ echo "Usage: $0 {lint|test|dist|clean|interop-data-generate|interop-data-test|profile-heap}" >&2
exit 1
esac
done