You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by kr...@apache.org on 2026/03/04 16:05:49 UTC
(avro-rs) branch feat/enums updated (84a36be -> c1adfee)
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a change to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
discard 84a36be wip: Full enum support
new 127aa7e chore: Make `writer` module a directory module (#497)
new f0bef72 chore: Split `writer` module into several submodules (#498)
new 5e69691 feat: Replace `to_datum*` functions with `GenericDatumWriter`
new 1dd524a fix: Improve documentation and fix resolving
new 3d3b1e9 fix: `Value::validate_schemata` should not set `enclosing_namespace`
new b3b93fa wip: Full enum support
new 939e171 all tests work again
new c1adfee Add test suite for deser
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (84a36be)
\
N -- N -- N refs/heads/feat/enums (c1adfee)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
avro/benches/single.rs | 67 +-
avro/src/duration.rs | 1 +
avro/src/error.rs | 2 +-
avro/src/lib.rs | 15 +-
avro/src/reader/datum.rs | 43 +-
avro/src/reader/single_object.rs | 6 +-
avro/src/schema/mod.rs | 13 +-
avro/src/serde/de.rs | 15 +-
avro/src/serde/deser_schema/array.rs | 16 +-
avro/src/serde/deser_schema/enums/mod.rs | 2 +
avro/src/serde/deser_schema/enums/plain.rs | 278 +----
avro/src/serde/deser_schema/enums/union.rs | 142 +++
avro/src/serde/deser_schema/identifier.rs | 255 +++++
avro/src/serde/deser_schema/map.rs | 16 +-
avro/src/serde/deser_schema/mod.rs | 1298 ++++++++++++++++++++---
avro/src/serde/deser_schema/record.rs | 275 +----
avro/src/serde/deser_schema/tuple.rs | 31 +-
avro/src/serde/deser_schema/union.rs | 450 --------
avro/src/serde/mod.rs | 4 +
avro/src/serde/ser_schema/mod.rs | 2 +-
avro/src/serde/with.rs | 7 +-
avro/src/types.rs | 10 +-
avro/src/writer/datum.rs | 503 +++++++++
avro/src/{writer.rs => writer/mod.rs} | 748 +------------
avro/src/writer/single_object.rs | 451 ++++++++
avro/tests/avro-3786.rs | 28 +-
avro/tests/avro-3787.rs | 12 +-
avro/tests/avro-rs-285-bytes_deserialization.rs | 4 +-
avro/tests/io.rs | 48 +-
avro/tests/schema.rs | 30 +-
avro/tests/to_from_avro_datum_schemata.rs | 17 +-
avro/tests/union_schema.rs | 4 +-
avro_derive/src/attributes/serde.rs | 4 +
avro_derive/tests/serde.rs | 6 +-
34 files changed, 2802 insertions(+), 2001 deletions(-)
create mode 100644 avro/src/serde/deser_schema/enums/union.rs
create mode 100644 avro/src/serde/deser_schema/identifier.rs
delete mode 100644 avro/src/serde/deser_schema/union.rs
create mode 100644 avro/src/writer/datum.rs
rename avro/src/{writer.rs => writer/mod.rs} (64%)
create mode 100644 avro/src/writer/single_object.rs
(avro-rs) 07/08: all tests work again
Posted by kr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a commit to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
commit 939e1712c62d79a9b5b42c45076b35b4a669da53
Author: default <ad...@kriskras99.nl>
AuthorDate: Wed Mar 4 13:44:27 2026 +0000
all tests work again
---
avro/src/duration.rs | 1 +
avro/src/error.rs | 2 +-
avro/src/lib.rs | 4 +
avro/src/reader/datum.rs | 41 ++-
avro/src/reader/single_object.rs | 6 +-
avro/src/schema/mod.rs | 1 +
avro/src/serde/de.rs | 1 +
avro/src/serde/deser_schema/array.rs | 16 +-
avro/src/serde/deser_schema/enums/mod.rs | 2 +
avro/src/serde/deser_schema/enums/plain.rs | 278 +--------------
avro/src/serde/deser_schema/enums/union.rs | 142 ++++++++
avro/src/serde/deser_schema/identifier.rs | 255 ++++++++++++++
avro/src/serde/deser_schema/map.rs | 16 +-
avro/src/serde/deser_schema/mod.rs | 346 ++++++++++--------
avro/src/serde/deser_schema/record.rs | 275 +--------------
avro/src/serde/deser_schema/tuple.rs | 31 +-
avro/src/serde/deser_schema/union.rs | 450 ------------------------
avro/src/serde/mod.rs | 4 +
avro/src/serde/ser_schema/mod.rs | 2 +-
avro/src/serde/with.rs | 7 +-
avro/tests/avro-rs-285-bytes_deserialization.rs | 4 +-
avro/tests/schema.rs | 8 +-
avro/tests/union_schema.rs | 4 +-
avro_derive/src/attributes/serde.rs | 4 +
avro_derive/tests/serde.rs | 6 +-
25 files changed, 708 insertions(+), 1198 deletions(-)
diff --git a/avro/src/duration.rs b/avro/src/duration.rs
index eecfca1..564ee72 100644
--- a/avro/src/duration.rs
+++ b/avro/src/duration.rs
@@ -209,6 +209,7 @@ mod tests {
use crate::types::Value;
use apache_avro_test_helper::TestResult;
+ #[expect(deprecated, reason = "This tests the deprecated function")]
#[test]
fn avro_rs_382_duration_from_value() -> TestResult {
let val = Value::Duration(Duration::new(Months::new(7), Days::new(4), Millis::new(45)));
diff --git a/avro/src/error.rs b/avro/src/error.rs
index 23db16b..4482280 100644
--- a/avro/src/error.rs
+++ b/avro/src/error.rs
@@ -586,7 +586,7 @@ pub enum Details {
},
#[error("Only expected `deserialize_identifier` to be called but `{0}` was called")]
- DeserializeKey(String),
+ DeserializeIdentifier(&'static str),
#[error("Failed to write buffer bytes during flush: {0}")]
WriteBytes(#[source] std::io::Error),
diff --git a/avro/src/lib.rs b/avro/src/lib.rs
index 314d596..e91464b 100644
--- a/avro/src/lib.rs
+++ b/avro/src/lib.rs
@@ -101,6 +101,10 @@ pub use reader::{
single_object::{GenericSingleObjectReader, SpecificSingleObjectReader},
};
pub use schema::Schema;
+#[expect(
+ deprecated,
+ reason = "Still need to export it until we remove it completely"
+)]
pub use serde::{AvroSchema, AvroSchemaComponent, from_value, to_value};
pub use uuid::Uuid;
#[expect(
diff --git a/avro/src/reader/datum.rs b/avro/src/reader/datum.rs
index cbf6fd7..f72b272 100644
--- a/avro/src/reader/datum.rs
+++ b/avro/src/reader/datum.rs
@@ -37,6 +37,7 @@ pub struct GenericDatumReader<'s> {
writer: &'s Schema,
resolved: ResolvedSchema<'s>,
reader: Option<(&'s Schema, ResolvedSchema<'s>)>,
+ human_readable: bool,
}
#[bon]
@@ -57,6 +58,7 @@ impl<'s> GenericDatumReader<'s> {
reader_schema: Option<&'s Schema>,
/// Already resolved schemata that will be used to resolve references in the reader's schema.
resolved_reader_schemata: Option<ResolvedSchema<'s>>,
+ #[builder(default)] human_readable: bool,
) -> AvroResult<Self> {
let resolved_writer_schemata = if let Some(resolved) = resolved_writer_schemata {
resolved
@@ -78,6 +80,7 @@ impl<'s> GenericDatumReader<'s> {
writer: writer_schema,
resolved: resolved_writer_schemata,
reader,
+ human_readable,
})
}
}
@@ -131,6 +134,17 @@ impl<'s> GenericDatumReader<'s> {
Ok(value)
}
}
+
+ pub fn read_deser<R: Read, T: DeserializeOwned>(&self, reader: &mut R) -> AvroResult<T> {
+ T::deserialize(SchemaAwareDeserializer::new(
+ reader,
+ self.writer,
+ Config {
+ names: self.resolved.get_names(),
+ human_readable: self.human_readable,
+ },
+ )?)
+ }
}
pub struct SpecificDatumReader<T: DeserializeOwned> {
@@ -253,7 +267,7 @@ mod tests {
use serde::Deserialize;
use crate::{
- Schema, from_value,
+ Schema,
reader::datum::GenericDatumReader,
types::{Record, Value},
};
@@ -298,7 +312,7 @@ mod tests {
const TEST_RECORD_SCHEMA_3240: &str = r#"
{
"type": "record",
- "name": "test",
+ "name": "TestRecord3240",
"fields": [
{
"name": "a",
@@ -340,24 +354,15 @@ mod tests {
let schema = Schema::parse_str(TEST_RECORD_SCHEMA_3240)?;
let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111];
- let expected_record: TestRecord3240 = TestRecord3240 {
- a: 27i64,
- b: String::from("foo"),
- a_nullable_array: None,
- a_nullable_string: None,
- };
-
- let avro_datum = GenericDatumReader::builder(&schema)
+ let error = GenericDatumReader::builder(&schema)
.build()?
- .read_value(&mut encoded)?;
- let parsed_record: TestRecord3240 = match &avro_datum {
- Value::Record(_) => from_value::<TestRecord3240>(&avro_datum)?,
- unexpected => {
- panic!("could not map avro data to struct, found unexpected: {unexpected:?}")
- }
- };
+ .read_deser::<_, TestRecord3240>(&mut encoded)
+ .unwrap_err();
- assert_eq!(parsed_record, expected_record);
+ assert_eq!(
+ error.to_string(),
+ "Failed to read bytes for decoding variable length integer: failed to fill whole buffer"
+ );
Ok(())
}
diff --git a/avro/src/reader/single_object.rs b/avro/src/reader/single_object.rs
index 6e0b744..2f1ef2b 100644
--- a/avro/src/reader/single_object.rs
+++ b/avro/src/reader/single_object.rs
@@ -21,7 +21,7 @@ use crate::headers::{HeaderBuilder, RabinFingerprintHeader};
use crate::schema::ResolvedOwnedSchema;
use crate::serde::deser_schema::{Config, SchemaAwareDeserializer};
use crate::types::Value;
-use crate::{AvroResult, AvroSchema, Schema, from_value};
+use crate::{AvroResult, AvroSchema, Schema};
use serde::de::DeserializeOwned;
use std::io::Read;
use std::marker::PhantomData;
@@ -129,7 +129,7 @@ where
T: AvroSchema + DeserializeOwned,
{
pub fn read<R: Read>(&self, reader: &mut R) -> AvroResult<T> {
- from_value::<T>(&self.inner.read_value(reader)?)
+ self.inner.read_deser(reader)
}
}
@@ -156,7 +156,7 @@ mod tests {
let schema = r#"
{
"type":"record",
- "name":"TestSingleObjectWrtierSerialize",
+ "name":"TestSingleObjectReader",
"fields":[
{
"name":"a",
diff --git a/avro/src/schema/mod.rs b/avro/src/schema/mod.rs
index 05908f3..7e17e60 100644
--- a/avro/src/schema/mod.rs
+++ b/avro/src/schema/mod.rs
@@ -3476,6 +3476,7 @@ mod tests {
Ok(())
}
+ #[expect(deprecated, reason = "Schema resolution is a WIP")]
#[test]
fn test_avro_3814_schema_resolution_failure() -> TestResult {
// Define a reader schema: a nested record with an optional field.
diff --git a/avro/src/serde/de.rs b/avro/src/serde/de.rs
index b05e065..aefe7b9 100644
--- a/avro/src/serde/de.rs
+++ b/avro/src/serde/de.rs
@@ -944,6 +944,7 @@ pub fn from_value<'de, D: Deserialize<'de>>(value: &'de Value) -> Result<D, Erro
D::deserialize(de)
}
+#[expect(deprecated, reason = "This tests the deprecated function")]
#[cfg(test)]
mod tests {
use num_bigint::BigInt;
diff --git a/avro/src/serde/deser_schema/array.rs b/avro/src/serde/deser_schema/array.rs
index 08cb055..615f0c4 100644
--- a/avro/src/serde/deser_schema/array.rs
+++ b/avro/src/serde/deser_schema/array.rs
@@ -21,7 +21,6 @@ use std::io::Read;
use serde::de::SeqAccess;
use super::Config;
-use crate::serde::deser_schema::union::UnionDeserializer;
use crate::{
Error, Schema, schema::ArraySchema, serde::deser_schema::SchemaAwareDeserializer, util::zag_i32,
};
@@ -71,16 +70,11 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> SeqAccess<'de> for ArrayDeserializ
self.next_element_seed(seed)
}
State::ReadingValue(mut remaining) => {
- let v = match self.schema.items.as_ref() {
- Schema::Union(union) => {
- seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
- }
- schema => seed.deserialize(SchemaAwareDeserializer::new(
- self.reader,
- schema,
- self.config,
- )?)?,
- };
+ let v = seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ &self.schema.items,
+ self.config,
+ )?)?;
remaining -= 1;
if remaining == 0 {
diff --git a/avro/src/serde/deser_schema/enums/mod.rs b/avro/src/serde/deser_schema/enums/mod.rs
index 65a8e66..7803cad 100644
--- a/avro/src/serde/deser_schema/enums/mod.rs
+++ b/avro/src/serde/deser_schema/enums/mod.rs
@@ -1,3 +1,5 @@
mod plain;
+mod union;
pub use plain::PlainEnumAccess;
+pub use union::UnionEnumAccess;
diff --git a/avro/src/serde/deser_schema/enums/plain.rs b/avro/src/serde/deser_schema/enums/plain.rs
index 65b5b81..f419de3 100644
--- a/avro/src/serde/deser_schema/enums/plain.rs
+++ b/avro/src/serde/deser_schema/enums/plain.rs
@@ -1,8 +1,8 @@
+use crate::Error;
use crate::error::Details;
use crate::schema::EnumSchema;
+use crate::serde::deser_schema::identifier::IdentifierDeserializer;
use crate::util::zag_i32;
-use crate::{Error, Schema};
-use serde::Deserializer;
use serde::de::{DeserializeSeed, EnumAccess, Unexpected, VariantAccess, Visitor};
use std::io::Read;
@@ -25,11 +25,21 @@ impl<'de, 's, 'r, R: Read> EnumAccess<'de> for PlainEnumAccess<'s, 'r, R> {
where
V: DeserializeSeed<'de>,
{
- let deserializer = EnumIdentifierDeserializer {
- reader: self.reader,
- schema: self.schema,
- };
- Ok((seed.deserialize(deserializer)?, self))
+ let orig_index = zag_i32(self.reader)?;
+ let index =
+ usize::try_from(orig_index).map_err(|e| Details::ConvertI32ToUsize(e, orig_index))?;
+ let symbol = self
+ .schema
+ .symbols
+ .get(index)
+ .ok_or(Details::EnumSymbolIndex {
+ index: index as usize,
+ num_variants: self.schema.symbols.len(),
+ })?;
+ Ok((
+ seed.deserialize(IdentifierDeserializer::string(symbol))?,
+ self,
+ ))
}
}
@@ -68,257 +78,3 @@ impl<'de, 's, 'r, R: Read> VariantAccess<'de> for PlainEnumAccess<'s, 'r, R> {
Err(serde::de::Error::invalid_type(unexp, &"newtype variant"))
}
}
-
-struct EnumIdentifierDeserializer<'s, 'r, R: Read> {
- schema: &'s EnumSchema,
- reader: &'r mut R,
-}
-
-impl<'s, 'r, R: Read> EnumIdentifierDeserializer<'s, 'r, R> {
- fn error(&self, error: impl Into<String>) -> Error {
- Error::new(Details::DeserializeValueWithSchema {
- value_type: "enum",
- value: error.into(),
- schema: Schema::Enum(self.schema.clone()),
- })
- }
-}
-
-impl<'de, 's, 'r, R: Read> Deserializer<'de> for EnumIdentifierDeserializer<'s, 'r, R> {
- type Error = Error;
-
- fn deserialize_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_any, expected deserialize_identifier"))
- }
-
- fn deserialize_bool<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_bool, expected deserialize_identifier"))
- }
-
- fn deserialize_i8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_i8, expected deserialize_identifier"))
- }
-
- fn deserialize_i16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_i16, expected deserialize_identifier"))
- }
-
- fn deserialize_i32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_i32, expected deserialize_identifier"))
- }
-
- fn deserialize_i64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_i64, expected deserialize_identifier"))
- }
-
- fn deserialize_u8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_u8, expected deserialize_identifier"))
- }
-
- fn deserialize_u16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_u16, expected deserialize_identifier"))
- }
-
- fn deserialize_u32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_u32, expected deserialize_identifier"))
- }
-
- fn deserialize_u64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_u64, expected deserialize_identifier"))
- }
-
- fn deserialize_f32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_f32, expected deserialize_identifier"))
- }
-
- fn deserialize_f64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_f64, expected deserialize_identifier"))
- }
-
- fn deserialize_char<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_char, expected deserialize_identifier"))
- }
-
- fn deserialize_str<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_str, expected deserialize_identifier"))
- }
-
- fn deserialize_string<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_string, expected deserialize_identifier"))
- }
-
- fn deserialize_bytes<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_bytes, expected deserialize_identifier"))
- }
-
- fn deserialize_byte_buf<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_byte_buf, expected deserialize_identifier"))
- }
-
- fn deserialize_option<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_option, expected deserialize_identifier"))
- }
-
- fn deserialize_unit<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_unit, expected deserialize_identifier"))
- }
-
- fn deserialize_unit_struct<V>(
- self,
- _name: &'static str,
- _visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_unit_struct, expected deserialize_identifier"))
- }
-
- fn deserialize_newtype_struct<V>(
- self,
- _name: &'static str,
- _visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_newtype_struct, expected deserialize_identifier"))
- }
-
- fn deserialize_seq<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_seq, expected deserialize_identifier"))
- }
-
- fn deserialize_tuple<V>(self, _len: usize, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_tuple, expected deserialize_identifier"))
- }
-
- fn deserialize_tuple_struct<V>(
- self,
- _name: &'static str,
- _len: usize,
- _visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_tuple_struct, expected deserialize_identifier"))
- }
-
- fn deserialize_map<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_map, expected deserialize_identifier"))
- }
-
- fn deserialize_struct<V>(
- self,
- _name: &'static str,
- _fields: &'static [&'static str],
- _visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_struct, expected deserialize_identifier"))
- }
-
- fn deserialize_enum<V>(
- self,
- _name: &'static str,
- _variants: &'static [&'static str],
- _visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- Err(self.error("Unexpected deserialize_enum, expected deserialize_identifier"))
- }
-
- fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- let index = zag_i32(self.reader)?;
- let symbol = self
- .schema
- .symbols
- .get(index as usize)
- .ok_or(Details::EnumSymbolIndex {
- index: index as usize,
- num_variants: self.schema.symbols.len(),
- })?;
- visitor.visit_str(symbol)
- }
-
- fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- self.deserialize_any(visitor)
- }
-}
diff --git a/avro/src/serde/deser_schema/enums/union.rs b/avro/src/serde/deser_schema/enums/union.rs
new file mode 100644
index 0000000..be09d2a
--- /dev/null
+++ b/avro/src/serde/deser_schema/enums/union.rs
@@ -0,0 +1,142 @@
+use std::{borrow::Borrow, io::Read};
+
+use serde::{
+ Deserializer,
+ de::{EnumAccess, Unexpected, VariantAccess},
+};
+
+use crate::{
+ Error, Schema,
+ error::Details,
+ schema::UnionSchema,
+ serde::deser_schema::{
+ Config, DESERIALIZE_ANY, SchemaAwareDeserializer, identifier::IdentifierDeserializer,
+ },
+ util::zag_i32,
+};
+
+pub struct UnionEnumAccess<'s, 'r, R: Read, S: Borrow<Schema>> {
+ schema: &'s UnionSchema,
+ reader: &'r mut R,
+ config: Config<'s, S>,
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> UnionEnumAccess<'s, 'r, R, S> {
+ pub fn new(schema: &'s UnionSchema, reader: &'r mut R, config: Config<'s, S>) -> Self {
+ Self {
+ schema,
+ reader,
+ config,
+ }
+ }
+}
+
+impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> EnumAccess<'de> for UnionEnumAccess<'s, 'r, R, S> {
+ type Error = Error;
+
+ type Variant = UnionVariantAccess<'s, 'r, R, S>;
+
+ fn variant_seed<V>(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error>
+ where
+ V: serde::de::DeserializeSeed<'de>,
+ {
+ let orig_index = zag_i32(self.reader)?;
+ let index =
+ usize::try_from(orig_index).map_err(|e| Details::ConvertI32ToUsize(e, orig_index))?;
+
+ let schema = self
+ .schema
+ .variants()
+ .get(index)
+ .ok_or(Details::GetUnionVariant {
+ index: orig_index as i64,
+ num_variants: self.schema.variants().len(),
+ })?;
+
+ Ok((
+ seed.deserialize(IdentifierDeserializer::index(index as u32))?,
+ UnionVariantAccess::new(schema, self.reader, self.config),
+ ))
+ }
+}
+
+pub struct UnionVariantAccess<'s, 'r, R: Read, S: Borrow<Schema>> {
+ schema: &'s Schema,
+ reader: &'r mut R,
+ config: Config<'s, S>,
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> UnionVariantAccess<'s, 'r, R, S> {
+ fn new(schema: &'s Schema, reader: &'r mut R, config: Config<'s, S>) -> Self {
+ Self {
+ schema,
+ reader,
+ config,
+ }
+ }
+}
+
+impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> VariantAccess<'de>
+ for UnionVariantAccess<'s, 'r, R, S>
+{
+ type Error = Error;
+
+ fn unit_variant(self) -> Result<(), Self::Error> {
+ if let Schema::Null = self.schema {
+ Ok(())
+ } else if let Schema::Record(record) = self.schema
+ && record.fields.is_empty()
+ {
+ Ok(())
+ } else {
+ let unexp = Unexpected::UnitVariant;
+ Err(serde::de::Error::invalid_type(unexp, &"other variant"))
+ }
+ }
+
+ fn newtype_variant_seed<T>(self, seed: T) -> Result<T::Value, Self::Error>
+ where
+ T: serde::de::DeserializeSeed<'de>,
+ {
+ if let Schema::Record(record) = self.schema
+ && record.fields.len() == 1
+ && record.fields[0].name == "field_0"
+ {
+ // Most likely a Union of Records
+ seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ &record.fields[0].schema,
+ self.config,
+ )?)
+ } else {
+ seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ self.schema,
+ self.config,
+ )?)
+ }
+ }
+
+ fn tuple_variant<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ SchemaAwareDeserializer::new(self.reader, self.schema, self.config)?
+ .deserialize_tuple(len, visitor)
+ }
+
+ fn struct_variant<V>(
+ self,
+ fields: &'static [&'static str],
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ SchemaAwareDeserializer::new(self.reader, self.schema, self.config)?.deserialize_struct(
+ DESERIALIZE_ANY,
+ fields,
+ visitor,
+ )
+ }
+}
diff --git a/avro/src/serde/deser_schema/identifier.rs b/avro/src/serde/deser_schema/identifier.rs
new file mode 100644
index 0000000..1082a18
--- /dev/null
+++ b/avro/src/serde/deser_schema/identifier.rs
@@ -0,0 +1,255 @@
+use crate::{Error, error::Details};
+use serde::{Deserializer, de::Visitor};
+
+/// Deserializer that only accepts `deserialize_identifier` calls.
+pub enum IdentifierDeserializer<'s> {
+ String(&'s str),
+ Index(u32),
+}
+
+impl<'s> IdentifierDeserializer<'s> {
+ pub fn string(name: &'s str) -> Self {
+ Self::String(name)
+ }
+
+ pub fn index(index: u32) -> Self {
+ Self::Index(index)
+ }
+
+ fn error(&self, error: &'static str) -> Error {
+ Error::new(Details::DeserializeIdentifier(error))
+ }
+}
+
+impl<'de, 's> Deserializer<'de> for IdentifierDeserializer<'s> {
+ type Error = Error;
+
+ fn deserialize_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_any"))
+ }
+
+ fn deserialize_bool<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_bool"))
+ }
+
+ fn deserialize_i8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_i8"))
+ }
+
+ fn deserialize_i16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_i16"))
+ }
+
+ fn deserialize_i32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_i32"))
+ }
+
+ fn deserialize_i64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_i64"))
+ }
+
+ fn deserialize_u8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_u8"))
+ }
+
+ fn deserialize_u16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_u16"))
+ }
+
+ fn deserialize_u32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_u32"))
+ }
+
+ fn deserialize_u64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_u64"))
+ }
+
+ fn deserialize_f32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_f32"))
+ }
+
+ fn deserialize_f64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_f64"))
+ }
+
+ fn deserialize_char<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_char"))
+ }
+
+ fn deserialize_str<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_str"))
+ }
+
+ fn deserialize_string<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_string"))
+ }
+
+ fn deserialize_bytes<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_bytes"))
+ }
+
+ fn deserialize_byte_buf<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_byte_buf"))
+ }
+
+ fn deserialize_option<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_option"))
+ }
+
+ fn deserialize_unit<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_unit"))
+ }
+
+ fn deserialize_unit_struct<V>(
+ self,
+ _name: &'static str,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_unit_struct"))
+ }
+
+ fn deserialize_newtype_struct<V>(
+ self,
+ _name: &'static str,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_newtype_struct"))
+ }
+
+ fn deserialize_seq<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_seq"))
+ }
+
+ fn deserialize_tuple<V>(self, _len: usize, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_tuple"))
+ }
+
+ fn deserialize_tuple_struct<V>(
+ self,
+ _name: &'static str,
+ _len: usize,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_tuple_struct"))
+ }
+
+ fn deserialize_map<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_map"))
+ }
+
+ fn deserialize_struct<V>(
+ self,
+ _name: &'static str,
+ _fields: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_struct"))
+ }
+
+ fn deserialize_enum<V>(
+ self,
+ _name: &'static str,
+ _variants: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_enum"))
+ }
+
+ fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ match self {
+ IdentifierDeserializer::String(string) => visitor.visit_str(string),
+ IdentifierDeserializer::Index(index) => visitor.visit_u32(index),
+ }
+ }
+
+ fn deserialize_ignored_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("deserialize_ignored_any"))
+ }
+}
diff --git a/avro/src/serde/deser_schema/map.rs b/avro/src/serde/deser_schema/map.rs
index 53b7ed5..fe12a7f 100644
--- a/avro/src/serde/deser_schema/map.rs
+++ b/avro/src/serde/deser_schema/map.rs
@@ -21,7 +21,6 @@ use std::io::Read;
use serde::de::MapAccess;
use super::Config;
-use crate::serde::deser_schema::union::UnionDeserializer;
use crate::{
Error, Schema, schema::MapSchema, serde::deser_schema::SchemaAwareDeserializer, util::zag_i32,
};
@@ -96,16 +95,11 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> MapAccess<'de> for MapDeserializer
let State::ReadingValue(mut remaining) = self.state else {
panic!("`next_key_seed` and `next_value_seed` where called in the wrong error")
};
- let v = match self.schema.types.as_ref() {
- Schema::Union(union) => {
- seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
- }
- schema => seed.deserialize(SchemaAwareDeserializer::new(
- self.reader,
- schema,
- self.config,
- )?)?,
- };
+ let v = seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ &self.schema.types,
+ self.config,
+ )?)?;
remaining -= 1;
if remaining == 0 {
diff --git a/avro/src/serde/deser_schema/mod.rs b/avro/src/serde/deser_schema/mod.rs
index 0e07518..fc6c560 100644
--- a/avro/src/serde/deser_schema/mod.rs
+++ b/avro/src/serde/deser_schema/mod.rs
@@ -17,10 +17,10 @@
mod array;
mod enums;
+mod identifier;
mod map;
mod record;
mod tuple;
-mod union;
use std::borrow::Borrow;
use std::collections::HashMap;
@@ -29,12 +29,11 @@ use std::io::Read;
use serde::Deserializer;
-use crate::schema::Name;
-use crate::serde::deser_schema::enums::PlainEnumAccess;
+use crate::schema::{Name, UnionSchema};
+use crate::serde::deser_schema::enums::{PlainEnumAccess, UnionEnumAccess};
use crate::serde::deser_schema::tuple::{
ManyTupleDeserializer, OneTupleDeserializer, UnitTupleDeserializer,
};
-use crate::serde::deser_schema::union::UnionDeserializer;
use crate::{
Error, Schema,
decode::decode_len,
@@ -56,13 +55,12 @@ pub struct Config<'s, S: Borrow<Schema>> {
pub human_readable: bool,
}
+// This needs to be implemented manually as the derive puts a bound on `S`
+// which is not necessary as a reference is always Copy.
impl<'s, S: Borrow<Schema>> Copy for Config<'s, S> {}
impl<'s, S: Borrow<Schema>> Clone for Config<'s, S> {
fn clone(&self) -> Self {
- Self {
- names: self.names,
- human_readable: self.human_readable,
- }
+ *self
}
}
@@ -127,6 +125,22 @@ impl<'s, 'r, R: Read, S: Borrow<Schema>> SchemaAwareDeserializer<'s, 'r, R, S> {
Ok(self)
}
+ /// Read the union and create a new deserializer with the existing reader and config.
+ ///
+ /// This will resolve the read schema if it is a reference.
+ fn with_union(self, schema: &'s UnionSchema) -> Result<Self, Error> {
+ let index = zag_i32(self.reader)?;
+ let variant =
+ schema
+ .variants()
+ .get(index as usize)
+ .ok_or_else(|| Details::GetUnionVariant {
+ index: index as i64,
+ num_variants: schema.variants().len(),
+ })?;
+ self.with_different_schema(variant)
+ }
+
fn read_int(&mut self, original_ty: &'static str) -> Result<i32, Error> {
match self.schema {
Schema::Int | Schema::Date | Schema::TimeMillis => zag_i32(self.reader),
@@ -203,9 +217,7 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
Schema::String | Schema::Uuid(UuidSchema::String) => self.deserialize_string(visitor),
Schema::Array(_) => self.deserialize_seq(visitor),
Schema::Map(_) => self.deserialize_map(visitor),
- Schema::Union(union) => {
- UnionDeserializer::new(self.reader, union, self.config)?.deserialize_any(visitor)
- }
+ Schema::Union(union) => self.with_union(union)?.deserialize_any(visitor),
Schema::Record(schema) => {
if schema.attributes.get("org.apache.avro.rust.tuple")
== Some(&serde_json::Value::Bool(true))
@@ -247,6 +259,7 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
};
visitor.visit_bool(boolean)
}
+ Schema::Union(union) => self.with_union(union)?.deserialize_bool(visitor),
_ => Err(self.error("bool", "Expected a Schema::Boolean")),
}
}
@@ -255,83 +268,113 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- let int = self.read_int("i8")?;
- let value = i8::try_from(int)
- .map_err(|_| self.error("i8", format!("Could not convert int ({int}) to an i8")))?;
- visitor.visit_i8(value)
+ if let Schema::Union(union) = self.schema {
+ self.with_union(union)?.deserialize_i8(visitor)
+ } else {
+ let int = self.read_int("i8")?;
+ let value = i8::try_from(int)
+ .map_err(|_| self.error("i8", format!("Could not convert int ({int}) to an i8")))?;
+ visitor.visit_i8(value)
+ }
}
fn deserialize_i16<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
where
V: serde::de::Visitor<'de>,
{
- let int = self.read_int("i16")?;
- let value = i16::try_from(int)
- .map_err(|_| self.error("i16", format!("Could not convert int ({int}) to an i16")))?;
- visitor.visit_i16(value)
+ if let Schema::Union(union) = self.schema {
+ self.with_union(union)?.deserialize_i16(visitor)
+ } else {
+ let int = self.read_int("i16")?;
+ let value = i16::try_from(int).map_err(|_| {
+ self.error("i16", format!("Could not convert int ({int}) to an i16"))
+ })?;
+ visitor.visit_i16(value)
+ }
}
fn deserialize_i32<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
where
V: serde::de::Visitor<'de>,
{
- let value = self.read_int("i32")?;
- visitor.visit_i32(value)
+ if let Schema::Union(union) = self.schema {
+ self.with_union(union)?.deserialize_i32(visitor)
+ } else {
+ let value = self.read_int("i32")?;
+ visitor.visit_i32(value)
+ }
}
fn deserialize_i64<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
where
V: serde::de::Visitor<'de>,
{
- let value = self.read_long("i64")?;
- visitor.visit_i64(value)
+ if let Schema::Union(union) = self.schema {
+ self.with_union(union)?.deserialize_i64(visitor)
+ } else {
+ let value = self.read_long("i64")?;
+ visitor.visit_i64(value)
+ }
}
fn deserialize_u8<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
where
V: serde::de::Visitor<'de>,
{
- let int = self.read_int("u8")?;
- let value = u8::try_from(int)
- .map_err(|_| self.error("u8", format!("Could not convert int ({int}) to an u8")))?;
- visitor.visit_u8(value)
+ if let Schema::Union(union) = self.schema {
+ self.with_union(union)?.deserialize_u8(visitor)
+ } else {
+ let int = self.read_int("u8")?;
+ let value = u8::try_from(int)
+ .map_err(|_| self.error("u8", format!("Could not convert int ({int}) to an u8")))?;
+ visitor.visit_u8(value)
+ }
}
fn deserialize_u16<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
where
V: serde::de::Visitor<'de>,
{
- let int = self.read_int("u16")?;
- let value = u16::try_from(int)
- .map_err(|_| self.error("u16", format!("Could not convert int ({int}) to an u16")))?;
- visitor.visit_u16(value)
+ if let Schema::Union(union) = self.schema {
+ self.with_union(union)?.deserialize_u16(visitor)
+ } else {
+ let int = self.read_int("u16")?;
+ let value = u16::try_from(int).map_err(|_| {
+ self.error("u16", format!("Could not convert int ({int}) to an u16"))
+ })?;
+ visitor.visit_u16(value)
+ }
}
fn deserialize_u32<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
where
V: serde::de::Visitor<'de>,
{
- let int = self.read_long("u32")?;
- let value = u32::try_from(int)
- .map_err(|_| self.error("u32", format!("Could not convert int ({int}) to an u32")))?;
- visitor.visit_u32(value)
+ if let Schema::Union(union) = self.schema {
+ self.with_union(union)?.deserialize_u32(visitor)
+ } else {
+ let int = self.read_long("u32")?;
+ let value = u32::try_from(int).map_err(|_| {
+ self.error("u32", format!("Could not convert int ({int}) to an u32"))
+ })?;
+ visitor.visit_u32(value)
+ }
}
fn deserialize_u64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::Fixed(fixed) = self.schema
- && fixed.size == 8
- && fixed.name.name() == "u64"
- {
- let mut buf = [0; 8];
- self.reader
- .read_exact(&mut buf)
- .map_err(Details::ReadBytes)?;
- visitor.visit_u64(u64::from_le_bytes(buf))
- } else {
- Err(self.error("u64", r#"Expected Schema::Fixed(name: "u64", size: 8)"#))
+ match self.schema {
+ Schema::Fixed(fixed) if fixed.size == 8 && fixed.name.name() == "u64" => {
+ let mut buf = [0; 8];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_u64(u64::from_le_bytes(buf))
+ }
+ Schema::Union(union) => self.with_union(union)?.deserialize_u64(visitor),
+ _ => Err(self.error("u64", r#"Expected Schema::Fixed(name: "u64", size: 8)"#)),
}
}
@@ -339,14 +382,16 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::Float = self.schema {
- let mut buf = [0; 4];
- self.reader
- .read_exact(&mut buf)
- .map_err(Details::ReadBytes)?;
- visitor.visit_f32(f32::from_le_bytes(buf))
- } else {
- Err(self.error("f32", r#"Expected Schema::Float)"#))
+ match self.schema {
+ Schema::Float => {
+ let mut buf = [0; 4];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_f32(f32::from_le_bytes(buf))
+ }
+ Schema::Union(union) => self.with_union(union)?.deserialize_f32(visitor),
+ _ => Err(self.error("f32", r#"Expected Schema::Float)"#)),
}
}
@@ -354,14 +399,16 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::Double = self.schema {
- let mut buf = [0; 8];
- self.reader
- .read_exact(&mut buf)
- .map_err(Details::ReadBytes)?;
- visitor.visit_f64(f64::from_le_bytes(buf))
- } else {
- Err(self.error("f64", r#"Expected Schema::Double)"#))
+ match self.schema {
+ Schema::Double => {
+ let mut buf = [0; 8];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_f64(f64::from_le_bytes(buf))
+ }
+ Schema::Union(union) => self.with_union(union)?.deserialize_f64(visitor),
+ _ => Err(self.error("f64", r#"Expected Schema::Double)"#)),
}
}
@@ -369,20 +416,22 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::String = self.schema {
- let string = self.read_string()?;
- let mut chars = string.chars();
- if let Some(character) = chars.next() {
- if chars.next().is_some() {
- Err(self.error("char", "String contains more than one character"))
+ match self.schema {
+ Schema::String => {
+ let string = self.read_string()?;
+ let mut chars = string.chars();
+ if let Some(character) = chars.next() {
+ if chars.next().is_some() {
+ Err(self.error("char", "String contains more than one character"))
+ } else {
+ visitor.visit_char(character)
+ }
} else {
- visitor.visit_char(character)
+ Err(self.error("char", "String is empty"))
}
- } else {
- Err(self.error("char", "String is empty"))
}
- } else {
- Err(self.error("char", "Expected Schema::String"))
+ Schema::Union(union) => self.with_union(union)?.deserialize_char(visitor),
+ _ => Err(self.error("char", "Expected Schema::String")),
}
}
@@ -397,11 +446,13 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::String = self.schema {
- let string = self.read_string()?;
- visitor.visit_string(string)
- } else {
- Err(self.error("string", "Expected Schema::String"))
+ match self.schema {
+ Schema::String => {
+ let string = self.read_string()?;
+ visitor.visit_string(string)
+ }
+ Schema::Union(union) => self.with_union(union)?.deserialize_string(visitor),
+ _ => Err(self.error("string", "Expected Schema::String")),
}
}
@@ -425,6 +476,9 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
let bytes = self.read_bytes(fixed.size)?;
visitor.visit_byte_buf(bytes)
}
+ Schema::Union(union) => {
+ self.with_union(union)?.deserialize_byte_buf(visitor)
+ }
_ => Err(self.error("bytes", "Expected Schema::Bytes | Schema::Fixed | Schema::BigDecimal | Schema::Decimal | Schema::Uuid(Fixed) | Schema::Duration"))
}
}
@@ -456,10 +510,10 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::Null = self.schema {
- visitor.visit_unit()
- } else {
- Err(self.error("unit", "Expected Schema::Null"))
+ match self.schema {
+ Schema::Null => visitor.visit_unit(),
+ Schema::Union(union) => self.with_union(union)?.deserialize_unit(visitor),
+ _ => Err(self.error("unit", "Expected Schema::Null")),
}
}
@@ -471,16 +525,15 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::Record(record) = self.schema
- && record.fields.is_empty()
- && record.name.name() == name
- {
- visitor.visit_unit()
- } else {
- Err(self.error(
+ match self.schema {
+ Schema::Record(record) if record.fields.is_empty() && record.name.name() == name => {
+ visitor.visit_unit()
+ }
+ Schema::Union(union) => self.with_union(union)?.deserialize_unit(visitor),
+ _ => Err(self.error(
"unit struct",
format!("Expected Schema::Record(name: {name}, fields.len() == 0)"),
- ))
+ )),
}
}
@@ -492,16 +545,17 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::Record(record) = self.schema
- && record.fields.len() == 1
- && record.name.name() == name
- {
- visitor.visit_newtype_struct(self.with_different_schema(&record.fields[0].schema)?)
- } else {
- Err(self.error(
- "unit struct",
+ match self.schema {
+ Schema::Record(record) if record.fields.len() == 1 && record.name.name() == name => {
+ visitor.visit_newtype_struct(self.with_different_schema(&record.fields[0].schema)?)
+ }
+ Schema::Union(union) => self
+ .with_union(union)?
+ .deserialize_newtype_struct(name, visitor),
+ _ => Err(self.error(
+ "newtype struct",
format!("Expected Schema::Record(name: {name}, fields.len() == 1)"),
- ))
+ )),
}
}
@@ -509,10 +563,12 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::Array(array) = self.schema {
- visitor.visit_seq(ArrayDeserializer::new(self.reader, array, self.config))
- } else {
- Err(self.error("array", "Expected Schema::Array"))
+ match self.schema {
+ Schema::Array(array) => {
+ visitor.visit_seq(ArrayDeserializer::new(self.reader, array, self.config))
+ }
+ Schema::Union(union) => self.with_union(union)?.deserialize_seq(visitor),
+ _ => Err(self.error("array", "Expected Schema::Array")),
}
}
@@ -529,6 +585,7 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
Schema::Record(record) if record.fields.len() == len => {
visitor.visit_seq(ManyTupleDeserializer::new(self.reader, record, self.config))
}
+ Schema::Union(union) => self.with_union(union)?.deserialize_tuple(len, visitor),
_ if len == 0 => Err(self.error("tuple", "Expected Schema::Null for unit tuple")),
_ => Err(self.error("tuple", format!("Expected Schema::Record for {len}-tuple"))),
}
@@ -543,16 +600,17 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::Record(record) = self.schema
- && record.name.name() == name
- && record.fields.len() == len
- {
- visitor.visit_map(RecordDeserializer::new(self.reader, record, self.config))
- } else {
- Err(self.error(
+ match self.schema {
+ Schema::Record(record) if record.name.name() == name && record.fields.len() == len => {
+ visitor.visit_map(RecordDeserializer::new(self.reader, record, self.config))
+ }
+ Schema::Union(union) => self
+ .with_union(union)?
+ .deserialize_tuple_struct(name, len, visitor),
+ _ => Err(self.error(
"tuple struct",
format!("Expected Schema::Record(fields.len() == {len})"),
- ))
+ )),
}
}
@@ -568,6 +626,7 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
// Needed for flattened structs which are (de)serialized as maps
visitor.visit_map(RecordDeserializer::new(self.reader, record, self.config))
}
+ Schema::Union(union) => self.with_union(union)?.deserialize_map(visitor),
_ => Err(self.error("map", "Expected Schema::Map")),
}
}
@@ -575,19 +634,22 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
fn deserialize_struct<V>(
self,
name: &'static str,
- _fields: &'static [&'static str],
+ fields: &'static [&'static str],
visitor: V,
) -> Result<V::Value, Self::Error>
where
V: serde::de::Visitor<'de>,
{
- println!("deserialize_struct(name: {name}, fields = {_fields:?}): {self:?}");
- if let Schema::Record(record) = self.schema
- && record.name.name() == name
- {
- visitor.visit_map(RecordDeserializer::new(self.reader, record, self.config))
- } else {
- Err(self.error("struct", format!("Expected Schema::Record(name: {name})")))
+ match self.schema {
+ Schema::Record(record)
+ if record.name.name() == name || name.as_ptr() == DESERIALIZE_ANY.as_ptr() =>
+ {
+ visitor.visit_map(RecordDeserializer::new(self.reader, record, self.config))
+ }
+ Schema::Union(union) => self
+ .with_union(union)?
+ .deserialize_struct(name, fields, visitor),
+ _ => Err(self.error("struct", format!("Expected Schema::Record(name: {name})"))),
}
}
@@ -600,9 +662,11 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- println!("deserialize_enum(name: {name}, variants: {variants:?}): {self:?}");
match self.schema {
Schema::Enum(schema) => visitor.visit_enum(PlainEnumAccess::new(self.reader, schema)),
+ Schema::Union(schema) => {
+ visitor.visit_enum(UnionEnumAccess::new(schema, self.reader, self.config))
+ }
_ => panic!(
"deserializing enum, name: {name}, variants: {variants:#?}, {:?}",
self.schema
@@ -614,17 +678,16 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::Fixed(fixed) = self.schema
- && fixed.size == 16
- && fixed.name.name() == "i128"
- {
- let mut buf = [0; 16];
- self.reader
- .read_exact(&mut buf)
- .map_err(Details::ReadBytes)?;
- visitor.visit_i128(i128::from_le_bytes(buf))
- } else {
- Err(self.error("i128", r#"Expected Schema::Fixed(name: "i128", size: 16)"#))
+ match self.schema {
+ Schema::Fixed(fixed) if fixed.size == 16 && fixed.name.name() == "i128" => {
+ let mut buf = [0; 16];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_i128(i128::from_le_bytes(buf))
+ }
+ Schema::Union(union) => self.with_union(union)?.deserialize_i128(visitor),
+ _ => Err(self.error("i128", r#"Expected Schema::Fixed(name: "i128", size: 16)"#)),
}
}
@@ -632,17 +695,16 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
where
V: serde::de::Visitor<'de>,
{
- if let Schema::Fixed(fixed) = self.schema
- && fixed.size == 16
- && fixed.name.name() == "u128"
- {
- let mut buf = [0; 16];
- self.reader
- .read_exact(&mut buf)
- .map_err(Details::ReadBytes)?;
- visitor.visit_u128(u128::from_le_bytes(buf))
- } else {
- Err(self.error("u128", r#"Expected Schema::Fixed(name: "u128", size: 16)"#))
+ match self.schema {
+ Schema::Fixed(fixed) if fixed.size == 16 && fixed.name.name() == "u128" => {
+ let mut buf = [0; 16];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_u128(u128::from_le_bytes(buf))
+ }
+ Schema::Union(union) => self.with_union(union)?.deserialize_u128(visitor),
+ _ => Err(self.error("u128", r#"Expected Schema::Fixed(name: "u128", size: 16)"#)),
}
}
diff --git a/avro/src/serde/deser_schema/record.rs b/avro/src/serde/deser_schema/record.rs
index 055dc37..86e6000 100644
--- a/avro/src/serde/deser_schema/record.rs
+++ b/avro/src/serde/deser_schema/record.rs
@@ -19,14 +19,11 @@ use std::borrow::Borrow;
use std::fmt::{Debug, Formatter};
use std::io::Read;
-use serde::{Deserializer, de::MapAccess};
+use serde::de::MapAccess;
use super::Config;
-use crate::serde::deser_schema::union::UnionDeserializer;
-use crate::{
- Error, Schema, error::Details, schema::RecordSchema,
- serde::deser_schema::SchemaAwareDeserializer,
-};
+use crate::serde::deser_schema::identifier::IdentifierDeserializer;
+use crate::{Error, Schema, schema::RecordSchema, serde::deser_schema::SchemaAwareDeserializer};
pub struct RecordDeserializer<'s, 'r, R: Read, S: Borrow<Schema>> {
reader: &'r mut R,
@@ -76,9 +73,9 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> MapAccess<'de> for RecordDeseriali
// Finished reading this record
Ok(None)
} else {
- let v = seed.deserialize(FieldName {
- name: &self.schema.fields[index].name,
- })?;
+ let v = seed.deserialize(IdentifierDeserializer::string(
+ &self.schema.fields[index].name,
+ ))?;
self.current_field = State::Value(index);
Ok(Some(v))
}
@@ -92,16 +89,11 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> MapAccess<'de> for RecordDeseriali
let State::Value(index) = self.current_field else {
panic!("`next_key_seed` and `next_value_seed` where called in the wrong error")
};
- let v = match &self.schema.fields[index].schema {
- Schema::Union(union) => {
- seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
- }
- schema => seed.deserialize(SchemaAwareDeserializer::new(
- self.reader,
- schema,
- self.config,
- )?)?,
- };
+ let v = seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ &self.schema.fields[index].schema,
+ self.config,
+ )?)?;
self.current_field = State::Key(index + 1);
Ok(v)
}
@@ -112,248 +104,3 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> MapAccess<'de> for RecordDeseriali
}
}
}
-
-/// "Deserializer" for the field name
-struct FieldName<'s> {
- name: &'s str,
-}
-
-impl<'de, 's> Deserializer<'de> for FieldName<'s> {
- type Error = Error;
-
- fn deserialize_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_any".into()).into())
- }
-
- fn deserialize_bool<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_bool".into()).into())
- }
-
- fn deserialize_i8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_i8".into()).into())
- }
-
- fn deserialize_i16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_i16".into()).into())
- }
-
- fn deserialize_i32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_i32".into()).into())
- }
-
- fn deserialize_i64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_i64".into()).into())
- }
-
- fn deserialize_u8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_u8".into()).into())
- }
-
- fn deserialize_u16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_u16".into()).into())
- }
-
- fn deserialize_u32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_u32".into()).into())
- }
-
- fn deserialize_u64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_u64".into()).into())
- }
-
- fn deserialize_f32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_f32".into()).into())
- }
-
- fn deserialize_f64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_f64".into()).into())
- }
-
- fn deserialize_char<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_char".into()).into())
- }
-
- fn deserialize_str<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_str".into()).into())
- }
-
- fn deserialize_string<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_string".into()).into())
- }
-
- fn deserialize_bytes<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_bytes".into()).into())
- }
-
- fn deserialize_byte_buf<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_byte_buf".into()).into())
- }
-
- fn deserialize_option<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_option".into()).into())
- }
-
- fn deserialize_unit<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_unit".into()).into())
- }
-
- fn deserialize_unit_struct<V>(
- self,
- name: &'static str,
- _visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey(format!("deserialize_unit_struct(name: {name})")).into())
- }
-
- fn deserialize_newtype_struct<V>(
- self,
- name: &'static str,
- _visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey(format!("deserialize_newtype_struct(name: {name})")).into())
- }
-
- fn deserialize_seq<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_seq".into()).into())
- }
-
- fn deserialize_tuple<V>(self, len: usize, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey(format!("deserialize_tuple(len: {len})")).into())
- }
-
- fn deserialize_tuple_struct<V>(
- self,
- name: &'static str,
- len: usize,
- _visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey(format!(
- "deserialize_tuple_struct(name: {name}, len: {len})"
- ))
- .into())
- }
-
- fn deserialize_map<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_map".into()).into())
- }
-
- fn deserialize_struct<V>(
- self,
- name: &'static str,
- fields: &'static [&'static str],
- _visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey(format!(
- "deserialize_struct(name: {name}, fields: {fields:?})"
- ))
- .into())
- }
-
- fn deserialize_enum<V>(
- self,
- name: &'static str,
- variants: &'static [&'static str],
- _visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey(format!(
- "deserialize_enum(name: {name}, variants: {variants:?})"
- ))
- .into())
- }
-
- fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- println!("deserializing_identifier: {}", self.name);
- visitor.visit_str(self.name)
- }
-
- fn deserialize_ignored_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
- where
- V: serde::de::Visitor<'de>,
- {
- Err(Details::DeserializeKey("deserialize_ignored_any".into()).into())
- }
-}
diff --git a/avro/src/serde/deser_schema/tuple.rs b/avro/src/serde/deser_schema/tuple.rs
index 6b9284d..bdf07de 100644
--- a/avro/src/serde/deser_schema/tuple.rs
+++ b/avro/src/serde/deser_schema/tuple.rs
@@ -1,5 +1,4 @@
use crate::schema::RecordSchema;
-use crate::serde::deser_schema::union::UnionDeserializer;
use crate::serde::deser_schema::{Config, SchemaAwareDeserializer};
use crate::{Error, Schema};
use serde::de::{DeserializeSeed, SeqAccess};
@@ -35,16 +34,11 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> SeqAccess<'de>
{
if self.current_field < self.schema.fields.len() {
let schema = &self.schema.fields[self.current_field].schema;
- let v = match schema {
- Schema::Union(union) => {
- seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
- }
- schema => seed.deserialize(SchemaAwareDeserializer::new(
- self.reader,
- schema,
- self.config,
- )?)?,
- };
+ let v = seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?;
self.current_field += 1;
Ok(Some(v))
} else {
@@ -95,16 +89,11 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> SeqAccess<'de>
if self.field_read {
Ok(None)
} else {
- let v = match self.schema {
- Schema::Union(union) => {
- seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
- }
- schema => seed.deserialize(SchemaAwareDeserializer::new(
- self.reader,
- schema,
- self.config,
- )?)?,
- };
+ let v = seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ self.schema,
+ self.config,
+ )?)?;
self.field_read = true;
Ok(Some(v))
}
diff --git a/avro/src/serde/deser_schema/union.rs b/avro/src/serde/deser_schema/union.rs
deleted file mode 100644
index 658eb2e..0000000
--- a/avro/src/serde/deser_schema/union.rs
+++ /dev/null
@@ -1,450 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::borrow::Borrow;
-use std::fmt::{Debug, Formatter};
-use std::io::Read;
-
-use serde::de::Visitor;
-
-use super::{Config, DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, SchemaAwareDeserializer};
-use crate::error::Details;
-use crate::schema::{SchemaKind, UuidSchema};
-use crate::util::zag_i32;
-use crate::{Error, Schema, schema::UnionSchema};
-
-pub struct UnionDeserializer<'s, 'r, R: Read, S: Borrow<Schema>> {
- reader: &'r mut R,
- schema: &'s UnionSchema,
- config: Config<'s, S>,
- variant: &'s Schema,
-}
-
-impl<'s, 'r, R: Read, S: Borrow<Schema>> Debug for UnionDeserializer<'s, 'r, R, S> {
- fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("UnionDeserializer")
- .field("schema", self.schema)
- .field("variant", &self.variant)
- .finish()
- }
-}
-
-impl<'s, 'r, R: Read, S: Borrow<Schema>> UnionDeserializer<'s, 'r, R, S> {
- pub fn new(
- reader: &'r mut R,
- schema: &'s UnionSchema,
- config: Config<'s, S>,
- ) -> Result<Self, Error> {
- let index = zag_i32(reader)?;
- let variant =
- schema
- .variants()
- .get(index as usize)
- .ok_or_else(|| Details::GetUnionVariant {
- index: index as i64,
- num_variants: schema.variants().len(),
- })?;
- Ok(Self {
- reader,
- schema,
- config,
- variant,
- })
- }
-
- fn error(&self, ty: &'static str, error: impl Into<String>) -> Error {
- Error::new(Details::DeserializeValueWithSchema {
- value_type: ty,
- value: error.into(),
- schema: Schema::Union(self.schema.clone()),
- })
- }
-}
-
-impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> serde::Deserializer<'de>
- for UnionDeserializer<'s, 'r, R, S>
-{
- type Error = Error;
-
- fn deserialize_any<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_any: {self:?}");
- match self.variant {
- Schema::Null => visitor.visit_unit(),
- Schema::Boolean => self.deserialize_bool(visitor),
- Schema::Int | Schema::Date | Schema::TimeMillis => self.deserialize_i32(visitor),
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => self.deserialize_i64(visitor),
- Schema::Float => self.deserialize_f32(visitor),
- Schema::Double => self.deserialize_f64(visitor),
- Schema::Bytes
- | Schema::Fixed(_)
- | Schema::Decimal(_)
- | Schema::BigDecimal
- | Schema::Uuid(UuidSchema::Fixed(_))
- | Schema::Duration(_) => self.deserialize_byte_buf(visitor),
- Schema::String | Schema::Uuid(UuidSchema::String) => self.deserialize_string(visitor),
- Schema::Array(_) => self.deserialize_seq(visitor),
- Schema::Map(_) => self.deserialize_map(visitor),
- Schema::Record(schema) => {
- if schema.attributes.get("org.apache.avro.rust.tuple")
- == Some(&serde_json::Value::Bool(true))
- {
- // This is needed because we can't tell the difference between a tuple and struct.
- // And a tuple needs to be deserialized as a sequence
- self.deserialize_tuple(schema.fields.len(), visitor)
- } else {
- self.deserialize_struct(DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, visitor)
- }
- }
- Schema::Enum(_) => {
- self.deserialize_enum(DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, visitor)
- }
- Schema::Ref { name } => {
- let schema = self
- .config
- .names
- .get(name)
- .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?
- .borrow();
- self.variant = schema;
- self.deserialize_any(visitor)
- }
- Schema::Union(_) => Err(self.error("any", "Nested unions are not supported")),
- Schema::Uuid(UuidSchema::Bytes) => panic!("Unsupported"),
- }
- }
-
- fn deserialize_bool<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_bool: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_bool(visitor)
- }
-
- fn deserialize_i8<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_i8: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_i8(visitor)
- }
-
- fn deserialize_i16<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_i16: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_i16(visitor)
- }
-
- fn deserialize_i32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_i32: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_i32(visitor)
- }
-
- fn deserialize_i64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_i64: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_i64(visitor)
- }
-
- fn deserialize_i128<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_i128: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_i128(visitor)
- }
-
- fn deserialize_u8<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_u8: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_u8(visitor)
- }
-
- fn deserialize_u16<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_u16: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_u16(visitor)
- }
-
- fn deserialize_u32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_u32: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_u32(visitor)
- }
-
- fn deserialize_u64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_u64: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_u64(visitor)
- }
-
- fn deserialize_u128<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_u128: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_u128(visitor)
- }
-
- fn deserialize_f32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_f32: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_f32(visitor)
- }
-
- fn deserialize_f64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_f64: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_f64(visitor)
- }
-
- fn deserialize_char<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_char: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_char(visitor)
- }
-
- fn deserialize_str<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_str: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_str(visitor)
- }
-
- fn deserialize_string<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_string: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_string(visitor)
- }
-
- fn deserialize_bytes<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_bytes: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_bytes(visitor)
- }
-
- fn deserialize_byte_buf<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_byte_buf: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_byte_buf(visitor)
- }
-
- fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_option: {self:?}");
- if self.schema.variants().len() == 2 && self.schema.index().get(&SchemaKind::Null).is_some()
- {
- match self.variant {
- Schema::Null => visitor.visit_none(),
- schema => visitor.visit_some(SchemaAwareDeserializer::new(
- self.reader,
- schema,
- self.config,
- )?),
- }
- } else {
- Err(self.error(
- "option",
- "Expected Schema::Union(variants.contains(Schema::Null), self.variants.len() == 2)",
- ))
- }
- }
-
- fn deserialize_unit<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_unit: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_unit(visitor)
- }
-
- fn deserialize_unit_struct<V>(
- self,
- name: &'static str,
- visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_unit_struct: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_unit_struct(name, visitor)
- }
-
- fn deserialize_newtype_struct<V>(
- self,
- name: &'static str,
- visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_newtype_struct: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_newtype_struct(name, visitor)
- }
-
- fn deserialize_seq<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_seq: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_seq(visitor)
- }
-
- fn deserialize_tuple<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_tuple: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_tuple(len, visitor)
- }
-
- fn deserialize_tuple_struct<V>(
- self,
- name: &'static str,
- len: usize,
- visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_tuple_struct: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_tuple_struct(name, len, visitor)
- }
-
- fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_map: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_map(visitor)
- }
-
- fn deserialize_struct<V>(
- self,
- name: &'static str,
- fields: &'static [&'static str],
- visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_struct: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_struct(name, fields, visitor)
- }
-
- fn deserialize_enum<V>(
- self,
- name: &'static str,
- variants: &'static [&'static str],
- visitor: V,
- ) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_enum: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_enum(name, variants, visitor)
- }
-
- fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- println!("deserialize_identifier: {self:?}");
- SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
- .deserialize_identifier(visitor)
- }
-
- fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
- where
- V: Visitor<'de>,
- {
- // TODO: We can do something more efficient here
- println!("deserialize_ignored_any: {self:?}");
- self.deserialize_any(visitor)
- }
-
- fn is_human_readable(&self) -> bool {
- self.config.human_readable
- }
-}
diff --git a/avro/src/serde/mod.rs b/avro/src/serde/mod.rs
index 96cf620..11ffd84 100644
--- a/avro/src/serde/mod.rs
+++ b/avro/src/serde/mod.rs
@@ -115,6 +115,10 @@ pub(crate) mod ser_schema;
mod util;
mod with;
+#[expect(
+ deprecated,
+ reason = "Still need to export it until we remove it completely"
+)]
pub use de::from_value;
pub use derive::{AvroSchema, AvroSchemaComponent};
pub use ser::to_value;
diff --git a/avro/src/serde/ser_schema/mod.rs b/avro/src/serde/ser_schema/mod.rs
index 9244910..8a1e511 100644
--- a/avro/src/serde/ser_schema/mod.rs
+++ b/avro/src/serde/ser_schema/mod.rs
@@ -455,7 +455,7 @@ impl<'s, 'w, W: Write> Serializer for SchemaAwareSerializer<'s, 'w, W> {
variant: &'static str,
) -> Result<Self::Ok, Self::Error> {
println!(
- "serialize_struct_variant(name: {name}, index: {variant_index}, variant: {variant}): {self:?}"
+ "serialize_unit_variant(name: {name}, index: {variant_index}, variant: {variant}): {self:?}"
);
match self.schema {
Schema::Enum(enum_schema) => {
diff --git a/avro/src/serde/with.rs b/avro/src/serde/with.rs
index 99a432e..ead124d 100644
--- a/avro/src/serde/with.rs
+++ b/avro/src/serde/with.rs
@@ -15,8 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-#![expect(clippy::ref_option, reason = "Required by the Serde API")]
-
use std::cell::Cell;
thread_local! {
@@ -814,7 +812,7 @@ pub mod array_opt {
#[cfg(test)]
mod tests {
- use crate::{Schema, from_value, to_value, types::Value};
+ use crate::{Schema, to_value, types::Value};
use serde::{Deserialize, Serialize};
#[test]
@@ -884,6 +882,7 @@ mod tests {
assert!(value.validate(&schema));
}
+ #[expect(deprecated, reason = "This tests the deprecated function")]
#[test]
fn avro_3631_deserialize_value_to_struct_with_byte_types() {
#[derive(Debug, Deserialize, PartialEq)]
@@ -1007,7 +1006,7 @@ mod tests {
Value::Union(1, Box::new(Value::Null)),
),
]);
- assert_eq!(expected, from_value(&value).unwrap());
+ assert_eq!(expected, crate::from_value(&value).unwrap());
}
#[test]
diff --git a/avro/tests/avro-rs-285-bytes_deserialization.rs b/avro/tests/avro-rs-285-bytes_deserialization.rs
index cf02567..da8abcd 100644
--- a/avro/tests/avro-rs-285-bytes_deserialization.rs
+++ b/avro/tests/avro-rs-285-bytes_deserialization.rs
@@ -72,13 +72,15 @@ fn avro_rs_285_bytes_deserialization_round_trip() -> TestResult {
// deserialize Avro binary data back into ExampleByteArray structs
let reader = apache_avro::Reader::new(&avro_data[..])?;
let deserialized_records: Vec<ExampleByteArray> = reader
- .map(|value| apache_avro::from_value::<ExampleByteArray>(&value.unwrap()).unwrap())
+ .into_serde_iter()
+ .map(|value| value.unwrap())
.collect();
assert_eq!(records, deserialized_records);
Ok(())
}
+#[expect(deprecated, reason = "Schema resolution is WIP")]
#[test]
fn avro_rs_285_bytes_deserialization_filtered_round_trip() -> TestResult {
let raw_schema = r#"
diff --git a/avro/tests/schema.rs b/avro/tests/schema.rs
index 89e328a..1273a3d 100644
--- a/avro/tests/schema.rs
+++ b/avro/tests/schema.rs
@@ -19,7 +19,6 @@ use apache_avro::writer::datum::GenericDatumWriter;
use apache_avro::{
Codec, Error, Reader, Schema, Writer,
error::Details,
- from_value,
reader::datum::GenericDatumReader,
schema::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema},
to_value,
@@ -837,7 +836,7 @@ fn avro_old_issue_47() -> TestResult {
let schema_str = r#"
{
"type": "record",
- "name": "my_record",
+ "name": "MyRecord",
"fields": [
{"name": "a", "type": "long"},
{"name": "b", "type": "string"}
@@ -863,10 +862,9 @@ fn avro_old_issue_47() -> TestResult {
.build()?
.write_value_to_vec(ser_value)?;
- let de_value = GenericDatumReader::builder(&schema)
+ let deserialized_record: MyRecord = GenericDatumReader::builder(&schema)
.build()?
- .read_value(&mut &*serialized_bytes)?;
- let deserialized_record = from_value::<MyRecord>(&de_value)?;
+ .read_deser(&mut &*serialized_bytes)?;
assert_eq!(record, deserialized_record);
Ok(())
diff --git a/avro/tests/union_schema.rs b/avro/tests/union_schema.rs
index 892b69b..d3c68c9 100644
--- a/avro/tests/union_schema.rs
+++ b/avro/tests/union_schema.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use apache_avro::{AvroResult, Codec, Reader, Schema, Writer, from_value};
+use apache_avro::{AvroResult, Codec, Reader, Schema, Writer};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
static SCHEMA_A_STR: &str = r#"{
@@ -80,7 +80,7 @@ where
.reader_schema(schema)
.schemata(schemata.iter().collect())
.build()?;
- from_value::<T>(&reader.next().expect("")?)
+ Ok(reader.next_deser::<T>()?.expect("Expected a value"))
}
#[test]
diff --git a/avro_derive/src/attributes/serde.rs b/avro_derive/src/attributes/serde.rs
index b0e4ae1..be3fe65 100644
--- a/avro_derive/src/attributes/serde.rs
+++ b/avro_derive/src/attributes/serde.rs
@@ -159,6 +159,10 @@ pub struct VariantAttributes {
#[darling(default)]
pub rename_all: RenameAll,
/// Do not serialize or deserialize this variant.
+ ///
+ /// Skip (and skip_{de,}serializing) should not be used.
+ /// Serde will remove the variant from the list of variants when deserializing but
+ /// will not update the index when serializing.
#[darling(default, rename = "skip")]
pub _skip: bool,
/// Do not serialize this variant.
diff --git a/avro_derive/tests/serde.rs b/avro_derive/tests/serde.rs
index 445e11c..7c4c2f4 100644
--- a/avro_derive/tests/serde.rs
+++ b/avro_derive/tests/serde.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use apache_avro::{AvroSchema, Error, Reader, Schema, Writer, from_value};
+use apache_avro::{AvroSchema, Error, Reader, Schema, Writer};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
/// Takes in a type that implements the right combination of traits and runs it through a Serde
@@ -66,8 +66,8 @@ where
let mut reader = Reader::builder(&encoded[..])
.reader_schema(&schema)
.build()?;
- if let Some(res) = reader.next() {
- return res.and_then(|v| from_value::<T>(&v));
+ if let Some(res) = reader.next_deser::<T>()? {
+ return Ok(res);
}
panic!("Nothing was encoded!")
}
(avro-rs) 04/08: fix: Improve documentation and fix resolving
Posted by kr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a commit to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
commit 1dd524a04bdea5ed48dd43dec05b06b3dfa5bf88
Author: default <ad...@kriskras99.nl>
AuthorDate: Wed Mar 4 09:33:50 2026 +0000
fix: Improve documentation and fix resolving
---
avro/src/writer/datum.rs | 55 +++++++++++++++++++++++++++++++++++++++++-------
1 file changed, 47 insertions(+), 8 deletions(-)
diff --git a/avro/src/writer/datum.rs b/avro/src/writer/datum.rs
index a45f704..69d833b 100644
--- a/avro/src/writer/datum.rs
+++ b/avro/src/writer/datum.rs
@@ -28,6 +28,11 @@ use crate::{
types::Value,
};
+/// Writer for writing raw Avro data.
+///
+/// This is most likely not what you need. Most users should use [`Writer`][crate::Writer],
+/// [`GenericSingleObjectWriter`][crate::GenericSingleObjectWriter], or
+/// [`SpecificSingleObjectWriter`][crate::SpecificSingleObjectWriter] instead.
pub struct GenericDatumWriter<'s> {
schema: &'s Schema,
resolved: ResolvedSchema<'s>,
@@ -36,11 +41,24 @@ pub struct GenericDatumWriter<'s> {
#[bon]
impl<'s> GenericDatumWriter<'s> {
+ /// Configure a new writer.
#[builder]
pub fn new(
- #[builder(start_fn)] schema: &'s Schema,
+ /// The schema for the data that will be written
+ #[builder(start_fn)]
+ schema: &'s Schema,
+ /// Already resolved schemata that will be used to resolve references in the writer's schema.
+ ///
+ /// You can also use [`Self::schemata`] instead.
resolved_schemata: Option<ResolvedSchema<'s>>,
- #[builder(default = true)] validate: bool,
+ /// Validate values against the writer schema before writing them.
+ ///
+ /// Defaults to `true`.
+ ///
+ /// Setting this to `false` and writing values that don't match the schema will make the
+ /// written data unreadable.
+ #[builder(default = true)]
+ validate: bool,
) -> AvroResult<Self> {
let resolved = if let Some(resolved) = resolved_schemata {
resolved
@@ -85,13 +103,12 @@ impl GenericDatumWriter<'_> {
self.write_value_ref(writer, &value)
}
+ /// Write a value to the writer.
pub fn write_value_ref<W: Write>(&self, writer: &mut W, value: &Value) -> AvroResult<usize> {
if self.validate
- && self.resolved.get_schemata().iter().all(|s| {
- value
- .validate_internal(s, self.resolved.get_names(), None)
- .is_some()
- })
+ && value
+ .validate_internal(self.schema, self.resolved.get_names(), None)
+ .is_some()
{
return Err(Details::Validation.into());
}
@@ -105,6 +122,7 @@ impl GenericDatumWriter<'_> {
Ok(vec)
}
+ /// Serialize `T` to the writer.
pub fn write_ser<W: Write, T: Serialize>(
&self,
writer: &mut W,
@@ -115,6 +133,7 @@ impl GenericDatumWriter<'_> {
value.serialize(&mut serializer)
}
+ /// Serialize `T` to a [`Vec`].
pub fn write_ser_to_vec<T: Serialize>(&self, value: &T) -> AvroResult<Vec<u8>> {
let mut vec = Vec::new();
self.write_ser(&mut vec, value)?;
@@ -122,6 +141,15 @@ impl GenericDatumWriter<'_> {
}
}
+/// Deprecated. Use [`GenericDatumWriter`] instead.
+///
+/// This is equivalent to:
+/// ```ignore
+/// GenericDatumWriter::builder(schema)
+/// .build()?
+/// .write_value_to_vec(value)
+/// ```
+///
/// Encode a value into raw Avro data, also performs schema validation.
///
/// **NOTE**: This function has a quite small niche of usage and does NOT generate headers and sync
@@ -136,12 +164,13 @@ pub fn to_avro_datum<T: Into<Value>>(schema: &Schema, value: T) -> AvroResult<Ve
/// Write the referenced [Serialize]able object to the provided [Write] object.
///
+/// It is recommended to use [`GenericDatumWriter`] instead.
+///
/// Returns a result with the number of bytes written.
///
/// **NOTE**: This function has a quite small niche of usage and does **NOT** generate headers and sync
/// markers; use [`append_ser`](Writer::append_ser) to be fully Avro-compatible
/// if you don't know what you are doing, instead.
-#[deprecated(since = "0.22.0", note = "Use GenericDatumWriter instead")]
pub fn write_avro_datum_ref<T: Serialize, W: Write>(
schema: &Schema,
names: &NamesRef,
@@ -152,6 +181,16 @@ pub fn write_avro_datum_ref<T: Serialize, W: Write>(
data.serialize(&mut serializer)
}
+/// Deprecated. Use [`GenericDatumWriter`] instead.
+///
+/// This is equivalent to:
+/// ```ignore
+/// GenericDatumWriter::builder(schema)
+/// .schemata(schemata)?
+/// .build()?
+/// .write_value_to_vec(value)
+/// ```
+///
/// Encode a value into raw Avro data, also performs schema validation.
///
/// If the provided `schema` is incomplete then its dependencies must be
(avro-rs) 06/08: wip: Full enum support
Posted by kr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a commit to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
commit b3b93fa0f906914180705baa16a84d1bba4e39fe
Author: default <ad...@kriskras99.nl>
AuthorDate: Tue Mar 3 12:30:18 2026 +0000
wip: Full enum support
---
Cargo.lock | 16 +
avro/src/bigdecimal.rs | 10 +-
avro/src/decode.rs | 2 +-
avro/src/error.rs | 11 +-
avro/src/reader/block.rs | 32 +
avro/src/reader/datum.rs | 30 +-
avro/src/reader/mod.rs | 37 +-
avro/src/reader/single_object.rs | 25 +
avro/src/schema/builders.rs | 6 +-
avro/src/schema/mod.rs | 4 +
avro/src/schema/union.rs | 4 +
avro/src/serde/de.rs | 1 +
avro/src/serde/derive.rs | 1 +
avro/src/serde/deser_schema/array.rs | 97 +
avro/src/serde/deser_schema/enums/mod.rs | 3 +
avro/src/serde/deser_schema/enums/plain.rs | 324 ++
avro/src/serde/deser_schema/map.rs | 119 +
avro/src/serde/deser_schema/mod.rs | 667 ++++
avro/src/serde/deser_schema/record.rs | 359 +++
avro/src/serde/deser_schema/tuple.rs | 112 +
avro/src/serde/deser_schema/union.rs | 450 +++
avro/src/serde/mod.rs | 6 +-
avro/src/serde/ser_schema/array.rs | 190 ++
avro/src/serde/ser_schema/map.rs | 219 ++
avro/src/serde/ser_schema/mod.rs | 3479 +++++++--------------
avro/src/serde/ser_schema/record/field_default.rs | 197 ++
avro/src/serde/ser_schema/record/mod.rs | 250 ++
avro/src/serde/ser_schema/tuple.rs | 262 ++
avro/src/serde/ser_schema/union.rs | 544 ++++
avro/src/serde/with.rs | 309 ++
avro/src/writer/datum.rs | 34 +-
avro/src/writer/mod.rs | 24 +-
avro/src/writer/single_object.rs | 40 +-
avro/tests/avro-rs-226.rs | 11 +-
avro/tests/avro-rs-285-bytes_deserialization.rs | 4 +-
avro/tests/serde_human_readable_false.rs | 256 +-
avro/tests/serde_human_readable_true.rs | 21 +-
avro_derive/Cargo.toml | 1 +
avro_derive/src/attributes/avro.rs | 17 +
avro_derive/src/attributes/mod.rs | 224 +-
avro_derive/src/attributes/serde.rs | 15 +-
avro_derive/src/case.rs | 8 +
avro_derive/src/enums/bare_union.rs | 93 +
avro_derive/src/enums/mod.rs | 34 +-
avro_derive/src/enums/plain.rs | 5 +-
avro_derive/src/enums/record_internally_tagged.rs | 79 +
avro_derive/src/enums/record_tag_content.rs | 127 +
avro_derive/src/enums/union_of_records.rs | 14 +
avro_derive/src/lib.rs | 120 +-
avro_derive/tests/derive.proptest-regressions | 9 +
avro_derive/tests/derive.rs | 61 +-
avro_derive/tests/enum.rs | 158 +
avro_derive/tests/serde.rs | 43 +-
53 files changed, 6566 insertions(+), 2598 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index b6354f4..670f8f6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -101,6 +101,7 @@ version = "0.22.0"
dependencies = [
"apache-avro",
"darling",
+ "hexdump",
"pretty_assertions",
"proc-macro2",
"proptest",
@@ -123,6 +124,12 @@ dependencies = [
"log",
]
+[[package]]
+name = "arrayvec"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
+
[[package]]
name = "async-trait"
version = "0.1.89"
@@ -676,6 +683,15 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1"
+[[package]]
+name = "hexdump"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf31ab66ed8145a1c7427bd8e9b42a6131bd74ccf444f69b9e620c2e73ded832"
+dependencies = [
+ "arrayvec",
+]
+
[[package]]
name = "id-arena"
version = "2.3.0"
diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs
index 7a6a379..7df71dc 100644
--- a/avro/src/bigdecimal.rs
+++ b/avro/src/bigdecimal.rs
@@ -70,7 +70,7 @@ pub(crate) fn deserialize_big_decimal(mut bytes: &[u8]) -> AvroResult<BigDecimal
#[cfg(test)]
mod tests {
use super::*;
- use crate::{Codec, Reader, Schema, Writer, error::Error, from_value, types::Record};
+ use crate::{Codec, Reader, Schema, Writer, error::Error, types::Record};
use apache_avro_test_helper::TestResult;
use bigdecimal::{One, Zero};
use pretty_assertions::assert_eq;
@@ -183,10 +183,14 @@ mod tests {
Ok(())
}
+ // TODO: Needs new schema aware deserializer
#[test]
+ #[ignore]
fn avro_rs_338_deserialize_serde_way() -> TestResult {
#[derive(Clone, PartialEq, Eq, Debug, Default, serde::Deserialize, serde::Serialize)]
+ #[serde(rename = "test")]
struct Test {
+ #[serde(with = "crate::serde::bigdecimal")]
big_decimal: BigDecimal,
}
@@ -218,9 +222,9 @@ mod tests {
// read record
let mut reader = Reader::new(&wrote_data[..])?;
- let value = reader.next().unwrap()?;
+ let value = reader.next_deser()?.unwrap();
- assert_eq!(test, from_value::<Test>(&value)?);
+ assert_eq!(test, value);
Ok(())
}
diff --git a/avro/src/decode.rs b/avro/src/decode.rs
index cab6797..add154a 100644
--- a/avro/src/decode.rs
+++ b/avro/src/decode.rs
@@ -39,7 +39,7 @@ pub(crate) fn decode_long<R: Read>(reader: &mut R) -> AvroResult<Value> {
}
#[inline]
-fn decode_int<R: Read>(reader: &mut R) -> AvroResult<Value> {
+pub(crate) fn decode_int<R: Read>(reader: &mut R) -> AvroResult<Value> {
zag_i32(reader).map(Value::Int)
}
diff --git a/avro/src/error.rs b/avro/src/error.rs
index 67ff13f..23db16b 100644
--- a/avro/src/error.rs
+++ b/avro/src/error.rs
@@ -156,7 +156,6 @@ pub enum Details {
#[error("Union index {index} out of bounds: {num_variants}")]
GetUnionVariant { index: i64, num_variants: usize },
- #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")]
#[error("Enum symbol index out of bounds: {num_variants}")]
EnumSymbolIndex { index: usize, num_variants: usize },
@@ -579,6 +578,16 @@ pub enum Details {
#[error("Failed to deserialize Avro value into value: {0}")]
DeserializeValue(String),
+ #[error("Failed to deserialize value of type {value_type} using schema {schema:?}: {value}")]
+ DeserializeValueWithSchema {
+ value_type: &'static str,
+ value: String,
+ schema: Schema,
+ },
+
+ #[error("Only expected `deserialize_identifier` to be called but `{0}` was called")]
+ DeserializeKey(String),
+
#[error("Failed to write buffer bytes during flush: {0}")]
WriteBytes(#[source] std::io::Error),
diff --git a/avro/src/reader/block.rs b/avro/src/reader/block.rs
index 06c07a2..2313281 100644
--- a/avro/src/reader/block.rs
+++ b/avro/src/reader/block.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use crate::serde::deser_schema::{Config, SchemaAwareDeserializer};
use crate::{
AvroResult, Codec, Error,
decode::{decode, decode_internal},
@@ -24,6 +25,7 @@ use crate::{
util,
};
use log::warn;
+use serde::de::DeserializeOwned;
use serde_json::from_slice;
use std::{
collections::HashMap,
@@ -205,6 +207,36 @@ impl<'r, R: Read> Block<'r, R> {
Ok(Some(item))
}
+ pub(super) fn read_next_deser<T: DeserializeOwned>(&mut self) -> AvroResult<Option<T>> {
+ if self.is_empty() {
+ self.read_block_next()?;
+ if self.is_empty() {
+ return Ok(None);
+ }
+ }
+
+ let mut block_bytes = &self.buf[self.buf_idx..];
+ let b_original = block_bytes.len();
+
+ let deserializer = SchemaAwareDeserializer::new(
+ &mut block_bytes,
+ &self.writer_schema,
+ Config {
+ names: &self.names_refs,
+ human_readable: false,
+ },
+ )?;
+ let item = T::deserialize(deserializer)?;
+
+ if b_original != 0 && b_original == block_bytes.len() {
+ // from_avro_datum did not consume any bytes, so return an error to avoid an infinite loop
+ return Err(Details::ReadBlock.into());
+ }
+ self.buf_idx += b_original - block_bytes.len();
+ self.message_count -= 1;
+ Ok(Some(item))
+ }
+
fn read_writer_schema(&mut self, metadata: &HashMap<String, Value>) -> AvroResult<()> {
let json: serde_json::Value = metadata
.get("avro.schema")
diff --git a/avro/src/reader/datum.rs b/avro/src/reader/datum.rs
index beed61b..cbf6fd7 100644
--- a/avro/src/reader/datum.rs
+++ b/avro/src/reader/datum.rs
@@ -15,11 +15,18 @@
// specific language governing permissions and limitations
// under the License.
-use std::io::Read;
+use std::{io::Read, marker::PhantomData};
use bon::bon;
+use serde::de::DeserializeOwned;
-use crate::{AvroResult, Schema, decode::decode_internal, schema::ResolvedSchema, types::Value};
+use crate::{
+ AvroResult, Schema,
+ decode::decode_internal,
+ schema::{ResolvedOwnedSchema, ResolvedSchema},
+ serde::deser_schema::{Config, SchemaAwareDeserializer},
+ types::Value,
+};
/// Reader for reading raw Avro data.
///
@@ -126,6 +133,25 @@ impl<'s> GenericDatumReader<'s> {
}
}
+pub struct SpecificDatumReader<T: DeserializeOwned> {
+ resolved: ResolvedOwnedSchema,
+ human_readable: bool,
+ phantom: PhantomData<T>,
+}
+
+impl<T: DeserializeOwned> SpecificDatumReader<T> {
+ pub fn deserialize<R: Read>(&self, reader: &mut R) -> AvroResult<T> {
+ T::deserialize(SchemaAwareDeserializer::new(
+ reader,
+ self.resolved.get_root_schema(),
+ Config {
+ names: self.resolved.get_names(),
+ human_readable: self.human_readable,
+ },
+ )?)
+ }
+}
+
/// Deprecated.
///
/// This is equivalent to
diff --git a/avro/src/reader/mod.rs b/avro/src/reader/mod.rs
index 94af909..8fa11ef 100644
--- a/avro/src/reader/mod.rs
+++ b/avro/src/reader/mod.rs
@@ -24,7 +24,8 @@ pub mod single_object;
use crate::{AvroResult, schema::Schema, types::Value};
use block::Block;
use bon::bon;
-use std::{collections::HashMap, io::Read};
+use serde::de::DeserializeOwned;
+use std::{collections::HashMap, io::Read, marker::PhantomData};
/// Main interface for reading Avro formatted values.
///
@@ -48,6 +49,29 @@ pub struct Reader<'a, R> {
should_resolve_schema: bool,
}
+pub struct ReaderSerde<'a, R, T: DeserializeOwned> {
+ inner: Reader<'a, R>,
+ phantom: PhantomData<T>,
+}
+
+impl<R: Read, T: DeserializeOwned> Iterator for ReaderSerde<'_, R, T> {
+ type Item = AvroResult<T>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ // to prevent keep on reading after the first error occurs
+ if self.inner.errored {
+ return None;
+ };
+ match self.inner.next_deser::<T>() {
+ Ok(opt) => opt.map(Ok),
+ Err(e) => {
+ self.inner.errored = true;
+ Some(Err(e))
+ }
+ }
+ }
+}
+
#[bon]
impl<'a, R: Read> Reader<'a, R> {
/// Creates a `Reader` given something implementing the `io::Read` trait to read from.
@@ -103,6 +127,13 @@ impl<'a, R: Read> Reader<'a, R> {
&self.block.user_metadata
}
+ pub fn into_serde_iter<T: DeserializeOwned>(self) -> ReaderSerde<'a, R, T> {
+ ReaderSerde {
+ inner: self,
+ phantom: PhantomData,
+ }
+ }
+
#[inline]
fn read_next(&mut self) -> AvroResult<Option<Value>> {
let read_schema = if self.should_resolve_schema {
@@ -113,6 +144,10 @@ impl<'a, R: Read> Reader<'a, R> {
self.block.read_next(read_schema)
}
+
+ pub fn next_deser<T: DeserializeOwned>(&mut self) -> AvroResult<Option<T>> {
+ self.block.read_next_deser()
+ }
}
impl<R: Read> Iterator for Reader<'_, R> {
diff --git a/avro/src/reader/single_object.rs b/avro/src/reader/single_object.rs
index c6151a3..6e0b744 100644
--- a/avro/src/reader/single_object.rs
+++ b/avro/src/reader/single_object.rs
@@ -19,6 +19,7 @@ use crate::decode::decode_internal;
use crate::error::Details;
use crate::headers::{HeaderBuilder, RabinFingerprintHeader};
use crate::schema::ResolvedOwnedSchema;
+use crate::serde::deser_schema::{Config, SchemaAwareDeserializer};
use crate::types::Value;
use crate::{AvroResult, AvroSchema, Schema, from_value};
use serde::de::DeserializeOwned;
@@ -68,6 +69,30 @@ impl GenericSingleObjectReader {
Err(io_error) => Err(Details::ReadHeader(io_error).into()),
}
}
+
+ pub fn read_deser<R: Read, T: DeserializeOwned>(&self, reader: &mut R) -> AvroResult<T> {
+ let mut header = vec![0; self.expected_header.len()];
+ match reader.read_exact(&mut header) {
+ Ok(_) => {
+ if self.expected_header == header {
+ T::deserialize(SchemaAwareDeserializer::new(
+ reader,
+ self.write_schema.get_root_schema(),
+ Config {
+ names: self.write_schema.get_names(),
+ human_readable: false,
+ },
+ )?)
+ } else {
+ Err(
+ Details::SingleObjectHeaderMismatch(self.expected_header.clone(), header)
+ .into(),
+ )
+ }
+ }
+ Err(io_error) => Err(Details::ReadHeader(io_error).into()),
+ }
+ }
}
pub struct SpecificSingleObjectReader<T>
diff --git a/avro/src/schema/builders.rs b/avro/src/schema/builders.rs
index dd16b3e..466b244 100644
--- a/avro/src/schema/builders.rs
+++ b/avro/src/schema/builders.rs
@@ -107,13 +107,11 @@ impl Schema {
#[builder(finish_fn = build)]
pub fn record(
#[builder(start_fn)] name: Name,
- fields: Option<Vec<RecordField>>,
+ #[builder(default)] fields: Vec<RecordField>,
aliases: Option<Vec<Alias>>,
doc: Option<String>,
- attributes: Option<BTreeMap<String, JsonValue>>,
+ #[builder(default)] attributes: BTreeMap<String, JsonValue>,
) -> Self {
- let fields = fields.unwrap_or_default();
- let attributes = attributes.unwrap_or_default();
let record_schema = RecordSchema::builder()
.name(name)
.fields(fields)
diff --git a/avro/src/schema/mod.rs b/avro/src/schema/mod.rs
index 97a5824..05908f3 100644
--- a/avro/src/schema/mod.rs
+++ b/avro/src/schema/mod.rs
@@ -28,6 +28,10 @@ pub(crate) use crate::schema::resolve::{
ResolvedOwnedSchema, resolve_names, resolve_names_with_schemata,
};
pub use crate::schema::{
+ builders::{
+ SchemaArrayBuilder, SchemaEnumBuilder, SchemaFixedBuilder, SchemaMapBuilder,
+ SchemaRecordBuilder,
+ },
name::{Alias, Aliases, Name, Names, NamesRef, Namespace, NamespaceRef},
record::{RecordField, RecordFieldBuilder, RecordSchema, RecordSchemaBuilder},
resolve::ResolvedSchema,
diff --git a/avro/src/schema/union.rs b/avro/src/schema/union.rs
index 31affe7..5a5cc9f 100644
--- a/avro/src/schema/union.rs
+++ b/avro/src/schema/union.rs
@@ -79,6 +79,10 @@ impl UnionSchema {
self.variant_index.contains_key(&SchemaKind::Null)
}
+ pub fn index(&self) -> &BTreeMap<SchemaKind, usize> {
+ &self.variant_index
+ }
+
/// Optionally returns a reference to the schema matched by this value, as well as its position
/// within this union.
///
diff --git a/avro/src/serde/de.rs b/avro/src/serde/de.rs
index b9900d5..b05e065 100644
--- a/avro/src/serde/de.rs
+++ b/avro/src/serde/de.rs
@@ -938,6 +938,7 @@ impl<'de> de::Deserializer<'de> for StringDeserializer {
///
/// This conversion can fail if the structure of the `Value` does not match the
/// structure expected by `D`.
+#[deprecated(since = "0.22.0", note = "Use the `deser` functions instead")]
pub fn from_value<'de, D: Deserialize<'de>>(value: &'de Value) -> Result<D, Error> {
let de = Deserializer::new(value);
D::deserialize(de)
diff --git a/avro/src/serde/derive.rs b/avro/src/serde/derive.rs
index 33c4f5b..a65dd9a 100644
--- a/avro/src/serde/derive.rs
+++ b/avro/src/serde/derive.rs
@@ -629,6 +629,7 @@ where
}
}
+// TODO: This does not match the Serde implementation and therefore does not work
impl AvroSchemaComponent for core::time::Duration {
/// The schema is [`Schema::Duration`] with the name `duration`.
///
diff --git a/avro/src/serde/deser_schema/array.rs b/avro/src/serde/deser_schema/array.rs
new file mode 100644
index 0000000..08cb055
--- /dev/null
+++ b/avro/src/serde/deser_schema/array.rs
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::borrow::Borrow;
+use std::io::Read;
+
+use serde::de::SeqAccess;
+
+use super::Config;
+use crate::serde::deser_schema::union::UnionDeserializer;
+use crate::{
+ Error, Schema, schema::ArraySchema, serde::deser_schema::SchemaAwareDeserializer, util::zag_i32,
+};
+
+pub struct ArrayDeserializer<'s, 'r, R: Read, S: Borrow<Schema>> {
+ reader: &'r mut R,
+ schema: &'s ArraySchema,
+ config: Config<'s, S>,
+ state: State,
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> ArrayDeserializer<'s, 'r, R, S> {
+ pub fn new(reader: &'r mut R, schema: &'s ArraySchema, config: Config<'s, S>) -> Self {
+ Self {
+ reader,
+ schema,
+ config,
+ state: State::EndOfBlock,
+ }
+ }
+}
+
+enum State {
+ EndOfBlock,
+ ReadingValue(u32),
+ Finished,
+}
+
+impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> SeqAccess<'de> for ArrayDeserializer<'s, 'r, R, S> {
+ type Error = Error;
+
+ fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>
+ where
+ T: serde::de::DeserializeSeed<'de>,
+ {
+ match self.state {
+ State::EndOfBlock => {
+ let remaining = zag_i32(self.reader)?;
+ if remaining < 0 {
+ let _bytes = zag_i32(self.reader)?;
+ }
+ if remaining == 0 {
+ self.state = State::Finished
+ } else {
+ self.state = State::ReadingValue(remaining.unsigned_abs())
+ }
+ self.next_element_seed(seed)
+ }
+ State::ReadingValue(mut remaining) => {
+ let v = match self.schema.items.as_ref() {
+ Schema::Union(union) => {
+ seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
+ }
+ schema => seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?,
+ };
+
+ remaining -= 1;
+ if remaining == 0 {
+ self.state = State::EndOfBlock;
+ } else {
+ self.state = State::ReadingValue(remaining);
+ }
+
+ Ok(Some(v))
+ }
+ State::Finished => Ok(None),
+ }
+ }
+}
diff --git a/avro/src/serde/deser_schema/enums/mod.rs b/avro/src/serde/deser_schema/enums/mod.rs
new file mode 100644
index 0000000..65a8e66
--- /dev/null
+++ b/avro/src/serde/deser_schema/enums/mod.rs
@@ -0,0 +1,3 @@
+mod plain;
+
+pub use plain::PlainEnumAccess;
diff --git a/avro/src/serde/deser_schema/enums/plain.rs b/avro/src/serde/deser_schema/enums/plain.rs
new file mode 100644
index 0000000..65b5b81
--- /dev/null
+++ b/avro/src/serde/deser_schema/enums/plain.rs
@@ -0,0 +1,324 @@
+use crate::error::Details;
+use crate::schema::EnumSchema;
+use crate::util::zag_i32;
+use crate::{Error, Schema};
+use serde::Deserializer;
+use serde::de::{DeserializeSeed, EnumAccess, Unexpected, VariantAccess, Visitor};
+use std::io::Read;
+
+pub struct PlainEnumAccess<'s, 'r, R: Read> {
+ schema: &'s EnumSchema,
+ reader: &'r mut R,
+}
+
+impl<'s, 'r, R: Read> PlainEnumAccess<'s, 'r, R> {
+ pub fn new(reader: &'r mut R, schema: &'s EnumSchema) -> Self {
+ Self { schema, reader }
+ }
+}
+
+impl<'de, 's, 'r, R: Read> EnumAccess<'de> for PlainEnumAccess<'s, 'r, R> {
+ type Error = Error;
+ type Variant = Self;
+
+ fn variant_seed<V>(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error>
+ where
+ V: DeserializeSeed<'de>,
+ {
+ let deserializer = EnumIdentifierDeserializer {
+ reader: self.reader,
+ schema: self.schema,
+ };
+ Ok((seed.deserialize(deserializer)?, self))
+ }
+}
+
+impl<'de, 's, 'r, R: Read> VariantAccess<'de> for PlainEnumAccess<'s, 'r, R> {
+ type Error = Error;
+
+ fn unit_variant(self) -> Result<(), Self::Error> {
+ Ok(())
+ }
+
+ fn newtype_variant_seed<T>(self, _seed: T) -> Result<T::Value, Self::Error>
+ where
+ T: DeserializeSeed<'de>,
+ {
+ let unexp = Unexpected::UnitVariant;
+ Err(serde::de::Error::invalid_type(unexp, &"newtype variant"))
+ }
+
+ fn tuple_variant<V>(self, _len: usize, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ let unexp = Unexpected::UnitVariant;
+ Err(serde::de::Error::invalid_type(unexp, &"newtype variant"))
+ }
+
+ fn struct_variant<V>(
+ self,
+ _fields: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ let unexp = Unexpected::UnitVariant;
+ Err(serde::de::Error::invalid_type(unexp, &"newtype variant"))
+ }
+}
+
+struct EnumIdentifierDeserializer<'s, 'r, R: Read> {
+ schema: &'s EnumSchema,
+ reader: &'r mut R,
+}
+
+impl<'s, 'r, R: Read> EnumIdentifierDeserializer<'s, 'r, R> {
+ fn error(&self, error: impl Into<String>) -> Error {
+ Error::new(Details::DeserializeValueWithSchema {
+ value_type: "enum",
+ value: error.into(),
+ schema: Schema::Enum(self.schema.clone()),
+ })
+ }
+}
+
+impl<'de, 's, 'r, R: Read> Deserializer<'de> for EnumIdentifierDeserializer<'s, 'r, R> {
+ type Error = Error;
+
+ fn deserialize_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_any, expected deserialize_identifier"))
+ }
+
+ fn deserialize_bool<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_bool, expected deserialize_identifier"))
+ }
+
+ fn deserialize_i8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_i8, expected deserialize_identifier"))
+ }
+
+ fn deserialize_i16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_i16, expected deserialize_identifier"))
+ }
+
+ fn deserialize_i32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_i32, expected deserialize_identifier"))
+ }
+
+ fn deserialize_i64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_i64, expected deserialize_identifier"))
+ }
+
+ fn deserialize_u8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_u8, expected deserialize_identifier"))
+ }
+
+ fn deserialize_u16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_u16, expected deserialize_identifier"))
+ }
+
+ fn deserialize_u32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_u32, expected deserialize_identifier"))
+ }
+
+ fn deserialize_u64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_u64, expected deserialize_identifier"))
+ }
+
+ fn deserialize_f32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_f32, expected deserialize_identifier"))
+ }
+
+ fn deserialize_f64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_f64, expected deserialize_identifier"))
+ }
+
+ fn deserialize_char<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_char, expected deserialize_identifier"))
+ }
+
+ fn deserialize_str<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_str, expected deserialize_identifier"))
+ }
+
+ fn deserialize_string<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_string, expected deserialize_identifier"))
+ }
+
+ fn deserialize_bytes<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_bytes, expected deserialize_identifier"))
+ }
+
+ fn deserialize_byte_buf<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_byte_buf, expected deserialize_identifier"))
+ }
+
+ fn deserialize_option<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_option, expected deserialize_identifier"))
+ }
+
+ fn deserialize_unit<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_unit, expected deserialize_identifier"))
+ }
+
+ fn deserialize_unit_struct<V>(
+ self,
+ _name: &'static str,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_unit_struct, expected deserialize_identifier"))
+ }
+
+ fn deserialize_newtype_struct<V>(
+ self,
+ _name: &'static str,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_newtype_struct, expected deserialize_identifier"))
+ }
+
+ fn deserialize_seq<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_seq, expected deserialize_identifier"))
+ }
+
+ fn deserialize_tuple<V>(self, _len: usize, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_tuple, expected deserialize_identifier"))
+ }
+
+ fn deserialize_tuple_struct<V>(
+ self,
+ _name: &'static str,
+ _len: usize,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_tuple_struct, expected deserialize_identifier"))
+ }
+
+ fn deserialize_map<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_map, expected deserialize_identifier"))
+ }
+
+ fn deserialize_struct<V>(
+ self,
+ _name: &'static str,
+ _fields: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_struct, expected deserialize_identifier"))
+ }
+
+ fn deserialize_enum<V>(
+ self,
+ _name: &'static str,
+ _variants: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_enum, expected deserialize_identifier"))
+ }
+
+ fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ let index = zag_i32(self.reader)?;
+ let symbol = self
+ .schema
+ .symbols
+ .get(index as usize)
+ .ok_or(Details::EnumSymbolIndex {
+ index: index as usize,
+ num_variants: self.schema.symbols.len(),
+ })?;
+ visitor.visit_str(symbol)
+ }
+
+ fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ self.deserialize_any(visitor)
+ }
+}
diff --git a/avro/src/serde/deser_schema/map.rs b/avro/src/serde/deser_schema/map.rs
new file mode 100644
index 0000000..53b7ed5
--- /dev/null
+++ b/avro/src/serde/deser_schema/map.rs
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::borrow::Borrow;
+use std::io::Read;
+
+use serde::de::MapAccess;
+
+use super::Config;
+use crate::serde::deser_schema::union::UnionDeserializer;
+use crate::{
+ Error, Schema, schema::MapSchema, serde::deser_schema::SchemaAwareDeserializer, util::zag_i32,
+};
+
+pub struct MapDeserializer<'s, 'r, R: Read, S: Borrow<Schema>> {
+ reader: &'r mut R,
+ schema: &'s MapSchema,
+ config: Config<'s, S>,
+ state: State,
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> MapDeserializer<'s, 'r, R, S> {
+ pub fn new(reader: &'r mut R, schema: &'s MapSchema, config: Config<'s, S>) -> Self {
+ Self {
+ reader,
+ schema,
+ config,
+ state: State::EndOfBlock,
+ }
+ }
+}
+
+enum State {
+ EndOfBlock,
+ ReadingKey(u32),
+ ReadingValue(u32),
+ Finished,
+}
+
+impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> MapAccess<'de> for MapDeserializer<'s, 'r, R, S> {
+ type Error = Error;
+
+ fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Self::Error>
+ where
+ K: serde::de::DeserializeSeed<'de>,
+ {
+ match self.state {
+ State::EndOfBlock => {
+ let remaining = zag_i32(self.reader)?;
+ if remaining < 0 {
+ let _bytes = zag_i32(self.reader)?;
+ }
+ if remaining == 0 {
+ self.state = State::Finished
+ } else {
+ self.state = State::ReadingKey(remaining.unsigned_abs())
+ }
+ self.next_key_seed(seed)
+ }
+ State::ReadingKey(remaining) => {
+ let v = seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ &Schema::String,
+ self.config,
+ )?)?;
+
+ self.state = State::ReadingValue(remaining);
+
+ Ok(Some(v))
+ }
+ State::Finished => Ok(None),
+ State::ReadingValue(_) => {
+ panic!("`next_key_seed` and `next_value_seed` where called in the wrong error")
+ }
+ }
+ }
+
+ fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::DeserializeSeed<'de>,
+ {
+ let State::ReadingValue(mut remaining) = self.state else {
+ panic!("`next_key_seed` and `next_value_seed` where called in the wrong error")
+ };
+ let v = match self.schema.types.as_ref() {
+ Schema::Union(union) => {
+ seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
+ }
+ schema => seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?,
+ };
+
+ remaining -= 1;
+ if remaining == 0 {
+ self.state = State::EndOfBlock;
+ } else {
+ self.state = State::ReadingKey(remaining);
+ }
+
+ Ok(v)
+ }
+}
diff --git a/avro/src/serde/deser_schema/mod.rs b/avro/src/serde/deser_schema/mod.rs
new file mode 100644
index 0000000..0e07518
--- /dev/null
+++ b/avro/src/serde/deser_schema/mod.rs
@@ -0,0 +1,667 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod array;
+mod enums;
+mod map;
+mod record;
+mod tuple;
+mod union;
+
+use std::borrow::Borrow;
+use std::collections::HashMap;
+use std::fmt::{Debug, Formatter};
+use std::io::Read;
+
+use serde::Deserializer;
+
+use crate::schema::Name;
+use crate::serde::deser_schema::enums::PlainEnumAccess;
+use crate::serde::deser_schema::tuple::{
+ ManyTupleDeserializer, OneTupleDeserializer, UnitTupleDeserializer,
+};
+use crate::serde::deser_schema::union::UnionDeserializer;
+use crate::{
+ Error, Schema,
+ decode::decode_len,
+ error::Details,
+ schema::{DecimalSchema, InnerDecimalSchema, SchemaKind, UuidSchema},
+ serde::deser_schema::{
+ array::ArrayDeserializer, map::MapDeserializer, record::RecordDeserializer,
+ },
+ util::{zag_i32, zag_i64},
+};
+
+#[derive(Debug)]
+pub struct Config<'s, S: Borrow<Schema>> {
+ /// All names that can be referenced in the schema being used for serialisation.
+ pub names: &'s HashMap<Name, S>,
+ /// Should `Serialize` implementations pick a human-readable format.
+ ///
+ /// It is recommended to set this to `false` as it results in compacter output.
+ pub human_readable: bool,
+}
+
+impl<'s, S: Borrow<Schema>> Copy for Config<'s, S> {}
+impl<'s, S: Borrow<Schema>> Clone for Config<'s, S> {
+ fn clone(&self) -> Self {
+ Self {
+ names: self.names,
+ human_readable: self.human_readable,
+ }
+ }
+}
+
+pub struct SchemaAwareDeserializer<'s, 'r, R: Read, S: Borrow<Schema>> {
+ reader: &'r mut R,
+ schema: &'s Schema,
+ config: Config<'s, S>,
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> Debug for SchemaAwareDeserializer<'s, 'r, R, S> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("SchemaAwareDeserializer")
+ .field("schema", &self.schema)
+ .finish()
+ }
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> SchemaAwareDeserializer<'s, 'r, R, S> {
+ pub fn new(
+ reader: &'r mut R,
+ schema: &'s Schema,
+ config: Config<'s, S>,
+ ) -> Result<Self, Error> {
+ if let Schema::Ref { name } = schema {
+ let schema = config
+ .names
+ .get(name)
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?
+ .borrow();
+ Self::new(reader, schema, config)
+ } else {
+ Ok(Self {
+ reader,
+ schema,
+ config,
+ })
+ }
+ }
+
+ fn error(&self, ty: &'static str, error: impl Into<String>) -> Error {
+ Error::new(Details::DeserializeValueWithSchema {
+ value_type: ty,
+ value: error.into(),
+ schema: self.schema.clone(),
+ })
+ }
+
+ /// Create a new deserializer with the existing reader and config.
+ ///
+ /// This will resolve the schema if it is a reference.
+ fn with_different_schema(mut self, schema: &'s Schema) -> Result<Self, Error> {
+ let schema = if let Schema::Ref { name } = schema {
+ self.config
+ .names
+ .get(name)
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?
+ .borrow()
+ } else {
+ schema
+ };
+ self.schema = schema;
+ Ok(self)
+ }
+
+ fn read_int(&mut self, original_ty: &'static str) -> Result<i32, Error> {
+ match self.schema {
+ Schema::Int | Schema::Date | Schema::TimeMillis => zag_i32(self.reader),
+ _ => Err(self.error(
+ original_ty,
+ "Expected Schema::Int | Schema::Date | Schema::TimeMillis",
+ )),
+ }
+ }
+
+ fn read_long(&mut self, original_ty: &'static str) -> Result<i64, Error> {
+ match self.schema {
+ Schema::Long | Schema::TimeMicros | Schema::TimestampMillis | Schema::TimestampMicros
+ | Schema::TimestampNanos | Schema::LocalTimestampMillis | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos => zag_i64(self.reader),
+ _ => Err(self.error(
+ original_ty,
+ "Expected Schema::Long | Schema::TimeMicros | Schema::{,Local}Timestamp{Millis,Micros,Nanos}",
+ )),
+ }
+ }
+
+ fn read_string(&mut self) -> Result<String, Error> {
+ let bytes = self.read_bytes_with_len()?;
+ Ok(String::from_utf8(bytes).map_err(Details::ConvertToUtf8)?)
+ }
+
+ fn read_bytes_with_len(&mut self) -> Result<Vec<u8>, Error> {
+ let length = decode_len(self.reader)?;
+ self.read_bytes(length)
+ }
+
+ fn read_bytes(&mut self, length: usize) -> Result<Vec<u8>, Error> {
+ let mut buf = vec![0; length];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ Ok(buf)
+ }
+}
+
+const DESERIALIZE_ANY: &str = "This value is compared by pointer value";
+const DESERIALIZE_ANY_FIELDS: &[&str] = &[];
+
+impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
+ for SchemaAwareDeserializer<'s, 'r, R, S>
+{
+ type Error = Error;
+
+ fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ match self.schema {
+ Schema::Null => self.deserialize_unit(visitor),
+ Schema::Boolean => self.deserialize_bool(visitor),
+ Schema::Int | Schema::Date | Schema::TimeMillis => self.deserialize_i32(visitor),
+ Schema::Long
+ | Schema::TimeMicros
+ | Schema::TimestampMillis
+ | Schema::TimestampMicros
+ | Schema::TimestampNanos
+ | Schema::LocalTimestampMillis
+ | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos => self.deserialize_i64(visitor),
+ Schema::Float => self.deserialize_f32(visitor),
+ Schema::Double => self.deserialize_f64(visitor),
+ Schema::Bytes
+ | Schema::Fixed(_)
+ | Schema::Decimal(_)
+ | Schema::BigDecimal
+ | Schema::Uuid(UuidSchema::Fixed(_))
+ | Schema::Duration(_) => self.deserialize_byte_buf(visitor),
+ Schema::String | Schema::Uuid(UuidSchema::String) => self.deserialize_string(visitor),
+ Schema::Array(_) => self.deserialize_seq(visitor),
+ Schema::Map(_) => self.deserialize_map(visitor),
+ Schema::Union(union) => {
+ UnionDeserializer::new(self.reader, union, self.config)?.deserialize_any(visitor)
+ }
+ Schema::Record(schema) => {
+ if schema.attributes.get("org.apache.avro.rust.tuple")
+ == Some(&serde_json::Value::Bool(true))
+ {
+ // This is needed because we can't tell the difference between a tuple and struct.
+ // And a tuple needs to be deserialized as a sequence
+ self.deserialize_tuple(schema.fields.len(), visitor)
+ } else {
+ self.deserialize_struct(DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, visitor)
+ }
+ }
+ Schema::Enum(_) => {
+ self.deserialize_enum(DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, visitor)
+ }
+ Schema::Ref { .. } => unreachable!("References are resolved on deserializer creation"),
+ Schema::Uuid(UuidSchema::Bytes) => panic!("Unsupported"),
+ }
+ }
+
+ fn deserialize_bool<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ match self.schema {
+ Schema::Boolean => {
+ let mut buf = [0];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ // TODO: The TryFrom implementation wasn't working??
+ let boolean = match buf[0] {
+ 0 => false,
+ 1 => true,
+ _ => {
+ return Err(
+ self.error("bool", format!("{} is not a valid boolean", buf[0]))
+ );
+ }
+ };
+ visitor.visit_bool(boolean)
+ }
+ _ => Err(self.error("bool", "Expected a Schema::Boolean")),
+ }
+ }
+
+ fn deserialize_i8<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_int("i8")?;
+ let value = i8::try_from(int)
+ .map_err(|_| self.error("i8", format!("Could not convert int ({int}) to an i8")))?;
+ visitor.visit_i8(value)
+ }
+
+ fn deserialize_i16<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_int("i16")?;
+ let value = i16::try_from(int)
+ .map_err(|_| self.error("i16", format!("Could not convert int ({int}) to an i16")))?;
+ visitor.visit_i16(value)
+ }
+
+ fn deserialize_i32<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let value = self.read_int("i32")?;
+ visitor.visit_i32(value)
+ }
+
+ fn deserialize_i64<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let value = self.read_long("i64")?;
+ visitor.visit_i64(value)
+ }
+
+ fn deserialize_u8<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_int("u8")?;
+ let value = u8::try_from(int)
+ .map_err(|_| self.error("u8", format!("Could not convert int ({int}) to an u8")))?;
+ visitor.visit_u8(value)
+ }
+
+ fn deserialize_u16<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_int("u16")?;
+ let value = u16::try_from(int)
+ .map_err(|_| self.error("u16", format!("Could not convert int ({int}) to an u16")))?;
+ visitor.visit_u16(value)
+ }
+
+ fn deserialize_u32<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_long("u32")?;
+ let value = u32::try_from(int)
+ .map_err(|_| self.error("u32", format!("Could not convert int ({int}) to an u32")))?;
+ visitor.visit_u32(value)
+ }
+
+ fn deserialize_u64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Fixed(fixed) = self.schema
+ && fixed.size == 8
+ && fixed.name.name() == "u64"
+ {
+ let mut buf = [0; 8];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_u64(u64::from_le_bytes(buf))
+ } else {
+ Err(self.error("u64", r#"Expected Schema::Fixed(name: "u64", size: 8)"#))
+ }
+ }
+
+ fn deserialize_f32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Float = self.schema {
+ let mut buf = [0; 4];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_f32(f32::from_le_bytes(buf))
+ } else {
+ Err(self.error("f32", r#"Expected Schema::Float)"#))
+ }
+ }
+
+ fn deserialize_f64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Double = self.schema {
+ let mut buf = [0; 8];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_f64(f64::from_le_bytes(buf))
+ } else {
+ Err(self.error("f64", r#"Expected Schema::Double)"#))
+ }
+ }
+
+ fn deserialize_char<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::String = self.schema {
+ let string = self.read_string()?;
+ let mut chars = string.chars();
+ if let Some(character) = chars.next() {
+ if chars.next().is_some() {
+ Err(self.error("char", "String contains more than one character"))
+ } else {
+ visitor.visit_char(character)
+ }
+ } else {
+ Err(self.error("char", "String is empty"))
+ }
+ } else {
+ Err(self.error("char", "Expected Schema::String"))
+ }
+ }
+
+ fn deserialize_str<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ self.deserialize_string(visitor)
+ }
+
+ fn deserialize_string<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::String = self.schema {
+ let string = self.read_string()?;
+ visitor.visit_string(string)
+ } else {
+ Err(self.error("string", "Expected Schema::String"))
+ }
+ }
+
+ fn deserialize_bytes<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ self.deserialize_byte_buf(visitor)
+ }
+
+ fn deserialize_byte_buf<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ match self.schema {
+ Schema::Bytes | Schema::BigDecimal | Schema::Decimal(DecimalSchema { inner: InnerDecimalSchema::Bytes, .. }) => {
+ let bytes = self.read_bytes_with_len()?;
+ visitor.visit_byte_buf(bytes)
+ }
+ Schema::Fixed(fixed) | Schema::Decimal(DecimalSchema { inner: InnerDecimalSchema::Fixed(fixed), ..}) | Schema::Uuid(UuidSchema::Fixed(fixed)) | Schema::Duration(fixed) => {
+ let bytes = self.read_bytes(fixed.size)?;
+ visitor.visit_byte_buf(bytes)
+ }
+ _ => Err(self.error("bytes", "Expected Schema::Bytes | Schema::Fixed | Schema::BigDecimal | Schema::Decimal | Schema::Uuid(Fixed) | Schema::Duration"))
+ }
+ }
+
+ fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Union(union) = self.schema
+ && union.variants().len() == 2
+ && let Some(null_index) = union.index().get(&SchemaKind::Null).copied()
+ {
+ let index = zag_i32(self.reader)?;
+ if !(0..=1).contains(&index) {
+ return Err(self.error("option", format!("Invalid union index {index}")));
+ }
+ let index = index as usize;
+ if index == null_index {
+ visitor.visit_none()
+ } else {
+ visitor.visit_some(self.with_different_schema(&union.variants()[index])?)
+ }
+ } else {
+ Err(self.error("option", "Expected Schema::Union([Null, _])"))
+ }
+ }
+
+ fn deserialize_unit<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Null = self.schema {
+ visitor.visit_unit()
+ } else {
+ Err(self.error("unit", "Expected Schema::Null"))
+ }
+ }
+
+ fn deserialize_unit_struct<V>(
+ self,
+ name: &'static str,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Record(record) = self.schema
+ && record.fields.is_empty()
+ && record.name.name() == name
+ {
+ visitor.visit_unit()
+ } else {
+ Err(self.error(
+ "unit struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == 0)"),
+ ))
+ }
+ }
+
+ fn deserialize_newtype_struct<V>(
+ self,
+ name: &'static str,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Record(record) = self.schema
+ && record.fields.len() == 1
+ && record.name.name() == name
+ {
+ visitor.visit_newtype_struct(self.with_different_schema(&record.fields[0].schema)?)
+ } else {
+ Err(self.error(
+ "unit struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == 1)"),
+ ))
+ }
+ }
+
+ fn deserialize_seq<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Array(array) = self.schema {
+ visitor.visit_seq(ArrayDeserializer::new(self.reader, array, self.config))
+ } else {
+ Err(self.error("array", "Expected Schema::Array"))
+ }
+ }
+
+ fn deserialize_tuple<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ println!("deserialize_tuple(len: {len}): {self:?}");
+ match self.schema {
+ Schema::Null if len == 0 => visitor.visit_seq(UnitTupleDeserializer),
+ schema if len == 1 => {
+ visitor.visit_seq(OneTupleDeserializer::new(self.reader, schema, self.config))
+ }
+ Schema::Record(record) if record.fields.len() == len => {
+ visitor.visit_seq(ManyTupleDeserializer::new(self.reader, record, self.config))
+ }
+ _ if len == 0 => Err(self.error("tuple", "Expected Schema::Null for unit tuple")),
+ _ => Err(self.error("tuple", format!("Expected Schema::Record for {len}-tuple"))),
+ }
+ }
+
+ fn deserialize_tuple_struct<V>(
+ self,
+ name: &'static str,
+ len: usize,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Record(record) = self.schema
+ && record.name.name() == name
+ && record.fields.len() == len
+ {
+ visitor.visit_map(RecordDeserializer::new(self.reader, record, self.config))
+ } else {
+ Err(self.error(
+ "tuple struct",
+ format!("Expected Schema::Record(fields.len() == {len})"),
+ ))
+ }
+ }
+
+ fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ match self.schema {
+ Schema::Map(map) => {
+ visitor.visit_map(MapDeserializer::new(self.reader, map, self.config))
+ }
+ Schema::Record(record) => {
+ // Needed for flattened structs which are (de)serialized as maps
+ visitor.visit_map(RecordDeserializer::new(self.reader, record, self.config))
+ }
+ _ => Err(self.error("map", "Expected Schema::Map")),
+ }
+ }
+
+ fn deserialize_struct<V>(
+ self,
+ name: &'static str,
+ _fields: &'static [&'static str],
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ println!("deserialize_struct(name: {name}, fields = {_fields:?}): {self:?}");
+ if let Schema::Record(record) = self.schema
+ && record.name.name() == name
+ {
+ visitor.visit_map(RecordDeserializer::new(self.reader, record, self.config))
+ } else {
+ Err(self.error("struct", format!("Expected Schema::Record(name: {name})")))
+ }
+ }
+
+ fn deserialize_enum<V>(
+ self,
+ name: &'static str,
+ variants: &'static [&'static str],
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ println!("deserialize_enum(name: {name}, variants: {variants:?}): {self:?}");
+ match self.schema {
+ Schema::Enum(schema) => visitor.visit_enum(PlainEnumAccess::new(self.reader, schema)),
+ _ => panic!(
+ "deserializing enum, name: {name}, variants: {variants:#?}, {:?}",
+ self.schema
+ ),
+ }
+ }
+
+ fn deserialize_i128<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Fixed(fixed) = self.schema
+ && fixed.size == 16
+ && fixed.name.name() == "i128"
+ {
+ let mut buf = [0; 16];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_i128(i128::from_le_bytes(buf))
+ } else {
+ Err(self.error("i128", r#"Expected Schema::Fixed(name: "i128", size: 16)"#))
+ }
+ }
+
+ fn deserialize_u128<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Fixed(fixed) = self.schema
+ && fixed.size == 16
+ && fixed.name.name() == "u128"
+ {
+ let mut buf = [0; 16];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_u128(u128::from_le_bytes(buf))
+ } else {
+ Err(self.error("u128", r#"Expected Schema::Fixed(name: "u128", size: 16)"#))
+ }
+ }
+
+ fn is_human_readable(&self) -> bool {
+ self.config.human_readable
+ }
+
+ fn deserialize_identifier<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ panic!("deserialize_identifier: {:?}", self.schema);
+ }
+
+ fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ // TODO: We can probably do something more efficient, but that might need the Seek trait bound
+ self.deserialize_any(visitor)
+ }
+}
diff --git a/avro/src/serde/deser_schema/record.rs b/avro/src/serde/deser_schema/record.rs
new file mode 100644
index 0000000..055dc37
--- /dev/null
+++ b/avro/src/serde/deser_schema/record.rs
@@ -0,0 +1,359 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::borrow::Borrow;
+use std::fmt::{Debug, Formatter};
+use std::io::Read;
+
+use serde::{Deserializer, de::MapAccess};
+
+use super::Config;
+use crate::serde::deser_schema::union::UnionDeserializer;
+use crate::{
+ Error, Schema, error::Details, schema::RecordSchema,
+ serde::deser_schema::SchemaAwareDeserializer,
+};
+
+pub struct RecordDeserializer<'s, 'r, R: Read, S: Borrow<Schema>> {
+ reader: &'r mut R,
+ schema: &'s RecordSchema,
+ config: Config<'s, S>,
+ current_field: State,
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> Debug for RecordDeserializer<'s, 'r, R, S> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("RecordDeserializer")
+ .field("schema", &self.schema)
+ .field("current_field", &self.current_field)
+ .finish()
+ }
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> RecordDeserializer<'s, 'r, R, S> {
+ pub fn new(reader: &'r mut R, schema: &'s RecordSchema, config: Config<'s, S>) -> Self {
+ Self {
+ reader,
+ schema,
+ config,
+ current_field: State::Key(0),
+ }
+ }
+}
+
+#[derive(Debug)]
+enum State {
+ Key(usize),
+ Value(usize),
+}
+
+impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> MapAccess<'de> for RecordDeserializer<'s, 'r, R, S> {
+ type Error = Error;
+
+ fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Self::Error>
+ where
+ K: serde::de::DeserializeSeed<'de>,
+ {
+ println!("next_key_seed: {self:?}");
+ let State::Key(index) = self.current_field else {
+ panic!("`next_key_seed` and `next_value_seed` where called in the wrong error")
+ };
+ if index >= self.schema.fields.len() {
+ // Finished reading this record
+ Ok(None)
+ } else {
+ let v = seed.deserialize(FieldName {
+ name: &self.schema.fields[index].name,
+ })?;
+ self.current_field = State::Value(index);
+ Ok(Some(v))
+ }
+ }
+
+ fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::DeserializeSeed<'de>,
+ {
+ println!("next_value_seed: {self:?}");
+ let State::Value(index) = self.current_field else {
+ panic!("`next_key_seed` and `next_value_seed` where called in the wrong error")
+ };
+ let v = match &self.schema.fields[index].schema {
+ Schema::Union(union) => {
+ seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
+ }
+ schema => seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?,
+ };
+ self.current_field = State::Key(index + 1);
+ Ok(v)
+ }
+
+ fn size_hint(&self) -> Option<usize> {
+ match self.current_field {
+ State::Key(index) | State::Value(index) => Some(self.schema.fields.len() - index),
+ }
+ }
+}
+
+/// "Deserializer" for the field name
+struct FieldName<'s> {
+ name: &'s str,
+}
+
+impl<'de, 's> Deserializer<'de> for FieldName<'s> {
+ type Error = Error;
+
+ fn deserialize_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_any".into()).into())
+ }
+
+ fn deserialize_bool<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_bool".into()).into())
+ }
+
+ fn deserialize_i8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_i8".into()).into())
+ }
+
+ fn deserialize_i16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_i16".into()).into())
+ }
+
+ fn deserialize_i32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_i32".into()).into())
+ }
+
+ fn deserialize_i64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_i64".into()).into())
+ }
+
+ fn deserialize_u8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_u8".into()).into())
+ }
+
+ fn deserialize_u16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_u16".into()).into())
+ }
+
+ fn deserialize_u32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_u32".into()).into())
+ }
+
+ fn deserialize_u64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_u64".into()).into())
+ }
+
+ fn deserialize_f32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_f32".into()).into())
+ }
+
+ fn deserialize_f64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_f64".into()).into())
+ }
+
+ fn deserialize_char<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_char".into()).into())
+ }
+
+ fn deserialize_str<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_str".into()).into())
+ }
+
+ fn deserialize_string<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_string".into()).into())
+ }
+
+ fn deserialize_bytes<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_bytes".into()).into())
+ }
+
+ fn deserialize_byte_buf<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_byte_buf".into()).into())
+ }
+
+ fn deserialize_option<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_option".into()).into())
+ }
+
+ fn deserialize_unit<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_unit".into()).into())
+ }
+
+ fn deserialize_unit_struct<V>(
+ self,
+ name: &'static str,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!("deserialize_unit_struct(name: {name})")).into())
+ }
+
+ fn deserialize_newtype_struct<V>(
+ self,
+ name: &'static str,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!("deserialize_newtype_struct(name: {name})")).into())
+ }
+
+ fn deserialize_seq<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_seq".into()).into())
+ }
+
+ fn deserialize_tuple<V>(self, len: usize, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!("deserialize_tuple(len: {len})")).into())
+ }
+
+ fn deserialize_tuple_struct<V>(
+ self,
+ name: &'static str,
+ len: usize,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!(
+ "deserialize_tuple_struct(name: {name}, len: {len})"
+ ))
+ .into())
+ }
+
+ fn deserialize_map<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_map".into()).into())
+ }
+
+ fn deserialize_struct<V>(
+ self,
+ name: &'static str,
+ fields: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!(
+ "deserialize_struct(name: {name}, fields: {fields:?})"
+ ))
+ .into())
+ }
+
+ fn deserialize_enum<V>(
+ self,
+ name: &'static str,
+ variants: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!(
+ "deserialize_enum(name: {name}, variants: {variants:?})"
+ ))
+ .into())
+ }
+
+ fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ println!("deserializing_identifier: {}", self.name);
+ visitor.visit_str(self.name)
+ }
+
+ fn deserialize_ignored_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_ignored_any".into()).into())
+ }
+}
diff --git a/avro/src/serde/deser_schema/tuple.rs b/avro/src/serde/deser_schema/tuple.rs
new file mode 100644
index 0000000..6b9284d
--- /dev/null
+++ b/avro/src/serde/deser_schema/tuple.rs
@@ -0,0 +1,112 @@
+use crate::schema::RecordSchema;
+use crate::serde::deser_schema::union::UnionDeserializer;
+use crate::serde::deser_schema::{Config, SchemaAwareDeserializer};
+use crate::{Error, Schema};
+use serde::de::{DeserializeSeed, SeqAccess};
+use std::borrow::Borrow;
+use std::io::Read;
+
+pub struct ManyTupleDeserializer<'s, 'r, R: Read, S: Borrow<Schema>> {
+ reader: &'r mut R,
+ schema: &'s RecordSchema,
+ config: Config<'s, S>,
+ current_field: usize,
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> ManyTupleDeserializer<'s, 'r, R, S> {
+ pub fn new(reader: &'r mut R, schema: &'s RecordSchema, config: Config<'s, S>) -> Self {
+ Self {
+ reader,
+ schema,
+ config,
+ current_field: 0,
+ }
+ }
+}
+
+impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> SeqAccess<'de>
+ for ManyTupleDeserializer<'s, 'r, R, S>
+{
+ type Error = Error;
+
+ fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>
+ where
+ T: DeserializeSeed<'de>,
+ {
+ if self.current_field < self.schema.fields.len() {
+ let schema = &self.schema.fields[self.current_field].schema;
+ let v = match schema {
+ Schema::Union(union) => {
+ seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
+ }
+ schema => seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?,
+ };
+ self.current_field += 1;
+ Ok(Some(v))
+ } else {
+ Ok(None)
+ }
+ }
+}
+
+pub struct UnitTupleDeserializer;
+impl<'de> SeqAccess<'de> for UnitTupleDeserializer {
+ type Error = Error;
+
+ fn next_element_seed<T>(&mut self, _seed: T) -> Result<Option<T::Value>, Self::Error>
+ where
+ T: DeserializeSeed<'de>,
+ {
+ Ok(None)
+ }
+}
+
+pub struct OneTupleDeserializer<'s, 'r, R: Read, S: Borrow<Schema>> {
+ reader: &'r mut R,
+ schema: &'s Schema,
+ config: Config<'s, S>,
+ field_read: bool,
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> OneTupleDeserializer<'s, 'r, R, S> {
+ pub fn new(reader: &'r mut R, schema: &'s Schema, config: Config<'s, S>) -> Self {
+ Self {
+ reader,
+ schema,
+ config,
+ field_read: false,
+ }
+ }
+}
+
+impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> SeqAccess<'de>
+ for OneTupleDeserializer<'s, 'r, R, S>
+{
+ type Error = Error;
+
+ fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>
+ where
+ T: DeserializeSeed<'de>,
+ {
+ if self.field_read {
+ Ok(None)
+ } else {
+ let v = match self.schema {
+ Schema::Union(union) => {
+ seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
+ }
+ schema => seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?,
+ };
+ self.field_read = true;
+ Ok(Some(v))
+ }
+ }
+}
diff --git a/avro/src/serde/deser_schema/union.rs b/avro/src/serde/deser_schema/union.rs
new file mode 100644
index 0000000..658eb2e
--- /dev/null
+++ b/avro/src/serde/deser_schema/union.rs
@@ -0,0 +1,450 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::borrow::Borrow;
+use std::fmt::{Debug, Formatter};
+use std::io::Read;
+
+use serde::de::Visitor;
+
+use super::{Config, DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, SchemaAwareDeserializer};
+use crate::error::Details;
+use crate::schema::{SchemaKind, UuidSchema};
+use crate::util::zag_i32;
+use crate::{Error, Schema, schema::UnionSchema};
+
+pub struct UnionDeserializer<'s, 'r, R: Read, S: Borrow<Schema>> {
+ reader: &'r mut R,
+ schema: &'s UnionSchema,
+ config: Config<'s, S>,
+ variant: &'s Schema,
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> Debug for UnionDeserializer<'s, 'r, R, S> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("UnionDeserializer")
+ .field("schema", self.schema)
+ .field("variant", &self.variant)
+ .finish()
+ }
+}
+
+impl<'s, 'r, R: Read, S: Borrow<Schema>> UnionDeserializer<'s, 'r, R, S> {
+ pub fn new(
+ reader: &'r mut R,
+ schema: &'s UnionSchema,
+ config: Config<'s, S>,
+ ) -> Result<Self, Error> {
+ let index = zag_i32(reader)?;
+ let variant =
+ schema
+ .variants()
+ .get(index as usize)
+ .ok_or_else(|| Details::GetUnionVariant {
+ index: index as i64,
+ num_variants: schema.variants().len(),
+ })?;
+ Ok(Self {
+ reader,
+ schema,
+ config,
+ variant,
+ })
+ }
+
+ fn error(&self, ty: &'static str, error: impl Into<String>) -> Error {
+ Error::new(Details::DeserializeValueWithSchema {
+ value_type: ty,
+ value: error.into(),
+ schema: Schema::Union(self.schema.clone()),
+ })
+ }
+}
+
+impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> serde::Deserializer<'de>
+ for UnionDeserializer<'s, 'r, R, S>
+{
+ type Error = Error;
+
+ fn deserialize_any<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_any: {self:?}");
+ match self.variant {
+ Schema::Null => visitor.visit_unit(),
+ Schema::Boolean => self.deserialize_bool(visitor),
+ Schema::Int | Schema::Date | Schema::TimeMillis => self.deserialize_i32(visitor),
+ Schema::Long
+ | Schema::TimeMicros
+ | Schema::TimestampMillis
+ | Schema::TimestampMicros
+ | Schema::TimestampNanos
+ | Schema::LocalTimestampMillis
+ | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos => self.deserialize_i64(visitor),
+ Schema::Float => self.deserialize_f32(visitor),
+ Schema::Double => self.deserialize_f64(visitor),
+ Schema::Bytes
+ | Schema::Fixed(_)
+ | Schema::Decimal(_)
+ | Schema::BigDecimal
+ | Schema::Uuid(UuidSchema::Fixed(_))
+ | Schema::Duration(_) => self.deserialize_byte_buf(visitor),
+ Schema::String | Schema::Uuid(UuidSchema::String) => self.deserialize_string(visitor),
+ Schema::Array(_) => self.deserialize_seq(visitor),
+ Schema::Map(_) => self.deserialize_map(visitor),
+ Schema::Record(schema) => {
+ if schema.attributes.get("org.apache.avro.rust.tuple")
+ == Some(&serde_json::Value::Bool(true))
+ {
+ // This is needed because we can't tell the difference between a tuple and struct.
+ // And a tuple needs to be deserialized as a sequence
+ self.deserialize_tuple(schema.fields.len(), visitor)
+ } else {
+ self.deserialize_struct(DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, visitor)
+ }
+ }
+ Schema::Enum(_) => {
+ self.deserialize_enum(DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, visitor)
+ }
+ Schema::Ref { name } => {
+ let schema = self
+ .config
+ .names
+ .get(name)
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?
+ .borrow();
+ self.variant = schema;
+ self.deserialize_any(visitor)
+ }
+ Schema::Union(_) => Err(self.error("any", "Nested unions are not supported")),
+ Schema::Uuid(UuidSchema::Bytes) => panic!("Unsupported"),
+ }
+ }
+
+ fn deserialize_bool<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_bool: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_bool(visitor)
+ }
+
+ fn deserialize_i8<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_i8: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_i8(visitor)
+ }
+
+ fn deserialize_i16<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_i16: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_i16(visitor)
+ }
+
+ fn deserialize_i32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_i32: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_i32(visitor)
+ }
+
+ fn deserialize_i64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_i64: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_i64(visitor)
+ }
+
+ fn deserialize_i128<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_i128: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_i128(visitor)
+ }
+
+ fn deserialize_u8<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_u8: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_u8(visitor)
+ }
+
+ fn deserialize_u16<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_u16: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_u16(visitor)
+ }
+
+ fn deserialize_u32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_u32: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_u32(visitor)
+ }
+
+ fn deserialize_u64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_u64: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_u64(visitor)
+ }
+
+ fn deserialize_u128<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_u128: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_u128(visitor)
+ }
+
+ fn deserialize_f32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_f32: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_f32(visitor)
+ }
+
+ fn deserialize_f64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_f64: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_f64(visitor)
+ }
+
+ fn deserialize_char<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_char: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_char(visitor)
+ }
+
+ fn deserialize_str<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_str: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_str(visitor)
+ }
+
+ fn deserialize_string<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_string: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_string(visitor)
+ }
+
+ fn deserialize_bytes<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_bytes: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_bytes(visitor)
+ }
+
+ fn deserialize_byte_buf<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_byte_buf: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_byte_buf(visitor)
+ }
+
+ fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_option: {self:?}");
+ if self.schema.variants().len() == 2 && self.schema.index().get(&SchemaKind::Null).is_some()
+ {
+ match self.variant {
+ Schema::Null => visitor.visit_none(),
+ schema => visitor.visit_some(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?),
+ }
+ } else {
+ Err(self.error(
+ "option",
+ "Expected Schema::Union(variants.contains(Schema::Null), self.variants.len() == 2)",
+ ))
+ }
+ }
+
+ fn deserialize_unit<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_unit: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_unit(visitor)
+ }
+
+ fn deserialize_unit_struct<V>(
+ self,
+ name: &'static str,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_unit_struct: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_unit_struct(name, visitor)
+ }
+
+ fn deserialize_newtype_struct<V>(
+ self,
+ name: &'static str,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_newtype_struct: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_newtype_struct(name, visitor)
+ }
+
+ fn deserialize_seq<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_seq: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_seq(visitor)
+ }
+
+ fn deserialize_tuple<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_tuple: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_tuple(len, visitor)
+ }
+
+ fn deserialize_tuple_struct<V>(
+ self,
+ name: &'static str,
+ len: usize,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_tuple_struct: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_tuple_struct(name, len, visitor)
+ }
+
+ fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_map: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_map(visitor)
+ }
+
+ fn deserialize_struct<V>(
+ self,
+ name: &'static str,
+ fields: &'static [&'static str],
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_struct: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_struct(name, fields, visitor)
+ }
+
+ fn deserialize_enum<V>(
+ self,
+ name: &'static str,
+ variants: &'static [&'static str],
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_enum: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_enum(name, variants, visitor)
+ }
+
+ fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_identifier: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_identifier(visitor)
+ }
+
+ fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ // TODO: We can do something more efficient here
+ println!("deserialize_ignored_any: {self:?}");
+ self.deserialize_any(visitor)
+ }
+
+ fn is_human_readable(&self) -> bool {
+ self.config.human_readable
+ }
+}
diff --git a/avro/src/serde/mod.rs b/avro/src/serde/mod.rs
index b3bfd2a..96cf620 100644
--- a/avro/src/serde/mod.rs
+++ b/avro/src/serde/mod.rs
@@ -109,6 +109,7 @@
mod de;
mod derive;
+pub mod deser_schema;
mod ser;
pub(crate) mod ser_schema;
mod util;
@@ -117,7 +118,10 @@ mod with;
pub use de::from_value;
pub use derive::{AvroSchema, AvroSchemaComponent};
pub use ser::to_value;
-pub use with::{bytes, bytes_opt, fixed, fixed_opt, slice, slice_opt};
+pub use with::{
+ array, array_opt, bigdecimal, bigdecimal_opt, bytes, bytes_opt, fixed, fixed_opt, slice,
+ slice_opt,
+};
#[doc(hidden)]
pub use derive::get_record_fields_in_ctxt;
diff --git a/avro/src/serde/ser_schema/array.rs b/avro/src/serde/ser_schema/array.rs
new file mode 100644
index 0000000..a7961af
--- /dev/null
+++ b/avro/src/serde/ser_schema/array.rs
@@ -0,0 +1,190 @@
+use std::io::Write;
+
+use serde::ser::SerializeSeq;
+
+use super::{Config, SchemaAwareSerializer};
+use crate::{Error, encode::encode_int, error::Details, schema::ArraySchema};
+
+#[expect(
+ private_interfaces,
+ reason = "Direct and Buffered should not be used directly"
+)]
+pub enum ArraySerializer<'s, 'w, W: Write> {
+ Direct(DirectArraySerializer<'s, 'w, W>),
+ Buffered(BufferedArraySerializer<'s, 'w, W>),
+}
+
+impl<'s, 'w, W: Write> ArraySerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ schema: &'s ArraySchema,
+ config: Config<'s>,
+ len: Option<usize>,
+ bytes_written: Option<usize>,
+ ) -> Result<Self, Error> {
+ if let Some(len) = len
+ && config.target_block_size.is_none()
+ {
+ Ok(Self::Direct(DirectArraySerializer::new(
+ writer,
+ schema,
+ config,
+ len,
+ bytes_written.unwrap_or(0),
+ )?))
+ } else {
+ let target_block_size = config.target_block_size.unwrap_or(1024);
+ Ok(Self::Buffered(BufferedArraySerializer::new(
+ writer,
+ schema,
+ config,
+ target_block_size,
+ bytes_written.unwrap_or(0),
+ )))
+ }
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeSeq for ArraySerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ match self {
+ ArraySerializer::Direct(direct) => direct.serialize_element(value),
+ ArraySerializer::Buffered(buffered) => buffered.serialize_element(value),
+ }
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ match self {
+ ArraySerializer::Direct(direct) => direct.end(),
+ ArraySerializer::Buffered(buffered) => buffered.end(),
+ }
+ }
+}
+
+struct DirectArraySerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ array: &'s ArraySchema,
+ config: Config<'s>,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> DirectArraySerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ array: &'s ArraySchema,
+ config: Config<'s>,
+ len: usize,
+ mut bytes_written: usize,
+ ) -> Result<Self, Error> {
+ if len != 0 {
+ // .end() always writes the zero block, so we only want to write
+ // the size for arrays that have at least one element
+ bytes_written += encode_int(len as i32, &mut *writer)?;
+ }
+ Ok(Self {
+ writer,
+ array,
+ config,
+ bytes_written,
+ })
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeSeq for DirectArraySerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(self.writer, &self.array.items, self.config)?;
+ self.bytes_written += value.serialize(ser)?;
+ Ok(())
+ }
+
+ fn end(mut self) -> Result<Self::Ok, Self::Error> {
+ self.writer.write_all(&[0]).map_err(Details::WriteBytes)?;
+ self.bytes_written += 1;
+
+ Ok(self.bytes_written)
+ }
+}
+struct BufferedArraySerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ buffer: Vec<u8>,
+ array: &'s ArraySchema,
+ config: Config<'s>,
+ bytes_written: usize,
+ items_written: usize,
+ target_block_size: usize,
+}
+
+impl<'s, 'w, W: Write> BufferedArraySerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ array: &'s ArraySchema,
+ config: Config<'s>,
+ target_block_size: usize,
+ bytes_written: usize,
+ ) -> Self {
+ Self {
+ writer,
+ buffer: Vec::new(),
+ array,
+ config,
+ bytes_written,
+ items_written: 0,
+ target_block_size,
+ }
+ }
+
+ fn write_block(&mut self) -> Result<(), Error> {
+ self.bytes_written += encode_int(0 - (self.items_written as i32), &mut *self.writer)?;
+ self.bytes_written += encode_int(self.buffer.len() as i32, &mut *self.writer)?;
+ self.writer
+ .write_all(&self.buffer)
+ .map_err(Details::WriteBytes)?;
+ self.bytes_written += self.buffer.len();
+ self.buffer.clear();
+ Ok(())
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeSeq for BufferedArraySerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(&mut self.buffer, &self.array.items, self.config)?;
+ value.serialize(ser)?;
+ self.items_written += 1;
+
+ if self.buffer.len() >= self.target_block_size {
+ self.write_block()?;
+ }
+
+ Ok(())
+ }
+
+ fn end(mut self) -> Result<Self::Ok, Self::Error> {
+ if self.items_written > 0 {
+ self.write_block()?;
+ }
+ assert_eq!(self.buffer.len(), 0, "Buffer must be empty at this point");
+
+ encode_int(0, self.writer)?;
+ self.bytes_written += 1;
+
+ Ok(self.bytes_written)
+ }
+}
diff --git a/avro/src/serde/ser_schema/map.rs b/avro/src/serde/ser_schema/map.rs
new file mode 100644
index 0000000..6870d00
--- /dev/null
+++ b/avro/src/serde/ser_schema/map.rs
@@ -0,0 +1,219 @@
+use std::io::Write;
+
+use serde::ser::SerializeMap;
+
+use super::{Config, SchemaAwareSerializer};
+use crate::{Error, Schema, encode::encode_int, error::Details, schema::MapSchema};
+
+#[expect(
+ private_interfaces,
+ reason = "Direct and Buffered should not be used directly"
+)]
+pub enum MapSerializer<'s, 'w, W: Write> {
+ Direct(DirectMapSerializer<'s, 'w, W>),
+ Buffered(BufferedMapSerializer<'s, 'w, W>),
+}
+
+impl<'s, 'w, W: Write> MapSerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ schema: &'s MapSchema,
+ config: Config<'s>,
+ len: Option<usize>,
+ bytes_written: Option<usize>,
+ ) -> Result<Self, Error> {
+ if let Some(len) = len
+ && config.target_block_size.is_none()
+ {
+ Ok(Self::Direct(DirectMapSerializer::new(
+ writer,
+ schema,
+ config,
+ len,
+ bytes_written.unwrap_or(0),
+ )?))
+ } else {
+ let target_block_size = config.target_block_size.unwrap_or(1024);
+ Ok(Self::Buffered(BufferedMapSerializer::new(
+ writer,
+ schema,
+ config,
+ target_block_size,
+ bytes_written.unwrap_or(0),
+ )))
+ }
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeMap for MapSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ match self {
+ MapSerializer::Direct(direct) => direct.serialize_key(key),
+ MapSerializer::Buffered(buffered) => buffered.serialize_key(key),
+ }
+ }
+
+ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ match self {
+ MapSerializer::Direct(direct) => direct.serialize_value(value),
+ MapSerializer::Buffered(buffered) => buffered.serialize_value(value),
+ }
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ match self {
+ MapSerializer::Direct(direct) => direct.end(),
+ MapSerializer::Buffered(buffered) => buffered.end(),
+ }
+ }
+}
+
+struct DirectMapSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ map: &'s MapSchema,
+ config: Config<'s>,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> DirectMapSerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ map: &'s MapSchema,
+ config: Config<'s>,
+ len: usize,
+ mut bytes_written: usize,
+ ) -> Result<Self, Error> {
+ if len != 0 {
+ // .end() always writes the zero block, so we only want to write
+ // the size for maps that have at least one entry
+ bytes_written += encode_int(len as i32, &mut *writer)?;
+ }
+ Ok(Self {
+ writer,
+ map,
+ config,
+ bytes_written,
+ })
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeMap for DirectMapSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(&mut *self.writer, &Schema::String, self.config)?;
+ self.bytes_written += key.serialize(ser)?;
+ Ok(())
+ }
+
+ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(&mut *self.writer, &self.map.types, self.config)?;
+ self.bytes_written += value.serialize(ser)?;
+ Ok(())
+ }
+
+ fn end(mut self) -> Result<Self::Ok, Self::Error> {
+ self.writer.write_all(&[0]).map_err(Details::WriteBytes)?;
+ self.bytes_written += 1;
+
+ Ok(self.bytes_written)
+ }
+}
+
+struct BufferedMapSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ buffer: Vec<u8>,
+ map: &'s MapSchema,
+ config: Config<'s>,
+ bytes_written: usize,
+ items_written: usize,
+ target_block_size: usize,
+}
+
+impl<'s, 'w, W: Write> BufferedMapSerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ map: &'s MapSchema,
+ config: Config<'s>,
+ target_block_size: usize,
+ bytes_written: usize,
+ ) -> Self {
+ Self {
+ writer,
+ buffer: Vec::new(),
+ map,
+ config,
+ bytes_written,
+ items_written: 0,
+ target_block_size,
+ }
+ }
+
+ fn write_block(&mut self) -> Result<(), Error> {
+ self.bytes_written += encode_int(0 - (self.items_written as i32), &mut *self.writer)?;
+ self.bytes_written += encode_int(self.buffer.len() as i32, &mut *self.writer)?;
+ self.writer
+ .write_all(&self.buffer)
+ .map_err(Details::WriteBytes)?;
+ self.bytes_written += self.buffer.len();
+ self.buffer.clear();
+ Ok(())
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeMap for BufferedMapSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(&mut self.buffer, &self.map.types, self.config)?;
+ self.bytes_written += key.serialize(ser)?;
+ Ok(())
+ }
+
+ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(&mut self.buffer, &self.map.types, self.config)?;
+ self.bytes_written += value.serialize(ser)?;
+ self.items_written += 1;
+
+ if self.buffer.len() >= self.target_block_size {
+ self.write_block()?;
+ }
+
+ Ok(())
+ }
+
+ fn end(mut self) -> Result<Self::Ok, Self::Error> {
+ if self.items_written > 0 {
+ self.write_block()?;
+ }
+ assert_eq!(self.buffer.len(), 0, "Buffer must be empty at this point");
+
+ encode_int(0, self.writer)?;
+ self.bytes_written += 1;
+
+ Ok(self.bytes_written)
+ }
+}
diff --git a/avro/src/serde/ser_schema/mod.rs b/avro/src/serde/ser_schema/mod.rs
index 070f562..9244910 100644
--- a/avro/src/serde/ser_schema/mod.rs
+++ b/avro/src/serde/ser_schema/mod.rs
@@ -17,1929 +17,435 @@
//! Logic for serde-compatible schema-aware serialization which writes directly to a writer.
-use crate::schema::{DecimalSchema, InnerDecimalSchema, UuidSchema};
-use crate::{
- bigdecimal::big_decimal_as_bytes,
- encode::{encode_int, encode_long},
- error::{Details, Error},
- schema::{Name, NamesRef, Namespace, RecordField, RecordSchema, Schema},
- serde::util::StringSerializer,
-};
-use bigdecimal::BigDecimal;
-use serde::{Serialize, ser};
-use std::{cmp::Ordering, collections::HashMap, io::Write, str::FromStr};
-
-const COLLECTION_SERIALIZER_ITEM_LIMIT: usize = 1024;
-const COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY: usize = 32;
-const SINGLE_VALUE_INIT_BUFFER_SIZE: usize = 128;
-
-/// The sequence serializer for [`SchemaAwareWriteSerializer`].
+mod array;
+mod map;
+mod record;
+mod tuple;
+mod union;
+
+use crate::encode::{encode_int, encode_long};
+use crate::error::Details;
+use crate::schema::{DecimalSchema, InnerDecimalSchema, NamesRef, SchemaKind, UuidSchema};
+use crate::serde::with::{BytesType, SER_BYTES_TYPE};
+use crate::{Error, Schema};
+use serde::ser::SerializeMap;
+use serde::{Serialize, Serializer};
+use std::fmt::{Debug, Formatter};
+use std::io::Write;
+
+use crate::serde::ser_schema::tuple::{OneTupleSerializer, TupleSerializer, UnitTupleSerializer};
+use array::ArraySerializer;
+use map::MapSerializer;
+use record::RecordSerializer;
+use tuple::ManyTupleSerializer;
+use union::UnionAwareSerializer;
+
+/// Indicate to the serializer that a record field default is being serialized.
///
-/// This may break large arrays up into multiple blocks to avoid having
-/// to obtain the length of the entire array before being able to write any data to the underlying
-/// writer (see the [Data Serialization and Deserialization] for more info).
-/// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization)
-pub struct SchemaAwareWriteSerializeSeq<'a, 's, W: Write> {
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- item_schema: &'s Schema,
- item_buffer_size: usize,
- item_buffers: Vec<Vec<u8>>,
- bytes_written: usize,
-}
-
-impl<'a, 's, W: Write> SchemaAwareWriteSerializeSeq<'a, 's, W> {
- fn new(
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- item_schema: &'s Schema,
- len: Option<usize>,
- ) -> SchemaAwareWriteSerializeSeq<'a, 's, W> {
- SchemaAwareWriteSerializeSeq {
- ser,
- item_schema,
- item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE,
- item_buffers: Vec::with_capacity(
- len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY),
- ),
- bytes_written: 0,
- }
- }
-
- fn write_buffered_items(&mut self) -> Result<(), Error> {
- if !self.item_buffers.is_empty() {
- self.bytes_written +=
- encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?;
- for item in self.item_buffers.drain(..) {
- self.bytes_written += self
- .ser
- .writer
- .write(item.as_slice())
- .map_err(Details::WriteBytes)?;
- }
- }
-
- Ok(())
- }
-
- fn serialize_element<T: ser::Serialize>(&mut self, value: &T) -> Result<(), Error> {
- let mut item_buffer: Vec<u8> = Vec::with_capacity(self.item_buffer_size);
- let mut item_ser = SchemaAwareWriteSerializer::new(
- &mut item_buffer,
- self.item_schema,
- self.ser.names,
- self.ser.enclosing_namespace.clone(),
- );
- value.serialize(&mut item_ser)?;
-
- self.item_buffer_size = std::cmp::max(self.item_buffer_size, item_buffer.len() + 16);
-
- self.item_buffers.push(item_buffer);
-
- if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT {
- self.write_buffered_items()?;
- }
-
- Ok(())
- }
-
- fn end(mut self) -> Result<usize, Error> {
- self.write_buffered_items()?;
- self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?;
-
- Ok(self.bytes_written)
- }
-}
-
-impl<W: Write> ser::SerializeSeq for SchemaAwareWriteSerializeSeq<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- self.serialize_element(&value)
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- self.end()
- }
-}
-
-impl<W: Write> ser::SerializeTuple for SchemaAwareWriteSerializeSeq<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- ser::SerializeSeq::serialize_element(self, value)
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- ser::SerializeSeq::end(self)
- }
-}
-
-/// The map serializer for [`SchemaAwareWriteSerializer`].
-///
-/// This may break large maps up into multiple blocks to avoid having to obtain the size of the entire
-/// map before being able to write any data to the underlying writer
-/// (see [Data Serialization and Deserialization] for more info)
-/// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization)
-pub struct SchemaAwareWriteSerializeMap<'a, 's, W: Write> {
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- item_schema: &'s Schema,
- item_buffer_size: usize,
- item_buffers: Vec<Vec<u8>>,
- bytes_written: usize,
-}
-
-impl<'a, 's, W: Write> SchemaAwareWriteSerializeMap<'a, 's, W> {
- fn new(
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- item_schema: &'s Schema,
- len: Option<usize>,
- ) -> SchemaAwareWriteSerializeMap<'a, 's, W> {
- SchemaAwareWriteSerializeMap {
- ser,
- item_schema,
- item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE,
- item_buffers: Vec::with_capacity(
- len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY),
- ),
- bytes_written: 0,
- }
- }
-
- fn write_buffered_items(&mut self) -> Result<(), Error> {
- if !self.item_buffers.is_empty() {
- self.bytes_written +=
- encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?;
- for item in self.item_buffers.drain(..) {
- self.bytes_written += self
- .ser
- .writer
- .write(item.as_slice())
- .map_err(Details::WriteBytes)?;
- }
- }
-
- Ok(())
- }
-}
-
-impl<W: Write> ser::SerializeMap for SchemaAwareWriteSerializeMap<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- let mut element_buffer: Vec<u8> = Vec::with_capacity(self.item_buffer_size);
- let string_schema = Schema::String;
- let mut key_ser = SchemaAwareWriteSerializer::new(
- &mut element_buffer,
- &string_schema,
- self.ser.names,
- self.ser.enclosing_namespace.clone(),
- );
- key.serialize(&mut key_ser)?;
-
- self.item_buffers.push(element_buffer);
-
- Ok(())
- }
-
- fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- let last_index = self.item_buffers.len() - 1;
- let element_buffer = &mut self.item_buffers[last_index];
- let mut val_ser = SchemaAwareWriteSerializer::new(
- element_buffer,
- self.item_schema,
- self.ser.names,
- self.ser.enclosing_namespace.clone(),
- );
- value.serialize(&mut val_ser)?;
-
- self.item_buffer_size = std::cmp::max(self.item_buffer_size, element_buffer.len() + 16);
-
- if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT {
- self.write_buffered_items()?;
- }
-
- Ok(())
- }
-
- fn end(mut self) -> Result<Self::Ok, Self::Error> {
- self.write_buffered_items()?;
- self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?;
-
- Ok(self.bytes_written)
- }
-}
-
-/// The struct serializer for [`SchemaAwareWriteSerializer`], which can serialize Avro records.
-///
-/// This can accept fields out of order, but doing so incurs a performance penalty, since it requires
-/// buffering serialized values in order to write them to the stream in order.
-pub struct SchemaAwareWriteSerializeStruct<'a, 's, W: Write> {
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- record_schema: &'s RecordSchema,
- /// Fields we received in the wrong order
- field_cache: HashMap<usize, Vec<u8>>,
- /// The current field name when serializing from a map (for `flatten` support).
- map_field_name: Option<String>,
- field_position: usize,
- bytes_written: usize,
-}
-
-impl<'a, 's, W: Write> SchemaAwareWriteSerializeStruct<'a, 's, W> {
- fn new(
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- record_schema: &'s RecordSchema,
- ) -> SchemaAwareWriteSerializeStruct<'a, 's, W> {
- SchemaAwareWriteSerializeStruct {
- ser,
- record_schema,
- field_cache: HashMap::new(),
- map_field_name: None,
- field_position: 0,
- bytes_written: 0,
- }
- }
-
- fn serialize_next_field<T>(
- &mut self,
- field: &RecordField,
- position: usize,
- value: &T,
- ) -> Result<(), Error>
- where
- T: ?Sized + ser::Serialize,
- {
- match self.field_position.cmp(&position) {
- Ordering::Equal => {
- // If we receive fields in order, write them directly to the main writer
- let mut value_ser = SchemaAwareWriteSerializer::new(
- &mut *self.ser.writer,
- &field.schema,
- self.ser.names,
- self.ser.enclosing_namespace.clone(),
- );
- self.bytes_written += value.serialize(&mut value_ser)?;
-
- self.field_position += 1;
- while let Some(bytes) = self.field_cache.remove(&self.field_position) {
- self.ser
- .writer
- .write_all(&bytes)
- .map_err(Details::WriteBytes)?;
- self.bytes_written += bytes.len();
- self.field_position += 1;
- }
- Ok(())
- }
- Ordering::Less => {
- // Current field position is smaller than this field position,
- // so we're still missing at least one field, save this field temporarily
- let mut bytes = Vec::new();
- let mut value_ser = SchemaAwareWriteSerializer::new(
- &mut bytes,
- &field.schema,
- self.ser.names,
- self.ser.enclosing_namespace.clone(),
- );
- value.serialize(&mut value_ser)?;
- if self.field_cache.insert(position, bytes).is_some() {
- Err(Details::FieldNameDuplicate(field.name.clone()).into())
- } else {
- Ok(())
- }
- }
- Ordering::Greater => {
- // Current field position is greater than this field position,
- // so we've already had this field
- Err(Details::FieldNameDuplicate(field.name.clone()).into())
- }
- }
- }
-
- fn end(mut self) -> Result<usize, Error> {
- // Write any fields that are `serde(skip)` or `serde(skip_serializing)`
- while self.field_position != self.record_schema.fields.len() {
- let field_info = &self.record_schema.fields[self.field_position];
- if let Some(bytes) = self.field_cache.remove(&self.field_position) {
- self.ser
- .writer
- .write_all(&bytes)
- .map_err(Details::WriteBytes)?;
- self.bytes_written += bytes.len();
- self.field_position += 1;
- } else if let Some(default) = &field_info.default {
- self.serialize_next_field(field_info, self.field_position, default)
- .map_err(|e| Details::SerializeRecordFieldWithSchema {
- field_name: field_info.name.clone(),
- record_schema: Schema::Record(self.record_schema.clone()),
- error: Box::new(e),
- })?;
- } else {
- return Err(Details::MissingDefaultForSkippedField {
- field_name: field_info.name.clone(),
- schema: Schema::Record(self.record_schema.clone()),
- }
- .into());
- }
- }
-
- debug_assert!(
- self.field_cache.is_empty(),
- "There should be no more unwritten fields at this point: {:?}",
- self.field_cache
- );
- debug_assert!(
- self.map_field_name.is_none(),
- "There should be no field name at this point: field {:?}",
- self.map_field_name
- );
- Ok(self.bytes_written)
- }
-}
-
-impl<W: Write> ser::SerializeStruct for SchemaAwareWriteSerializeStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- if let Some(position) = self.record_schema.lookup.get(key).copied() {
- let field = self
- .record_schema
- .fields
- .get(position)
- .ok_or_else(|| Details::FieldName(String::from(key)))?;
- self.serialize_next_field(field, position, value)
- .map_err(|e| {
- Details::SerializeRecordFieldWithSchema {
- field_name: key.to_string(),
- record_schema: Schema::Record(self.record_schema.clone()),
- error: Box::new(e),
- }
- .into()
- })
- } else {
- Err(Details::FieldName(String::from(key)).into())
- }
- }
-
- fn skip_field(&mut self, key: &'static str) -> Result<(), Self::Error> {
- if let Some(position) = self.record_schema.lookup.get(key).copied() {
- let field = &self.record_schema.fields[position];
- if let Some(default) = &field.default {
- self.serialize_next_field(field, position, default)
- .map_err(|e| {
- Details::SerializeRecordFieldWithSchema {
- field_name: key.to_string(),
- record_schema: Schema::Record(self.record_schema.clone()),
- error: Box::new(e),
- }
- .into()
- })
- } else {
- Err(Details::MissingDefaultForSkippedField {
- field_name: key.to_string(),
- schema: Schema::Record(self.record_schema.clone()),
- }
- .into())
- }
- } else {
- Err(Details::GetField(key.to_string()).into())
- }
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- self.end()
- }
-}
-
-/// This implementation is used to support `#[serde(flatten)]` as that uses [`SerializeMap`] instead of [`SerializeStruct`].
-///
-/// [`SerializeMap`](ser::SerializeMap)
-/// [`SerializeStruct`](ser::SerializeStruct)
-impl<W: Write> ser::SerializeMap for SchemaAwareWriteSerializeStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + Serialize,
- {
- let name = key.serialize(StringSerializer)?;
- let old = self.map_field_name.replace(name);
- debug_assert!(
- old.is_none(),
- "Expected a value instead of a key: old key: {old:?}, new key: {:?}",
- self.map_field_name
- );
- Ok(())
- }
-
- fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + Serialize,
- {
- let key = self.map_field_name.take().ok_or(Details::MapNoKey)?;
- if let Some(position) = self.record_schema.lookup.get(&key).copied() {
- let field = &self.record_schema.fields[position];
- self.serialize_next_field(field, position, value)
- .map_err(|e| {
- Details::SerializeRecordFieldWithSchema {
- field_name: key.to_string(),
- record_schema: Schema::Record(self.record_schema.clone()),
- error: Box::new(e),
- }
- .into()
- })
- } else {
- Err(Details::FieldName(key).into())
- }
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- self.end()
- }
-}
-
-impl<W: Write> ser::SerializeStructVariant for SchemaAwareWriteSerializeStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- ser::SerializeStruct::serialize_field(self, key, value)
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- ser::SerializeStruct::end(self)
- }
-}
-
-/// Map serializer that switches between Struct or Map.
-///
-/// This exists because when `#[serde(flatten)]` is used, struct fields are serialized as a map.
-pub enum SchemaAwareWriteSerializeMapOrStruct<'a, 's, W: Write> {
- Struct(SchemaAwareWriteSerializeStruct<'a, 's, W>),
- Map(SchemaAwareWriteSerializeMap<'a, 's, W>),
-}
-
-impl<W: Write> ser::SerializeMap for SchemaAwareWriteSerializeMapOrStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + Serialize,
- {
- match self {
- Self::Struct(s) => s.serialize_key(key),
- Self::Map(s) => s.serialize_key(key),
- }
- }
-
- fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + Serialize,
- {
- match self {
- Self::Struct(s) => s.serialize_value(value),
- Self::Map(s) => s.serialize_value(value),
- }
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- match self {
- Self::Struct(s) => s.end(),
- Self::Map(s) => s.end(),
- }
- }
-}
-
-/// The tuple struct serializer for [`SchemaAwareWriteSerializer`].
-///
-/// This can serialize to an Avro array, record, or big-decimal.
-/// When serializing to a record, fields must be provided in the correct order, since no names are provided.
-pub enum SchemaAwareWriteSerializeTupleStruct<'a, 's, W: Write> {
- Record(SchemaAwareWriteSerializeStruct<'a, 's, W>),
- Array(SchemaAwareWriteSerializeSeq<'a, 's, W>),
-}
-
-impl<W: Write> SchemaAwareWriteSerializeTupleStruct<'_, '_, W> {
- fn serialize_field<T>(&mut self, value: &T) -> Result<(), Error>
- where
- T: ?Sized + ser::Serialize,
- {
- use SchemaAwareWriteSerializeTupleStruct::*;
- match self {
- Record(_record_ser) => {
- unimplemented!("Tuple struct serialization to record is not supported!");
- }
- Array(array_ser) => array_ser.serialize_element(&value),
- }
- }
-
- fn end(self) -> Result<usize, Error> {
- use SchemaAwareWriteSerializeTupleStruct::*;
- match self {
- Record(record_ser) => record_ser.end(),
- Array(array_ser) => array_ser.end(),
- }
- }
-}
-
-impl<W: Write> ser::SerializeTupleStruct for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- self.serialize_field(&value)
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- self.end()
- }
-}
-
-impl<W: Write> ser::SerializeTupleVariant for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- self.serialize_field(&value)
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- self.end()
- }
-}
-
-/// A [`Serializer`](ser::Serializer) implementation that serializes directly to raw Avro data.
-///
-/// If data does not match with the schema it will return an error.
-///
-/// This does not keep state and can therefore be reused to write to the same writer.
-pub struct SchemaAwareWriteSerializer<'s, W: Write> {
- writer: &'s mut W,
- root_schema: &'s Schema,
- names: &'s NamesRef<'s>,
- enclosing_namespace: Namespace,
-}
-
-impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> {
- /// Create a new [`SchemaAwareWriteSerializer`].
- ///
- /// `writer` is the [`std::fmt::Write`] stream to be written to.
- ///
- /// `schema` is the schema of the value to be written.
- ///
- /// `names` is the mapping of schema names to schemas, to be used for type reference lookups
- ///
- /// `enclosing_namespace` is the enclosing namespace to be used for type reference lookups
- pub fn new(
- writer: &'s mut W,
- schema: &'s Schema,
- names: &'s NamesRef<'s>,
- enclosing_namespace: Namespace,
- ) -> SchemaAwareWriteSerializer<'s, W> {
- SchemaAwareWriteSerializer {
- writer,
- root_schema: schema,
- names,
- enclosing_namespace,
- }
- }
-
- fn get_ref_schema(&self, name: &'s Name) -> Result<&'s Schema, Error> {
- let full_name = name.fully_qualified_name(self.enclosing_namespace.as_deref());
-
- let ref_schema = self.names.get(full_name.as_ref()).copied();
-
- ref_schema.ok_or_else(|| Details::SchemaResolutionError(full_name.as_ref().clone()).into())
- }
-
- fn write_bytes(&mut self, bytes: &[u8]) -> Result<usize, Error> {
- let mut bytes_written: usize = 0;
-
- bytes_written += encode_long(bytes.len() as i64, &mut self.writer)?;
- // write_all() will retry when the error is ErrorKind::Interrupted (happens mostly on network storage)
- self.writer.write_all(bytes).map_err(Details::WriteBytes)?;
- bytes_written += bytes.len();
-
- Ok(bytes_written)
- }
-
- fn serialize_bool_with_schema(&mut self, value: bool, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "bool",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Boolean => self
- .writer
- .write(&[u8::from(value)])
- .map_err(|e| Details::WriteBytes(e).into()),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Boolean => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bool_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "No matching Schema::Bool found in {:?}",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: Bool"))),
- }
- }
-
- fn serialize_i32_with_schema(&mut self, value: i32, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "int (i8 | i16 | i32)",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Int | Schema::TimeMillis | Schema::Date => encode_int(value, &mut self.writer),
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Int
- | Schema::TimeMillis
- | Schema::Date
- | Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_i32_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching int-like schema in {union_schema:?}"
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: Int/Long"))),
- }
- }
-
- fn serialize_i64_with_schema(&mut self, value: i64, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "i64",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Int | Schema::TimeMillis | Schema::Date => {
- let int_value =
- i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?;
- encode_int(int_value, &mut self.writer)
- }
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => encode_long(value, &mut self.writer),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Int
- | Schema::TimeMillis
- | Schema::Date
- | Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_i64_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching int/long-like schema in {:?}",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))),
- }
- }
-
- fn serialize_i128_with_schema(&mut self, value: i128, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "i128",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Fixed(fixed) if fixed.size == 16 && fixed.name.name() == "i128" => {
- self.writer
- .write_all(&value.to_le_bytes())
- .map_err(Details::WriteBytes)?;
- Ok(16)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Fixed(fixed) if fixed.size == 16 && fixed.name.name() == "i128" => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_i128_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- r#"Cannot find a Fixed(size = 16, name = "i128") schema in {:?}"#,
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: i128"))),
- }
- }
-
- fn serialize_u8_with_schema(&mut self, value: u8, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "u8",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Int | Schema::TimeMillis | Schema::Date => {
- encode_int(value as i32, &mut self.writer)
- }
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer),
- Schema::Bytes => self.write_bytes(&[value]),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Int
- | Schema::TimeMillis
- | Schema::Date
- | Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos
- | Schema::Bytes => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_u8_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching Int-like, Long-like or Bytes schema in {union_schema:?}"
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Int"))),
- }
- }
-
- fn serialize_u32_with_schema(&mut self, value: u32, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "unsigned int (u16 | u32)",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Int | Schema::TimeMillis | Schema::Date => {
- let int_value =
- i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?;
- encode_int(int_value, &mut self.writer)
- }
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Int
- | Schema::TimeMillis
- | Schema::Date
- | Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_u32_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching Int-like or Long-like schema in {union_schema:?}"
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))),
- }
- }
-
- fn serialize_u64_with_schema(&mut self, value: u64, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "u64",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Int | Schema::TimeMillis | Schema::Date => {
- let int_value =
- i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?;
- encode_int(int_value, &mut self.writer)
- }
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => {
- let long_value =
- i64::try_from(value).map_err(|cause| create_error(cause.to_string()))?;
- encode_long(long_value, &mut self.writer)
- }
- Schema::Fixed(fixed) if fixed.size == 8 && fixed.name.name() == "u64" => {
- self.writer
- .write_all(&value.to_le_bytes())
- .map_err(Details::WriteBytes)?;
- Ok(8)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Int
- | Schema::TimeMillis
- | Schema::Date
- | Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_u64_with_schema(value, variant_schema);
- }
- Schema::Fixed(fixed) if fixed.size == 8 && fixed.name.name() == "u64" => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_u64_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- r#"Cannot find a matching Int-like, Long-like or Fixed(size = 8, name "u64") schema in {:?}"#,
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: u64"))),
- }
- }
-
- fn serialize_u128_with_schema(&mut self, value: u128, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "u128",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Fixed(fixed) if fixed.size == 16 && fixed.name.name() == "u128" => {
- self.writer
- .write_all(&value.to_le_bytes())
- .map_err(Details::WriteBytes)?;
- Ok(16)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Fixed(fixed) if fixed.size == 16 && fixed.name.name() == "u128" => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_u128_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- r#"Cannot find a Fixed(size = 16, name = "u128") schema in {:?}"#,
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: u128"))),
- }
- }
-
- fn serialize_f32_with_schema(&mut self, value: f32, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "f32",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Float => self
- .writer
- .write(&value.to_le_bytes())
- .map_err(|e| Details::WriteBytes(e).into()),
- Schema::Double => self
- .writer
- .write(&(value as f64).to_le_bytes())
- .map_err(|e| Details::WriteBytes(e).into()),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Float | Schema::Double => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_f32_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a Float schema in {:?}",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Float"))),
- }
- }
-
- fn serialize_f64_with_schema(&mut self, value: f64, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "f64",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Float => self
- .writer
- .write(&(value as f32).to_le_bytes())
- .map_err(|e| Details::WriteBytes(e).into()),
- Schema::Double => self
- .writer
- .write(&value.to_le_bytes())
- .map_err(|e| Details::WriteBytes(e).into()),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Float | Schema::Double => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_f64_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a Double schema in {:?}",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Double"))),
- }
- }
-
- fn serialize_char_with_schema(&mut self, value: char, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "char",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::String | Schema::Bytes => self.write_bytes(String::from(value).as_bytes()),
- Schema::Fixed(fixed) if fixed.size == 4 && fixed.name.name() == "char" => {
- self.writer
- .write_all(&u32::from(value).to_le_bytes())
- .map_err(Details::WriteBytes)?;
- Ok(4)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::String | Schema::Bytes => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_char_with_schema(value, variant_schema);
- }
- Schema::Fixed(fixed) if fixed.size == 4 && fixed.name.name() == "char" => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_char_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- r#"Cannot find a matching String, Bytes or Fixed(size = 4, name = "char") schema in {union_schema:?}"#
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: char"))),
- }
- }
-
- fn serialize_str_with_schema(&mut self, value: &str, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "string",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::String | Schema::Bytes | Schema::Uuid(UuidSchema::String) => {
- self.write_bytes(value.as_bytes())
- }
- Schema::Uuid(UuidSchema::Bytes | UuidSchema::Fixed(_)) => {
- Err(create_error("Expected bytes but got a string. Did you mean to use `Schema::Uuid(UuidSchema::String)` or `utils::serde_set_human_readable(false)`?".to_string()))
- }
- Schema::BigDecimal => {
- // If we get a string for a `BigDecimal` type, expect a display string representation, such as "12.75"
- let decimal_val =
- BigDecimal::from_str(value).map_err(|e| create_error(e.to_string()))?;
- let decimal_bytes = big_decimal_as_bytes(&decimal_val)?;
- self.write_bytes(decimal_bytes.as_slice())
- }
- Schema::Fixed(fixed_schema) => {
- if value.len() == fixed_schema.size {
- self.writer
- .write(value.as_bytes())
- .map_err(|e| Details::WriteBytes(e).into())
- } else {
- Err(create_error(format!(
- "Fixed schema size ({}) does not match the value length ({})",
- fixed_schema.size,
- value.len()
- )))
- }
- }
- Schema::Ref { name } => {
- let ref_schema = self.get_ref_schema(name)?;
- self.serialize_str_with_schema(value, ref_schema)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::String
- | Schema::Bytes
- | Schema::Uuid(UuidSchema::String)
- | Schema::Fixed(_)
- | Schema::Ref { name: _ } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_str_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected one of the union variants {:?}. Got: String",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: String"))),
- }
- }
-
- fn serialize_bytes_with_schema(
- &mut self,
- value: &[u8],
- schema: &Schema,
- ) -> Result<usize, Error> {
- let create_error = |cause: String| {
- use std::fmt::Write;
- let mut v_str = String::with_capacity(value.len());
- for b in value {
- if write!(&mut v_str, "{b:x}").is_err() {
- v_str.push_str("??");
- }
- }
- Error::new(Details::SerializeValueWithSchema {
- value_type: "bytes",
- value: format!("{v_str}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::String | Schema::Bytes | Schema::BigDecimal => self.write_bytes(value),
- Schema::Uuid(UuidSchema::Bytes) => {
- if value.len() == 16 {
- self.write_bytes(value)
- } else {
- Err(create_error(format!(
- "Expected 16 bytes for `Schema::Uuid(Bytes)` but got {} bytes",
- value.len()
- )))
- }
- }
- Schema::Uuid(UuidSchema::String) if value.len() == 16 => {
- Err(create_error("Expected a string, but got 16 bytes. Did you mean to use `Schema::Uuid(UuidSchema::Fixed)` or `utils::serde_set_human_readable(true)`?".to_string()))
- }
- Schema::Fixed(fixed_schema) | Schema::Uuid(UuidSchema::Fixed(fixed_schema)) => {
- if value.len() == fixed_schema.size {
- self.writer
- .write(value)
- .map_err(|e| Details::WriteBytes(e).into())
- } else {
- Err(create_error(format!(
- "Fixed schema size ({}) does not match the value length ({})",
- fixed_schema.size,
- value.len()
- )))
- }
- }
- Schema::Duration(_) => {
- if value.len() == 12 {
- self.writer
- .write(value)
- .map_err(|e| Details::WriteBytes(e).into())
- } else {
- Err(create_error(format!(
- "Duration length must be 12! Got ({})",
- value.len()
- )))
- }
- }
- Schema::Decimal(decimal_schema) => match &decimal_schema.inner {
- InnerDecimalSchema::Bytes => self.write_bytes(value),
- InnerDecimalSchema::Fixed(fixed_schema) => {
- match fixed_schema.size.checked_sub(value.len()) {
- Some(pad) => {
- let pad_val = match value.len() {
- 0 => 0,
- _ => value[0],
- };
- let padding = vec![pad_val; pad];
- let mut bytes_written = self
- .writer
- .write(padding.as_slice())
- .map_err(Details::WriteBytes)?;
- bytes_written +=
- self.writer.write(value).map_err(Details::WriteBytes)?;
- Ok(bytes_written)
- }
- None => Err(Details::CompareFixedSizes {
- size: fixed_schema.size,
- n: value.len(),
- }
- .into()),
- }
- }
- },
- Schema::Ref { name } => {
- let ref_schema = self.get_ref_schema(name)?;
- self.serialize_bytes_with_schema(value, ref_schema)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::String
- | Schema::Bytes
- | Schema::BigDecimal
- | Schema::Decimal(DecimalSchema {
- inner: InnerDecimalSchema::Bytes,
- ..
- })
- | Schema::Ref { name: _ } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bytes_with_schema(value, variant_schema);
- }
- Schema::Uuid(UuidSchema::Bytes) if value.len() == 16 => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bytes_with_schema(value, variant_schema);
- }
- Schema::Fixed(fixed) | Schema::Uuid(UuidSchema::Fixed(fixed))
- if fixed.size == value.len() =>
- {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bytes_with_schema(value, variant_schema);
- }
- Schema::Decimal(DecimalSchema {
- inner: InnerDecimalSchema::Fixed(fixed),
- ..
- }) if fixed.size >= value.len() => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bytes_with_schema(value, variant_schema);
- }
- Schema::Duration(_) if value.len() == 12 => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bytes_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal or Ref schema in {union_schema:?}"
- )))
- }
- unsupported => Err(create_error(format!(
- "Expected String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal, Ref or Union schema. Got: {unsupported}"
- ))),
- }
- }
+/// This is needed because the serializer takes a `&'static str` for the enum name and variant name.
+/// When this value is encountered, the serializer will blindly trust the variant index.
+///
+/// To prevent users from abusing this fact, the string is compared by pointer value. Because the static
+/// is not public, there is no way for a user to obtain that value.
+static SERIALIZING_SCHEMA_DEFAULT: &str = "The pointer value is used, not the string itself";
- fn serialize_none_with_schema(&mut self, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "none",
- value: format!("None. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
+pub enum MapOrRecordSerializer<'s, 'w, W: Write> {
+ Map(MapSerializer<'s, 'w, W>),
+ Record(RecordSerializer<'s, 'w, W>),
+}
- match schema {
- Schema::Null => Ok(0),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Null => {
- return encode_int(i as i32, &mut *self.writer);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching Null schema in {:?}",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Null"))),
- }
- }
+impl<'s, 'w, W: Write> SerializeMap for MapOrRecordSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
- fn serialize_some_with_schema<T>(&mut self, value: &T, schema: &Schema) -> Result<usize, Error>
+ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
where
- T: ?Sized + ser::Serialize,
+ T: ?Sized + Serialize,
{
- let mut inner_ser = SchemaAwareWriteSerializer::new(
- &mut *self.writer,
- schema,
- self.names,
- self.enclosing_namespace.clone(),
- );
- value.serialize(&mut inner_ser)
- }
-
- fn serialize_unit_struct_with_schema(
- &mut self,
- name: &'static str,
- schema: &Schema,
- ) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "unit struct",
- value: format!("{name}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Record(sch) => match sch.fields.len() {
- 0 => Ok(0),
- too_many => Err(create_error(format!(
- "Too many fields: {too_many}. Expected: 0"
- ))),
- },
- Schema::Null => Ok(0),
- Schema::Ref { name: ref_name } => {
- let ref_schema = self.get_ref_schema(ref_name)?;
- self.serialize_unit_struct_with_schema(name, ref_schema)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Record(record_schema) if record_schema.fields.is_empty() => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_unit_struct_with_schema(name, variant_schema);
- }
- Schema::Null | Schema::Ref { name: _ } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_unit_struct_with_schema(name, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching Null schema in {union_schema:?}"
- )))
- }
- unsupported => Err(create_error(format!(
- "Expected Null or Union schema. Got: {unsupported}"
- ))),
- }
- }
-
- fn serialize_unit_variant_with_schema(
- &mut self,
- name: &'static str,
- variant_index: u32,
- variant: &'static str,
- schema: &Schema,
- ) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "unit variant",
- value: format!("{name}::{variant} (index={variant_index}). Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Enum(enum_schema) => {
- if variant_index as usize >= enum_schema.symbols.len() {
- return Err(create_error(format!(
- "Variant index out of bounds: {}. The Enum schema has '{}' symbols",
- variant_index,
- enum_schema.symbols.len()
- )));
- }
-
- encode_int(variant_index as i32, &mut self.writer)
- }
- Schema::Union(union_schema) => {
- if variant_index as usize >= union_schema.schemas.len() {
- return Err(create_error(format!(
- "Variant index out of bounds: {}. The union schema has '{}' schemas",
- variant_index,
- union_schema.schemas.len()
- )));
- }
-
- encode_int(variant_index as i32, &mut self.writer)?;
- self.serialize_unit_struct_with_schema(
- name,
- &union_schema.schemas[variant_index as usize],
- )
- }
- Schema::Ref { name: ref_name } => {
- let ref_schema = self.get_ref_schema(ref_name)?;
- self.serialize_unit_variant_with_schema(name, variant_index, variant, ref_schema)
- }
- unsupported => Err(create_error(format!(
- "Unsupported schema: {unsupported:?}. Expected: Enum, Union or Ref"
- ))),
+ match self {
+ MapOrRecordSerializer::Map(map) => map.serialize_key(key),
+ MapOrRecordSerializer::Record(record) => record.serialize_key(key),
}
}
- fn serialize_newtype_struct_with_schema<T>(
- &mut self,
- _name: &'static str,
- value: &T,
- schema: &Schema,
- ) -> Result<usize, Error>
+ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
where
- T: ?Sized + ser::Serialize,
+ T: ?Sized + Serialize,
{
- let mut inner_ser = SchemaAwareWriteSerializer::new(
- &mut *self.writer,
- schema,
- self.names,
- self.enclosing_namespace.clone(),
- );
- // Treat any newtype struct as a transparent wrapper around the contained type
- value.serialize(&mut inner_ser)
+ match self {
+ MapOrRecordSerializer::Map(map) => map.serialize_value(value),
+ MapOrRecordSerializer::Record(record) => record.serialize_value(value),
+ }
}
- fn serialize_newtype_variant_with_schema<T>(
- &mut self,
- name: &'static str,
- variant_index: u32,
- variant: &'static str,
- value: &T,
- schema: &Schema,
- ) -> Result<usize, Error>
- where
- T: ?Sized + ser::Serialize,
- {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "newtype variant",
- value: format!("{name}::{variant}(?) (index={variant_index}). Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Union(union_schema) => {
- let variant_schema = union_schema
- .schemas
- .get(variant_index as usize)
- .ok_or_else(|| {
- create_error(format!(
- "No variant schema at position {variant_index} for {union_schema:?}"
- ))
- })?;
-
- encode_int(variant_index as i32, &mut self.writer)?;
- self.serialize_newtype_struct_with_schema(variant, value, variant_schema)
- }
- _ => Err(create_error(format!(
- "Expected Union schema. Got: {schema}"
- ))),
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ match self {
+ MapOrRecordSerializer::Map(map) => map.end(),
+ MapOrRecordSerializer::Record(record) => record.end(),
}
}
+}
- fn serialize_seq_with_schema<'a>(
- &'a mut self,
- len: Option<usize>,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeSeq<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- let len_str = len
- .map(|l| format!("{l}"))
- .unwrap_or_else(|| String::from("?"));
+#[derive(Clone, Copy)]
+pub struct Config<'s> {
+ /// All names that can be referenced in the schema being used for serialisation.
+ pub names: &'s NamesRef<'s>,
+ /// At what block size to start a new block.
+ ///
+ /// This is a minimum value, the block size will always be larger than this except for the last
+ /// block.
+ ///
+ /// When set to `None` all values will be written in a single block. This can be faster as no
+ /// intermediate buffer is used, but deserialization can be slower as the block size is not written.
+ pub target_block_size: Option<usize>,
+ /// Should `Serialize` implementations pick a human-readable format.
+ ///
+ /// It is recommended to set this to `false` as it results in compacter output.
+ pub human_readable: bool,
+}
- Error::new(Details::SerializeValueWithSchema {
- value_type: "sequence",
- value: format!("sequence (len={len_str}). Cause: {cause}"),
- schema: schema.clone(),
- })
- };
+pub struct SchemaAwareSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ schema: &'s Schema,
+ config: Config<'s>,
+}
- match schema {
- Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new(
- self,
- array_schema.items.as_ref(),
- len,
- )),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Array(_) => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_seq_with_schema(len, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected Array schema in {union_schema:?}"
- )))
- }
- _ => Err(create_error(format!("Expected: {schema}. Got: Array"))),
- }
+impl<'s, 'w, W: Write> Debug for SchemaAwareSerializer<'s, 'w, W> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("SchemaAwareSerializer")
+ .field("schema", &self.schema)
+ .finish()
}
+}
- fn serialize_tuple_with_schema<'a>(
- &'a mut self,
- len: usize,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeSeq<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "tuple",
- value: format!("tuple (len={len}). Cause: {cause}"),
- schema: schema.clone(),
+impl<'s, 'w, W: Write> SchemaAwareSerializer<'s, 'w, W> {
+ pub fn new(writer: &'w mut W, schema: &'s Schema, config: Config<'s>) -> Result<Self, Error> {
+ if let Schema::Ref { name } = schema {
+ let schema = config
+ .names
+ .get(name)
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?;
+ Self::new(writer, schema, config)
+ } else {
+ Ok(Self {
+ writer,
+ schema,
+ config,
})
- };
-
- match schema {
- Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new(
- self,
- array_schema.items.as_ref(),
- Some(len),
- )),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Array(_) => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_tuple_with_schema(len, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected Array schema in {union_schema:?}"
- )))
- }
- _ => Err(create_error(format!("Expected: {schema}. Got: Array"))),
}
}
- fn serialize_tuple_struct_with_schema<'a>(
- &'a mut self,
- name: &'static str,
- len: usize,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeTupleStruct<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "tuple struct",
- value: format!(
- "{name}({}). Cause: {cause}",
- vec!["?"; len].as_slice().join(",")
- ),
- schema: schema.clone(),
- })
+ /// Create a new serializer with the existing writer and config.
+ ///
+ /// This will resolve the schema if it is a reference.
+ fn with_different_schema(mut self, schema: &'s Schema) -> Result<Self, Error> {
+ let schema = if let Schema::Ref { name } = schema {
+ self.config
+ .names
+ .get(name)
+ .copied()
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?
+ } else {
+ schema
};
-
- match schema {
- Schema::Array(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Array(
- SchemaAwareWriteSerializeSeq::new(self, &sch.items, Some(len)),
+ self.schema = schema;
+ Ok(self)
+ }
+
+ fn error(&self, ty: &'static str, error: impl Into<String>) -> Error {
+ Error::new(Details::SerializeValueWithSchema {
+ value_type: ty,
+ value: error.into(),
+ schema: self.schema.clone(),
+ })
+ }
+
+ fn serialize_int(self, original_ty: &'static str, v: i32) -> Result<usize, Error> {
+ match self.schema {
+ Schema::Int | Schema::Date | Schema::TimeMillis => encode_int(v, &mut *self.writer),
+ Schema::Union(union) => UnionAwareSerializer::new(self.writer, union, self.config)
+ .serialize_int(original_ty, v),
+ _ => Err(self.error(
+ original_ty,
+ "Expected Schema::Int | Schema::Date | Schema::TimeMillis",
)),
- Schema::Record(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Record(
- SchemaAwareWriteSerializeStruct::new(self, sch),
- )),
- Schema::Ref { name: ref_name } => {
- let ref_schema = self.get_ref_schema(ref_name)?;
- self.serialize_tuple_struct_with_schema(name, len, ref_schema)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Record(inner) => {
- if inner.fields.len() == len {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_tuple_struct_with_schema(
- name,
- len,
- variant_schema,
- );
- }
- }
- Schema::Array(_) | Schema::Ref { name: _ } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_tuple_struct_with_schema(
- name,
- len,
- variant_schema,
- );
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected Record, Array or Ref schema in {union_schema:?}"
- )))
- }
- _ => Err(create_error(format!(
- "Expected Record, Array, Ref or Union schema. Got: {schema}"
- ))),
}
}
- fn serialize_tuple_variant_with_schema<'a>(
- &'a mut self,
- name: &'static str,
- variant_index: u32,
- variant: &'static str,
- len: usize,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeTupleStruct<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "tuple variant",
- value: format!(
- "{name}::{variant}({}) (index={variant_index}). Cause: {cause}",
- vec!["?"; len].as_slice().join(",")
- ),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Union(union_schema) => {
- let variant_schema = union_schema
- .schemas
- .get(variant_index as usize)
- .ok_or_else(|| {
- create_error(format!(
- "Cannot find a variant at position {variant_index} in {union_schema:?}"
- ))
- })?;
-
- encode_int(variant_index as i32, &mut self.writer)?;
- self.serialize_tuple_struct_with_schema(variant, len, variant_schema)
+ fn serialize_long(self, original_ty: &'static str, v: i64) -> Result<usize, Error> {
+ match self.schema {
+ Schema::Long | Schema::TimeMicros | Schema::TimestampMillis | Schema::TimestampMicros
+ | Schema::TimestampNanos | Schema::LocalTimestampMillis | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos => {
+ encode_long(v, &mut *self.writer)
}
- _ => Err(create_error(format!(
- "Expected Union schema. Got: {schema}"
- ))),
- }
- }
-
- fn serialize_map_with_schema<'a>(
- &'a mut self,
- len: Option<usize>,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeMapOrStruct<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- let len_str = len
- .map(|l| format!("{l}"))
- .unwrap_or_else(|| String::from("?"));
-
- Error::new(Details::SerializeValueWithSchema {
- value_type: "map",
- value: format!("map (size={len_str}). Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Map(map_schema) => Ok(SchemaAwareWriteSerializeMapOrStruct::Map(
- SchemaAwareWriteSerializeMap::new(self, map_schema.types.as_ref(), len),
- )),
- Schema::Ref { name: ref_name } => {
- let ref_schema = self.get_ref_schema(ref_name)?;
- self.serialize_map_with_schema(len, ref_schema)
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_long(original_ty, v)
}
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Map(_) | Schema::Record(_) | Schema::Ref { .. } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_map_with_schema(len, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected a Map schema in {union_schema:?}"
- )))
+ _ => {
+ Err(self.error(original_ty, "Expected Schema::Long | Schema::TimeMicros | Schema::{,Local}Timestamp{Millis,Micros,Nanos}"))
}
- Schema::Record(record_schema) => Ok(SchemaAwareWriteSerializeMapOrStruct::Struct(
- SchemaAwareWriteSerializeStruct::new(self, record_schema),
- )),
- _ => Err(create_error(format!(
- "Expected Map or Union schema. Got: {schema}"
- ))),
}
}
- fn serialize_struct_with_schema<'a>(
- &'a mut self,
- name: &'static str,
- len: usize,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeStruct<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "struct",
- value: format!("{name}{{ ... }}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
+ fn write_bytes(&mut self, bytes: &[u8]) -> Result<usize, Error> {
+ self.writer.write_all(bytes).map_err(Details::WriteBytes)?;
- match schema {
- Schema::Record(record_schema) => {
- Ok(SchemaAwareWriteSerializeStruct::new(self, record_schema))
- }
- Schema::Ref { name: ref_name } => {
- let ref_schema = self.get_ref_schema(ref_name)?;
- self.serialize_struct_with_schema(name, len, ref_schema)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Record(inner)
- if inner.fields.len() == len && inner.name.name() == name =>
- {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_struct_with_schema(name, len, variant_schema);
- }
- Schema::Ref { name: _ } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_struct_with_schema(name, len, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected Record or Ref schema in {union_schema:?}"
- )))
- }
- _ => Err(create_error(format!(
- "Expected Record, Ref or Union schema. Got: {schema}"
- ))),
- }
+ Ok(bytes.len())
}
- fn serialize_struct_variant_with_schema<'a>(
- &'a mut self,
- name: &'static str,
- variant_index: u32,
- variant: &'static str,
- len: usize,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeStruct<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "struct variant",
- value: format!("{name}::{variant}{{ ... }} (size={len}. Cause: {cause})"),
- schema: schema.clone(),
- })
- };
+ fn write_bytes_with_len(&mut self, bytes: &[u8]) -> Result<usize, Error> {
+ let mut bytes_written: usize = 0;
- match schema {
- Schema::Union(union_schema) => {
- let variant_schema = union_schema
- .schemas
- .get(variant_index as usize)
- .ok_or_else(|| {
- create_error(format!(
- "Cannot find variant at position {variant_index} in {union_schema:?}"
- ))
- })?;
-
- encode_int(variant_index as i32, &mut self.writer)?;
- self.serialize_struct_with_schema(variant, len, variant_schema)
- }
- _ => Err(create_error(format!(
- "Expected Union schema. Got: {schema}"
- ))),
- }
+ bytes_written += encode_long(bytes.len() as i64, &mut *self.writer)?;
+ bytes_written += self.write_bytes(bytes)?;
+
+ Ok(bytes_written)
}
}
-impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s, W> {
+impl<'s, 'w, W: Write> Serializer for SchemaAwareSerializer<'s, 'w, W> {
+ /// Amount of bytes written
type Ok = usize;
type Error = Error;
- type SerializeSeq = SchemaAwareWriteSerializeSeq<'a, 's, W>;
- type SerializeTuple = SchemaAwareWriteSerializeSeq<'a, 's, W>;
- type SerializeTupleStruct = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>;
- type SerializeTupleVariant = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>;
- type SerializeMap = SchemaAwareWriteSerializeMapOrStruct<'a, 's, W>;
- type SerializeStruct = SchemaAwareWriteSerializeStruct<'a, 's, W>;
- type SerializeStructVariant = SchemaAwareWriteSerializeStruct<'a, 's, W>;
+ type SerializeSeq = ArraySerializer<'s, 'w, W>;
+ type SerializeTuple = TupleSerializer<'s, 'w, W>;
+ type SerializeTupleStruct = ManyTupleSerializer<'s, 'w, W>;
+ type SerializeTupleVariant = ManyTupleSerializer<'s, 'w, W>;
+ type SerializeMap = MapOrRecordSerializer<'s, 'w, W>;
+ type SerializeStruct = RecordSerializer<'s, 'w, W>;
+ type SerializeStructVariant = RecordSerializer<'s, 'w, W>;
fn serialize_bool(self, v: bool) -> Result<Self::Ok, Self::Error> {
- self.serialize_bool_with_schema(v, self.root_schema)
+ println!("serialize_bool({v}): {self:?}");
+ match self.schema {
+ Schema::Boolean => {
+ self.writer
+ .write_all(&[v as u8])
+ .map_err(Details::WriteBytes)?;
+ Ok(1)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_bool(v)
+ }
+ _ => Err(self.error("bool", "Expected Schema::Boolean")),
+ }
}
fn serialize_i8(self, v: i8) -> Result<Self::Ok, Self::Error> {
- self.serialize_i32(v as i32)
+ println!("serialize_i8({v}): {self:?}");
+ self.serialize_int("i8", i32::from(v))
}
fn serialize_i16(self, v: i16) -> Result<Self::Ok, Self::Error> {
- self.serialize_i32(v as i32)
+ println!("serialize_i16({v}): {self:?}");
+ self.serialize_int("i16", i32::from(v))
}
fn serialize_i32(self, v: i32) -> Result<Self::Ok, Self::Error> {
- self.serialize_i32_with_schema(v, self.root_schema)
+ println!("serialize_i32({v}): {self:?}");
+ self.serialize_int("i32", v)
}
fn serialize_i64(self, v: i64) -> Result<Self::Ok, Self::Error> {
- self.serialize_i64_with_schema(v, self.root_schema)
+ println!("serialize_i64({v}): {self:?}");
+ self.serialize_long("i64", v)
}
- fn serialize_i128(self, v: i128) -> Result<Self::Ok, Self::Error> {
- self.serialize_i128_with_schema(v, self.root_schema)
+ fn serialize_i128(mut self, v: i128) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_i128({v}): {self:?}");
+ match self.schema {
+ Schema::Fixed(fixed) if fixed.name.name() == "i128" && fixed.size == 16 => {
+ let bytes = v.to_le_bytes();
+ self.write_bytes(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_i128(v)
+ }
+ _ => Err(self.error("i128", r#"Expected Schema::Fixed(name: "i128", size: 16)"#)),
+ }
}
fn serialize_u8(self, v: u8) -> Result<Self::Ok, Self::Error> {
- self.serialize_u8_with_schema(v, self.root_schema)
+ println!("serialize_u8({v}): {self:?}");
+ self.serialize_int("u8", i32::from(v))
}
fn serialize_u16(self, v: u16) -> Result<Self::Ok, Self::Error> {
- self.serialize_u32(v as u32)
+ println!("serialize_u16({v}): {self:?}");
+ self.serialize_int("u16", i32::from(v))
}
fn serialize_u32(self, v: u32) -> Result<Self::Ok, Self::Error> {
- self.serialize_u32_with_schema(v, self.root_schema)
+ println!("serialize_u32({v}): {self:?}");
+ self.serialize_long("u32", i64::from(v))
}
- fn serialize_u64(self, v: u64) -> Result<Self::Ok, Self::Error> {
- self.serialize_u64_with_schema(v, self.root_schema)
+ fn serialize_u64(mut self, v: u64) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_u64({v}): {self:?}");
+ match self.schema {
+ Schema::Fixed(fixed) if fixed.name.name() == "u64" && fixed.size == 8 => {
+ let bytes = v.to_le_bytes();
+ self.write_bytes(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_u64(v)
+ }
+ _ => Err(self.error("u64", r#"Expected Schema::Fixed(name: "u64", size: 8)"#)),
+ }
}
- fn serialize_u128(self, v: u128) -> Result<Self::Ok, Self::Error> {
- self.serialize_u128_with_schema(v, self.root_schema)
+ fn serialize_u128(mut self, v: u128) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_u128({v}): {self:?}");
+ match self.schema {
+ Schema::Fixed(fixed) if fixed.name.name() == "u128" && fixed.size == 16 => {
+ let bytes = v.to_le_bytes();
+ self.write_bytes(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_u128(v)
+ }
+ _ => Err(self.error("u128", r#"Expected Schema::Fixed(name: "u128", size: 16)"#)),
+ }
}
- fn serialize_f32(self, v: f32) -> Result<Self::Ok, Self::Error> {
- self.serialize_f32_with_schema(v, self.root_schema)
+ fn serialize_f32(mut self, v: f32) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_f32({v}): {self:?}");
+ match self.schema {
+ Schema::Float => {
+ let bytes = v.to_le_bytes();
+ self.write_bytes(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_f32(v)
+ }
+ _ => Err(self.error("f32", r"Expected Schema::Float")),
+ }
}
- fn serialize_f64(self, v: f64) -> Result<Self::Ok, Self::Error> {
- self.serialize_f64_with_schema(v, self.root_schema)
+ fn serialize_f64(mut self, v: f64) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_f64({v}): {self:?}");
+ match self.schema {
+ Schema::Double => {
+ let bytes = v.to_le_bytes();
+ self.write_bytes(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_f64(v)
+ }
+ _ => Err(self.error("f64", r"Expected Schema::Double")),
+ }
}
- fn serialize_char(self, v: char) -> Result<Self::Ok, Self::Error> {
- self.serialize_char_with_schema(v, self.root_schema)
+ fn serialize_char(mut self, v: char) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_char({v}): {self:?}");
+ match self.schema {
+ Schema::String => {
+ let bytes = v.to_string().into_bytes();
+ self.write_bytes_with_len(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_char(v)
+ }
+ _ => Err(self.error("char", r"Expected Schema::String")),
+ }
}
- fn serialize_str(self, v: &str) -> Result<Self::Ok, Self::Error> {
- self.serialize_str_with_schema(v, self.root_schema)
+ fn serialize_str(mut self, v: &str) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_str({v}): {self:?}");
+ match self.schema {
+ Schema::String | Schema::Uuid(UuidSchema::String) => {
+ self.write_bytes_with_len(v.as_bytes())
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_str(v)
+ }
+ _ => Err(self.error("str", "Expected Schema::String | Schema::Uuid(String)")),
+ }
}
- fn serialize_bytes(self, v: &[u8]) -> Result<Self::Ok, Self::Error> {
- self.serialize_bytes_with_schema(v, self.root_schema)
+ fn serialize_bytes(mut self, v: &[u8]) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_bytes({v:?}): {self:?}");
+ match (SER_BYTES_TYPE.get(), self.schema) {
+ (BytesType::Unset | BytesType::Bytes, Schema::Bytes | Schema::BigDecimal | Schema::Decimal(DecimalSchema { inner: InnerDecimalSchema::Bytes, .. })) => {
+ self.write_bytes_with_len(v)
+ }
+ (BytesType::Unset | BytesType::Fixed, Schema::Fixed(fixed) | Schema::Decimal(DecimalSchema { inner: InnerDecimalSchema::Fixed(fixed), ..}) | Schema::Uuid(UuidSchema::Fixed(fixed)) | Schema::Duration(fixed)) => {
+ if fixed.size != v.len() {
+ Err(self.error("bytes", format!("Fixed size ({}) does not match value length ({})", fixed.size, v.len())))
+ } else {
+ self.write_bytes(v)
+ }
+ }
+ (_, Schema::Union(union)) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_bytes(v)
+ }
+ _ => Err(self.error("bytes", "Expected Schema::Bytes | Schema::BigDecimal | Schema::Decimal(Bytes | Fixed) | Schema::Fixed | Schema::Uuid(Fixed) | Schema::Duration")),
+ }
}
fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
- self.serialize_none_with_schema(self.root_schema)
+ println!("serialize_none: {self:?}");
+ if let Schema::Union(union) = self.schema
+ && union.variants().len() == 2
+ && let Some(index) = union.index().get(&SchemaKind::Null).copied()
+ {
+ encode_int(index as i32, &mut *self.writer)
+ } else {
+ Err(self.error(
+ "None",
+ "Expected Schema::Union(variants.len() == 2 && variants.contains(Schema::Null))",
+ ))
+ }
}
fn serialize_some<T>(self, value: &T) -> Result<Self::Ok, Self::Error>
where
- T: ?Sized + ser::Serialize,
+ T: ?Sized + Serialize,
{
- self.serialize_some_with_schema(value, self.root_schema)
+ println!("serialize_some: {self:?}");
+ if let Schema::Union(union) = self.schema
+ && union.variants().len() == 2
+ && let Some(index) = union.index().get(&SchemaKind::Null).copied()
+ {
+ // Convert the index of null to the other index
+ let index = (index + 1) & 1;
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let ser = self.with_different_schema(&union.variants()[index])?;
+ bytes_written += value.serialize(ser)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "None",
+ "Expected Schema::Union(variants.len() == 2 && variants.contains(Schema::Null))",
+ ))
+ }
}
fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
- self.serialize_none()
+ println!("serialize_unit: {self:?}");
+ match self.schema {
+ Schema::Null => Ok(0),
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_unit()
+ }
+ _ => Err(self.error("unit", "Expected Schema::Null")),
+ }
}
fn serialize_unit_struct(self, name: &'static str) -> Result<Self::Ok, Self::Error> {
- self.serialize_unit_struct_with_schema(name, self.root_schema)
+ println!("serialize_unit_struct(name: {name}): {self:?}");
+ match self.schema {
+ Schema::Record(record) if record.name.name() == name && record.fields.is_empty() => {
+ Ok(0)
+ }
+ Schema::Union(union) => UnionAwareSerializer::new(self.writer, union, self.config)
+ .serialize_unit_struct(name),
+ _ => Err(self.error(
+ "unit struct",
+ format!("Expected Schema::Record(name: {name}, fields: [])"),
+ )),
+ }
}
fn serialize_unit_variant(
@@ -1948,7 +454,46 @@ impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s
variant_index: u32,
variant: &'static str,
) -> Result<Self::Ok, Self::Error> {
- self.serialize_unit_variant_with_schema(name, variant_index, variant, self.root_schema)
+ println!(
+ "serialize_struct_variant(name: {name}, index: {variant_index}, variant: {variant}): {self:?}"
+ );
+ match self.schema {
+ Schema::Enum(enum_schema) => {
+ if name.as_ptr() == SERIALIZING_SCHEMA_DEFAULT.as_ptr() {
+ encode_int(variant_index as i32, &mut *self.writer)
+ } else {
+ if enum_schema.symbols[variant_index as usize] != variant {
+ return Err(self.error(
+ "unit variant",
+ format!(
+ "Enum variant ({variant}) is not at index {variant_index} in symbols"
+ ),
+ ));
+ }
+ encode_int(variant_index as i32, &mut *self.writer)
+ }
+ }
+ Schema::Union(union) => {
+ if let Some(index) = union.index().get(&SchemaKind::Null).copied() {
+ // Bare union
+ encode_int(index as i32, &mut *self.writer)
+ } else {
+ // Union of records
+ let Some(Schema::Record(record)) = union.variants().get(variant_index as usize)
+ else {
+ return Err(self.error("unit variant", format!("Union does not contain null and variant at index {variant_index} is not a record")));
+ };
+ if record.name.name() != variant {
+ return Err(self.error("unit variant", format!("Union does not contain null and record at index {variant_index} is not named {variant}")));
+ }
+ if !record.fields.is_empty() {
+ return Err(self.error("unit variant", format!("Union does not contain null and record at index {variant_index} is not empty")));
+ }
+ encode_int(variant_index as i32, &mut *self.writer)
+ }
+ }
+ _ => Err(self.error("unit variant", "Expected Schema::Enum | Schema::Union")),
+ }
}
fn serialize_newtype_struct<T>(
@@ -1957,36 +502,93 @@ impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s
value: &T,
) -> Result<Self::Ok, Self::Error>
where
- T: ?Sized + ser::Serialize,
+ T: ?Sized + Serialize,
{
- self.serialize_newtype_struct_with_schema(name, value, self.root_schema)
+ println!("serialize_newtype_struct(name: {name}): {self:?}");
+ match self.schema {
+ Schema::Record(record) if record.name.name() == name && record.fields.len() == 1 => {
+ let schema = &record.fields[0].schema;
+ let ser = self.with_different_schema(schema)?;
+ value.serialize(ser)
+ }
+ Schema::Union(union) => UnionAwareSerializer::new(self.writer, union, self.config)
+ .serialize_newtype_struct(name, value),
+ _ => Err(self.error(
+ "newtype struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == 1)"),
+ )),
+ }
}
fn serialize_newtype_variant<T>(
self,
- name: &'static str,
+ _name: &'static str,
variant_index: u32,
variant: &'static str,
value: &T,
) -> Result<Self::Ok, Self::Error>
where
- T: ?Sized + ser::Serialize,
+ T: ?Sized + Serialize,
{
- self.serialize_newtype_variant_with_schema(
- name,
- variant_index,
- variant,
- value,
- self.root_schema,
- )
+ println!(
+ "serialize_newtype_variant(name: {_name}, index: {variant_index}, variant: {variant}): {self:?}"
+ );
+ match self.schema {
+ Schema::Union(union) => {
+ if let Some(Schema::Record(record)) = union.variants().get(variant_index as usize)
+ && record.name.name() == variant
+ && record.fields.len() == 1
+ {
+ // Union of records
+ let mut bytes_written = encode_int(variant_index as i32, &mut *self.writer)?;
+
+ let schema = &record.fields[0].schema;
+ let ser = self.with_different_schema(schema)?;
+ bytes_written += value.serialize(ser)?;
+ Ok(bytes_written)
+ } else {
+ // Bare union, UnionAwareSerializer will write the correct index
+ let ser = UnionAwareSerializer::new(self.writer, union, self.config);
+ value.serialize(ser)
+ }
+ }
+ _ => Err(self.error("newtype variant", "Expected Schema::Union")),
+ }
}
fn serialize_seq(self, len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
- self.serialize_seq_with_schema(len, self.root_schema)
+ println!("serialize_seq(len: {len:?}): {self:?}");
+ match self.schema {
+ Schema::Array(array) => {
+ ArraySerializer::new(self.writer, array, self.config, len, None)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_seq(len)
+ }
+ _ => Err(self.error("seq", "Expected Schema::Array")),
+ }
}
fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, Self::Error> {
- self.serialize_tuple_with_schema(len, self.root_schema)
+ println!("serialize_tuple(len: {len}): {self:?}");
+ match self.schema {
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_tuple(len)
+ }
+ Schema::Null if len == 0 => Ok(TupleSerializer::Unit(UnitTupleSerializer::new(None))),
+ schema if len == 1 => Ok(TupleSerializer::One(OneTupleSerializer::new(
+ self.writer,
+ schema,
+ self.config,
+ ))),
+ Schema::Record(record) if record.fields.len() == len => Ok(TupleSerializer::Many(
+ ManyTupleSerializer::new(self.writer, record, self.config, None),
+ )),
+ _ => Err(self.error(
+ "tuple",
+ format!("Expected Schema::Record(fields.len() == {len})"),
+ )),
+ }
}
fn serialize_tuple_struct(
@@ -1994,27 +596,101 @@ impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s
name: &'static str,
len: usize,
) -> Result<Self::SerializeTupleStruct, Self::Error> {
- self.serialize_tuple_struct_with_schema(name, len, self.root_schema)
+ println!("serialize_tuple_struct(name: {name}, len: {len}): {self:?}");
+ match self.schema {
+ Schema::Record(record) if record.name.name() == name && record.fields.len() == len => {
+ Ok(ManyTupleSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ None,
+ ))
+ }
+ Schema::Union(union) => UnionAwareSerializer::new(self.writer, union, self.config)
+ .serialize_tuple_struct(name, len),
+ _ => Err(self.error(
+ "tuple",
+ format!("Expected Schema::Record(name: {name}, fields.len() == {len})"),
+ )),
+ }
}
fn serialize_tuple_variant(
self,
- name: &'static str,
+ _name: &'static str,
variant_index: u32,
variant: &'static str,
len: usize,
) -> Result<Self::SerializeTupleVariant, Self::Error> {
- self.serialize_tuple_variant_with_schema(
- name,
- variant_index,
- variant,
- len,
- self.root_schema,
- )
+ println!(
+ "serialize_tuple_variant(name: {_name}, index: {variant_index}, variant: {variant}, len: {len}): {self:?}"
+ );
+ match self.schema {
+ Schema::Union(union) => {
+ if let Some(Schema::Record(record)) = union.variants().get(variant_index as usize)
+ && record.name.name() == variant
+ && record.fields.len() == len
+ {
+ // Union of records
+ let bytes_written = encode_int(variant_index as i32, &mut *self.writer)?;
+
+ Ok(ManyTupleSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else if let Some((index, Schema::Record(record))) = union
+ .variants()
+ .iter()
+ .enumerate()
+ .find(|(_i, s)| s.name().is_some_and(|n| n.name() == variant))
+ && record.fields.len() == len
+ {
+ // Bare union
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+
+ Ok(ManyTupleSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else {
+ Err(self.error(
+ "tuple variant",
+ format!("Expected Schema::Union(variants.contains(Schema::Record(name: {variant}, fields.len() == {len})))"),
+ ))
+ }
+ }
+ _ => Err(self.error("tuple variant", "Expected Schema::Union")),
+ }
}
fn serialize_map(self, len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
- self.serialize_map_with_schema(len, self.root_schema)
+ println!("serialize_map(len: {len:?}): {self:?}");
+ match self.schema {
+ Schema::Map(map) => Ok(MapOrRecordSerializer::Map(MapSerializer::new(
+ self.writer,
+ map,
+ self.config,
+ len,
+ None,
+ )?)),
+ Schema::Record(record) => {
+ // Structs with flattened fields will be serialized as a map
+ Ok(MapOrRecordSerializer::Record(RecordSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ len,
+ )))
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_map(len)
+ }
+ _ => Err(self.error("map", "Expected Schema::Map")),
+ }
}
fn serialize_struct(
@@ -2022,64 +698,135 @@ impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s
name: &'static str,
len: usize,
) -> Result<Self::SerializeStruct, Self::Error> {
- self.serialize_struct_with_schema(name, len, self.root_schema)
+ println!("serialize_struct(name: {name}, len: {len}): {self:?}");
+ match self.schema {
+ // For unit variants with tag,content `len` will be 1 but we expect 2.
+ Schema::Record(record) if record.name.name() == name => Ok(RecordSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ None,
+ )),
+ Schema::Union(union) => UnionAwareSerializer::new(self.writer, union, self.config)
+ .serialize_struct(name, len),
+ _ => Err(self.error(
+ "struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == {len})"),
+ )),
+ }
}
fn serialize_struct_variant(
self,
- name: &'static str,
+ _name: &'static str,
variant_index: u32,
variant: &'static str,
len: usize,
) -> Result<Self::SerializeStructVariant, Self::Error> {
- self.serialize_struct_variant_with_schema(
- name,
- variant_index,
- variant,
- len,
- self.root_schema,
- )
+ println!(
+ "serialize_struct_variant(name: {_name}, index: {variant_index}, variant: {variant}, len: {len}): {self:?}"
+ );
+ match self.schema {
+ Schema::Union(union) => {
+ if let Some(Schema::Record(record)) = union.variants().get(variant_index as usize)
+ && record.name.name() == variant
+ && record.fields.len() == len
+ {
+ // Union of records
+ let bytes_written = encode_int(variant_index as i32, &mut *self.writer)?;
+
+ Ok(RecordSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else if let Some((index, Schema::Record(record))) = union
+ .variants()
+ .iter()
+ .enumerate()
+ .find(|(_i, s)| s.name().is_some_and(|n| n.name() == variant))
+ && record.fields.len() == len
+ {
+ // Bare union
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+
+ Ok(RecordSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else {
+ Err(self.error(
+ "struct variant",
+ format!("Expected Schema::Union(variants.contains(Schema::Record(name: {variant}, fields.len() == {len})))"),
+ ))
+ }
+ }
+ _ => Err(self.error("struct variant", "Expected Schema::Union")),
+ }
}
fn is_human_readable(&self) -> bool {
- crate::util::is_human_readable()
+ self.config.human_readable
}
}
#[cfg(test)]
mod tests {
- use super::*;
- use crate::schema::FixedSchema;
+ use super::{Config, SchemaAwareSerializer};
+ use crate::schema::{FixedSchema, Name};
use crate::{
- Days, Duration, Millis, Months, Reader, Writer, decimal::Decimal, error::Details,
- from_value, schema::ResolvedSchema,
+ Days, Duration, Error, Millis, Months, Reader, Schema, Writer, decimal::Decimal,
+ error::Details, schema::ResolvedSchema,
};
use apache_avro_test_helper::TestResult;
use bigdecimal::BigDecimal;
use num_bigint::{BigInt, Sign};
use serde::{Deserialize, Serialize};
use serde_bytes::{ByteArray, Bytes};
- use std::{
- collections::{BTreeMap, HashMap},
- marker::PhantomData,
- };
+ use std::collections::HashMap;
+ use std::{collections::BTreeMap, marker::PhantomData};
use uuid::Uuid;
#[test]
fn test_serialize_null() -> TestResult {
let schema = Schema::Null;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- ().serialize(&mut serializer)?;
- None::<()>.serialize(&mut serializer)?;
- None::<i32>.serialize(&mut serializer)?;
- None::<String>.serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ None::<()>
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ None::<i32>
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ None::<String>
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
- assert_eq!(buffer.as_slice(), Vec::<u8>::new().as_slice());
+ assert!(buffer.is_empty());
Ok(())
}
@@ -2088,13 +835,23 @@ mod tests {
fn test_serialize_bool() -> TestResult {
let schema = Schema::Boolean;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- true.serialize(&mut serializer)?;
- false.serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ true.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ false.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
assert_eq!(buffer.as_slice(), &[1, 0]);
@@ -2105,19 +862,33 @@ mod tests {
fn test_serialize_int() -> TestResult {
let schema = Schema::Int;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 4u8.serialize(&mut serializer)?;
- 31u16.serialize(&mut serializer)?;
- 13u32.serialize(&mut serializer)?;
- 7i8.serialize(&mut serializer)?;
- (-57i16).serialize(&mut serializer)?;
- 129i32.serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ 4u8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 31u16.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 13u32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 7i8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ (-57i16).serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 129i32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
- assert_eq!(buffer.as_slice(), &[8, 62, 26, 14, 113, 130, 2]);
+ assert_eq!(buffer.as_slice(), &[8, 62, 14, 113, 130, 2]);
Ok(())
}
@@ -2126,25 +897,54 @@ mod tests {
fn test_serialize_long() -> TestResult {
let schema = Schema::Long;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 4u8.serialize(&mut serializer)?;
- 31u16.serialize(&mut serializer)?;
- 13u32.serialize(&mut serializer)?;
- 291u64.serialize(&mut serializer)?;
- 7i8.serialize(&mut serializer)?;
- (-57i16).serialize(&mut serializer)?;
- 129i32.serialize(&mut serializer)?;
- (-432i64).serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- assert_eq!(
- buffer.as_slice(),
- &[8, 62, 26, 198, 4, 14, 113, 130, 2, 223, 6]
+ assert!(
+ 4u8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 31u16
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 13u32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 291u64
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 7i8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ (-57i16)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 129i32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ (-432i64).serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
);
+ assert_eq!(buffer.as_slice(), &[26, 223, 6]);
+
Ok(())
}
@@ -2152,15 +952,29 @@ mod tests {
fn test_serialize_float() -> TestResult {
let schema = Schema::Float;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 4.7f32.serialize(&mut serializer)?;
- (-14.1f64).serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ 4.7f32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ (-14.1f64)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
- assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]);
+ assert_eq!(buffer.as_slice(), &[102, 102, 150, 64]);
Ok(())
}
@@ -2169,13 +983,28 @@ mod tests {
fn test_serialize_double() -> TestResult {
let schema = Schema::Float;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 4.7f32.serialize(&mut serializer)?;
- (-14.1f64).serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ 4.7f32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ (-14.1f32).serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ (-14.1f64)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]);
@@ -2186,19 +1015,37 @@ mod tests {
fn test_serialize_bytes() -> TestResult {
let schema = Schema::Bytes;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 'a'.serialize(&mut serializer)?;
- "test".serialize(&mut serializer)?;
- Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?;
- assert!(().serialize(&mut serializer).is_err());
- assert!(PhantomData::<String>.serialize(&mut serializer).is_err());
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- assert_eq!(
- buffer.as_slice(),
- &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4]
+ assert!(
+ 'a'.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ "test"
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ Bytes::new(&[12, 3, 7, 91, 4]).serialize(SchemaAwareSerializer::new(
+ &mut buffer,
+ &schema,
+ config,
+ )?)?;
+ assert!(
+ ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
);
+ assert!(
+ PhantomData::<String>
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+
+ assert_eq!(buffer.as_slice(), &[10, 12, 3, 7, 91, 4]);
Ok(())
}
@@ -2207,20 +1054,31 @@ mod tests {
fn test_serialize_string() -> TestResult {
let schema = Schema::String;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 'a'.serialize(&mut serializer)?;
- "test".serialize(&mut serializer)?;
- Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?;
- assert!(().serialize(&mut serializer).is_err());
- assert!(PhantomData::<String>.serialize(&mut serializer).is_err());
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- assert_eq!(
- buffer.as_slice(),
- &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4]
+ 'a'.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ "test".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ Bytes::new(&[12, 3, 7, 91, 4])
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config,)?)
+ .is_err()
+ );
+ assert!(
+ ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ PhantomData::<String>
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
);
+ assert_eq!(buffer.as_slice(), &[2, b'a', 8, b't', b'e', b's', b't']);
+
Ok(())
}
@@ -2238,37 +1096,51 @@ mod tests {
)?;
#[derive(Serialize)]
- #[serde(rename_all = "camelCase")]
+ #[serde(rename_all = "camelCase", rename = "TestRecord")]
struct GoodTestRecord {
string_field: String,
int_field: i32,
}
#[derive(Serialize)]
- #[serde(rename_all = "camelCase")]
+ #[serde(rename_all = "camelCase", rename = "TestRecord")]
struct BadTestRecord {
foo_string_field: String,
bar_int_field: i32,
}
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let good_record = GoodTestRecord {
string_field: String::from("test"),
int_field: 10,
};
- good_record.serialize(&mut serializer)?;
+ good_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let bad_record = BadTestRecord {
foo_string_field: String::from("test"),
bar_int_field: 10,
};
- assert!(bad_record.serialize(&mut serializer).is_err());
+ assert!(
+ bad_record
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
assert_eq!(buffer.as_slice(), &[8, b't', b'e', b's', b't', 20]);
@@ -2286,14 +1158,18 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
#[derive(Serialize)]
struct EmptyRecord;
- EmptyRecord.serialize(&mut serializer)?;
+ EmptyRecord.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
#[derive(Serialize)]
+ #[serde(rename = "EmptyRecord")]
struct NonEmptyRecord {
foo: String,
}
@@ -2301,21 +1177,21 @@ mod tests {
foo: "bar".to_string(),
};
match record
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::FieldName(field_name)) if field_name == "foo" => (),
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- match ().serialize(&mut serializer).map_err(Error::into_details) {
+ match ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?).map_err(Error::into_details) {
Err(Details::SerializeValueWithSchema {
value_type,
value,
schema,
}) => {
- assert_eq!(value_type, "none"); // serialize_unit() delegates to serialize_none()
- assert_eq!(value, "None. Cause: Expected: Record. Got: Null");
+ assert_eq!(value_type, "unit");
+ assert_eq!(value, "Expected Schema::Null");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2337,6 +1213,7 @@ mod tests {
)?;
#[derive(Serialize)]
+ #[serde(rename_all = "UPPERCASE")]
enum Suit {
Spades,
Hearts,
@@ -2345,15 +1222,18 @@ mod tests {
}
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- Suit::Spades.serialize(&mut serializer)?;
- Suit::Hearts.serialize(&mut serializer)?;
- Suit::Diamonds.serialize(&mut serializer)?;
- Suit::Clubs.serialize(&mut serializer)?;
+ Suit::Spades.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ Suit::Hearts.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ Suit::Diamonds.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ Suit::Clubs.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match None::<()>
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2361,8 +1241,11 @@ mod tests {
value,
schema,
}) => {
- assert_eq!(value_type, "none");
- assert_eq!(value, "None. Cause: Expected: Enum. Got: Null");
+ assert_eq!(value_type, "None");
+ assert_eq!(
+ value,
+ "Expected Schema::Union(variants.len() == 2 && variants.contains(Schema::Null))"
+ );
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2383,14 +1266,17 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let arr: Vec<i64> = vec![10, 5, 400];
- arr.serialize(&mut serializer)?;
+ arr.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match vec![1_f32]
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2399,13 +1285,15 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "f32");
- assert_eq!(value, "1. Cause: Expected: Long. Got: Float");
+ assert_eq!(value, "Expected Schema::Float");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- assert_eq!(buffer.as_slice(), &[6, 20, 10, 160, 6, 0]);
+ // The 2 at the end is because the DirectArraySerializer immediately writes the length and doesn't
+ // know the values will be invalid yet
+ assert_eq!(buffer.as_slice(), &[6, 20, 10, 160, 6, 0, 2]);
Ok(())
}
@@ -2420,35 +1308,43 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let mut map: BTreeMap<String, i64> = BTreeMap::new();
map.insert(String::from("item1"), 10);
map.insert(String::from("item2"), 400);
- map.serialize(&mut serializer)?;
+ map.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let mut map: BTreeMap<String, &str> = BTreeMap::new();
map.insert(String::from("item1"), "value1");
- match map.serialize(&mut serializer).map_err(Error::into_details) {
+ match map
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .map_err(Error::into_details)
+ {
Err(Details::SerializeValueWithSchema {
value_type,
value,
schema,
}) => {
- assert_eq!(value_type, "string");
- assert_eq!(value, "value1. Cause: Expected: Long. Got: String");
+ assert_eq!(value_type, "str");
+ assert_eq!(value, "Expected Schema::String | Schema::Uuid(String)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
+ // The last 7 bytes are written because the DirectMapSerializer will write immediately and doesn't
+ // know yet that the value is not a long
assert_eq!(
buffer.as_slice(),
&[
4, 10, b'i', b't', b'e', b'm', b'1', 20, 10, b'i', b't', b'e', b'm', b'2', 160, 6,
- 0
+ 0, 2, 10, 105, 116, 101, 109, 49
]
);
@@ -2470,16 +1366,23 @@ mod tests {
}
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- Some(10i64).serialize(&mut serializer)?;
- None::<i64>.serialize(&mut serializer)?;
- NullableLong::Long(400).serialize(&mut serializer)?;
- NullableLong::Null.serialize(&mut serializer)?;
+ Some(10i64).serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ None::<i64>.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ NullableLong::Long(400).serialize(SchemaAwareSerializer::new(
+ &mut buffer,
+ &schema,
+ config,
+ )?)?;
+ NullableLong::Null.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match "invalid"
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2487,11 +1390,8 @@ mod tests {
value,
schema,
}) => {
- assert_eq!(value_type, "string");
- assert_eq!(
- value,
- "invalid. Cause: Expected one of the union variants [Null, Long]. Got: String"
- );
+ assert_eq!(value_type, "str");
+ assert_eq!(value, "Expected Schema::String in variants");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2518,15 +1418,26 @@ mod tests {
}
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- LongOrString::Null.serialize(&mut serializer)?;
- LongOrString::Long(400).serialize(&mut serializer)?;
- LongOrString::Str(String::from("test")).serialize(&mut serializer)?;
+ LongOrString::Null.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ LongOrString::Long(400).serialize(SchemaAwareSerializer::new(
+ &mut buffer,
+ &schema,
+ config,
+ )?)?;
+ LongOrString::Str(String::from("test")).serialize(SchemaAwareSerializer::new(
+ &mut buffer,
+ &schema,
+ config,
+ )?)?;
match 1_f64
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2535,10 +1446,7 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "f64");
- assert_eq!(
- value,
- "1. Cause: Cannot find a Double schema in [Null, Long, String]"
- );
+ assert_eq!(value, "Expected Schema::Double in variants");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2563,14 +1471,21 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- Bytes::new(&[10, 124, 31, 97, 14, 201, 3, 88]).serialize(&mut serializer)?;
+ Bytes::new(&[10, 124, 31, 97, 14, 201, 3, 88]).serialize(SchemaAwareSerializer::new(
+ &mut buffer,
+ &schema,
+ config,
+ )?)?;
// non-8 size
match Bytes::new(&[123])
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2579,10 +1494,7 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "bytes");
- assert_eq!(
- value,
- "7b. Cause: Fixed schema size (8) does not match the value length (1)"
- ); // Bytes represents its values as hexadecimals: '7b' is 123
+ assert_eq!(value, "Fixed size (8) does not match value length (1)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2590,7 +1502,7 @@ mod tests {
// array
match [1; 8]
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2598,8 +1510,9 @@ mod tests {
value,
schema,
}) => {
- assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?!
- assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array");
+ // Arrays are serialized as a tuple by Serde
+ assert_eq!(value_type, "tuple");
+ assert_eq!(value, "Expected Schema::Record(fields.len() == 8)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2607,7 +1520,7 @@ mod tests {
// slice
match &[1, 2, 3, 4, 5, 6, 7, 8]
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2615,8 +1528,9 @@ mod tests {
value,
schema,
}) => {
- assert_eq!(*value_type, "tuple"); // TODO: why is this 'tuple' ?!
- assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array");
+ // This is a tuple as Serde serializes array as a tuple
+ assert_eq!(*value_type, "tuple");
+ assert_eq!(value, "Expected Schema::Record(fields.len() == 8)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2639,20 +1553,23 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let val = Decimal::from(&[251, 155]);
- val.serialize(&mut serializer)?;
+ val.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
- match ().serialize(&mut serializer).map_err(Error::into_details) {
+ match ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?).map_err(Error::into_details) {
Err(Details::SerializeValueWithSchema {
value_type,
value,
schema,
}) => {
- assert_eq!(value_type, "none");
- assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null");
+ assert_eq!(value_type, "unit");
+ assert_eq!(value, "Expected Schema::Null");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2677,20 +1594,23 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let val = Decimal::from(&[0, 0, 0, 0, 0, 0, 251, 155]);
- val.serialize(&mut serializer)?;
+ val.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
- match ().serialize(&mut serializer).map_err(Error::into_details) {
+ match ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?).map_err(Error::into_details) {
Err(Details::SerializeValueWithSchema {
value_type,
value,
schema,
}) => {
- assert_eq!(value_type, "none");
- assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null");
+ assert_eq!(value_type, "unit");
+ assert_eq!(value, "Expected Schema::Null");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2711,11 +1631,17 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let val = BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2);
- val.serialize(&mut serializer)?;
+ crate::serde::bigdecimal::serialize(
+ &val,
+ SchemaAwareSerializer::new(&mut buffer, &schema, config)?,
+ )?;
assert_eq!(buffer.as_slice(), &[10, 6, 0, 195, 104, 4]);
@@ -2735,21 +1661,30 @@ mod tests {
assert!(!crate::util::is_human_readable());
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
"8c28da81-238c-4326-bddd-4e3d00cc5099"
.parse::<Uuid>()?
- .serialize(&mut serializer)?;
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
- match 1_u8.serialize(&mut serializer).map_err(Error::into_details) {
+ match 1_u8
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .map_err(Error::into_details)
+ {
Err(Details::SerializeValueWithSchema {
value_type,
value,
schema,
}) => {
assert_eq!(value_type, "u8");
- assert_eq!(value, "1. Cause: Expected: Uuid. Got: Int");
+ assert_eq!(
+ value,
+ "Expected Schema::Int | Schema::Date | Schema::TimeMillis"
+ );
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2775,17 +1710,24 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 100_u8.serialize(&mut serializer)?;
- 1000_u16.serialize(&mut serializer)?;
- 10000_u32.serialize(&mut serializer)?;
- 1000_i16.serialize(&mut serializer)?;
- 10000_i32.serialize(&mut serializer)?;
+ 100_u8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 1000_u16.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 10000_u32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 1000_i16.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 10000_i32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match 10000_f32
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2794,16 +1736,13 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "f32");
- assert_eq!(value, "10000. Cause: Expected: Date. Got: Float");
+ assert_eq!(value, "Expected Schema::Float");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- assert_eq!(
- buffer.as_slice(),
- &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1]
- );
+ assert_eq!(buffer.as_slice(), &[200, 1, 208, 15, 208, 15, 160, 156, 1]);
Ok(())
}
@@ -2818,17 +1757,24 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 100_u8.serialize(&mut serializer)?;
- 1000_u16.serialize(&mut serializer)?;
- 10000_u32.serialize(&mut serializer)?;
- 1000_i16.serialize(&mut serializer)?;
- 10000_i32.serialize(&mut serializer)?;
+ 100_u8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 1000_u16.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 10000_u32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 1000_i16.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 10000_i32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match 10000_f32
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2837,16 +1783,13 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "f32");
- assert_eq!(value, "10000. Cause: Expected: TimeMillis. Got: Float");
+ assert_eq!(value, "Expected Schema::Float");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- assert_eq!(
- buffer.as_slice(),
- &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1]
- );
+ assert_eq!(buffer.as_slice(), &[200, 1, 208, 15, 208, 15, 160, 156, 1]);
Ok(())
}
@@ -2861,18 +1804,37 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 100_u8.serialize(&mut serializer)?;
- 1000_u16.serialize(&mut serializer)?;
- 10000_u32.serialize(&mut serializer)?;
- 1000_i16.serialize(&mut serializer)?;
- 10000_i32.serialize(&mut serializer)?;
- 10000_i64.serialize(&mut serializer)?;
+ assert!(
+ 100_u8
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 1000_u16
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 10000_u32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 1000_i16
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 10000_i32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 10000_i64.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match 10000_f32
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2881,18 +1843,13 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "f32");
- assert_eq!(value, "10000. Cause: Expected: TimeMicros. Got: Float");
+ assert_eq!(value, "Expected Schema::Float");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- assert_eq!(
- buffer.as_slice(),
- &[
- 200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1, 160, 156, 1
- ]
- );
+ assert_eq!(buffer.as_slice(), &[160, 156, 1, 160, 156, 1,]);
Ok(())
}
@@ -2908,19 +1865,37 @@ mod tests {
))?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer =
- SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 100_u8.serialize(&mut serializer)?;
- 1000_u16.serialize(&mut serializer)?;
- 10000_u32.serialize(&mut serializer)?;
- 1000_i16.serialize(&mut serializer)?;
- 10000_i32.serialize(&mut serializer)?;
- 10000_i64.serialize(&mut serializer)?;
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
+
+ assert!(
+ 100_u8
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 1000_u16
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 10000_u32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 1000_i16
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 10000_i32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 10000_i64.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match 10000_f64
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2933,23 +1908,13 @@ mod tests {
capital_precision.replace_range(..1, &c.to_uppercase().to_string());
}
assert_eq!(value_type, "f64");
- assert_eq!(
- value,
- format!(
- "10000. Cause: Expected: Timestamp{capital_precision}. Got: Double"
- )
- );
+ assert_eq!(value, format!("Expected Schema::Double"));
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- assert_eq!(
- buffer.as_slice(),
- &[
- 200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1, 160, 156, 1
- ]
- );
+ assert_eq!(buffer.as_slice(), &[160, 156, 1, 160, 156, 1,]);
}
Ok(())
@@ -2967,15 +1932,18 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let duration_bytes =
ByteArray::new(Duration::new(Months::new(3), Days::new(2), Millis::new(1200)).into());
- duration_bytes.serialize(&mut serializer)?;
+ duration_bytes.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match [1; 12]
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2983,11 +1951,9 @@ mod tests {
value,
schema,
}) => {
- assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?!
- assert_eq!(
- value,
- "tuple (len=12). Cause: Expected: Duration. Got: Array"
- );
+ // This is a tuple because Serde serializes arrays [T; N] as tuples
+ assert_eq!(value_type, "tuple");
+ assert_eq!(value, "Expected Schema::Record(fields.len() == 12)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -3019,6 +1985,7 @@ mod tests {
struct TestRecord {
string_field: String,
int_field: i32,
+ #[serde(with = "crate::serde::bigdecimal")]
big_decimal_field: BigDecimal,
uuid_field: Uuid,
// #[serde(skip_serializing_if = "Option::is_none")] => Never ignore None!
@@ -3028,8 +1995,11 @@ mod tests {
assert!(!crate::util::is_human_readable());
let mut buffer: Vec<u8> = Vec::new();
let rs = ResolvedSchema::try_from(&schema)?;
- let mut serializer =
- SchemaAwareWriteSerializer::new(&mut buffer, &schema, rs.get_names(), None);
+ let config = Config {
+ names: rs.get_names(),
+ target_block_size: None,
+ human_readable: false,
+ };
let good_record = TestRecord {
string_field: String::from("test"),
@@ -3044,7 +2014,7 @@ mod tests {
inner_record: None,
})),
};
- good_record.serialize(&mut serializer)?;
+ good_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
assert_eq!(
buffer.as_slice(),
@@ -3059,7 +2029,9 @@ mod tests {
Ok(())
}
+ // TODO: Figure out what to do with Option<Enum> mapping to Union([Null, ..])
#[test]
+ #[ignore]
fn avro_rs_337_serialize_union_record_variant() -> TestResult {
let schema = Schema::parse_str(
r#"{
@@ -3109,33 +2081,38 @@ mod tests {
let mut buffer: Vec<u8> = Vec::new();
let rs = ResolvedSchema::try_from(&schema)?;
- let mut serializer =
- SchemaAwareWriteSerializer::new(&mut buffer, &schema, rs.get_names(), None);
+ let config = Config {
+ names: rs.get_names(),
+ target_block_size: None,
+ human_readable: false,
+ };
let foo_record = TestRecord {
inner_union: InnerUnion::InnerVariantFoo(InnerRecordFoo {
foo: String::from("foo"),
}),
};
- foo_record.serialize(&mut serializer)?;
+ foo_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let bar_record = TestRecord {
inner_union: InnerUnion::InnerVariantBar(InnerRecordBar {
bar: String::from("bar"),
}),
};
- bar_record.serialize(&mut serializer)?;
+ bar_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let int_record = TestRecord {
inner_union: InnerUnion::IntField(1),
};
- int_record.serialize(&mut serializer)?;
+ int_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let string_record = TestRecord {
inner_union: InnerUnion::StringField(String::from("string")),
};
- string_record.serialize(&mut serializer)?;
+ string_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
Ok(())
}
+ // TODO: Figure out what to do with Option<Enum> mapping to Union([Null, ..])
#[test]
+ #[ignore]
fn avro_rs_337_serialize_option_union_record_variant() -> TestResult {
let schema = Schema::parse_str(
r#"{
@@ -3152,7 +2129,7 @@ mod tests {
]},
{"name": "intField", "type": "int"},
{"name": "stringField", "type": "string"}
- ]
+ ],
}]
}"#,
)?;
@@ -3186,31 +2163,34 @@ mod tests {
let mut buffer: Vec<u8> = Vec::new();
let rs = ResolvedSchema::try_from(&schema)?;
- let mut serializer =
- SchemaAwareWriteSerializer::new(&mut buffer, &schema, rs.get_names(), None);
+ let config = Config {
+ names: rs.get_names(),
+ target_block_size: None,
+ human_readable: false,
+ };
let null_record = TestRecord { inner_union: None };
- null_record.serialize(&mut serializer)?;
+ null_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let foo_record = TestRecord {
inner_union: Some(InnerUnion::InnerVariantFoo(InnerRecordFoo {
foo: String::from("foo"),
})),
};
- foo_record.serialize(&mut serializer)?;
+ foo_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let bar_record = TestRecord {
inner_union: Some(InnerUnion::InnerVariantBar(InnerRecordBar {
bar: String::from("bar"),
})),
};
- bar_record.serialize(&mut serializer)?;
+ bar_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let int_record = TestRecord {
inner_union: Some(InnerUnion::IntField(1)),
};
- int_record.serialize(&mut serializer)?;
+ int_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let string_record = TestRecord {
inner_union: Some(InnerUnion::StringField(String::from("string"))),
};
- string_record.serialize(&mut serializer)?;
+ string_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
Ok(())
}
@@ -3220,9 +2200,9 @@ mod tests {
struct Foo {
a: String,
b: String,
- c: usize,
+ c: i64,
d: f64,
- e: usize,
+ e: i64,
}
let schema = Schema::parse_str(
r#"
@@ -3267,7 +2247,7 @@ mod tests {
let mut reader = Reader::builder(&encoded[..])
.reader_schema(&schema)
.build()?;
- let decoded = from_value::<Foo>(&reader.next().unwrap()?)?;
+ let decoded: Foo = reader.next_deser()?.unwrap();
assert_eq!(
decoded,
Foo {
@@ -3286,149 +2266,37 @@ mod tests {
let schema = Schema::String;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 'a'.serialize(&mut serializer)?;
-
- assert_eq!(buffer.as_slice(), &[2, b'a']);
-
- Ok(())
- }
-
- #[test]
- fn avro_rs_414_serialize_char_as_bytes() -> TestResult {
- let schema = Schema::Bytes;
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 'a'.serialize(&mut serializer)?;
+ 'a'.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
assert_eq!(buffer.as_slice(), &[2, b'a']);
Ok(())
}
- #[test]
- fn avro_rs_414_serialize_char_as_fixed() -> TestResult {
- let schema = Schema::Fixed(FixedSchema {
- name: Name::new("char")?,
- aliases: None,
- doc: None,
- size: 4,
- attributes: Default::default(),
- });
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 'a'.serialize(&mut serializer)?;
-
- assert_eq!(buffer.as_slice(), &[b'a', 0, 0, 0]);
-
- Ok(())
- }
-
#[test]
fn avro_rs_414_serialize_emoji_char_as_string() -> TestResult {
let schema = Schema::String;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- '👹'.serialize(&mut serializer)?;
-
- assert_eq!(buffer.as_slice(), &[8, 240, 159, 145, 185]);
-
- Ok(())
- }
-
- #[test]
- fn avro_rs_414_serialize_emoji_char_as_bytes() -> TestResult {
- let schema = Schema::Bytes;
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- '👹'.serialize(&mut serializer)?;
+ '👹'.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
assert_eq!(buffer.as_slice(), &[8, 240, 159, 145, 185]);
Ok(())
}
- #[test]
- fn avro_rs_414_serialize_emoji_char_as_fixed() -> TestResult {
- let schema = Schema::Fixed(FixedSchema {
- name: Name::new("char")?,
- aliases: None,
- doc: None,
- size: 4,
- attributes: Default::default(),
- });
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- '👹'.serialize(&mut serializer)?;
-
- // This is a different byte value than the tests above. This is because by creating a String
- // the unicode value is normalized by Rust
- assert_eq!(buffer.as_slice(), &[121, 244, 1, 0]);
-
- Ok(())
- }
-
- #[test]
- fn avro_rs_414_serialize_char_as_fixed_wrong_name() -> TestResult {
- let schema = Schema::Fixed(FixedSchema {
- name: Name::new("characters")?,
- aliases: None,
- doc: None,
- size: 4,
- attributes: Default::default(),
- });
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- assert!(matches!(
- 'a'.serialize(&mut serializer).unwrap_err().details(),
- Details::SerializeValueWithSchema { .. }
- ));
-
- Ok(())
- }
-
- #[test]
- fn avro_rs_414_serialize_char_as_fixed_wrong_size() -> TestResult {
- let schema = Schema::Fixed(FixedSchema {
- name: Name::new("char")?,
- aliases: None,
- doc: None,
- size: 1,
- attributes: Default::default(),
- });
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- assert!(matches!(
- 'a'.serialize(&mut serializer).unwrap_err().details(),
- Details::SerializeValueWithSchema { .. }
- ));
-
- Ok(())
- }
-
#[test]
fn avro_rs_414_serialize_i128_as_fixed() -> TestResult {
let schema = Schema::Fixed(FixedSchema {
@@ -3440,10 +2308,14 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- let bytes_written = i128::MAX.serialize(&mut serializer)?;
+ let bytes_written =
+ i128::MAX.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
assert_eq!(bytes_written, 16);
assert_eq!(
@@ -3468,11 +2340,17 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
assert!(matches!(
- i128::MAX.serialize(&mut serializer).unwrap_err().details(),
+ i128::MAX
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .unwrap_err()
+ .details(),
Details::SerializeValueWithSchema { .. }
));
@@ -3490,11 +2368,17 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
assert!(matches!(
- i128::MAX.serialize(&mut serializer).unwrap_err().details(),
+ i128::MAX
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .unwrap_err()
+ .details(),
Details::SerializeValueWithSchema { .. }
));
@@ -3512,10 +2396,14 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- let bytes_written = u128::MAX.serialize(&mut serializer)?;
+ let bytes_written =
+ u128::MAX.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
assert_eq!(bytes_written, 16);
assert_eq!(
@@ -3540,11 +2428,17 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
assert!(matches!(
- u128::MAX.serialize(&mut serializer).unwrap_err().details(),
+ u128::MAX
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .unwrap_err()
+ .details(),
Details::SerializeValueWithSchema { .. }
));
@@ -3562,11 +2456,17 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
assert!(matches!(
- u128::MAX.serialize(&mut serializer).unwrap_err().details(),
+ u128::MAX
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .unwrap_err()
+ .details(),
Details::SerializeValueWithSchema { .. }
));
@@ -3580,17 +2480,30 @@ mod tests {
{ "name": "fixed4", "type": "fixed", "size": 4 },
{ "name": "fixed8", "type": "fixed", "size": 8 }
]"#,
- )
- .unwrap();
+ )?;
+
+ #[derive(Serialize)]
+ enum UnionOfFixed {
+ #[serde(with = "crate::serde::fixed")]
+ Four([u8; 4]),
+ #[serde(with = "crate::serde::fixed")]
+ Eight([u8; 8]),
+ }
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
- let bytes_written = crate::serde::fixed::serialize(&[0, 1, 2, 3], &mut serializer)?;
- assert_eq!(bytes_written, 4);
- let bytes_written =
- crate::serde::fixed::serialize(&[4, 5, 6, 7, 8, 9, 10, 11], &mut serializer)?;
- assert_eq!(bytes_written, 8);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
+
+ let bytes_written = UnionOfFixed::Four([0, 1, 2, 3])
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert_eq!(bytes_written, 5);
+
+ let bytes_written = UnionOfFixed::Eight([4, 5, 6, 7, 8, 9, 10, 11])
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert_eq!(bytes_written, 9);
assert_eq!(buffer, &[0, 0, 1, 2, 3, 2, 4, 5, 6, 7, 8, 9, 10, 11][..]);
diff --git a/avro/src/serde/ser_schema/record/field_default.rs b/avro/src/serde/ser_schema/record/field_default.rs
new file mode 100644
index 0000000..89e48e1
--- /dev/null
+++ b/avro/src/serde/ser_schema/record/field_default.rs
@@ -0,0 +1,197 @@
+use crate::schema::SchemaKind;
+use crate::{Schema, serde::ser_schema::SERIALIZING_SCHEMA_DEFAULT};
+use serde::Serialize;
+use serde_json::Value;
+
+#[derive(Debug)]
+pub struct SchemaAwareDefault<'v, 's> {
+ value: &'v Value,
+ schema: &'s Schema,
+}
+
+impl<'v, 's> SchemaAwareDefault<'v, 's> {
+ pub fn new(value: &'v Value, schema: &'s Schema) -> Self {
+ Self { value, schema }
+ }
+
+ fn serialize_as_newtype_variant<S>(
+ &self,
+ serializer: S,
+ index: usize,
+ ) -> Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ let Schema::Union(union) = self.schema else {
+ panic!("This function should only be called if self.schema is a Union");
+ };
+ let value = Self::new(self.value, &union.variants()[index]);
+ let index = index as u32;
+ serializer.serialize_newtype_variant(
+ SERIALIZING_SCHEMA_DEFAULT,
+ index,
+ SERIALIZING_SCHEMA_DEFAULT,
+ &value,
+ )
+ }
+
+ fn recursive_type_check(value: &Value, schema: &Schema) -> bool {
+ match (value, schema) {
+ (Value::Null, Schema::Null)
+ | (Value::Bool(_), Schema::Boolean)
+ | (Value::String(_), Schema::Bytes | Schema::String) => true,
+ (Value::Number(n), Schema::Int | Schema::Date | Schema::TimeMillis) if n.is_i64() => {
+ let long = n.as_i64().unwrap();
+ i32::try_from(long).is_ok()
+ }
+ (
+ Value::Number(n),
+ Schema::Long
+ | Schema::TimeMicros
+ | Schema::TimestampMillis
+ | Schema::TimestampMicros
+ | Schema::TimestampNanos
+ | Schema::LocalTimestampMillis
+ | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos,
+ ) if n.is_i64() => true,
+ (Value::Number(n), Schema::Float | Schema::Double) if n.is_f64() => true,
+ (Value::String(s), Schema::Fixed(fixed)) => s.len() == fixed.size,
+ (Value::String(s), Schema::Enum(enum_schema)) => enum_schema.symbols.contains(s),
+ (Value::Object(o), Schema::Record(record)) => record.fields.iter().all(|field| {
+ if let Some(value) = o.get(&field.name) {
+ Self::recursive_type_check(value, &field.schema)
+ } else {
+ field.default.is_some()
+ }
+ }),
+ (Value::Object(o), Schema::Map(map)) => o
+ .values()
+ .all(|value| Self::recursive_type_check(value, &map.types)),
+ (Value::Array(a), Schema::Array(array)) => a
+ .iter()
+ .all(|value| Self::recursive_type_check(value, &array.items)),
+ (_, Schema::Union(union)) => union
+ .variants()
+ .iter()
+ .any(|variant| Self::recursive_type_check(value, variant)),
+ _ => false,
+ }
+ }
+}
+
+impl<'v, 's> Serialize for SchemaAwareDefault<'v, 's> {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ use serde::ser::Error;
+ match (&self.value, self.schema) {
+ (Value::Null, Schema::Null) => serializer.serialize_unit(),
+ (Value::Bool(boolean), Schema::Boolean) => serializer.serialize_bool(*boolean),
+ (Value::Number(n), Schema::Int | Schema::Date | Schema::TimeMillis) if n.is_i64() => {
+ let long = n.as_i64().unwrap();
+ let int = i32::try_from(long).map_err(|_| {
+ S::Error::custom(format!("Default {long} is too large for {:?}", self.schema))
+ })?;
+ serializer.serialize_i32(int)
+ }
+ (
+ Value::Number(n),
+ Schema::Long
+ | Schema::TimeMicros
+ | Schema::TimestampMillis
+ | Schema::TimestampMicros
+ | Schema::TimestampNanos
+ | Schema::LocalTimestampMillis
+ | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos,
+ ) if n.is_i64() => {
+ let long = n.as_i64().unwrap();
+ serializer.serialize_i64(long)
+ }
+ (Value::Number(n), Schema::Float) if n.is_f64() => {
+ let float = n.as_f64().unwrap();
+ serializer.serialize_f32(float as f32)
+ }
+ (Value::Number(n), Schema::Double) if n.is_f64() => {
+ let double = n.as_f64().unwrap();
+ serializer.serialize_f64(double)
+ }
+ (Value::String(s), Schema::Bytes | Schema::Fixed(_)) => {
+ serializer.serialize_bytes(s.as_bytes())
+ }
+ (Value::String(s), Schema::String) => serializer.serialize_str(s),
+ (Value::String(s), Schema::Enum(enum_schema)) => {
+ let Some((variant_index, _)) = enum_schema
+ .symbols
+ .iter()
+ .enumerate()
+ .find(|(_i, symbol)| *symbol == s)
+ else {
+ return Err(S::Error::custom(format!(
+ "Could not find `{s}` in enum: {enum_schema:?}"
+ )));
+ };
+
+ serializer.serialize_unit_variant(
+ SERIALIZING_SCHEMA_DEFAULT,
+ variant_index as u32,
+ SERIALIZING_SCHEMA_DEFAULT,
+ )
+ }
+ // This abuses the support for flattened fields, which are also serialized as a map.
+ (Value::Object(o), Schema::Record(record)) => serializer.collect_map(
+ o.iter()
+ .enumerate()
+ .map(|(i, (k, v))| (k, Self::new(v, &record.fields[i].schema))),
+ ),
+ (Value::Object(o), Schema::Map(map)) => {
+ serializer.collect_map(o.iter().map(|(k, v)| (k, Self::new(v, &map.types))))
+ }
+ (Value::Array(a), Schema::Array(array)) => {
+ serializer.collect_seq(a.iter().map(|v| Self::new(v, &array.items)))
+ }
+ (_, Schema::Union(union)) => {
+ if union.variants().len() == 2
+ && let Some(index) = union.index().get(&SchemaKind::Null)
+ {
+ // Fast path for options
+ if self.value == &Value::Null {
+ serializer.serialize_none()
+ } else {
+ let other_index = (index + 1) & 1;
+ let value = Self::new(self.value, &union.variants()[other_index]);
+ serializer.serialize_some(&value)
+ }
+ } else {
+ // Find the first variant that can match this value
+ for (index, variant) in union.variants().iter().enumerate() {
+ match (self.value, variant) {
+ (Value::Null, Schema::Null) => {
+ let index = index as u32;
+ return serializer.serialize_unit_variant(
+ SERIALIZING_SCHEMA_DEFAULT,
+ index,
+ SERIALIZING_SCHEMA_DEFAULT,
+ );
+ }
+ _ if Self::recursive_type_check(self.value, variant) => {
+ return self.serialize_as_newtype_variant(serializer, index);
+ }
+ _ => {}
+ }
+ }
+ Err(S::Error::custom(format!(
+ "Could not match default to any variant of {:?}, default: {:?}",
+ self.schema, self.value
+ )))
+ }
+ }
+ _ => Err(S::Error::custom(format!(
+ "Unexpected default for {:?}, default: {:?}",
+ self.schema, self.value
+ ))),
+ }
+ }
+}
diff --git a/avro/src/serde/ser_schema/record/mod.rs b/avro/src/serde/ser_schema/record/mod.rs
new file mode 100644
index 0000000..12e268f
--- /dev/null
+++ b/avro/src/serde/ser_schema/record/mod.rs
@@ -0,0 +1,250 @@
+use std::{cmp::Ordering, collections::HashMap, io::Write};
+
+use serde::ser::{Serialize, SerializeMap, SerializeStruct, SerializeStructVariant};
+
+use super::{Config, SchemaAwareSerializer};
+use crate::serde::ser_schema::record::field_default::SchemaAwareDefault;
+use crate::{
+ Error, Schema,
+ error::Details,
+ schema::{RecordField, RecordSchema},
+ serde::util::StringSerializer,
+};
+
+mod field_default;
+
+pub struct RecordSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ schema: &'s RecordSchema,
+ config: Config<'s>,
+ /// Fields we received in the wrong order
+ field_cache: HashMap<usize, Vec<u8>>,
+ /// The current field name when serializing from a map (for `flatten` support).
+ map_field_name: Option<String>,
+ field_position: usize,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> RecordSerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ schema: &'s RecordSchema,
+ config: Config<'s>,
+ bytes_written: Option<usize>,
+ ) -> Self {
+ Self {
+ writer,
+ schema,
+ config,
+ field_cache: HashMap::new(),
+ map_field_name: None,
+ field_position: 0,
+ bytes_written: bytes_written.unwrap_or(0),
+ }
+ }
+
+ fn serialize_next_field<T>(
+ &mut self,
+ field: &RecordField,
+ position: usize,
+ value: &T,
+ ) -> Result<(), Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ match self.field_position.cmp(&position) {
+ Ordering::Equal => {
+ // If we receive fields in order, write them directly to the main writer
+ let value_ser =
+ SchemaAwareSerializer::new(&mut *self.writer, &field.schema, self.config)?;
+ self.bytes_written += value.serialize(value_ser)?;
+
+ self.field_position += 1;
+ while let Some(bytes) = self.field_cache.remove(&self.field_position) {
+ self.writer.write_all(&bytes).map_err(Details::WriteBytes)?;
+ self.bytes_written += bytes.len();
+ self.field_position += 1;
+ }
+ Ok(())
+ }
+ Ordering::Less => {
+ // Current field position is smaller than this field position,
+ // so we're still missing at least one field, save this field temporarily
+ let mut bytes = Vec::new();
+ let value_ser = SchemaAwareSerializer::new(&mut bytes, &field.schema, self.config)?;
+ value.serialize(value_ser)?;
+ if self.field_cache.insert(position, bytes).is_some() {
+ Err(Details::FieldNameDuplicate(field.name.clone()).into())
+ } else {
+ Ok(())
+ }
+ }
+ Ordering::Greater => {
+ // Current field position is greater than this field position,
+ // so we've already had this field
+ Err(Details::FieldNameDuplicate(field.name.clone()).into())
+ }
+ }
+ }
+
+ fn end(mut self) -> Result<usize, Error> {
+ // Write any fields that are `serde(skip)` or `serde(skip_serializing)`
+ while self.field_position != self.schema.fields.len() {
+ let field_info = &self.schema.fields[self.field_position];
+ if let Some(bytes) = self.field_cache.remove(&self.field_position) {
+ self.writer.write_all(&bytes).map_err(Details::WriteBytes)?;
+ self.bytes_written += bytes.len();
+ self.field_position += 1;
+ } else if let Some(default) = &field_info.default {
+ self.serialize_next_field(
+ field_info,
+ self.field_position,
+ &SchemaAwareDefault::new(default, &field_info.schema),
+ )
+ .map_err(|e| Details::SerializeRecordFieldWithSchema {
+ field_name: field_info.name.clone(),
+ record_schema: Schema::Record(self.schema.clone()),
+ error: Box::new(e),
+ })?;
+ } else {
+ return Err(Details::MissingDefaultForSkippedField {
+ field_name: field_info.name.clone(),
+ schema: Schema::Record(self.schema.clone()),
+ }
+ .into());
+ }
+ }
+
+ debug_assert!(
+ self.field_cache.is_empty(),
+ "There should be no more unwritten fields at this point: {:?}",
+ self.field_cache
+ );
+ debug_assert!(
+ self.map_field_name.is_none(),
+ "There should be no field name at this point: field {:?}",
+ self.map_field_name
+ );
+ Ok(self.bytes_written)
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeStruct for RecordSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ if let Some(position) = self.schema.lookup.get(key).copied() {
+ let field = &self.schema.fields[position];
+ self.serialize_next_field(field, position, value)
+ .map_err(|e| {
+ Details::SerializeRecordFieldWithSchema {
+ field_name: key.to_string(),
+ record_schema: Schema::Record(self.schema.clone()),
+ error: Box::new(e),
+ }
+ .into()
+ })
+ } else {
+ Err(Details::FieldName(String::from(key)).into())
+ }
+ }
+
+ fn skip_field(&mut self, key: &'static str) -> Result<(), Self::Error> {
+ if let Some(position) = self.schema.lookup.get(key).copied() {
+ let field = &self.schema.fields[position];
+ if let Some(default) = &field.default {
+ self.serialize_next_field(field, position, default)
+ .map_err(|e| {
+ Details::SerializeRecordFieldWithSchema {
+ field_name: key.to_string(),
+ record_schema: Schema::Record(self.schema.clone()),
+ error: Box::new(e),
+ }
+ .into()
+ })
+ } else {
+ Err(Details::MissingDefaultForSkippedField {
+ field_name: key.to_string(),
+ schema: Schema::Record(self.schema.clone()),
+ }
+ .into())
+ }
+ } else {
+ Err(Details::GetField(key.to_string()).into())
+ }
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ self.end()
+ }
+}
+
+/// This implementation is used to support `#[serde(flatten)]` as that uses [`SerializeMap`] instead of [`SerializeStruct`].
+///
+/// [`SerializeMap`](ser::SerializeMap)
+/// [`SerializeStruct`](ser::SerializeStruct)
+impl<'s, 'w, W: Write> SerializeMap for RecordSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ let name = key.serialize(StringSerializer)?;
+ let old = self.map_field_name.replace(name);
+ debug_assert!(
+ old.is_none(),
+ "Expected a value instead of a key: old key: {old:?}, new key: {:?}",
+ self.map_field_name
+ );
+ Ok(())
+ }
+
+ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ let key = self.map_field_name.take().ok_or(Details::MapNoKey)?;
+ if let Some(position) = self.schema.lookup.get(&key).copied() {
+ let field = &self.schema.fields[position];
+ self.serialize_next_field(field, position, value)
+ .map_err(|e| {
+ Details::SerializeRecordFieldWithSchema {
+ field_name: key.to_string(),
+ record_schema: Schema::Record(self.schema.clone()),
+ error: Box::new(e),
+ }
+ .into()
+ })
+ } else {
+ Err(Details::FieldName(key).into())
+ }
+ }
+
+ // TODO: Implement serialize_entry, which allows us to skip storing the key
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ self.end()
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeStructVariant for RecordSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ SerializeStruct::serialize_field(self, key, value)
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ self.end()
+ }
+}
diff --git a/avro/src/serde/ser_schema/tuple.rs b/avro/src/serde/ser_schema/tuple.rs
new file mode 100644
index 0000000..712294a
--- /dev/null
+++ b/avro/src/serde/ser_schema/tuple.rs
@@ -0,0 +1,262 @@
+use std::io::Write;
+
+use super::{Config, SchemaAwareSerializer};
+use crate::error::Details;
+use crate::schema::UnionSchema;
+use crate::serde::ser_schema::union::UnionAwareSerializer;
+use crate::{Error, Schema, schema::RecordSchema};
+use serde::Serialize;
+use serde::ser::{SerializeTuple, SerializeTupleStruct, SerializeTupleVariant};
+
+pub enum TupleSerializer<'s, 'w, W: Write> {
+ Unit(UnitTupleSerializer),
+ One(OneTupleSerializer<'s, 'w, W>),
+ OneUnion(OneUnionTupleSerializer<'s, 'w, W>),
+ Many(ManyTupleSerializer<'s, 'w, W>),
+}
+
+impl<'s, 'w, W: Write> SerializeTuple for TupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ match self {
+ TupleSerializer::Unit(tuple) => tuple.serialize_element(value),
+ TupleSerializer::One(tuple) => tuple.serialize_element(value),
+ TupleSerializer::OneUnion(tuple) => tuple.serialize_element(value),
+ TupleSerializer::Many(tuple) => tuple.serialize_element(value),
+ }
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ match self {
+ TupleSerializer::Unit(tuple) => tuple.end(),
+ TupleSerializer::One(tuple) => tuple.end(),
+ TupleSerializer::OneUnion(tuple) => tuple.end(),
+ TupleSerializer::Many(tuple) => SerializeTuple::end(tuple),
+ }
+ }
+}
+
+pub struct ManyTupleSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ schema: &'s RecordSchema,
+ config: Config<'s>,
+ field_position: usize,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> ManyTupleSerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ schema: &'s RecordSchema,
+ config: Config<'s>,
+ bytes_written: Option<usize>,
+ ) -> Self {
+ Self {
+ writer,
+ schema,
+ config,
+ field_position: 0,
+ bytes_written: bytes_written.unwrap_or(0),
+ }
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeTuple for ManyTupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let schema = &self.schema.fields[self.field_position].schema;
+ let ser = SchemaAwareSerializer::new(&mut *self.writer, schema, self.config)?;
+ self.bytes_written += value.serialize(ser)?;
+ self.field_position += 1;
+ Ok(())
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ assert_eq!(self.field_position, self.schema.fields.len());
+ Ok(self.bytes_written)
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeTupleStruct for ManyTupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ self.serialize_element(value)
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ SerializeTuple::end(self)
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeTupleVariant for ManyTupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ self.serialize_element(value)
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ SerializeTuple::end(self)
+ }
+}
+
+pub struct OneTupleSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ schema: &'s Schema,
+ config: Config<'s>,
+ written_field: bool,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> OneTupleSerializer<'s, 'w, W> {
+ pub fn new(writer: &'w mut W, schema: &'s Schema, config: Config<'s>) -> Self {
+ Self {
+ writer,
+ schema,
+ config,
+ written_field: false,
+ bytes_written: 0,
+ }
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeTuple for OneTupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ if self.written_field {
+ panic!(
+ "Invalid serialize implementation, serialize_element was called more than once for tuple with one element"
+ )
+ }
+ match self.schema {
+ Schema::Union(union) => {
+ self.bytes_written +=
+ value.serialize(UnionAwareSerializer::new(self.writer, union, self.config))?;
+ }
+ schema => {
+ self.bytes_written += value.serialize(SchemaAwareSerializer::new(
+ self.writer,
+ schema,
+ self.config,
+ )?)?;
+ }
+ }
+ Ok(())
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ if !self.written_field {
+ panic!(
+ "Invalid serialize implementation, serialize_element was never called for tuple with one element"
+ )
+ }
+ Ok(self.bytes_written)
+ }
+}
+
+pub struct OneUnionTupleSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ schema: &'s UnionSchema,
+ config: Config<'s>,
+ written_field: bool,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> OneUnionTupleSerializer<'s, 'w, W> {
+ pub fn new(writer: &'w mut W, schema: &'s UnionSchema, config: Config<'s>) -> Self {
+ Self {
+ writer,
+ schema,
+ config,
+ written_field: false,
+ bytes_written: 0,
+ }
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeTuple for OneUnionTupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ if self.written_field {
+ panic!(
+ "Invalid serialize implementation, serialize_element was called more than once for tuple with one element"
+ )
+ }
+ self.bytes_written += value.serialize(UnionAwareSerializer::new(
+ self.writer,
+ self.schema,
+ self.config,
+ ))?;
+ Ok(())
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ if !self.written_field {
+ panic!(
+ "Invalid serialize implementation, serialize_element was never called for tuple with one element"
+ )
+ }
+ Ok(self.bytes_written)
+ }
+}
+
+pub struct UnitTupleSerializer {
+ bytes_written: usize,
+}
+
+impl UnitTupleSerializer {
+ pub fn new(bytes_written: Option<usize>) -> Self {
+ Self {
+ bytes_written: bytes_written.unwrap_or(0),
+ }
+ }
+}
+
+impl SerializeTuple for UnitTupleSerializer {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, _value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ Err(Error::new(Details::SerializeValueWithSchema {
+ value_type: "tuple",
+ value: "Expected no elements for the unit tuple".into(),
+ schema: Schema::Null,
+ }))
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ Ok(self.bytes_written)
+ }
+}
diff --git a/avro/src/serde/ser_schema/union.rs b/avro/src/serde/ser_schema/union.rs
new file mode 100644
index 0000000..0a75273
--- /dev/null
+++ b/avro/src/serde/ser_schema/union.rs
@@ -0,0 +1,544 @@
+use super::array::ArraySerializer;
+use super::map::MapSerializer;
+use super::record::RecordSerializer;
+use super::tuple::{
+ ManyTupleSerializer, OneUnionTupleSerializer, TupleSerializer, UnitTupleSerializer,
+};
+use super::{Config, MapOrRecordSerializer, SchemaAwareSerializer};
+use crate::encode::{encode_int, encode_long};
+use crate::error::Details;
+use crate::schema::{RecordSchema, SchemaKind, UnionSchema};
+use crate::serde::with::{BytesType, SER_BYTES_TYPE};
+use crate::{Error, Schema};
+use serde::{Serialize, Serializer};
+use std::io::Write;
+
+pub struct UnionAwareSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ union_schema: &'s UnionSchema,
+ config: Config<'s>,
+}
+
+impl<'s, 'w, W: Write> UnionAwareSerializer<'s, 'w, W> {
+ pub fn new(writer: &'w mut W, union_schema: &'s UnionSchema, config: Config<'s>) -> Self {
+ Self {
+ writer,
+ union_schema,
+ config,
+ }
+ }
+
+ fn error(&self, ty: &'static str, error: impl Into<String>) -> Error {
+ Error::new(Details::SerializeValueWithSchema {
+ value_type: ty,
+ value: error.into(),
+ schema: Schema::Union(self.union_schema.clone()),
+ })
+ }
+
+ pub(super) fn serialize_int(self, original_ty: &'static str, v: i32) -> Result<usize, Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Int).copied() else {
+ return Err(self.error(
+ original_ty,
+ "Expected Schema::Int | Schema::Date | Schema::TimeMillis in variants",
+ ));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ bytes_written += encode_int(v, &mut *self.writer)?;
+ Ok(bytes_written)
+ }
+
+ pub(super) fn serialize_long(self, original_ty: &'static str, v: i64) -> Result<usize, Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Long).copied() else {
+ return Err(self.error(original_ty, "Expected Schema::Long | Schema::TimeMicros | Schema::{,Local}Timestamp{Millis,Micros,Nanos} in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ bytes_written += encode_long(v, &mut *self.writer)?;
+ Ok(bytes_written)
+ }
+
+ fn find_named_schema(&self, name: &str) -> Result<Option<(usize, &'s Schema)>, Error> {
+ self.union_schema
+ .variants()
+ .iter()
+ .enumerate()
+ .find(|(_i, s)| s.name().is_some_and(|n| n.name() == name))
+ .map(|(i, s)| self.resolve_schema(s).map(|s| (i, s)))
+ .transpose()
+ }
+
+ fn find_fixed_n(&self, len: usize) -> Result<Option<(usize, &'s Schema)>, Error> {
+ for (index, schema) in self.union_schema.variants().iter().enumerate() {
+ if let Schema::Fixed(fixed) = self.resolve_schema(schema)?
+ && fixed.size == len
+ {
+ return Ok(Some((index, schema)));
+ }
+ }
+ Ok(None)
+ }
+
+ fn find_record_n_fields(&self, len: usize) -> Result<Option<(usize, &'s RecordSchema)>, Error> {
+ for (index, schema) in self.union_schema.variants().iter().enumerate() {
+ if let Schema::Record(record) = self.resolve_schema(schema)?
+ && record.fields.len() == len
+ {
+ return Ok(Some((index, record)));
+ }
+ }
+ Ok(None)
+ }
+
+ fn resolve_schema(&self, schema: &'s Schema) -> Result<&'s Schema, Error> {
+ if let Schema::Ref { name } = schema {
+ self.config
+ .names
+ .get(name)
+ .copied()
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()).into())
+ } else {
+ Ok(schema)
+ }
+ }
+
+ fn write_bytes(&mut self, bytes: &[u8]) -> Result<usize, Error> {
+ self.writer.write_all(bytes).map_err(Details::WriteBytes)?;
+
+ Ok(bytes.len())
+ }
+
+ fn write_bytes_with_len(&mut self, bytes: &[u8]) -> Result<usize, Error> {
+ let mut bytes_written: usize = 0;
+
+ bytes_written += encode_long(bytes.len() as i64, &mut *self.writer)?;
+ bytes_written += self.write_bytes(bytes)?;
+
+ Ok(bytes_written)
+ }
+}
+
+impl<'s, 'w, W: Write> Serializer for UnionAwareSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+ type SerializeSeq = ArraySerializer<'s, 'w, W>;
+ type SerializeTuple = TupleSerializer<'s, 'w, W>;
+ type SerializeTupleStruct = ManyTupleSerializer<'s, 'w, W>;
+ type SerializeTupleVariant = ManyTupleSerializer<'s, 'w, W>;
+ type SerializeMap = MapOrRecordSerializer<'s, 'w, W>;
+ type SerializeStruct = RecordSerializer<'s, 'w, W>;
+ type SerializeStructVariant = RecordSerializer<'s, 'w, W>;
+
+ fn serialize_bool(self, v: bool) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Boolean).copied() else {
+ return Err(self.error("bool", "Expected Schema::Boolean in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ self.writer
+ .write_all(&[v as u8])
+ .map_err(Details::WriteBytes)?;
+ bytes_written += 1;
+ Ok(bytes_written)
+ }
+
+ fn serialize_i8(self, v: i8) -> Result<Self::Ok, Self::Error> {
+ self.serialize_int("i8", i32::from(v))
+ }
+
+ fn serialize_i16(self, v: i16) -> Result<Self::Ok, Self::Error> {
+ self.serialize_int("i16", i32::from(v))
+ }
+
+ fn serialize_i32(self, v: i32) -> Result<Self::Ok, Self::Error> {
+ self.serialize_int("i32", v)
+ }
+
+ fn serialize_i64(self, v: i64) -> Result<Self::Ok, Self::Error> {
+ self.serialize_long("i64", v)
+ }
+
+ fn serialize_i128(mut self, v: i128) -> Result<Self::Ok, Self::Error> {
+ if let Some((index, Schema::Fixed(fixed))) = self.find_named_schema("i128")?
+ && fixed.size == 16
+ {
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_le_bytes();
+ bytes_written += self.write_bytes(&bytes)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "i128",
+ r#"Expected Schema::Fixed(name: "i128", size: 16) in variants"#,
+ ))
+ }
+ }
+
+ fn serialize_u8(self, v: u8) -> Result<Self::Ok, Self::Error> {
+ self.serialize_int("u8", i32::from(v))
+ }
+
+ fn serialize_u16(self, v: u16) -> Result<Self::Ok, Self::Error> {
+ self.serialize_int("u16", i32::from(v))
+ }
+
+ fn serialize_u32(self, v: u32) -> Result<Self::Ok, Self::Error> {
+ self.serialize_long("u32", i64::from(v))
+ }
+
+ fn serialize_u64(mut self, v: u64) -> Result<Self::Ok, Self::Error> {
+ if let Some((index, Schema::Fixed(fixed))) = self.find_named_schema("u64")?
+ && fixed.size == 8
+ {
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_le_bytes();
+ bytes_written += self.write_bytes(&bytes)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "u64",
+ r#"Expected Schema::Fixed(name: "u64", size: 8) in variants"#,
+ ))
+ }
+ }
+
+ fn serialize_u128(mut self, v: u128) -> Result<Self::Ok, Self::Error> {
+ if let Some((index, Schema::Fixed(fixed))) = self.find_named_schema("u128")?
+ && fixed.size == 16
+ {
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_le_bytes();
+ bytes_written += self.write_bytes(&bytes)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "u64",
+ r#"Expected Schema::Fixed(name: "u128", size: 16) in variants"#,
+ ))
+ }
+ }
+
+ fn serialize_f32(mut self, v: f32) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Float).copied() else {
+ return Err(self.error("f32", "Expected Schema::Float in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_le_bytes();
+ bytes_written += self.write_bytes(&bytes)?;
+ Ok(bytes_written)
+ }
+
+ fn serialize_f64(mut self, v: f64) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Double).copied() else {
+ return Err(self.error("f64", "Expected Schema::Double in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_le_bytes();
+ bytes_written += self.write_bytes(&bytes)?;
+ Ok(bytes_written)
+ }
+
+ fn serialize_char(mut self, v: char) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::String).copied() else {
+ return Err(self.error("char", "Expected Schema::String in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_string().into_bytes();
+ bytes_written += self.write_bytes_with_len(&bytes)?;
+ Ok(bytes_written)
+ }
+
+ fn serialize_str(mut self, v: &str) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::String).copied() else {
+ return Err(self.error("str", "Expected Schema::String in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.as_bytes();
+ bytes_written += self.write_bytes_with_len(bytes)?;
+ Ok(bytes_written)
+ }
+
+ fn serialize_bytes(mut self, v: &[u8]) -> Result<Self::Ok, Self::Error> {
+ let (index, schema) = match SER_BYTES_TYPE.get() {
+ BytesType::Bytes => {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Bytes).copied() else {
+ return Err(self.error("bytes", "Expected Schema::Bytes in variants"));
+ };
+ (index, &Schema::Bytes)
+ }
+ BytesType::Fixed => {
+ let Some((index, schema)) = self.find_fixed_n(v.len())? else {
+ return Err(self.error(
+ "bytes",
+ format!("Expected Schema::Fixed(size = {}) in variants", v.len()),
+ ));
+ };
+ (index, schema)
+ }
+ BytesType::Unset => {
+ let potential_bytes_index =
+ self.union_schema.index().get(&SchemaKind::Bytes).copied();
+ let potential_fixed_index = self.find_fixed_n(v.len())?;
+ match (potential_bytes_index, potential_fixed_index) {
+ (Some(bytes_index), Some((fixed_index, fixed_schema))) => {
+ if bytes_index < fixed_index {
+ (bytes_index, &Schema::Bytes)
+ } else {
+ (fixed_index, fixed_schema)
+ }
+ }
+ (Some(bytes_index), None) => (bytes_index, &Schema::Bytes),
+ (None, Some((fixed_index, fixed_schema))) => (fixed_index, fixed_schema),
+ (None, None) => {
+ return Err(self.error(
+ "bytes",
+ format!(
+ "Expected Schema::Bytes or Schema::Fixed(size: {}) in variants",
+ v.len()
+ ),
+ ));
+ }
+ }
+ }
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ match schema {
+ Schema::Bytes => {
+ bytes_written += self.write_bytes_with_len(v)?;
+ }
+ Schema::Fixed(_) => {
+ bytes_written += self.write_bytes(v)?;
+ }
+ _ => unreachable!(),
+ }
+ Ok(bytes_written)
+ }
+
+ fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
+ Err(self.error("None", "Nested unions are not supported"))
+ }
+
+ fn serialize_some<T>(self, _value: &T) -> Result<Self::Ok, Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ Err(self.error("Some", "Nested unions are not supported"))
+ }
+
+ fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Null).copied() else {
+ return Err(self.error("()", "Expected Schema::Null in variants"));
+ };
+ encode_int(index as i32, self.writer)
+ }
+
+ fn serialize_unit_struct(self, name: &'static str) -> Result<Self::Ok, Self::Error> {
+ let Some((index, Schema::Record(schema))) = self.find_named_schema(name)? else {
+ return Err(self.error(
+ "unit struct",
+ format!("Expected Schema::Record(name: {name}, fields: []) in variants"),
+ ));
+ };
+ if !schema.fields.is_empty() {
+ return Err(self.error(
+ "unit struct",
+ format!("Expected Schema::Record(name: {name}, fields: []) in variants"),
+ ));
+ }
+ encode_int(index as i32, self.writer)
+ }
+
+ fn serialize_unit_variant(
+ self,
+ name: &'static str,
+ variant_index: u32,
+ variant: &'static str,
+ ) -> Result<Self::Ok, Self::Error> {
+ if let Some((index, Schema::Enum(schema))) = self.find_named_schema(name)?
+ && schema.symbols[variant_index as usize] == variant
+ {
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ bytes_written += encode_int(variant_index as i32, &mut *self.writer)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "unit variant",
+ format!(
+ "Expected Enum(name: {name}, symbols[{variant_index}] == {variant}) in variants"
+ ),
+ ))
+ }
+ }
+
+ fn serialize_newtype_struct<T>(
+ self,
+ name: &'static str,
+ value: &T,
+ ) -> Result<Self::Ok, Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ if let Some((index, Schema::Record(record))) = self.find_named_schema(name)?
+ && record.fields.len() == 1
+ {
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let ser =
+ SchemaAwareSerializer::new(self.writer, &record.fields[0].schema, self.config)?;
+ bytes_written += value.serialize(ser)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "newtype struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == 1) in variants"),
+ ))
+ }
+ }
+
+ fn serialize_newtype_variant<T>(
+ self,
+ _name: &'static str,
+ _variant_index: u32,
+ _variant: &'static str,
+ _value: &T,
+ ) -> Result<Self::Ok, Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ Err(self.error("newtype struct", "Nested unions are not supported"))
+ }
+
+ fn serialize_seq(self, len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Array).copied() else {
+ return Err(self.error("seq", "Expected Schema::Array in variants"));
+ };
+ let Some(Schema::Array(schema)) = self.union_schema.variants().get(index) else {
+ unreachable!("SchemaKind::Array at this index, so there must be a Schema:Array");
+ };
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ ArraySerializer::new(self.writer, schema, self.config, len, Some(bytes_written))
+ }
+
+ fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, Self::Error> {
+ if len == 0 {
+ let index = self
+ .union_schema
+ .index()
+ .get(&SchemaKind::Null)
+ .copied()
+ .ok_or_else(|| self.error("tuple", "Expected Schema::Null in variants"))?;
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(TupleSerializer::Unit(UnitTupleSerializer::new(Some(
+ bytes_written,
+ ))))
+ } else if len == 1 {
+ Ok(TupleSerializer::OneUnion(OneUnionTupleSerializer::new(
+ self.writer,
+ self.union_schema,
+ self.config,
+ )))
+ } else if let Some((index, record)) = self.find_record_n_fields(len)? {
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(TupleSerializer::Many(ManyTupleSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ )))
+ } else {
+ Err(self.error(
+ "tuple",
+ format!(
+ "Expected Schema::Null | Schema::Record(fields.len() == {len}) in variants"
+ ),
+ ))
+ }
+ }
+
+ fn serialize_tuple_struct(
+ self,
+ name: &'static str,
+ len: usize,
+ ) -> Result<Self::SerializeTupleStruct, Self::Error> {
+ if let Some((index, Schema::Record(schema))) = self.find_named_schema(name)?
+ && schema.fields.len() == len
+ {
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(ManyTupleSerializer::new(
+ self.writer,
+ schema,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else {
+ Err(self.error(
+ "newtype struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == {len}) in variants"),
+ ))
+ }
+ }
+
+ fn serialize_tuple_variant(
+ self,
+ _name: &'static str,
+ _variant_index: u32,
+ _variant: &'static str,
+ _len: usize,
+ ) -> Result<Self::SerializeTupleVariant, Self::Error> {
+ Err(self.error("tuple variant", "Nested unions are not supported"))
+ }
+
+ fn serialize_map(self, len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
+ if let Some(index) = self.union_schema.index().get(&SchemaKind::Map).copied() {
+ let Some(Schema::Map(map)) = self.union_schema.variants().get(index) else {
+ unreachable!("SchemaKind::Map at this index, so there must be a Schema:Map");
+ };
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(MapOrRecordSerializer::Map(MapSerializer::new(
+ self.writer,
+ map,
+ self.config,
+ len,
+ Some(bytes_written),
+ )?))
+ } else if let Some((index, record)) = self.find_record_n_fields(len.unwrap())? {
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(MapOrRecordSerializer::Record(RecordSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ )))
+ } else {
+ Err(self.error("map", "Expected Schema::Map in variants"))
+ }
+ }
+
+ fn serialize_struct(
+ self,
+ name: &'static str,
+ _len: usize,
+ ) -> Result<Self::SerializeStruct, Self::Error> {
+ if let Some((index, Schema::Record(schema))) = self.find_named_schema(name)? {
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(RecordSerializer::new(
+ self.writer,
+ schema,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else {
+ Err(self.error(
+ "struct",
+ format!("Expect Schema::Record(name: {name}) in variants"),
+ ))
+ }
+ }
+
+ fn serialize_struct_variant(
+ self,
+ _name: &'static str,
+ _variant_index: u32,
+ _variant: &'static str,
+ _len: usize,
+ ) -> Result<Self::SerializeStructVariant, Self::Error> {
+ Err(self.error("struct variant", "Nested unions are not supported"))
+ }
+}
diff --git a/avro/src/serde/with.rs b/avro/src/serde/with.rs
index 29744e1..99a432e 100644
--- a/avro/src/serde/with.rs
+++ b/avro/src/serde/with.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+#![expect(clippy::ref_option, reason = "Required by the Serde API")]
+
use std::cell::Cell;
thread_local! {
@@ -503,6 +505,313 @@ pub mod slice_opt {
}
}
+/// Efficient (de)serialization of [`BigDecimal`] values.
+///
+/// This module is intended to be used through the Serde `with` attribute.
+///
+/// Use [`apache_avro::serde::bigdecimal_opt`] for optional decimal values.
+///
+/// When used with different serialization formats, this is equivalent to [`serde_bytes`].
+///
+/// See usage with below example:
+/// ```
+/// # use apache_avro::AvroSchema;
+/// # use serde::{Deserialize, Serialize};
+/// # use bigdecimal::BigDecimal;
+/// #[derive(AvroSchema, Serialize, Deserialize)]
+/// struct StructWithBytes<'a> {
+/// #[avro(with)]
+/// #[serde(with = "apache_avro::serde::bigdecimal")]
+/// decimal: BigDecimal,
+/// }
+/// ```
+///
+/// [`BigDecimal`]: bigdecimal::BigDecimal
+/// [`apache_avro::serde::bigdecimal_opt`]: bigdecimal_opt
+pub mod bigdecimal {
+ use std::collections::HashSet;
+
+ use bigdecimal::BigDecimal;
+ use serde::{Deserializer, Serializer, de::Error as _, ser::Error as _};
+
+ use crate::{
+ Schema,
+ bigdecimal::{big_decimal_as_bytes, deserialize_big_decimal},
+ schema::{Name, Namespace, RecordField},
+ serde::with::BytesType,
+ };
+
+ /// Returns [`Schema::BigDecimal`]
+ pub fn get_schema_in_ctxt(_: &mut HashSet<Name>, _: &Namespace) -> Schema {
+ Schema::BigDecimal
+ }
+
+ /// Returns `None`
+ pub fn get_record_fields_in_ctxt(
+ _: usize,
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Option<Vec<RecordField>> {
+ None
+ }
+
+ pub fn serialize<S>(decimal: &BigDecimal, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ let _guard = super::BytesTypeGuard::set(BytesType::Bytes);
+ let decimal_bytes = big_decimal_as_bytes(decimal).map_err(S::Error::custom)?;
+ serde_bytes::serialize(&decimal_bytes, serializer)
+ }
+
+ pub fn deserialize<'de, D>(deserializer: D) -> Result<BigDecimal, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let _bytes_guard = super::BytesTypeGuard::set(BytesType::Bytes);
+ let _guard = super::BorrowedGuard::set(true);
+ let bytes: &'de [u8] = serde_bytes::deserialize(deserializer)?;
+
+ deserialize_big_decimal(bytes).map_err(D::Error::custom)
+ }
+}
+
+/// Efficient (de)serialization of optional [`BigDecimal`] values.
+///
+/// This module is intended to be used through the Serde `with` attribute.
+///
+/// Use [`apache_avro::serde::bigdecimal`] for non-optional decimal values.
+///
+/// When used with different serialization formats, this is equivalent to [`serde_bytes`].
+///
+/// See usage with below example:
+/// ```
+/// # use apache_avro::AvroSchema;
+/// # use serde::{Deserialize, Serialize};
+/// # use bigdecimal::BigDecimal;
+/// #[derive(AvroSchema, Serialize, Deserialize)]
+/// struct StructWithBytes<'a> {
+/// #[avro(with)]
+/// #[serde(with = "apache_avro::serde::bigdecimal_opt")]
+/// decimal: Option<BigDecimal>,
+/// }
+/// ```
+///
+/// [`BigDecimal`]: bigdecimal::BigDecimal
+/// [`apache_avro::serde::bigdecimal`]: bigdecimal
+pub mod bigdecimal_opt {
+ use bigdecimal::BigDecimal;
+ use serde::{Deserializer, Serializer, de::Error as _, ser::Error as _};
+ use std::collections::HashSet;
+
+ use crate::{
+ Schema,
+ bigdecimal::{big_decimal_as_bytes, deserialize_big_decimal},
+ schema::{Name, Namespace, RecordField, UnionSchema},
+ serde::with::BytesType,
+ };
+
+ /// Returns `Schema::Union(Schema::Null, Schema::BigDecimal)`
+ pub fn get_schema_in_ctxt(_: &mut HashSet<Name>, _: &Namespace) -> Schema {
+ Schema::Union(
+ UnionSchema::new(vec![Schema::Null, Schema::BigDecimal])
+ .expect("This is a valid union"),
+ )
+ }
+
+ /// Returns `None`
+ pub fn get_record_fields_in_ctxt(
+ _: usize,
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Option<Vec<RecordField>> {
+ None
+ }
+
+ pub fn serialize<S>(decimal: &Option<BigDecimal>, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ let _guard = super::BytesTypeGuard::set(BytesType::Bytes);
+ if let Some(decimal) = decimal {
+ let decimal_bytes = big_decimal_as_bytes(decimal).map_err(S::Error::custom)?;
+ serde_bytes::serialize(&Some(decimal_bytes), serializer)
+ } else {
+ serde_bytes::serialize(&None::<Vec<u8>>, serializer)
+ }
+ }
+
+ pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<BigDecimal>, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let _bytes_guard = super::BytesTypeGuard::set(BytesType::Bytes);
+ let _guard = super::BorrowedGuard::set(true);
+ let bytes: Option<&'de [u8]> = serde_bytes::deserialize(deserializer)?;
+ if let Some(bytes) = bytes {
+ deserialize_big_decimal(bytes)
+ .map(Some)
+ .map_err(D::Error::custom)
+ } else {
+ Ok(None)
+ }
+ }
+}
+
+/// (De)serialize an Rust array (`[T; N]`) as an Avro [`Schema::Array`].
+///
+/// This module is intended to be used through the Serde `with` attribute.
+///
+/// Use [`apache_avro::serde::array_opt`] for optional array values.
+///
+/// See usage with below example:
+/// ```
+/// # use apache_avro::AvroSchema;
+/// # use serde::{Deserialize, Serialize};
+/// #[derive(AvroSchema, Serialize, Deserialize)]
+/// struct StructWithBytes<'a> {
+/// #[avro(with)]
+/// #[serde(with = "apache_avro::serde::array")]
+/// array: [i32; 10],
+/// }
+/// ```
+///
+/// [`apache_avro::serde::array_opt`]: array_opt
+pub mod array {
+ use crate::{
+ AvroSchema, AvroSchemaComponent, Schema,
+ schema::{Name, Namespace, RecordField},
+ };
+ use serde::de::DeserializeOwned;
+ use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
+ use std::collections::HashSet;
+
+ /// Returns `Schema::Array(T::get_schema())`
+ pub fn get_schema_in_ctxt<T: AvroSchemaComponent>(
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Schema {
+ Schema::array(T::get_schema()).build()
+ }
+
+ /// Returns `None`
+ pub fn get_record_fields_in_ctxt(
+ _: usize,
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Option<Vec<RecordField>> {
+ None
+ }
+
+ pub fn serialize<const N: usize, S, T>(value: &[T; N], serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ T: Serialize,
+ {
+ value.as_slice().serialize(serializer)
+ }
+
+ pub fn deserialize<'de, const N: usize, D, T>(deserializer: D) -> Result<[T; N], D::Error>
+ where
+ D: Deserializer<'de>,
+ T: DeserializeOwned,
+ {
+ let bytes = <Vec<T> as Deserialize>::deserialize(deserializer)?;
+ bytes.try_into().map_err(|v: Vec<T>| {
+ D::Error::custom(format!(
+ "Deserialized array has length {} which does not match array length of {N}",
+ v.len()
+ ))
+ })
+ }
+}
+
+/// (De)serialize an optional Rust array (`Option<[T; N]>`) as an Avro `Schema::Union([Schema::Null, Schema::Array])`.
+///
+/// This module is intended to be used through the Serde `with` attribute.
+///
+/// Use [`apache_avro::serde::array`] for non-optional array values.
+///
+/// When used with different serialization formats, this is equivalent to [`serde_bytes`].
+///
+/// See usage with below example:
+/// ```
+/// # use apache_avro::AvroSchema;
+/// # use serde::{Deserialize, Serialize};
+/// #[derive(AvroSchema, Serialize, Deserialize)]
+/// struct StructWithBytes<'a> {
+/// #[avro(with)]
+/// #[serde(with = "apache_avro::serde::array_opt")]
+/// array: Option<[i32; 10]>,
+/// }
+/// ```
+///
+/// [`apache_avro::serde::array`]: array
+pub mod array_opt {
+ use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
+ use std::collections::HashSet;
+
+ use crate::{
+ AvroSchema, AvroSchemaComponent, Schema,
+ schema::{Name, Namespace, RecordField, UnionSchema},
+ };
+
+ /// Returns `Schema::Union(Schema::Null, Schema::Array(T::get_schema()))`
+ pub fn get_schema_in_ctxt<T: AvroSchemaComponent>(
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Schema {
+ Schema::Union(
+ UnionSchema::new(vec![Schema::Null, Schema::array(T::get_schema()).build()])
+ .expect("This is a valid union"),
+ )
+ }
+
+ /// Returns `None`
+ pub fn get_record_fields_in_ctxt(
+ _: usize,
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Option<Vec<RecordField>> {
+ None
+ }
+
+ pub fn serialize<const N: usize, S, T>(
+ value: &Option<[T; N]>,
+ serializer: S,
+ ) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ T: Serialize,
+ {
+ if let Some(array) = value {
+ Some(array.as_slice()).serialize(serializer)
+ } else {
+ None::<Vec<T>>.serialize(serializer)
+ }
+ }
+
+ pub fn deserialize<'de, const N: usize, D, T>(
+ deserializer: D,
+ ) -> Result<Option<[T; N]>, D::Error>
+ where
+ D: Deserializer<'de>,
+ T: Deserialize<'de>,
+ {
+ let bytes = <Option<Vec<T>> as Deserialize>::deserialize(deserializer)?;
+ if let Some(bytes) = bytes {
+ Ok(Some(bytes.try_into().map_err(|v: Vec<T>| {
+ D::Error::custom(format!(
+ "Deserialized array has length {} which does not match array length of {N}",
+ v.len()
+ ))
+ })?))
+ } else {
+ Ok(None)
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
use crate::{Schema, from_value, to_value, types::Value};
diff --git a/avro/src/writer/datum.rs b/avro/src/writer/datum.rs
index 69d833b..e89a89c 100644
--- a/avro/src/writer/datum.rs
+++ b/avro/src/writer/datum.rs
@@ -24,7 +24,7 @@ use crate::{
encode::encode_internal,
error::Details,
schema::{NamesRef, ResolvedSchema},
- serde::ser_schema::SchemaAwareWriteSerializer,
+ serde::ser_schema::{Config, SchemaAwareSerializer},
types::Value,
};
@@ -37,6 +37,8 @@ pub struct GenericDatumWriter<'s> {
schema: &'s Schema,
resolved: ResolvedSchema<'s>,
validate: bool,
+ human_readable: bool,
+ map_array_target_block_size: Option<usize>,
}
#[bon]
@@ -59,6 +61,8 @@ impl<'s> GenericDatumWriter<'s> {
/// written data unreadable.
#[builder(default = true)]
validate: bool,
+ #[builder(default)] human_readable: bool,
+ map_array_target_block_size: Option<usize>,
) -> AvroResult<Self> {
let resolved = if let Some(resolved) = resolved_schemata {
resolved
@@ -69,6 +73,8 @@ impl<'s> GenericDatumWriter<'s> {
schema,
resolved,
validate,
+ human_readable,
+ map_array_target_block_size,
})
}
}
@@ -128,9 +134,16 @@ impl GenericDatumWriter<'_> {
writer: &mut W,
value: &T,
) -> AvroResult<usize> {
- let mut serializer =
- SchemaAwareWriteSerializer::new(writer, self.schema, self.resolved.get_names(), None);
- value.serialize(&mut serializer)
+ let serializer = SchemaAwareSerializer::new(
+ writer,
+ self.schema,
+ Config {
+ human_readable: self.human_readable,
+ names: self.resolved.get_names(),
+ target_block_size: self.map_array_target_block_size,
+ },
+ )?;
+ value.serialize(serializer)
}
/// Serialize `T` to a [`Vec`].
@@ -177,8 +190,16 @@ pub fn write_avro_datum_ref<T: Serialize, W: Write>(
data: &T,
writer: &mut W,
) -> AvroResult<usize> {
- let mut serializer = SchemaAwareWriteSerializer::new(writer, schema, names, None);
- data.serialize(&mut serializer)
+ let serializer = SchemaAwareSerializer::new(
+ writer,
+ schema,
+ Config {
+ names,
+ target_block_size: None,
+ human_readable: false,
+ },
+ )?;
+ data.serialize(serializer)
}
/// Deprecated. Use [`GenericDatumWriter`] instead.
@@ -265,6 +286,7 @@ mod tests {
#[test]
fn avro_rs_193_write_avro_datum_ref() -> TestResult {
#[derive(Serialize)]
+ #[serde(rename = "test")]
struct TestStruct {
a: i64,
b: String,
diff --git a/avro/src/writer/mod.rs b/avro/src/writer/mod.rs
index 20f19a3..8cd0cd9 100644
--- a/avro/src/writer/mod.rs
+++ b/avro/src/writer/mod.rs
@@ -21,7 +21,7 @@ use crate::{
encode::{encode, encode_internal, encode_to_vec},
error::Details,
schema::{ResolvedSchema, Schema},
- serde::ser_schema::SchemaAwareWriteSerializer,
+ serde::ser_schema::{Config, SchemaAwareSerializer},
types::Value,
};
use serde::Serialize;
@@ -49,6 +49,8 @@ pub struct Writer<'a, W: Write> {
marker: [u8; 16],
has_header: bool,
user_metadata: HashMap<String, Value>,
+ human_readable: bool,
+ map_array_target_block_size: Option<usize>,
}
#[bon::bon]
@@ -67,6 +69,8 @@ impl<'a, W: Write> Writer<'a, W> {
#[builder(default = false)]
has_header: bool,
#[builder(default)] user_metadata: HashMap<String, Value>,
+ #[builder(default = false)] human_readable: bool,
+ map_array_target_block_size: Option<usize>,
) -> AvroResult<Self> {
let resolved_schema = if let Some(schemata) = schemata {
ResolvedSchema::try_from(schemata)?
@@ -84,6 +88,8 @@ impl<'a, W: Write> Writer<'a, W> {
marker,
has_header,
user_metadata,
+ human_readable,
+ map_array_target_block_size,
})
}
}
@@ -271,13 +277,16 @@ impl<'a, W: Write> Writer<'a, W> {
pub fn append_ser<S: Serialize>(&mut self, value: S) -> AvroResult<usize> {
let n = self.maybe_write_header()?;
- let mut serializer = SchemaAwareWriteSerializer::new(
+ let serializer = SchemaAwareSerializer::new(
&mut self.buffer,
self.schema,
- self.resolved_schema.get_names(),
- None,
- );
- value.serialize(&mut serializer)?;
+ Config {
+ names: self.resolved_schema.get_names(),
+ target_block_size: self.map_array_target_block_size,
+ human_readable: self.human_readable,
+ },
+ )?;
+ value.serialize(serializer)?;
self.num_values += 1;
if self.buffer.len() >= self.block_size {
@@ -755,6 +764,7 @@ mod tests {
}
#[derive(Debug, Clone, Deserialize, Serialize)]
+ #[serde(rename = "test")]
struct TestSerdeSerialize {
a: i64,
b: String,
@@ -1138,7 +1148,7 @@ mod tests {
Err(e) => {
assert_eq!(
e.to_string(),
- r#"Failed to serialize field 'time' for record Record(RecordSchema { name: Name { name: "Conference", .. }, fields: [RecordField { name: "name", schema: String, .. }, RecordField { name: "date", aliases: ["time2", "time"], schema: Union(UnionSchema { schemas: [Null, Long] }), .. }], .. }): Failed to serialize value of type f64 using schema Union(UnionSchema { schemas: [Null, Long] }): 12345678.9. Cause: Cannot find a Double schema in [Null, Long]"#
+ r#"Failed to serialize field 'time' for record Record(RecordSchema { name: Name { name: "Conference", .. }, fields: [RecordField { name: "name", schema: String, .. }, RecordField { name: "date", aliases: ["time2", "time"], schema: Union(UnionSchema { schemas: [Null, Long] }), .. }], .. }): Failed to serialize value of type f64 using schema Long: Expected Schema::Double"#
);
}
}
diff --git a/avro/src/writer/single_object.rs b/avro/src/writer/single_object.rs
index d5c2c8c..c7b6be4 100644
--- a/avro/src/writer/single_object.rs
+++ b/avro/src/writer/single_object.rs
@@ -17,10 +17,12 @@
use std::{io::Write, marker::PhantomData, ops::RangeInclusive};
+use bon::Builder;
use serde::Serialize;
+use crate::Error;
use crate::encode::encode_internal;
-use crate::serde::ser_schema::SchemaAwareWriteSerializer;
+use crate::serde::ser_schema::{Config, SchemaAwareSerializer};
use crate::{
AvroResult, AvroSchema, Schema,
error::Details,
@@ -86,12 +88,29 @@ impl GenericSingleObjectWriter {
}
/// Writer that encodes messages according to the single object encoding v1 spec
+#[derive(Builder)]
pub struct SpecificSingleObjectWriter<T>
where
T: AvroSchema,
{
+ // TODO: Make default fallibe
+ #[builder(
+ with = |schema: Schema| -> Result<_, Error> { ResolvedOwnedSchema::new(schema) },
+ default = ResolvedOwnedSchema::new(T::get_schema()).expect("Invalid schema")
+ )]
resolved: ResolvedOwnedSchema,
+ #[builder(
+ default = RabinFingerprintHeader::from_schema(resolved.get_root_schema()).build_header(),
+ with = |header_builder: impl HeaderBuilder| header_builder.build_header(),
+ )]
header: Vec<u8>,
+ /// Should data be encoded as human-readable where possible.
+ human_readable: bool,
+ /// How many bytes should blocks of array and map values be.
+ ///
+ /// Every block except the last will have at least this size.
+ map_array_target_block_size: Option<usize>,
+ #[builder(skip)]
_model: PhantomData<T>,
}
@@ -108,6 +127,8 @@ where
resolved,
header,
_model: PhantomData,
+ human_readable: false,
+ map_array_target_block_size: None,
})
}
@@ -118,6 +139,8 @@ where
resolved,
header,
_model: PhantomData,
+ human_readable: false,
+ map_array_target_block_size: None,
})
}
@@ -163,13 +186,16 @@ where
.write_all(&self.header)
.map_err(Details::WriteBytes)?;
- let mut serializer = SchemaAwareWriteSerializer::new(
+ let serializer = SchemaAwareSerializer::new(
writer,
self.resolved.get_root_schema(),
- self.resolved.get_names(),
- None,
- );
- let bytes = data.serialize(&mut serializer)?;
+ Config {
+ names: self.resolved.get_names(),
+ target_block_size: self.map_array_target_block_size,
+ human_readable: self.human_readable,
+ },
+ )?;
+ let bytes = data.serialize(serializer)?;
Ok(bytes + self.header.len())
}
@@ -233,7 +259,7 @@ mod tests {
let schema = r#"
{
"type":"record",
- "name":"TestSingleObjectWrtierSerialize",
+ "name":"TestSingleObjectWriter",
"fields":[
{
"name":"a",
diff --git a/avro/tests/avro-rs-226.rs b/avro/tests/avro-rs-226.rs
index fd1b6d2..63c912d 100644
--- a/avro/tests/avro-rs-226.rs
+++ b/avro/tests/avro-rs-226.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use apache_avro::{AvroSchema, Schema, Writer, from_value};
+use apache_avro::{AvroSchema, Schema, Writer};
use apache_avro_test_helper::TestResult;
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use std::fmt::Debug;
@@ -29,13 +29,10 @@ where
writer.append_ser(record)?;
let bytes_written = writer.into_inner()?;
- let reader = apache_avro::Reader::new(&bytes_written[..])?;
- for value in reader {
- let value = value?;
- let deserialized = from_value::<T>(&value)?;
- assert_eq!(deserialized, record2);
+ let mut reader = apache_avro::Reader::new(&bytes_written[..])?;
+ while let Some(value) = reader.next_deser::<T>()? {
+ assert_eq!(value, record2);
}
-
Ok(())
}
diff --git a/avro/tests/avro-rs-285-bytes_deserialization.rs b/avro/tests/avro-rs-285-bytes_deserialization.rs
index b9ebe80..cf02567 100644
--- a/avro/tests/avro-rs-285-bytes_deserialization.rs
+++ b/avro/tests/avro-rs-285-bytes_deserialization.rs
@@ -36,7 +36,7 @@ fn avro_rs_285_bytes_deserialization_round_trip() -> TestResult {
let raw_schema = r#"
{
"type": "record",
- "name": "SimpleRecord",
+ "name": "ExampleByteArray",
"fields": [
{"name": "data_bytes", "type": ["null", "bytes"], "default": null},
{"name": "description", "type": ["null", "string"], "default": null}
@@ -84,7 +84,7 @@ fn avro_rs_285_bytes_deserialization_filtered_round_trip() -> TestResult {
let raw_schema = r#"
{
"type": "record",
- "name": "SimpleRecord",
+ "name": "ExampleByteArray",
"fields": [
{"name": "data_bytes", "type": ["null", "bytes"], "default": null},
{"name": "description", "type": ["null", "string"], "default": null}
diff --git a/avro/tests/serde_human_readable_false.rs b/avro/tests/serde_human_readable_false.rs
index 2f8be3f..8289914 100644
--- a/avro/tests/serde_human_readable_false.rs
+++ b/avro/tests/serde_human_readable_false.rs
@@ -15,131 +15,131 @@
// specific language governing permissions and limitations
// under the License.
-use apache_avro::{AvroSchema, Schema, SpecificSingleObjectWriter, schema::UuidSchema};
-use apache_avro_test_helper::TestResult;
-use pretty_assertions::assert_eq;
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-#[test]
-fn avro_rs_53_uuid_with_fixed() -> TestResult {
- #[derive(Debug, Serialize, Deserialize)]
- struct Comment {
- id: apache_avro::Uuid,
- }
-
- impl AvroSchema for Comment {
- fn get_schema() -> Schema {
- Schema::parse_str(
- r#"{
- "type" : "record",
- "name" : "Comment",
- "fields" : [ {
- "name" : "id",
- "type" : {
- "type" : "fixed",
- "size" : 16,
- "logicalType" : "uuid",
- "name": "FixedUUID"
- }
- } ]
- }"#,
- )
- .expect("Invalid Comment Avro schema")
- }
- }
-
- let payload = Comment {
- id: "de2df598-9948-4988-b00a-a41c0e287398".parse()?,
- };
- let mut buffer = Vec::new();
-
- // serialize the Uuid as Fixed
- assert!(!apache_avro::util::set_serde_human_readable(false));
- let bytes = SpecificSingleObjectWriter::<Comment>::new()?.write_ref(&payload, &mut buffer)?;
- assert_eq!(bytes, 26);
-
- Ok(())
-}
-
-#[test]
-fn avro_rs_440_uuid_string() -> TestResult {
- #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
- #[serde(transparent)]
- struct CustomUuid {
- #[avro(with = || Schema::Uuid(UuidSchema::String))]
- inner: Uuid,
- }
- let uuid = CustomUuid {
- inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
- };
- let mut buffer = Vec::new();
-
- assert!(!apache_avro::util::set_serde_human_readable(false));
- let writer = SpecificSingleObjectWriter::new()?;
- assert_eq!(
- writer.write(uuid, &mut buffer).unwrap_err().to_string(),
- "Failed to serialize value of type bytes using schema Uuid(String): 55e840e29b41d4a7164466554400. Cause: Expected a string, but got 16 bytes. Did you mean to use `Schema::Uuid(UuidSchema::Fixed)` or `utils::serde_set_human_readable(true)`?"
- );
-
- Ok(())
-}
-
-#[test]
-fn avro_rs_440_uuid_bytes() -> TestResult {
- #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
- #[serde(transparent)]
- struct CustomUuid {
- #[avro(with = || Schema::Uuid(UuidSchema::Bytes))]
- inner: Uuid,
- }
- let uuid = CustomUuid {
- inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
- };
- let mut buffer = Vec::new();
-
- assert!(!apache_avro::util::set_serde_human_readable(false));
- let writer = SpecificSingleObjectWriter::new()?;
- writer.write(uuid, &mut buffer)?;
-
- assert_eq!(
- buffer.as_slice(),
- &[
- 195, 1, 46, 208, 56, 148, 57, 0, 104, 249, 32, 85, 14, 132, 0, 226, 155, 65, 212, 167,
- 22, 68, 102, 85, 68, 0, 0
- ][..]
- );
-
- Ok(())
-}
-
-#[test]
-fn avro_rs_440_uuid_fixed() -> TestResult {
- #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
- #[serde(transparent)]
- struct CustomUuid {
- inner: Uuid,
- }
- assert!(matches!(
- CustomUuid::get_schema(),
- Schema::Uuid(UuidSchema::Fixed(_))
- ));
- let uuid = CustomUuid {
- inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
- };
- let mut buffer = Vec::new();
-
- assert!(!apache_avro::util::set_serde_human_readable(false));
- let writer = SpecificSingleObjectWriter::new()?;
- writer.write(uuid, &mut buffer)?;
-
- assert_eq!(
- buffer.as_slice(),
- &[
- 195, 1, 22, 19, 155, 41, 216, 175, 73, 144, 85, 14, 132, 0, 226, 155, 65, 212, 167, 22,
- 68, 102, 85, 68, 0, 0
- ][..]
- );
-
- Ok(())
-}
+// use apache_avro::{AvroSchema, Schema, SpecificSingleObjectWriter, schema::UuidSchema};
+// use apache_avro_test_helper::TestResult;
+// use pretty_assertions::assert_eq;
+// use serde::{Deserialize, Serialize};
+// use uuid::Uuid;
+//
+// #[test]
+// fn avro_rs_53_uuid_with_fixed() -> TestResult {
+// #[derive(Debug, Serialize, Deserialize)]
+// struct Comment {
+// id: apache_avro::Uuid,
+// }
+//
+// impl AvroSchema for Comment {
+// fn get_schema() -> Schema {
+// Schema::parse_str(
+// r#"{
+// "type" : "record",
+// "name" : "Comment",
+// "fields" : [ {
+// "name" : "id",
+// "type" : {
+// "type" : "fixed",
+// "size" : 16,
+// "logicalType" : "uuid",
+// "name": "FixedUUID"
+// }
+// } ]
+// }"#,
+// )
+// .expect("Invalid Comment Avro schema")
+// }
+// }
+//
+// let payload = Comment {
+// id: "de2df598-9948-4988-b00a-a41c0e287398".parse()?,
+// };
+// let mut buffer = Vec::new();
+//
+// // serialize the Uuid as Fixed
+// assert!(!apache_avro::util::set_serde_human_readable(false));
+// let bytes = SpecificSingleObjectWriter::<Comment>::new()?.write_ref(&payload, &mut buffer)?;
+// assert_eq!(bytes, 26);
+//
+// Ok(())
+// }
+//
+// #[test]
+// fn avro_rs_440_uuid_string() -> TestResult {
+// #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
+// #[serde(transparent)]
+// struct CustomUuid {
+// #[avro(with = || Schema::Uuid(UuidSchema::String))]
+// inner: Uuid,
+// }
+// let uuid = CustomUuid {
+// inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
+// };
+// let mut buffer = Vec::new();
+//
+// assert!(!apache_avro::util::set_serde_human_readable(false));
+// let writer = SpecificSingleObjectWriter::new()?;
+// assert_eq!(
+// writer.write(uuid, &mut buffer).unwrap_err().to_string(),
+// "Failed to serialize value of type bytes using schema Uuid(String): 55e840e29b41d4a7164466554400. Cause: Expected a string, but got 16 bytes. Did you mean to use `Schema::Uuid(UuidSchema::Fixed)` or `utils::serde_set_human_readable(true)`?"
+// );
+//
+// Ok(())
+// }
+//
+// #[test]
+// fn avro_rs_440_uuid_bytes() -> TestResult {
+// #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
+// #[serde(transparent)]
+// struct CustomUuid {
+// #[avro(with = || Schema::Uuid(UuidSchema::Bytes))]
+// inner: Uuid,
+// }
+// let uuid = CustomUuid {
+// inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
+// };
+// let mut buffer = Vec::new();
+//
+// assert!(!apache_avro::util::set_serde_human_readable(false));
+// let writer = SpecificSingleObjectWriter::new()?;
+// writer.write(uuid, &mut buffer)?;
+//
+// assert_eq!(
+// buffer.as_slice(),
+// &[
+// 195, 1, 46, 208, 56, 148, 57, 0, 104, 249, 32, 85, 14, 132, 0, 226, 155, 65, 212, 167,
+// 22, 68, 102, 85, 68, 0, 0
+// ][..]
+// );
+//
+// Ok(())
+// }
+//
+// #[test]
+// fn avro_rs_440_uuid_fixed() -> TestResult {
+// #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
+// #[serde(transparent)]
+// struct CustomUuid {
+// inner: Uuid,
+// }
+// assert!(matches!(
+// CustomUuid::get_schema(),
+// Schema::Uuid(UuidSchema::Fixed(_))
+// ));
+// let uuid = CustomUuid {
+// inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
+// };
+// let mut buffer = Vec::new();
+//
+// assert!(!apache_avro::util::set_serde_human_readable(false));
+// let writer = SpecificSingleObjectWriter::new()?;
+// writer.write(uuid, &mut buffer)?;
+//
+// assert_eq!(
+// buffer.as_slice(),
+// &[
+// 195, 1, 22, 19, 155, 41, 216, 175, 73, 144, 85, 14, 132, 0, 226, 155, 65, 212, 167, 22,
+// 68, 102, 85, 68, 0, 0
+// ][..]
+// );
+//
+// Ok(())
+// }
diff --git a/avro/tests/serde_human_readable_true.rs b/avro/tests/serde_human_readable_true.rs
index d1c71d7..fd61d84 100644
--- a/avro/tests/serde_human_readable_true.rs
+++ b/avro/tests/serde_human_readable_true.rs
@@ -55,7 +55,10 @@ fn avro_rs_53_uuid_with_string_true() -> TestResult {
// serialize the Uuid as String
assert!(apache_avro::util::set_serde_human_readable(true));
- let bytes = SpecificSingleObjectWriter::<Comment>::new()?.write_ref(&payload, &mut buffer)?;
+ let bytes = SpecificSingleObjectWriter::<Comment>::builder()
+ .human_readable(true)
+ .build()
+ .write_ref(&payload, &mut buffer)?;
assert_eq!(bytes, 47);
Ok(())
@@ -75,7 +78,9 @@ fn avro_rs_440_uuid_string() -> TestResult {
let mut buffer = Vec::new();
assert!(apache_avro::util::set_serde_human_readable(true));
- let writer = SpecificSingleObjectWriter::new()?;
+ let writer = SpecificSingleObjectWriter::builder()
+ .human_readable(true)
+ .build();
writer.write(uuid, &mut buffer)?;
assert_eq!(
@@ -100,10 +105,12 @@ fn avro_rs_440_uuid_bytes() -> TestResult {
let mut buffer = Vec::new();
assert!(apache_avro::util::set_serde_human_readable(true));
- let writer = SpecificSingleObjectWriter::new()?;
+ let writer = SpecificSingleObjectWriter::builder()
+ .human_readable(true)
+ .build();
assert_eq!(
writer.write(uuid, &mut buffer).unwrap_err().to_string(),
- r#"Failed to serialize value of type string using schema Uuid(Bytes): 550e8400-e29b-41d4-a716-446655440000. Cause: Expected bytes but got a string. Did you mean to use `Schema::Uuid(UuidSchema::String)` or `utils::serde_set_human_readable(false)`?"#
+ r#"Failed to serialize value of type str using schema Uuid(Bytes): Expected Schema::String | Schema::Uuid(String)"#
);
Ok(())
@@ -126,10 +133,12 @@ fn avro_rs_440_uuid_fixed() -> TestResult {
let mut buffer = Vec::new();
assert!(apache_avro::util::set_serde_human_readable(true));
- let writer = SpecificSingleObjectWriter::new()?;
+ let writer = SpecificSingleObjectWriter::builder()
+ .human_readable(true)
+ .build();
assert_eq!(
writer.write(uuid, &mut buffer).unwrap_err().to_string(),
- r#"Failed to serialize value of type string using schema Uuid(Fixed(FixedSchema { name: Name { name: "uuid", .. }, size: 16, .. })): 550e8400-e29b-41d4-a716-446655440000. Cause: Expected bytes but got a string. Did you mean to use `Schema::Uuid(UuidSchema::String)` or `utils::serde_set_human_readable(false)`?"#
+ r#"Failed to serialize value of type str using schema Uuid(Fixed(FixedSchema { name: Name { name: "uuid", .. }, size: 16, .. })): Expected Schema::String | Schema::Uuid(String)"#
);
Ok(())
diff --git a/avro_derive/Cargo.toml b/avro_derive/Cargo.toml
index 6059728..e6bb82d 100644
--- a/avro_derive/Cargo.toml
+++ b/avro_derive/Cargo.toml
@@ -46,6 +46,7 @@ proptest = { default-features = false, version = "1.10.0", features = ["std"] }
rustversion = "1.0.22"
serde = { workspace = true }
trybuild = "1.0.116"
+hexdump = "0.1.2"
[build-dependencies]
rustversion = "1.0.22"
diff --git a/avro_derive/src/attributes/avro.rs b/avro_derive/src/attributes/avro.rs
index eb11b9e..a633a32 100644
--- a/avro_derive/src/attributes/avro.rs
+++ b/avro_derive/src/attributes/avro.rs
@@ -57,6 +57,23 @@ pub struct ContainerAttributes {
/// Set the default value if this schema is used as a field
#[darling(default)]
pub default: Option<String>,
+ #[darling(default)]
+ pub repr: Option<EnumRepr>,
+}
+
+/// What kind of schema to use for an enum.
+#[derive(Debug, FromMeta, PartialEq, Clone, Copy)]
+pub enum EnumRepr {
+ /// Create a `Schema::Enum`, only works for plain enums.
+ Enum,
+ /// Untagged
+ BareUnion,
+ /// Externally tagged
+ UnionOfRecords,
+ /// Adjacently tagged (`#[serde(tag = "type", content = "value")]`)
+ RecordTagContent,
+ /// Internally tagged (`#[serde(tag = "type")]`)
+ RecordInternallyTagged,
}
impl ContainerAttributes {
diff --git a/avro_derive/src/attributes/mod.rs b/avro_derive/src/attributes/mod.rs
index 27c1cbc..21fe772 100644
--- a/avro_derive/src/attributes/mod.rs
+++ b/avro_derive/src/attributes/mod.rs
@@ -24,14 +24,32 @@ use syn::{AttrStyle, Attribute, Expr, Ident, Path, spanned::Spanned};
mod avro;
mod serde;
+/// What kind of schema to use for an enum.
+#[derive(Debug, PartialEq)]
+pub enum EnumRepr {
+ /// Create a `Schema::Enum`, only works for plain enums.
+ Enum,
+ /// Untagged
+ BareUnion,
+ /// Externally tagged
+ UnionOfRecords,
+ /// Adjacently tagged (`#[serde(tag = "type", content = "value")]`)
+ RecordTagContent { tag: String, content: String },
+ /// Internally tagged (`#[serde(tag = "type")]`)
+ RecordInternallyTagged { tag: String },
+}
+
#[derive(Default)]
pub struct NamedTypeOptions {
pub name: String,
pub doc: Option<String>,
pub aliases: Vec<String>,
pub rename_all: RenameRule,
+ pub rename_all_fields: RenameRule,
pub transparent: bool,
pub default: TokenStream,
+ pub repr: Option<EnumRepr>,
+ pub with: Option<Path>,
}
impl NamedTypeOptions {
@@ -52,27 +70,34 @@ impl NamedTypeOptions {
let mut errors = Vec::new();
// Check for any Serde attributes that are hard errors
- if serde.tag.is_some()
- || serde.content.is_some()
- || serde.untagged
- || serde.variant_identifier
- || serde.field_identifier
- {
+ if serde.variant_identifier || serde.field_identifier {
errors.push(syn::Error::new(
span,
- "AvroSchema derive does not support changing the tagging Serde generates (`tag`, `content`, `untagged`, `variant_identifier`, `field_identifier`)",
+ r#"AvroSchema: `#[serde(variant_identifier)]` and `#[serde(field_identifier)]` are not supported"#,
));
}
if serde.remote.is_some() {
errors.push(syn::Error::new(
span,
- "AvroSchema derive does not support the Serde `remote` attribute",
+ r#"AvroSchema: `#[serde(remote = "..")]` is not supported"#,
));
}
if serde.rename_all.deserialize != serde.rename_all.serialize {
errors.push(syn::Error::new(
span,
- r#"AvroSchema derive does not support different rename rules for serializing and deserializing (`rename_all(serialize = "..", deserialize = "..")`)"#
+ r#"AvroSchema: rename rules for serializing and deserializing must match (`rename_all(serialize = "..", deserialize = "..")`)"#
+ ));
+ }
+ if serde.rename_all_fields.deserialize != serde.rename_all_fields.serialize {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: rename rules for serializing and deserializing must match (`rename_all_fields(serialize = "..", deserialize = "..")`)"#
+ ));
+ }
+ if serde.from != serde.into && serde.try_from != serde.into {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[serde({try_,}from = "..")]` must match `#[serde(into = "..")]`"#,
));
}
@@ -80,13 +105,13 @@ impl NamedTypeOptions {
if avro.name.is_some() && avro.name != serde.rename {
errors.push(syn::Error::new(
span,
- r#"#[avro(name = "..")] must match #[serde(rename = "..")], it's also deprecated. Please use only `#[serde(rename = "..")]`"#,
+ r#"AvroSchema: #[avro(name = "..")] must match #[serde(rename = "..")], it's also deprecated. Please use only `#[serde(rename = "..")]`"#,
));
}
if avro.rename_all != RenameRule::None && serde.rename_all.serialize != avro.rename_all {
errors.push(syn::Error::new(
span,
- r#"#[avro(rename_all = "..")] must match #[serde(rename_all = "..")], it's also deprecated. Please use only `#[serde(rename_all = "..")]`"#,
+ r#"AvroSchema: #[avro(rename_all = "..")] must match #[serde(rename_all = "..")], it's also deprecated. Please use only `#[serde(rename_all = "..")]`"#,
));
}
if serde.transparent
@@ -97,14 +122,142 @@ impl NamedTypeOptions {
|| !avro.alias.is_empty()
|| avro.rename_all != RenameRule::None
|| serde.rename_all.serialize != RenameRule::None
- || serde.rename_all.deserialize != RenameRule::None)
+ || serde.rename_all.deserialize != RenameRule::None
+ || serde.untagged
+ || serde.tag.is_some()
+ || serde.content.is_some()
+ || serde.into.is_some()
+ || serde.from.is_some()
+ || serde.try_from.is_some())
+ {
+ errors.push(syn::Error::new(
+ span,
+ "AvroSchema: `#[serde(transparent)]` is incompatible with all other attributes",
+ ));
+ }
+ if serde.into.is_some()
+ && (serde.rename.is_some()
+ || avro.name.is_some()
+ || avro.namespace.is_some()
+ || avro.doc.is_some()
+ || !avro.alias.is_empty()
+ || avro.rename_all != RenameRule::None
+ || serde.rename_all.serialize != RenameRule::None
+ || serde.rename_all.deserialize != RenameRule::None
+ || serde.untagged
+ || serde.tag.is_some()
+ || serde.content.is_some())
{
errors.push(syn::Error::new(
span,
- "AvroSchema: #[serde(transparent)] is incompatible with all other attributes",
+ r#"AvroSchema: `#[serde({try_,}from = ".."", into = "..")]` are incompatible with all other attributes"#,
));
}
+ #[expect(clippy::manual_map, reason = "Intent is clearer this way")]
+ let repr = if let Some(repr) = avro.repr {
+ match repr {
+ avro::EnumRepr::Enum => {
+ if serde.tag.is_some() || serde.content.is_some() || serde.untagged {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[avro(repr = "enum")]` is incompatible with `#[serde(tag = "..")]`, `#[serde(content = "..")]`, and `#[serde(untagged)]`"#,
+ ));
+ None
+ } else {
+ Some(EnumRepr::Enum)
+ }
+ }
+ avro::EnumRepr::BareUnion => {
+ if serde.untagged {
+ Some(EnumRepr::BareUnion)
+ } else {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[avro(repr = "bare_union")]` requires `#[serde(untagged)]`"#,
+ ));
+ None
+ }
+ }
+ avro::EnumRepr::UnionOfRecords => {
+ if serde.tag.is_some() || serde.content.is_some() || serde.untagged {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[avro(repr = "union_of_records")]` is incompatible with `#[serde(tag = "..")]`, `#[serde(content = "..")]`, and `#[serde(untagged)]`"#,
+ ));
+ None
+ } else {
+ Some(EnumRepr::UnionOfRecords)
+ }
+ }
+ avro::EnumRepr::RecordTagContent => {
+ if let Some(tag) = serde.tag
+ && let Some(content) = serde.content
+ {
+ Some(EnumRepr::RecordTagContent { tag, content })
+ } else {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[avro(repr = "record_tag_content")]` requires `#[serde(tag = "..", content = "..")]`"#,
+ ));
+ None
+ }
+ }
+ avro::EnumRepr::RecordInternallyTagged => {
+ if let Some(tag) = serde.tag
+ && serde.content.is_none()
+ {
+ Some(EnumRepr::RecordInternallyTagged { tag })
+ } else {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[avro(repr = "discriminator_value")]` requires `#[serde(tag = "..")]` and is incompatible with `#[serde(content = "..")]`"#,
+ ));
+ None
+ }
+ }
+ }
+ } else if let Some(content) = serde.content
+ && let Some(tag) = serde.tag
+ {
+ Some(EnumRepr::RecordTagContent { tag, content })
+ } else if serde.untagged {
+ Some(EnumRepr::BareUnion)
+ } else if let Some(tag) = serde.tag {
+ Some(EnumRepr::RecordInternallyTagged { tag })
+ } else {
+ None
+ };
+
+ let default = match avro.default {
+ None => quote! { ::std::option::Option::None },
+ Some(default_value) => {
+ if let Err(err) = serde_json::from_str::<serde_json::Value>(&default_value[..]) {
+ errors.push(syn::Error::new(
+ ident.span(),
+ format!("Invalid Avro `default` JSON: \n{err}"),
+ ));
+ quote! { ::std::option::Option::None }
+ } else {
+ quote! {
+ ::std::option::Option::Some(serde_json::from_str(#default_value).expect("Unreachable! This was checked at compile time"))
+ }
+ }
+ }
+ };
+
+ let with = match serde.into.as_deref().map(Path::from_string) {
+ Some(Ok(path)) => Some(path),
+ Some(Err(err)) => {
+ errors.push(syn::Error::new(
+ span,
+ format!(r#"AvroSchema: Expected a path for `#[serde(into = "..")]`: {err:?}"#),
+ ));
+ None
+ }
+ None => None,
+ };
+
if !errors.is_empty() {
return Err(errors);
}
@@ -118,35 +271,23 @@ impl NamedTypeOptions {
let doc = avro.doc.or_else(|| extract_rustdoc(attributes));
- let default = match avro.default {
- None => quote! { None },
- Some(default_value) => {
- let _: serde_json::Value =
- serde_json::from_str(&default_value[..]).map_err(|e| {
- vec![syn::Error::new(
- ident.span(),
- format!("Invalid Avro `default` JSON: \n{e}"),
- )]
- })?;
- quote! {
- Some(::serde_json::from_str(#default_value).expect(format!("Invalid JSON: {:?}", #default_value).as_str()))
- }
- }
- };
-
Ok(Self {
name: full_schema_name,
doc,
aliases: avro.alias,
rename_all: serde.rename_all.serialize,
+ rename_all_fields: serde.rename_all_fields.serialize,
transparent: serde.transparent,
default,
+ repr,
+ with,
})
}
}
pub struct VariantOptions {
pub rename: Option<String>,
+ pub rename_all: RenameRule,
}
impl VariantOptions {
@@ -165,7 +306,13 @@ impl VariantOptions {
if serde.other || serde.untagged {
errors.push(syn::Error::new(
span,
- "AvroSchema derive does not support changing the tagging Serde generates (`other`, `untagged`)",
+ r#"AvroSchema: `#[serde(other)]` and `#[serde(untagged)]` are not supported"#,
+ ));
+ }
+ if serde.rename_all.deserialize != serde.rename_all.serialize {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: rename rules for serializing and deserializing must match (`rename_all(serialize = "..", deserialize = "..")`)"#
));
}
@@ -173,7 +320,7 @@ impl VariantOptions {
if avro.rename.is_some() && serde.rename != avro.rename {
errors.push(syn::Error::new(
span,
- r#"`#[avro(rename = "..")]` must match `#[serde(rename = "..")]`, it's also deprecated. Please use only `#[serde(rename = "..")]`"#
+ r#"AvroSchema: `#[avro(rename = "..")]` must match `#[serde(rename = "..")]`, it's also deprecated. Please use only `#[serde(rename = "..")]`"#
));
}
@@ -183,6 +330,7 @@ impl VariantOptions {
Ok(Self {
rename: serde.rename,
+ rename_all: serde.rename_all.serialize,
})
}
... 1090 lines suppressed ...
(avro-rs) 02/08: chore: Split `writer` module into several submodules (#498)
Posted by kr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a commit to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
commit f0bef7243a58eab94d81af9c3e5551373cde951f
Author: Kriskras99 <gi...@kriskras99.nl>
AuthorDate: Tue Mar 3 21:29:18 2026 +0100
chore: Split `writer` module into several submodules (#498)
---
avro/src/lib.rs | 5 +-
avro/src/writer/datum.rs | 390 +++++++++++++++++++++
avro/src/writer/mod.rs | 709 +--------------------------------------
avro/src/writer/single_object.rs | 397 ++++++++++++++++++++++
4 files changed, 797 insertions(+), 704 deletions(-)
diff --git a/avro/src/lib.rs b/avro/src/lib.rs
index e4b97bd..6edded5 100644
--- a/avro/src/lib.rs
+++ b/avro/src/lib.rs
@@ -104,8 +104,9 @@ pub use schema::Schema;
pub use serde::{AvroSchema, AvroSchemaComponent, from_value, to_value};
pub use uuid::Uuid;
pub use writer::{
- Clearable, GenericSingleObjectWriter, SpecificSingleObjectWriter, Writer, WriterBuilder,
- to_avro_datum, to_avro_datum_schemata, write_avro_datum_ref,
+ Clearable, Writer, WriterBuilder,
+ datum::{to_avro_datum, to_avro_datum_schemata, write_avro_datum_ref},
+ single_object::{GenericSingleObjectWriter, SpecificSingleObjectWriter},
};
#[cfg(feature = "derive")]
diff --git a/avro/src/writer/datum.rs b/avro/src/writer/datum.rs
new file mode 100644
index 0000000..5708eed
--- /dev/null
+++ b/avro/src/writer/datum.rs
@@ -0,0 +1,390 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io::Write;
+
+use serde::Serialize;
+
+use crate::{
+ AvroResult, Schema,
+ encode::{encode, encode_internal},
+ error::Details,
+ schema::{NamesRef, ResolvedOwnedSchema, ResolvedSchema},
+ serde::ser_schema::SchemaAwareWriteSerializer,
+ types::Value,
+};
+
+/// Encode a value into raw Avro data, also performs schema validation.
+///
+/// **NOTE**: This function has a quite small niche of usage and does NOT generate headers and sync
+/// markers; use [`Writer`] to be fully Avro-compatible if you don't know what
+/// you are doing, instead.
+pub fn to_avro_datum<T: Into<Value>>(schema: &Schema, value: T) -> AvroResult<Vec<u8>> {
+ let mut buffer = Vec::new();
+ write_avro_datum(schema, value, &mut buffer)?;
+ Ok(buffer)
+}
+
+/// Write the referenced [Serialize]able object to the provided [Write] object.
+///
+/// Returns a result with the number of bytes written.
+///
+/// **NOTE**: This function has a quite small niche of usage and does **NOT** generate headers and sync
+/// markers; use [`append_ser`](Writer::append_ser) to be fully Avro-compatible
+/// if you don't know what you are doing, instead.
+pub fn write_avro_datum_ref<T: Serialize, W: Write>(
+ schema: &Schema,
+ names: &NamesRef,
+ data: &T,
+ writer: &mut W,
+) -> AvroResult<usize> {
+ let mut serializer = SchemaAwareWriteSerializer::new(writer, schema, names, None);
+ data.serialize(&mut serializer)
+}
+
+/// Encode a value into raw Avro data, also performs schema validation.
+///
+/// If the provided `schema` is incomplete then its dependencies must be
+/// provided in `schemata`
+pub fn to_avro_datum_schemata<T: Into<Value>>(
+ schema: &Schema,
+ schemata: Vec<&Schema>,
+ value: T,
+) -> AvroResult<Vec<u8>> {
+ let mut buffer = Vec::new();
+ write_avro_datum_schemata(schema, schemata, value, &mut buffer)?;
+ Ok(buffer)
+}
+
+/// Encode a value into raw Avro data, also performs schema validation.
+///
+/// This is an internal function which gets the bytes buffer where to write as parameter instead of
+/// creating a new one like `to_avro_datum`.
+pub(super) fn write_avro_datum<T: Into<Value>, W: Write>(
+ schema: &Schema,
+ value: T,
+ writer: &mut W,
+) -> AvroResult<()> {
+ let avro = value.into();
+ if !avro.validate(schema) {
+ return Err(Details::Validation.into());
+ }
+ encode(&avro, schema, writer)?;
+ Ok(())
+}
+
+pub(super) fn write_avro_datum_schemata<T: Into<Value>>(
+ schema: &Schema,
+ schemata: Vec<&Schema>,
+ value: T,
+ buffer: &mut Vec<u8>,
+) -> AvroResult<usize> {
+ let avro = value.into();
+ let rs = ResolvedSchema::try_from(schemata)?;
+ let names = rs.get_names();
+ let enclosing_namespace = schema.namespace();
+ if let Some(_err) = avro.validate_internal(schema, names, enclosing_namespace) {
+ return Err(Details::Validation.into());
+ }
+ encode_internal(&avro, schema, names, enclosing_namespace, buffer)
+}
+
+pub(super) fn write_value_ref_owned_resolved<W: Write>(
+ resolved_schema: &ResolvedOwnedSchema,
+ value: &Value,
+ writer: &mut W,
+) -> AvroResult<usize> {
+ let root_schema = resolved_schema.get_root_schema();
+ if let Some(reason) = value.validate_internal(
+ root_schema,
+ resolved_schema.get_names(),
+ root_schema.namespace(),
+ ) {
+ return Err(Details::ValidationWithReason {
+ value: value.clone(),
+ schema: root_schema.clone(),
+ reason,
+ }
+ .into());
+ }
+ encode_internal(
+ value,
+ root_schema,
+ resolved_schema.get_names(),
+ root_schema.namespace(),
+ writer,
+ )
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::HashMap;
+
+ use apache_avro_test_helper::TestResult;
+
+ use crate::{
+ Days, Decimal, Duration, Millis, Months,
+ reader::datum::GenericDatumReader,
+ schema::{DecimalSchema, FixedSchema, InnerDecimalSchema, Name},
+ types::Record,
+ util::zig_i64,
+ };
+
+ use super::*;
+
+ const SCHEMA: &str = r#"
+ {
+ "type": "record",
+ "name": "test",
+ "fields": [
+ {
+ "name": "a",
+ "type": "long",
+ "default": 42
+ },
+ {
+ "name": "b",
+ "type": "string"
+ }
+ ]
+ }
+ "#;
+
+ const UNION_SCHEMA: &str = r#"["null", "long"]"#;
+
+ #[test]
+ fn test_to_avro_datum() -> TestResult {
+ let schema = Schema::parse_str(SCHEMA)?;
+ let mut record = Record::new(&schema).unwrap();
+ record.put("a", 27i64);
+ record.put("b", "foo");
+
+ let mut expected = Vec::new();
+ zig_i64(27, &mut expected)?;
+ zig_i64(3, &mut expected)?;
+ expected.extend([b'f', b'o', b'o']);
+
+ assert_eq!(to_avro_datum(&schema, record)?, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_193_write_avro_datum_ref() -> TestResult {
+ #[derive(Serialize)]
+ struct TestStruct {
+ a: i64,
+ b: String,
+ }
+
+ let schema = Schema::parse_str(SCHEMA)?;
+ let mut writer: Vec<u8> = Vec::new();
+ let data = TestStruct {
+ a: 27,
+ b: "foo".to_string(),
+ };
+
+ let mut expected = Vec::new();
+ zig_i64(27, &mut expected)?;
+ zig_i64(3, &mut expected)?;
+ expected.extend([b'f', b'o', b'o']);
+
+ let bytes = write_avro_datum_ref(&schema, &HashMap::new(), &data, &mut writer)?;
+
+ assert_eq!(bytes, expected.len());
+ assert_eq!(writer, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_union_not_null() -> TestResult {
+ let schema = Schema::parse_str(UNION_SCHEMA)?;
+ let union = Value::Union(1, Box::new(Value::Long(3)));
+
+ let mut expected = Vec::new();
+ zig_i64(1, &mut expected)?;
+ zig_i64(3, &mut expected)?;
+
+ assert_eq!(to_avro_datum(&schema, union)?, expected);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_union_null() -> TestResult {
+ let schema = Schema::parse_str(UNION_SCHEMA)?;
+ let union = Value::Union(0, Box::new(Value::Null));
+
+ let mut expected = Vec::new();
+ zig_i64(0, &mut expected)?;
+
+ assert_eq!(to_avro_datum(&schema, union)?, expected);
+
+ Ok(())
+ }
+
+ fn logical_type_test<T: Into<Value> + Clone>(
+ schema_str: &'static str,
+
+ expected_schema: &Schema,
+ value: Value,
+
+ raw_schema: &Schema,
+ raw_value: T,
+ ) -> TestResult {
+ let schema = Schema::parse_str(schema_str)?;
+ assert_eq!(&schema, expected_schema);
+ // The serialized format should be the same as the schema.
+ let ser = to_avro_datum(&schema, value.clone())?;
+ let raw_ser = to_avro_datum(raw_schema, raw_value)?;
+ assert_eq!(ser, raw_ser);
+
+ // Should deserialize from the schema into the logical type.
+ let mut r = ser.as_slice();
+ let de = GenericDatumReader::builder(&schema)
+ .build()?
+ .read_value(&mut r)?;
+ assert_eq!(de, value);
+ Ok(())
+ }
+
+ #[test]
+ fn date() -> TestResult {
+ logical_type_test(
+ r#"{"type": "int", "logicalType": "date"}"#,
+ &Schema::Date,
+ Value::Date(1_i32),
+ &Schema::Int,
+ 1_i32,
+ )
+ }
+
+ #[test]
+ fn time_millis() -> TestResult {
+ logical_type_test(
+ r#"{"type": "int", "logicalType": "time-millis"}"#,
+ &Schema::TimeMillis,
+ Value::TimeMillis(1_i32),
+ &Schema::Int,
+ 1_i32,
+ )
+ }
+
+ #[test]
+ fn time_micros() -> TestResult {
+ logical_type_test(
+ r#"{"type": "long", "logicalType": "time-micros"}"#,
+ &Schema::TimeMicros,
+ Value::TimeMicros(1_i64),
+ &Schema::Long,
+ 1_i64,
+ )
+ }
+
+ #[test]
+ fn timestamp_millis() -> TestResult {
+ logical_type_test(
+ r#"{"type": "long", "logicalType": "timestamp-millis"}"#,
+ &Schema::TimestampMillis,
+ Value::TimestampMillis(1_i64),
+ &Schema::Long,
+ 1_i64,
+ )
+ }
+
+ #[test]
+ fn timestamp_micros() -> TestResult {
+ logical_type_test(
+ r#"{"type": "long", "logicalType": "timestamp-micros"}"#,
+ &Schema::TimestampMicros,
+ Value::TimestampMicros(1_i64),
+ &Schema::Long,
+ 1_i64,
+ )
+ }
+
+ #[test]
+ fn decimal_fixed() -> TestResult {
+ let size = 30;
+ let fixed = FixedSchema {
+ name: Name::new("decimal")?,
+ aliases: None,
+ doc: None,
+ size,
+ attributes: Default::default(),
+ };
+ let inner = InnerDecimalSchema::Fixed(fixed.clone());
+ let value = vec![0u8; size];
+ logical_type_test(
+ r#"{"type": {"type": "fixed", "size": 30, "name": "decimal"}, "logicalType": "decimal", "precision": 20, "scale": 5}"#,
+ &Schema::Decimal(DecimalSchema {
+ precision: 20,
+ scale: 5,
+ inner,
+ }),
+ Value::Decimal(Decimal::from(value.clone())),
+ &Schema::Fixed(fixed),
+ Value::Fixed(size, value),
+ )
+ }
+
+ #[test]
+ fn decimal_bytes() -> TestResult {
+ let value = vec![0u8; 10];
+ logical_type_test(
+ r#"{"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 3}"#,
+ &Schema::Decimal(DecimalSchema {
+ precision: 4,
+ scale: 3,
+ inner: InnerDecimalSchema::Bytes,
+ }),
+ Value::Decimal(Decimal::from(value.clone())),
+ &Schema::Bytes,
+ value,
+ )
+ }
+
+ #[test]
+ fn duration() -> TestResult {
+ let inner = Schema::Fixed(FixedSchema {
+ name: Name::new("duration")?,
+ aliases: None,
+ doc: None,
+ size: 12,
+ attributes: Default::default(),
+ });
+ let value = Value::Duration(Duration::new(
+ Months::new(256),
+ Days::new(512),
+ Millis::new(1024),
+ ));
+ logical_type_test(
+ r#"{"type": {"type": "fixed", "name": "duration", "size": 12}, "logicalType": "duration"}"#,
+ &Schema::Duration(FixedSchema {
+ name: Name::try_from("duration").expect("Name is valid"),
+ aliases: None,
+ doc: None,
+ size: 12,
+ attributes: Default::default(),
+ }),
+ value,
+ &inner,
+ Value::Fixed(12, vec![0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0]),
+ )
+ }
+}
diff --git a/avro/src/writer/mod.rs b/avro/src/writer/mod.rs
index 02b883f..20f19a3 100644
--- a/avro/src/writer/mod.rs
+++ b/avro/src/writer/mod.rs
@@ -20,15 +20,15 @@ use crate::{
AvroResult, Codec, Error,
encode::{encode, encode_internal, encode_to_vec},
error::Details,
- headers::{HeaderBuilder, RabinFingerprintHeader},
- schema::{NamesRef, ResolvedOwnedSchema, ResolvedSchema, Schema},
- serde::{AvroSchema, ser_schema::SchemaAwareWriteSerializer},
+ schema::{ResolvedSchema, Schema},
+ serde::ser_schema::SchemaAwareWriteSerializer,
types::Value,
};
use serde::Serialize;
-use std::{
- collections::HashMap, io::Write, marker::PhantomData, mem::ManuallyDrop, ops::RangeInclusive,
-};
+use std::{collections::HashMap, io::Write, mem::ManuallyDrop};
+
+pub mod datum;
+pub mod single_object;
const DEFAULT_BLOCK_SIZE: usize = 16000;
const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01";
@@ -614,263 +614,6 @@ impl<W: Write> Drop for Writer<'_, W> {
}
}
-/// Encode a value into raw Avro data, also performs schema validation.
-///
-/// This is an internal function which gets the bytes buffer where to write as parameter instead of
-/// creating a new one like `to_avro_datum`.
-fn write_avro_datum<T: Into<Value>, W: Write>(
- schema: &Schema,
- value: T,
- writer: &mut W,
-) -> Result<(), Error> {
- let avro = value.into();
- if !avro.validate(schema) {
- return Err(Details::Validation.into());
- }
- encode(&avro, schema, writer)?;
- Ok(())
-}
-
-fn write_avro_datum_schemata<T: Into<Value>>(
- schema: &Schema,
- schemata: Vec<&Schema>,
- value: T,
- buffer: &mut Vec<u8>,
-) -> AvroResult<usize> {
- let avro = value.into();
- let rs = ResolvedSchema::try_from(schemata)?;
- let names = rs.get_names();
- let enclosing_namespace = schema.namespace();
- if let Some(_err) = avro.validate_internal(schema, names, enclosing_namespace) {
- return Err(Details::Validation.into());
- }
- encode_internal(&avro, schema, names, enclosing_namespace, buffer)
-}
-
-/// Writer that encodes messages according to the single object encoding v1 spec
-/// Uses an API similar to the current File Writer
-/// Writes all object bytes at once, and drains internal buffer
-pub struct GenericSingleObjectWriter {
- buffer: Vec<u8>,
- resolved: ResolvedOwnedSchema,
-}
-
-impl GenericSingleObjectWriter {
- pub fn new_with_capacity(
- schema: &Schema,
- initial_buffer_cap: usize,
- ) -> AvroResult<GenericSingleObjectWriter> {
- let header_builder = RabinFingerprintHeader::from_schema(schema);
- Self::new_with_capacity_and_header_builder(schema, initial_buffer_cap, header_builder)
- }
-
- pub fn new_with_capacity_and_header_builder<HB: HeaderBuilder>(
- schema: &Schema,
- initial_buffer_cap: usize,
- header_builder: HB,
- ) -> AvroResult<GenericSingleObjectWriter> {
- let mut buffer = Vec::with_capacity(initial_buffer_cap);
- let header = header_builder.build_header();
- buffer.extend_from_slice(&header);
-
- Ok(GenericSingleObjectWriter {
- buffer,
- resolved: ResolvedOwnedSchema::try_from(schema.clone())?,
- })
- }
-
- const HEADER_LENGTH_RANGE: RangeInclusive<usize> = 10_usize..=20_usize;
-
- /// Write the referenced Value to the provided Write object. Returns a result with the number of bytes written including the header
- pub fn write_value_ref<W: Write>(&mut self, v: &Value, writer: &mut W) -> AvroResult<usize> {
- let original_length = self.buffer.len();
- if !Self::HEADER_LENGTH_RANGE.contains(&original_length) {
- Err(Details::IllegalSingleObjectWriterState.into())
- } else {
- write_value_ref_owned_resolved(&self.resolved, v, &mut self.buffer)?;
- writer
- .write_all(&self.buffer)
- .map_err(Details::WriteBytes)?;
- let len = self.buffer.len();
- self.buffer.truncate(original_length);
- Ok(len)
- }
- }
-
- /// Write the Value to the provided Write object. Returns a result with the number of bytes written including the header
- pub fn write_value<W: Write>(&mut self, v: Value, writer: &mut W) -> AvroResult<usize> {
- self.write_value_ref(&v, writer)
- }
-}
-
-/// Writer that encodes messages according to the single object encoding v1 spec
-pub struct SpecificSingleObjectWriter<T>
-where
- T: AvroSchema,
-{
- resolved: ResolvedOwnedSchema,
- header: Vec<u8>,
- _model: PhantomData<T>,
-}
-
-impl<T> SpecificSingleObjectWriter<T>
-where
- T: AvroSchema,
-{
- pub fn new() -> AvroResult<Self> {
- let schema = T::get_schema();
- let header = RabinFingerprintHeader::from_schema(&schema).build_header();
- let resolved = ResolvedOwnedSchema::new(schema)?;
- // We don't use Self::new_with_header_builder as that would mean calling T::get_schema() twice
- Ok(Self {
- resolved,
- header,
- _model: PhantomData,
- })
- }
-
- pub fn new_with_header_builder(header_builder: impl HeaderBuilder) -> AvroResult<Self> {
- let header = header_builder.build_header();
- let resolved = ResolvedOwnedSchema::new(T::get_schema())?;
- Ok(Self {
- resolved,
- header,
- _model: PhantomData,
- })
- }
-
- /// Deprecated. Use [`SpecificSingleObjectWriter::new`] instead.
- #[deprecated(since = "0.22.0", note = "Use new() instead")]
- pub fn with_capacity(_buffer_cap: usize) -> AvroResult<Self> {
- Self::new()
- }
-}
-
-impl<T> SpecificSingleObjectWriter<T>
-where
- T: AvroSchema + Into<Value>,
-{
- /// Write the value to the writer
- ///
- /// Returns the number of bytes written.
- ///
- /// Each call writes a complete single-object encoded message (header + data),
- /// making each message independently decodable.
- pub fn write_value<W: Write>(&self, data: T, writer: &mut W) -> AvroResult<usize> {
- writer
- .write_all(&self.header)
- .map_err(Details::WriteBytes)?;
- let value: Value = data.into();
- let bytes = write_value_ref_owned_resolved(&self.resolved, &value, writer)?;
- Ok(bytes + self.header.len())
- }
-}
-
-impl<T> SpecificSingleObjectWriter<T>
-where
- T: AvroSchema + Serialize,
-{
- /// Write the object to the writer.
- ///
- /// Returns the number of bytes written.
- ///
- /// Each call writes a complete single-object encoded message (header + data),
- /// making each message independently decodable.
- pub fn write_ref<W: Write>(&self, data: &T, writer: &mut W) -> AvroResult<usize> {
- writer
- .write_all(&self.header)
- .map_err(Details::WriteBytes)?;
-
- let bytes = write_avro_datum_ref(
- self.resolved.get_root_schema(),
- self.resolved.get_names(),
- data,
- writer,
- )?;
-
- Ok(bytes + self.header.len())
- }
-
- /// Write the object to the writer.
- ///
- /// Returns the number of bytes written.
- ///
- /// Each call writes a complete single-object encoded message (header + data),
- /// making each message independently decodable.
- pub fn write<W: Write>(&self, data: T, writer: &mut W) -> AvroResult<usize> {
- self.write_ref(&data, writer)
- }
-}
-
-fn write_value_ref_owned_resolved<W: Write>(
- resolved_schema: &ResolvedOwnedSchema,
- value: &Value,
- writer: &mut W,
-) -> AvroResult<usize> {
- let root_schema = resolved_schema.get_root_schema();
- if let Some(reason) = value.validate_internal(
- root_schema,
- resolved_schema.get_names(),
- root_schema.namespace(),
- ) {
- return Err(Details::ValidationWithReason {
- value: value.clone(),
- schema: root_schema.clone(),
- reason,
- }
- .into());
- }
- encode_internal(
- value,
- root_schema,
- resolved_schema.get_names(),
- root_schema.namespace(),
- writer,
- )
-}
-
-/// Encode a value into raw Avro data, also performs schema validation.
-///
-/// **NOTE**: This function has a quite small niche of usage and does NOT generate headers and sync
-/// markers; use [`Writer`] to be fully Avro-compatible if you don't know what
-/// you are doing, instead.
-pub fn to_avro_datum<T: Into<Value>>(schema: &Schema, value: T) -> AvroResult<Vec<u8>> {
- let mut buffer = Vec::new();
- write_avro_datum(schema, value, &mut buffer)?;
- Ok(buffer)
-}
-
-/// Write the referenced [Serialize]able object to the provided [Write] object.
-///
-/// Returns a result with the number of bytes written.
-///
-/// **NOTE**: This function has a quite small niche of usage and does **NOT** generate headers and sync
-/// markers; use [`append_ser`](Writer::append_ser) to be fully Avro-compatible
-/// if you don't know what you are doing, instead.
-pub fn write_avro_datum_ref<T: Serialize, W: Write>(
- schema: &Schema,
- names: &NamesRef,
- data: &T,
- writer: &mut W,
-) -> AvroResult<usize> {
- let mut serializer = SchemaAwareWriteSerializer::new(writer, schema, names, None);
- data.serialize(&mut serializer)
-}
-
-/// Encode a value into raw Avro data, also performs schema validation.
-///
-/// If the provided `schema` is incomplete then its dependencies must be
-/// provided in `schemata`
-pub fn to_avro_datum_schemata<T: Into<Value>>(
- schema: &Schema,
- schemata: Vec<&Schema>,
- value: T,
-) -> AvroResult<Vec<u8>> {
- let mut buffer = Vec::new();
- write_avro_datum_schemata(schema, schemata, value, &mut buffer)?;
- Ok(buffer)
-}
-
#[cfg(not(target_arch = "wasm32"))]
fn generate_sync_marker() -> [u8; 16] {
rand::random()
@@ -892,22 +635,10 @@ mod tests {
use std::{cell::RefCell, rc::Rc};
use super::*;
- use crate::{
- Reader,
- decimal::Decimal,
- duration::{Days, Duration, Millis, Months},
- headers::GlueSchemaUuidHeader,
- rabin::Rabin,
- reader::datum::GenericDatumReader,
- schema::{DecimalSchema, FixedSchema, Name},
- types::Record,
- util::zig_i64,
- };
+ use crate::{Reader, types::Record, util::zig_i64};
use pretty_assertions::assert_eq;
use serde::{Deserialize, Serialize};
- use uuid::Uuid;
- use crate::schema::InnerDecimalSchema;
use crate::{codec::DeflateSettings, error::Details};
use apache_avro_test_helper::TestResult;
@@ -931,53 +662,6 @@ mod tests {
}
"#;
- const UNION_SCHEMA: &str = r#"["null", "long"]"#;
-
- #[test]
- fn test_to_avro_datum() -> TestResult {
- let schema = Schema::parse_str(SCHEMA)?;
- let mut record = Record::new(&schema).unwrap();
- record.put("a", 27i64);
- record.put("b", "foo");
-
- let mut expected = Vec::new();
- zig_i64(27, &mut expected)?;
- zig_i64(3, &mut expected)?;
- expected.extend([b'f', b'o', b'o']);
-
- assert_eq!(to_avro_datum(&schema, record)?, expected);
-
- Ok(())
- }
-
- #[test]
- fn avro_rs_193_write_avro_datum_ref() -> TestResult {
- #[derive(Serialize)]
- struct TestStruct {
- a: i64,
- b: String,
- }
-
- let schema = Schema::parse_str(SCHEMA)?;
- let mut writer: Vec<u8> = Vec::new();
- let data = TestStruct {
- a: 27,
- b: "foo".to_string(),
- };
-
- let mut expected = Vec::new();
- zig_i64(27, &mut expected)?;
- zig_i64(3, &mut expected)?;
- expected.extend([b'f', b'o', b'o']);
-
- let bytes = write_avro_datum_ref(&schema, &HashMap::new(), &data, &mut writer)?;
-
- assert_eq!(bytes, expected.len());
- assert_eq!(writer, expected);
-
- Ok(())
- }
-
#[test]
fn avro_rs_220_flush_write_header() -> TestResult {
let schema = Schema::parse_str(SCHEMA)?;
@@ -1001,183 +685,6 @@ mod tests {
Ok(())
}
- #[test]
- fn test_union_not_null() -> TestResult {
- let schema = Schema::parse_str(UNION_SCHEMA)?;
- let union = Value::Union(1, Box::new(Value::Long(3)));
-
- let mut expected = Vec::new();
- zig_i64(1, &mut expected)?;
- zig_i64(3, &mut expected)?;
-
- assert_eq!(to_avro_datum(&schema, union)?, expected);
-
- Ok(())
- }
-
- #[test]
- fn test_union_null() -> TestResult {
- let schema = Schema::parse_str(UNION_SCHEMA)?;
- let union = Value::Union(0, Box::new(Value::Null));
-
- let mut expected = Vec::new();
- zig_i64(0, &mut expected)?;
-
- assert_eq!(to_avro_datum(&schema, union)?, expected);
-
- Ok(())
- }
-
- fn logical_type_test<T: Into<Value> + Clone>(
- schema_str: &'static str,
-
- expected_schema: &Schema,
- value: Value,
-
- raw_schema: &Schema,
- raw_value: T,
- ) -> TestResult {
- let schema = Schema::parse_str(schema_str)?;
- assert_eq!(&schema, expected_schema);
- // The serialized format should be the same as the schema.
- let ser = to_avro_datum(&schema, value.clone())?;
- let raw_ser = to_avro_datum(raw_schema, raw_value)?;
- assert_eq!(ser, raw_ser);
-
- // Should deserialize from the schema into the logical type.
- let mut r = ser.as_slice();
- let de = GenericDatumReader::builder(&schema)
- .build()?
- .read_value(&mut r)?;
- assert_eq!(de, value);
- Ok(())
- }
-
- #[test]
- fn date() -> TestResult {
- logical_type_test(
- r#"{"type": "int", "logicalType": "date"}"#,
- &Schema::Date,
- Value::Date(1_i32),
- &Schema::Int,
- 1_i32,
- )
- }
-
- #[test]
- fn time_millis() -> TestResult {
- logical_type_test(
- r#"{"type": "int", "logicalType": "time-millis"}"#,
- &Schema::TimeMillis,
- Value::TimeMillis(1_i32),
- &Schema::Int,
- 1_i32,
- )
- }
-
- #[test]
- fn time_micros() -> TestResult {
- logical_type_test(
- r#"{"type": "long", "logicalType": "time-micros"}"#,
- &Schema::TimeMicros,
- Value::TimeMicros(1_i64),
- &Schema::Long,
- 1_i64,
- )
- }
-
- #[test]
- fn timestamp_millis() -> TestResult {
- logical_type_test(
- r#"{"type": "long", "logicalType": "timestamp-millis"}"#,
- &Schema::TimestampMillis,
- Value::TimestampMillis(1_i64),
- &Schema::Long,
- 1_i64,
- )
- }
-
- #[test]
- fn timestamp_micros() -> TestResult {
- logical_type_test(
- r#"{"type": "long", "logicalType": "timestamp-micros"}"#,
- &Schema::TimestampMicros,
- Value::TimestampMicros(1_i64),
- &Schema::Long,
- 1_i64,
- )
- }
-
- #[test]
- fn decimal_fixed() -> TestResult {
- let size = 30;
- let fixed = FixedSchema {
- name: Name::new("decimal")?,
- aliases: None,
- doc: None,
- size,
- attributes: Default::default(),
- };
- let inner = InnerDecimalSchema::Fixed(fixed.clone());
- let value = vec![0u8; size];
- logical_type_test(
- r#"{"type": {"type": "fixed", "size": 30, "name": "decimal"}, "logicalType": "decimal", "precision": 20, "scale": 5}"#,
- &Schema::Decimal(DecimalSchema {
- precision: 20,
- scale: 5,
- inner,
- }),
- Value::Decimal(Decimal::from(value.clone())),
- &Schema::Fixed(fixed),
- Value::Fixed(size, value),
- )
- }
-
- #[test]
- fn decimal_bytes() -> TestResult {
- let value = vec![0u8; 10];
- logical_type_test(
- r#"{"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 3}"#,
- &Schema::Decimal(DecimalSchema {
- precision: 4,
- scale: 3,
- inner: InnerDecimalSchema::Bytes,
- }),
- Value::Decimal(Decimal::from(value.clone())),
- &Schema::Bytes,
- value,
- )
- }
-
- #[test]
- fn duration() -> TestResult {
- let inner = Schema::Fixed(FixedSchema {
- name: Name::new("duration")?,
- aliases: None,
- doc: None,
- size: 12,
- attributes: Default::default(),
- });
- let value = Value::Duration(Duration::new(
- Months::new(256),
- Days::new(512),
- Millis::new(1024),
- ));
- logical_type_test(
- r#"{"type": {"type": "fixed", "name": "duration", "size": 12}, "logicalType": "duration"}"#,
- &Schema::Duration(FixedSchema {
- name: Name::try_from("duration").expect("Name is valid"),
- aliases: None,
- doc: None,
- size: 12,
- attributes: Default::default(),
- }),
- value,
- &inner,
- Value::Fixed(12, vec![0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0]),
- )
- }
-
#[test]
fn test_writer_append() -> TestResult {
let schema = Schema::parse_str(SCHEMA)?;
@@ -1567,163 +1074,6 @@ mod tests {
Ok(())
}
- #[derive(Serialize, Clone)]
- struct TestSingleObjectWriter {
- a: i64,
- b: f64,
- c: Vec<String>,
- }
-
- impl AvroSchema for TestSingleObjectWriter {
- fn get_schema() -> Schema {
- let schema = r#"
- {
- "type":"record",
- "name":"TestSingleObjectWrtierSerialize",
- "fields":[
- {
- "name":"a",
- "type":"long"
- },
- {
- "name":"b",
- "type":"double"
- },
- {
- "name":"c",
- "type":{
- "type":"array",
- "items":"string"
- }
- }
- ]
- }
- "#;
- Schema::parse_str(schema).unwrap()
- }
- }
-
- impl From<TestSingleObjectWriter> for Value {
- fn from(obj: TestSingleObjectWriter) -> Value {
- Value::Record(vec![
- ("a".into(), obj.a.into()),
- ("b".into(), obj.b.into()),
- (
- "c".into(),
- Value::Array(obj.c.into_iter().map(|s| s.into()).collect()),
- ),
- ])
- }
- }
-
- #[test]
- fn test_single_object_writer() -> TestResult {
- let mut buf: Vec<u8> = Vec::new();
- let obj = TestSingleObjectWriter {
- a: 300,
- b: 34.555,
- c: vec!["cat".into(), "dog".into()],
- };
- let mut writer = GenericSingleObjectWriter::new_with_capacity(
- &TestSingleObjectWriter::get_schema(),
- 1024,
- )
- .expect("Should resolve schema");
- let value = obj.into();
- let written_bytes = writer
- .write_value_ref(&value, &mut buf)
- .expect("Error serializing properly");
-
- assert!(buf.len() > 10, "no bytes written");
- assert_eq!(buf.len(), written_bytes);
- assert_eq!(buf[0], 0xC3);
- assert_eq!(buf[1], 0x01);
- assert_eq!(
- &buf[2..10],
- &TestSingleObjectWriter::get_schema()
- .fingerprint::<Rabin>()
- .bytes[..]
- );
- let mut msg_binary = Vec::new();
- encode(
- &value,
- &TestSingleObjectWriter::get_schema(),
- &mut msg_binary,
- )
- .expect("encode should have failed by here as a dependency of any writing");
- assert_eq!(&buf[10..], &msg_binary[..]);
-
- Ok(())
- }
-
- #[test]
- fn test_single_object_writer_with_header_builder() -> TestResult {
- let mut buf: Vec<u8> = Vec::new();
- let obj = TestSingleObjectWriter {
- a: 300,
- b: 34.555,
- c: vec!["cat".into(), "dog".into()],
- };
- let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?;
- let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid);
- let mut writer = GenericSingleObjectWriter::new_with_capacity_and_header_builder(
- &TestSingleObjectWriter::get_schema(),
- 1024,
- header_builder,
- )
- .expect("Should resolve schema");
- let value = obj.into();
- writer
- .write_value_ref(&value, &mut buf)
- .expect("Error serializing properly");
-
- assert_eq!(buf[0], 0x03);
- assert_eq!(buf[1], 0x00);
- assert_eq!(buf[2..18], schema_uuid.into_bytes()[..]);
- Ok(())
- }
-
- #[test]
- fn test_writer_parity() -> TestResult {
- let obj1 = TestSingleObjectWriter {
- a: 300,
- b: 34.555,
- c: vec!["cat".into(), "dog".into()],
- };
-
- let mut buf1: Vec<u8> = Vec::new();
- let mut buf2: Vec<u8> = Vec::new();
- let mut buf3: Vec<u8> = Vec::new();
- let mut buf4: Vec<u8> = Vec::new();
-
- let mut generic_writer = GenericSingleObjectWriter::new_with_capacity(
- &TestSingleObjectWriter::get_schema(),
- 1024,
- )
- .expect("Should resolve schema");
- let specific_writer = SpecificSingleObjectWriter::<TestSingleObjectWriter>::new()
- .expect("Resolved should pass");
- specific_writer
- .write_ref(&obj1, &mut buf1)
- .expect("Serialization expected");
- specific_writer
- .write_ref(&obj1, &mut buf2)
- .expect("Serialization expected");
- specific_writer
- .write_value(obj1.clone(), &mut buf3)
- .expect("Serialization expected");
-
- generic_writer
- .write_value(obj1.into(), &mut buf4)
- .expect("Serialization expected");
-
- assert_eq!(buf1, buf2);
- assert_eq!(buf2, buf3);
- assert_eq!(buf3, buf4);
-
- Ok(())
- }
-
#[test]
fn avro_3894_take_aliases_into_account_when_serializing() -> TestResult {
const SCHEMA: &str = r#"
@@ -1849,51 +1199,6 @@ mod tests {
Ok(())
}
- #[test]
- fn avro_rs_439_specific_single_object_writer_ref() -> TestResult {
- #[derive(Serialize)]
- struct Recursive {
- field: bool,
- recurse: Option<Box<Recursive>>,
- }
-
- impl AvroSchema for Recursive {
- fn get_schema() -> Schema {
- Schema::parse_str(
- r#"{
- "name": "Recursive",
- "type": "record",
- "fields": [
- { "name": "field", "type": "boolean" },
- { "name": "recurse", "type": ["null", "Recursive"] }
- ]
- }"#,
- )
- .unwrap()
- }
- }
-
- let mut buffer = Vec::new();
- let writer = SpecificSingleObjectWriter::new()?;
-
- writer.write(
- Recursive {
- field: true,
- recurse: Some(Box::new(Recursive {
- field: false,
- recurse: None,
- })),
- },
- &mut buffer,
- )?;
- assert_eq!(
- buffer,
- &[195, 1, 83, 223, 43, 26, 181, 179, 227, 224, 1, 2, 0, 0][..]
- );
-
- Ok(())
- }
-
#[test]
fn avro_rs_310_append_unvalidated_value() -> TestResult {
let schema = Schema::String;
diff --git a/avro/src/writer/single_object.rs b/avro/src/writer/single_object.rs
new file mode 100644
index 0000000..136986a
--- /dev/null
+++ b/avro/src/writer/single_object.rs
@@ -0,0 +1,397 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::{io::Write, marker::PhantomData, ops::RangeInclusive};
+
+use serde::Serialize;
+
+use crate::{
+ AvroResult, AvroSchema, Schema,
+ error::Details,
+ headers::{HeaderBuilder, RabinFingerprintHeader},
+ schema::ResolvedOwnedSchema,
+ types::Value,
+ write_avro_datum_ref,
+ writer::datum::write_value_ref_owned_resolved,
+};
+
+/// Writer that encodes messages according to the single object encoding v1 spec
+/// Uses an API similar to the current File Writer
+/// Writes all object bytes at once, and drains internal buffer
+pub struct GenericSingleObjectWriter {
+ buffer: Vec<u8>,
+ resolved: ResolvedOwnedSchema,
+}
+
+impl GenericSingleObjectWriter {
+ pub fn new_with_capacity(
+ schema: &Schema,
+ initial_buffer_cap: usize,
+ ) -> AvroResult<GenericSingleObjectWriter> {
+ let header_builder = RabinFingerprintHeader::from_schema(schema);
+ Self::new_with_capacity_and_header_builder(schema, initial_buffer_cap, header_builder)
+ }
+
+ pub fn new_with_capacity_and_header_builder<HB: HeaderBuilder>(
+ schema: &Schema,
+ initial_buffer_cap: usize,
+ header_builder: HB,
+ ) -> AvroResult<GenericSingleObjectWriter> {
+ let mut buffer = Vec::with_capacity(initial_buffer_cap);
+ let header = header_builder.build_header();
+ buffer.extend_from_slice(&header);
+
+ Ok(GenericSingleObjectWriter {
+ buffer,
+ resolved: ResolvedOwnedSchema::try_from(schema.clone())?,
+ })
+ }
+
+ const HEADER_LENGTH_RANGE: RangeInclusive<usize> = 10_usize..=20_usize;
+
+ /// Write the referenced Value to the provided Write object. Returns a result with the number of bytes written including the header
+ pub fn write_value_ref<W: Write>(&mut self, v: &Value, writer: &mut W) -> AvroResult<usize> {
+ let original_length = self.buffer.len();
+ if !Self::HEADER_LENGTH_RANGE.contains(&original_length) {
+ Err(Details::IllegalSingleObjectWriterState.into())
+ } else {
+ write_value_ref_owned_resolved(&self.resolved, v, &mut self.buffer)?;
+ writer
+ .write_all(&self.buffer)
+ .map_err(Details::WriteBytes)?;
+ let len = self.buffer.len();
+ self.buffer.truncate(original_length);
+ Ok(len)
+ }
+ }
+
+ /// Write the Value to the provided Write object. Returns a result with the number of bytes written including the header
+ pub fn write_value<W: Write>(&mut self, v: Value, writer: &mut W) -> AvroResult<usize> {
+ self.write_value_ref(&v, writer)
+ }
+}
+
+/// Writer that encodes messages according to the single object encoding v1 spec
+pub struct SpecificSingleObjectWriter<T>
+where
+ T: AvroSchema,
+{
+ resolved: ResolvedOwnedSchema,
+ header: Vec<u8>,
+ _model: PhantomData<T>,
+}
+
+impl<T> SpecificSingleObjectWriter<T>
+where
+ T: AvroSchema,
+{
+ pub fn new() -> AvroResult<Self> {
+ let schema = T::get_schema();
+ let header = RabinFingerprintHeader::from_schema(&schema).build_header();
+ let resolved = ResolvedOwnedSchema::new(schema)?;
+ // We don't use Self::new_with_header_builder as that would mean calling T::get_schema() twice
+ Ok(Self {
+ resolved,
+ header,
+ _model: PhantomData,
+ })
+ }
+
+ pub fn new_with_header_builder(header_builder: impl HeaderBuilder) -> AvroResult<Self> {
+ let header = header_builder.build_header();
+ let resolved = ResolvedOwnedSchema::new(T::get_schema())?;
+ Ok(Self {
+ resolved,
+ header,
+ _model: PhantomData,
+ })
+ }
+
+ /// Deprecated. Use [`SpecificSingleObjectWriter::new`] instead.
+ #[deprecated(since = "0.22.0", note = "Use new() instead")]
+ pub fn with_capacity(_buffer_cap: usize) -> AvroResult<Self> {
+ Self::new()
+ }
+}
+
+impl<T> SpecificSingleObjectWriter<T>
+where
+ T: AvroSchema + Into<Value>,
+{
+ /// Write the value to the writer
+ ///
+ /// Returns the number of bytes written.
+ ///
+ /// Each call writes a complete single-object encoded message (header + data),
+ /// making each message independently decodable.
+ pub fn write_value<W: Write>(&self, data: T, writer: &mut W) -> AvroResult<usize> {
+ writer
+ .write_all(&self.header)
+ .map_err(Details::WriteBytes)?;
+ let value: Value = data.into();
+ let bytes = write_value_ref_owned_resolved(&self.resolved, &value, writer)?;
+ Ok(bytes + self.header.len())
+ }
+}
+
+impl<T> SpecificSingleObjectWriter<T>
+where
+ T: AvroSchema + Serialize,
+{
+ /// Write the object to the writer.
+ ///
+ /// Returns the number of bytes written.
+ ///
+ /// Each call writes a complete single-object encoded message (header + data),
+ /// making each message independently decodable.
+ pub fn write_ref<W: Write>(&self, data: &T, writer: &mut W) -> AvroResult<usize> {
+ writer
+ .write_all(&self.header)
+ .map_err(Details::WriteBytes)?;
+
+ let bytes = write_avro_datum_ref(
+ self.resolved.get_root_schema(),
+ self.resolved.get_names(),
+ data,
+ writer,
+ )?;
+
+ Ok(bytes + self.header.len())
+ }
+
+ /// Write the object to the writer.
+ ///
+ /// Returns the number of bytes written.
+ ///
+ /// Each call writes a complete single-object encoded message (header + data),
+ /// making each message independently decodable.
+ pub fn write<W: Write>(&self, data: T, writer: &mut W) -> AvroResult<usize> {
+ self.write_ref(&data, writer)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use apache_avro_test_helper::TestResult;
+ use uuid::Uuid;
+
+ use crate::{encode::encode, headers::GlueSchemaUuidHeader, rabin::Rabin};
+
+ use super::*;
+
+ #[derive(Serialize, Clone)]
+ struct TestSingleObjectWriter {
+ a: i64,
+ b: f64,
+ c: Vec<String>,
+ }
+
+ impl AvroSchema for TestSingleObjectWriter {
+ fn get_schema() -> Schema {
+ let schema = r#"
+ {
+ "type":"record",
+ "name":"TestSingleObjectWrtierSerialize",
+ "fields":[
+ {
+ "name":"a",
+ "type":"long"
+ },
+ {
+ "name":"b",
+ "type":"double"
+ },
+ {
+ "name":"c",
+ "type":{
+ "type":"array",
+ "items":"string"
+ }
+ }
+ ]
+ }
+ "#;
+ Schema::parse_str(schema).unwrap()
+ }
+ }
+
+ impl From<TestSingleObjectWriter> for Value {
+ fn from(obj: TestSingleObjectWriter) -> Value {
+ Value::Record(vec![
+ ("a".into(), obj.a.into()),
+ ("b".into(), obj.b.into()),
+ (
+ "c".into(),
+ Value::Array(obj.c.into_iter().map(|s| s.into()).collect()),
+ ),
+ ])
+ }
+ }
+
+ #[test]
+ fn test_single_object_writer() -> TestResult {
+ let mut buf: Vec<u8> = Vec::new();
+ let obj = TestSingleObjectWriter {
+ a: 300,
+ b: 34.555,
+ c: vec!["cat".into(), "dog".into()],
+ };
+ let mut writer = GenericSingleObjectWriter::new_with_capacity(
+ &TestSingleObjectWriter::get_schema(),
+ 1024,
+ )
+ .expect("Should resolve schema");
+ let value = obj.into();
+ let written_bytes = writer
+ .write_value_ref(&value, &mut buf)
+ .expect("Error serializing properly");
+
+ assert!(buf.len() > 10, "no bytes written");
+ assert_eq!(buf.len(), written_bytes);
+ assert_eq!(buf[0], 0xC3);
+ assert_eq!(buf[1], 0x01);
+ assert_eq!(
+ &buf[2..10],
+ &TestSingleObjectWriter::get_schema()
+ .fingerprint::<Rabin>()
+ .bytes[..]
+ );
+ let mut msg_binary = Vec::new();
+ encode(
+ &value,
+ &TestSingleObjectWriter::get_schema(),
+ &mut msg_binary,
+ )
+ .expect("encode should have failed by here as a dependency of any writing");
+ assert_eq!(&buf[10..], &msg_binary[..]);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_single_object_writer_with_header_builder() -> TestResult {
+ let mut buf: Vec<u8> = Vec::new();
+ let obj = TestSingleObjectWriter {
+ a: 300,
+ b: 34.555,
+ c: vec!["cat".into(), "dog".into()],
+ };
+ let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?;
+ let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid);
+ let mut writer = GenericSingleObjectWriter::new_with_capacity_and_header_builder(
+ &TestSingleObjectWriter::get_schema(),
+ 1024,
+ header_builder,
+ )
+ .expect("Should resolve schema");
+ let value = obj.into();
+ writer
+ .write_value_ref(&value, &mut buf)
+ .expect("Error serializing properly");
+
+ assert_eq!(buf[0], 0x03);
+ assert_eq!(buf[1], 0x00);
+ assert_eq!(buf[2..18], schema_uuid.into_bytes()[..]);
+ Ok(())
+ }
+
+ #[test]
+ fn test_writer_parity() -> TestResult {
+ let obj1 = TestSingleObjectWriter {
+ a: 300,
+ b: 34.555,
+ c: vec!["cat".into(), "dog".into()],
+ };
+
+ let mut buf1: Vec<u8> = Vec::new();
+ let mut buf2: Vec<u8> = Vec::new();
+ let mut buf3: Vec<u8> = Vec::new();
+ let mut buf4: Vec<u8> = Vec::new();
+
+ let mut generic_writer = GenericSingleObjectWriter::new_with_capacity(
+ &TestSingleObjectWriter::get_schema(),
+ 1024,
+ )
+ .expect("Should resolve schema");
+ let specific_writer = SpecificSingleObjectWriter::<TestSingleObjectWriter>::new()
+ .expect("Resolved should pass");
+ specific_writer
+ .write_ref(&obj1, &mut buf1)
+ .expect("Serialization expected");
+ specific_writer
+ .write_ref(&obj1, &mut buf2)
+ .expect("Serialization expected");
+ specific_writer
+ .write_value(obj1.clone(), &mut buf3)
+ .expect("Serialization expected");
+
+ generic_writer
+ .write_value(obj1.into(), &mut buf4)
+ .expect("Serialization expected");
+
+ assert_eq!(buf1, buf2);
+ assert_eq!(buf2, buf3);
+ assert_eq!(buf3, buf4);
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_439_specific_single_object_writer_ref() -> TestResult {
+ #[derive(Serialize)]
+ struct Recursive {
+ field: bool,
+ recurse: Option<Box<Recursive>>,
+ }
+
+ impl AvroSchema for Recursive {
+ fn get_schema() -> Schema {
+ Schema::parse_str(
+ r#"{
+ "name": "Recursive",
+ "type": "record",
+ "fields": [
+ { "name": "field", "type": "boolean" },
+ { "name": "recurse", "type": ["null", "Recursive"] }
+ ]
+ }"#,
+ )
+ .unwrap()
+ }
+ }
+
+ let mut buffer = Vec::new();
+ let writer = SpecificSingleObjectWriter::new()?;
+
+ writer.write(
+ Recursive {
+ field: true,
+ recurse: Some(Box::new(Recursive {
+ field: false,
+ recurse: None,
+ })),
+ },
+ &mut buffer,
+ )?;
+ assert_eq!(
+ buffer,
+ &[195, 1, 83, 223, 43, 26, 181, 179, 227, 224, 1, 2, 0, 0][..]
+ );
+
+ Ok(())
+ }
+}
(avro-rs) 03/08: feat: Replace `to_datum*` functions with `GenericDatumWriter`
Posted by kr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a commit to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
commit 5e69691a604d8397a5ced93bdfad445cc7949fe6
Author: Kriskras99 <gi...@kriskras99.nl>
AuthorDate: Tue Mar 3 22:32:41 2026 +0100
feat: Replace `to_datum*` functions with `GenericDatumWriter`
---
avro/benches/single.rs | 67 +++++++++-
avro/src/lib.rs | 6 +-
avro/src/schema/mod.rs | 12 +-
avro/src/serde/de.rs | 14 +-
avro/src/writer/datum.rs | 208 +++++++++++++++++++-----------
avro/src/writer/single_object.rs | 40 +++++-
avro/tests/avro-3786.rs | 28 +++-
avro/tests/avro-3787.rs | 12 +-
avro/tests/io.rs | 48 +++++--
avro/tests/schema.rs | 22 ++--
avro/tests/to_from_avro_datum_schemata.rs | 17 ++-
11 files changed, 339 insertions(+), 135 deletions(-)
diff --git a/avro/benches/single.rs b/avro/benches/single.rs
index 05d8453..862a15f 100644
--- a/avro/benches/single.rs
+++ b/avro/benches/single.rs
@@ -15,12 +15,13 @@
// specific language governing permissions and limitations
// under the License.
+use apache_avro::writer::datum::GenericDatumWriter;
use apache_avro::{
schema::Schema,
- to_avro_datum,
types::{Record, Value},
};
use criterion::{Criterion, criterion_group, criterion_main};
+use std::time::Duration;
const RAW_SMALL_SCHEMA: &str = r#"
{
@@ -174,20 +175,74 @@ fn make_big_record() -> Result<(Schema, Value), Box<dyn std::error::Error>> {
fn bench_small_schema_write_record(c: &mut Criterion) {
let (schema, record) = make_small_record().unwrap();
c.bench_function("small record", |b| {
- b.iter(|| to_avro_datum(&schema, record.clone()))
+ b.iter(|| {
+ GenericDatumWriter::builder(&schema)
+ .build()
+ .unwrap()
+ .write_value_to_vec(record.clone())
+ })
});
}
fn bench_big_schema_write_record(c: &mut Criterion) {
let (schema, record) = make_big_record().unwrap();
c.bench_function("big record", |b| {
- b.iter(|| to_avro_datum(&schema, record.clone()))
+ b.iter(|| {
+ GenericDatumWriter::builder(&schema)
+ .build()
+ .unwrap()
+ .write_value_to_vec(record.clone())
+ })
+ });
+}
+
+fn bench_small_schema_write_record_reuse_datum_writer(c: &mut Criterion) {
+ let (schema, record) = make_small_record().unwrap();
+ let writer = GenericDatumWriter::builder(&schema).build().unwrap();
+ c.bench_function("small record (reused writer)", |b| {
+ b.iter(|| writer.write_value_ref(&mut Vec::new(), &record))
+ });
+}
+
+fn bench_big_schema_write_record_reuse_datum_writer(c: &mut Criterion) {
+ let (schema, record) = make_big_record().unwrap();
+ let writer = GenericDatumWriter::builder(&schema).build().unwrap();
+ c.bench_function("big record (reused writer)", |b| {
+ b.iter(|| writer.write_value_ref(&mut Vec::new(), &record))
+ });
+}
+
+fn bench_small_schema_write_record_no_validation(c: &mut Criterion) {
+ let (schema, record) = make_small_record().unwrap();
+ let writer = GenericDatumWriter::builder(&schema)
+ .validate(false)
+ .build()
+ .unwrap();
+ c.bench_function("small record (no validation)", |b| {
+ b.iter(|| writer.write_value_ref(&mut Vec::new(), &record))
+ });
+}
+
+fn bench_big_schema_write_record_no_validation(c: &mut Criterion) {
+ let (schema, record) = make_big_record().unwrap();
+ let writer = GenericDatumWriter::builder(&schema)
+ .validate(false)
+ .build()
+ .unwrap();
+ c.bench_function("big record (no validation)", |b| {
+ b.iter(|| writer.write_value_ref(&mut Vec::new(), &record))
});
}
criterion_group!(
- benches,
- bench_small_schema_write_record,
- bench_big_schema_write_record
+ name = benches;
+ config = Criterion::default().sample_size(200).measurement_time(Duration::from_secs(10));
+ targets =
+ bench_small_schema_write_record,
+ bench_big_schema_write_record,
+ bench_small_schema_write_record_reuse_datum_writer,
+ bench_big_schema_write_record_reuse_datum_writer,
+ bench_small_schema_write_record_no_validation,
+ bench_big_schema_write_record_no_validation,
);
criterion_main!(benches);
diff --git a/avro/src/lib.rs b/avro/src/lib.rs
index 6edded5..314d596 100644
--- a/avro/src/lib.rs
+++ b/avro/src/lib.rs
@@ -56,7 +56,6 @@ mod decimal;
mod decode;
mod duration;
mod encode;
-mod writer;
#[cfg(doc)]
pub mod documentation;
@@ -71,6 +70,7 @@ pub mod serde;
pub mod types;
pub mod util;
pub mod validator;
+pub mod writer;
#[expect(deprecated)]
pub use crate::{
@@ -103,6 +103,10 @@ pub use reader::{
pub use schema::Schema;
pub use serde::{AvroSchema, AvroSchemaComponent, from_value, to_value};
pub use uuid::Uuid;
+#[expect(
+ deprecated,
+ reason = "Still need to export it until we remove it completely"
+)]
pub use writer::{
Clearable, Writer, WriterBuilder,
datum::{to_avro_datum, to_avro_datum_schemata, write_avro_datum_ref},
diff --git a/avro/src/schema/mod.rs b/avro/src/schema/mod.rs
index 98600fa..97a5824 100644
--- a/avro/src/schema/mod.rs
+++ b/avro/src/schema/mod.rs
@@ -1168,7 +1168,9 @@ fn field_ordering_position(field: &str) -> Option<usize> {
#[cfg(test)]
mod tests {
use super::*;
- use crate::{error::Details, rabin::Rabin, reader::datum::GenericDatumReader};
+ use crate::reader::datum::GenericDatumReader;
+ use crate::writer::datum::GenericDatumWriter;
+ use crate::{error::Details, rabin::Rabin};
use apache_avro_test_helper::{
TestResult,
logger::{assert_logged, assert_not_logged},
@@ -2992,7 +2994,9 @@ mod tests {
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = crate::to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
let mut x = &datum[..];
let reader_schema = Schema::parse_str(reader_schema)?;
let deser_value = GenericDatumReader::builder(&writer_schema)
@@ -3576,7 +3580,9 @@ mod tests {
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = crate::to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
// Now, attempt to deserialize using the reader schema.
let reader_schema = Schema::parse(&reader_schema)?;
diff --git a/avro/src/serde/de.rs b/avro/src/serde/de.rs
index 3144fac..b9900d5 100644
--- a/avro/src/serde/de.rs
+++ b/avro/src/serde/de.rs
@@ -953,7 +953,9 @@ mod tests {
use apache_avro_test_helper::TestResult;
use super::*;
- use crate::{Decimal, reader::datum::GenericDatumReader};
+ use crate::Decimal;
+ use crate::reader::datum::GenericDatumReader;
+ use crate::writer::datum::GenericDatumWriter;
#[derive(PartialEq, Eq, Serialize, Deserialize, Debug)]
pub struct StringEnum {
@@ -989,7 +991,11 @@ mod tests {
// encode into avro
let value = crate::to_value(&data)?;
- let mut buf = std::io::Cursor::new(crate::to_avro_datum(&schema, value)?);
+ let mut buf = std::io::Cursor::new(
+ GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_value_to_vec(value)?,
+ );
// decode from avro
let value = GenericDatumReader::builder(&schema)
@@ -1033,7 +1039,9 @@ mod tests {
let value = crate::to_value(data)?;
// The following sentence have to fail has the data is wrong.
- let encoded_data = crate::to_avro_datum(&schema, value);
+ let encoded_data = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_value_to_vec(value);
assert!(encoded_data.is_err());
diff --git a/avro/src/writer/datum.rs b/avro/src/writer/datum.rs
index 5708eed..a45f704 100644
--- a/avro/src/writer/datum.rs
+++ b/avro/src/writer/datum.rs
@@ -15,28 +15,123 @@
// specific language governing permissions and limitations
// under the License.
-use std::io::Write;
-
+use bon::bon;
use serde::Serialize;
+use std::io::Write;
use crate::{
AvroResult, Schema,
- encode::{encode, encode_internal},
+ encode::encode_internal,
error::Details,
- schema::{NamesRef, ResolvedOwnedSchema, ResolvedSchema},
+ schema::{NamesRef, ResolvedSchema},
serde::ser_schema::SchemaAwareWriteSerializer,
types::Value,
};
+pub struct GenericDatumWriter<'s> {
+ schema: &'s Schema,
+ resolved: ResolvedSchema<'s>,
+ validate: bool,
+}
+
+#[bon]
+impl<'s> GenericDatumWriter<'s> {
+ #[builder]
+ pub fn new(
+ #[builder(start_fn)] schema: &'s Schema,
+ resolved_schemata: Option<ResolvedSchema<'s>>,
+ #[builder(default = true)] validate: bool,
+ ) -> AvroResult<Self> {
+ let resolved = if let Some(resolved) = resolved_schemata {
+ resolved
+ } else {
+ ResolvedSchema::try_from(schema)?
+ };
+ Ok(Self {
+ schema,
+ resolved,
+ validate,
+ })
+ }
+}
+
+impl<'s, S: generic_datum_writer_builder::State> GenericDatumWriterBuilder<'s, S> {
+ /// Set the schemata that will be used to resolve any references in the schema.
+ ///
+ /// This is equivalent to `.resolved_schemata(ResolvedSchema::new_with_schemata(schemata)?)`.
+ /// If you already have a [`ResolvedSchema`], use that function instead.
+ pub fn schemata(
+ self,
+ schemata: Vec<&'s Schema>,
+ ) -> AvroResult<
+ GenericDatumWriterBuilder<'s, generic_datum_writer_builder::SetResolvedSchemata<S>>,
+ >
+ where
+ S::ResolvedSchemata: generic_datum_writer_builder::IsUnset,
+ {
+ let resolved = ResolvedSchema::new_with_schemata(schemata)?;
+ Ok(self.resolved_schemata(resolved))
+ }
+}
+
+impl GenericDatumWriter<'_> {
+ /// Write a value to the writer.
+ pub fn write_value<W: Write, V: Into<Value>>(
+ &self,
+ writer: &mut W,
+ value: V,
+ ) -> AvroResult<usize> {
+ let value = value.into();
+ self.write_value_ref(writer, &value)
+ }
+
+ pub fn write_value_ref<W: Write>(&self, writer: &mut W, value: &Value) -> AvroResult<usize> {
+ if self.validate
+ && self.resolved.get_schemata().iter().all(|s| {
+ value
+ .validate_internal(s, self.resolved.get_names(), None)
+ .is_some()
+ })
+ {
+ return Err(Details::Validation.into());
+ }
+ encode_internal(value, self.schema, self.resolved.get_names(), None, writer)
+ }
+
+ /// Write a value to a [`Vec`].
+ pub fn write_value_to_vec<V: Into<Value>>(&self, value: V) -> AvroResult<Vec<u8>> {
+ let mut vec = Vec::new();
+ self.write_value(&mut vec, value)?;
+ Ok(vec)
+ }
+
+ pub fn write_ser<W: Write, T: Serialize>(
+ &self,
+ writer: &mut W,
+ value: &T,
+ ) -> AvroResult<usize> {
+ let mut serializer =
+ SchemaAwareWriteSerializer::new(writer, self.schema, self.resolved.get_names(), None);
+ value.serialize(&mut serializer)
+ }
+
+ pub fn write_ser_to_vec<T: Serialize>(&self, value: &T) -> AvroResult<Vec<u8>> {
+ let mut vec = Vec::new();
+ self.write_ser(&mut vec, value)?;
+ Ok(vec)
+ }
+}
+
/// Encode a value into raw Avro data, also performs schema validation.
///
/// **NOTE**: This function has a quite small niche of usage and does NOT generate headers and sync
/// markers; use [`Writer`] to be fully Avro-compatible if you don't know what
/// you are doing, instead.
+#[deprecated(since = "0.22.0", note = "Use GenericDatumWriter instead")]
pub fn to_avro_datum<T: Into<Value>>(schema: &Schema, value: T) -> AvroResult<Vec<u8>> {
- let mut buffer = Vec::new();
- write_avro_datum(schema, value, &mut buffer)?;
- Ok(buffer)
+ GenericDatumWriter::builder(schema)
+ .build()?
+ .write_value_to_vec(value)
}
/// Write the referenced [Serialize]able object to the provided [Write] object.
@@ -46,6 +141,7 @@ pub fn to_avro_datum<T: Into<Value>>(schema: &Schema, value: T) -> AvroResult<Ve
/// **NOTE**: This function has a quite small niche of usage and does **NOT** generate headers and sync
/// markers; use [`append_ser`](Writer::append_ser) to be fully Avro-compatible
/// if you don't know what you are doing, instead.
+#[deprecated(since = "0.22.0", note = "Use GenericDatumWriter instead")]
pub fn write_avro_datum_ref<T: Serialize, W: Write>(
schema: &Schema,
names: &NamesRef,
@@ -60,80 +156,20 @@ pub fn write_avro_datum_ref<T: Serialize, W: Write>(
///
/// If the provided `schema` is incomplete then its dependencies must be
/// provided in `schemata`
+#[deprecated(since = "0.22.0", note = "Use GenericDatumWriter instead")]
pub fn to_avro_datum_schemata<T: Into<Value>>(
schema: &Schema,
schemata: Vec<&Schema>,
value: T,
) -> AvroResult<Vec<u8>> {
- let mut buffer = Vec::new();
- write_avro_datum_schemata(schema, schemata, value, &mut buffer)?;
- Ok(buffer)
-}
-
-/// Encode a value into raw Avro data, also performs schema validation.
-///
-/// This is an internal function which gets the bytes buffer where to write as parameter instead of
-/// creating a new one like `to_avro_datum`.
-pub(super) fn write_avro_datum<T: Into<Value>, W: Write>(
- schema: &Schema,
- value: T,
- writer: &mut W,
-) -> AvroResult<()> {
- let avro = value.into();
- if !avro.validate(schema) {
- return Err(Details::Validation.into());
- }
- encode(&avro, schema, writer)?;
- Ok(())
-}
-
-pub(super) fn write_avro_datum_schemata<T: Into<Value>>(
- schema: &Schema,
- schemata: Vec<&Schema>,
- value: T,
- buffer: &mut Vec<u8>,
-) -> AvroResult<usize> {
- let avro = value.into();
- let rs = ResolvedSchema::try_from(schemata)?;
- let names = rs.get_names();
- let enclosing_namespace = schema.namespace();
- if let Some(_err) = avro.validate_internal(schema, names, enclosing_namespace) {
- return Err(Details::Validation.into());
- }
- encode_internal(&avro, schema, names, enclosing_namespace, buffer)
-}
-
-pub(super) fn write_value_ref_owned_resolved<W: Write>(
- resolved_schema: &ResolvedOwnedSchema,
- value: &Value,
- writer: &mut W,
-) -> AvroResult<usize> {
- let root_schema = resolved_schema.get_root_schema();
- if let Some(reason) = value.validate_internal(
- root_schema,
- resolved_schema.get_names(),
- root_schema.namespace(),
- ) {
- return Err(Details::ValidationWithReason {
- value: value.clone(),
- schema: root_schema.clone(),
- reason,
- }
- .into());
- }
- encode_internal(
- value,
- root_schema,
- resolved_schema.get_names(),
- root_schema.namespace(),
- writer,
- )
+ GenericDatumWriter::builder(schema)
+ .schemata(schemata)?
+ .build()?
+ .write_value_to_vec(value)
}
#[cfg(test)]
mod tests {
- use std::collections::HashMap;
-
use apache_avro_test_helper::TestResult;
use crate::{
@@ -178,7 +214,11 @@ mod tests {
zig_i64(3, &mut expected)?;
expected.extend([b'f', b'o', b'o']);
- assert_eq!(to_avro_datum(&schema, record)?, expected);
+ let written = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_value_to_vec(record)?;
+
+ assert_eq!(written, expected);
Ok(())
}
@@ -203,7 +243,9 @@ mod tests {
zig_i64(3, &mut expected)?;
expected.extend([b'f', b'o', b'o']);
- let bytes = write_avro_datum_ref(&schema, &HashMap::new(), &data, &mut writer)?;
+ let bytes = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_ser(&mut writer, &data)?;
assert_eq!(bytes, expected.len());
assert_eq!(writer, expected);
@@ -220,7 +262,10 @@ mod tests {
zig_i64(1, &mut expected)?;
zig_i64(3, &mut expected)?;
- assert_eq!(to_avro_datum(&schema, union)?, expected);
+ let written = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_value_to_vec(union)?;
+ assert_eq!(written, expected);
Ok(())
}
@@ -233,7 +278,10 @@ mod tests {
let mut expected = Vec::new();
zig_i64(0, &mut expected)?;
- assert_eq!(to_avro_datum(&schema, union)?, expected);
+ let written = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_value_to_vec(union)?;
+ assert_eq!(written, expected);
Ok(())
}
@@ -250,8 +298,12 @@ mod tests {
let schema = Schema::parse_str(schema_str)?;
assert_eq!(&schema, expected_schema);
// The serialized format should be the same as the schema.
- let ser = to_avro_datum(&schema, value.clone())?;
- let raw_ser = to_avro_datum(raw_schema, raw_value)?;
+ let ser = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_value_to_vec(value.clone())?;
+ let raw_ser = GenericDatumWriter::builder(raw_schema)
+ .build()?
+ .write_value_to_vec(raw_value)?;
assert_eq!(ser, raw_ser);
// Should deserialize from the schema into the logical type.
diff --git a/avro/src/writer/single_object.rs b/avro/src/writer/single_object.rs
index 136986a..d5c2c8c 100644
--- a/avro/src/writer/single_object.rs
+++ b/avro/src/writer/single_object.rs
@@ -19,14 +19,14 @@ use std::{io::Write, marker::PhantomData, ops::RangeInclusive};
use serde::Serialize;
+use crate::encode::encode_internal;
+use crate::serde::ser_schema::SchemaAwareWriteSerializer;
use crate::{
AvroResult, AvroSchema, Schema,
error::Details,
headers::{HeaderBuilder, RabinFingerprintHeader},
schema::ResolvedOwnedSchema,
types::Value,
- write_avro_datum_ref,
- writer::datum::write_value_ref_owned_resolved,
};
/// Writer that encodes messages according to the single object encoding v1 spec
@@ -163,12 +163,13 @@ where
.write_all(&self.header)
.map_err(Details::WriteBytes)?;
- let bytes = write_avro_datum_ref(
+ let mut serializer = SchemaAwareWriteSerializer::new(
+ writer,
self.resolved.get_root_schema(),
self.resolved.get_names(),
- data,
- writer,
- )?;
+ None,
+ );
+ let bytes = data.serialize(&mut serializer)?;
Ok(bytes + self.header.len())
}
@@ -184,6 +185,33 @@ where
}
}
+fn write_value_ref_owned_resolved<W: Write>(
+ resolved_schema: &ResolvedOwnedSchema,
+ value: &Value,
+ writer: &mut W,
+) -> AvroResult<usize> {
+ let root_schema = resolved_schema.get_root_schema();
+ if let Some(reason) = value.validate_internal(
+ root_schema,
+ resolved_schema.get_names(),
+ root_schema.namespace(),
+ ) {
+ return Err(Details::ValidationWithReason {
+ value: value.clone(),
+ schema: root_schema.clone(),
+ reason,
+ }
+ .into());
+ }
+ encode_internal(
+ value,
+ root_schema,
+ resolved_schema.get_names(),
+ root_schema.namespace(),
+ writer,
+ )
+}
+
#[cfg(test)]
mod tests {
use apache_avro_test_helper::TestResult;
diff --git a/avro/tests/avro-3786.rs b/avro/tests/avro-3786.rs
index 42c5464..f9f95b9 100644
--- a/avro/tests/avro-3786.rs
+++ b/avro/tests/avro-3786.rs
@@ -15,7 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-use apache_avro::{Schema, reader::datum::GenericDatumReader, to_avro_datum, to_value, types};
+use apache_avro::reader::datum::GenericDatumReader;
+use apache_avro::writer::datum::GenericDatumWriter;
+use apache_avro::{Schema, to_value, types};
use apache_avro_test_helper::TestResult;
#[test]
@@ -132,7 +134,9 @@ fn avro_3786_deserialize_union_with_different_enum_order() -> TestResult {
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
let mut x = &datum[..];
let reader_schema = Schema::parse_str(reader_schema)?;
let deser_value = GenericDatumReader::builder(&writer_schema)
@@ -259,7 +263,9 @@ fn avro_3786_deserialize_union_with_different_enum_order_defined_in_record() ->
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
let mut x = &datum[..];
let reader_schema = Schema::parse_str(reader_schema)?;
let deser_value = GenericDatumReader::builder(&writer_schema)
@@ -375,7 +381,9 @@ fn test_avro_3786_deserialize_union_with_different_enum_order_defined_in_record_
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
let mut x = &datum[..];
let reader_schema = Schema::parse_str(reader_schema)?;
let deser_value = GenericDatumReader::builder(&writer_schema)
@@ -491,7 +499,9 @@ fn test_avro_3786_deserialize_union_with_different_enum_order_defined_in_record_
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
let mut x = &datum[..];
let reader_schema = Schema::parse_str(reader_schema)?;
let deser_value = GenericDatumReader::builder(&writer_schema)
@@ -607,7 +617,9 @@ fn deserialize_union_with_different_enum_order_defined_in_record() -> TestResult
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
let mut x = &datum[..];
let reader_schema = Schema::parse_str(reader_schema)?;
let deser_value = GenericDatumReader::builder(&writer_schema)
@@ -884,7 +896,9 @@ fn deserialize_union_with_record_with_enum_defined_inline_reader_has_different_i
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
let mut x = &datum[..];
let reader_schema = Schema::parse_str(reader_schema)?;
let deser_value = GenericDatumReader::builder(&writer_schema)
diff --git a/avro/tests/avro-3787.rs b/avro/tests/avro-3787.rs
index 4fc2f3d..f50ae96 100644
--- a/avro/tests/avro-3787.rs
+++ b/avro/tests/avro-3787.rs
@@ -15,7 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-use apache_avro::{Schema, reader::datum::GenericDatumReader, to_avro_datum, to_value, types};
+use apache_avro::reader::datum::GenericDatumReader;
+use apache_avro::writer::datum::GenericDatumWriter;
+use apache_avro::{Schema, to_value, types};
use apache_avro_test_helper::TestResult;
#[test]
@@ -133,7 +135,9 @@ fn avro_3787_deserialize_union_with_unknown_symbol() -> TestResult {
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
let mut x = &datum[..];
let reader_schema = Schema::parse_str(reader_schema)?;
let deser_value = GenericDatumReader::builder(&writer_schema)
@@ -267,7 +271,9 @@ fn avro_3787_deserialize_union_with_unknown_symbol_no_ref() -> TestResult {
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
let mut x = &datum[..];
let reader_schema = Schema::parse_str(reader_schema)?;
let deser_value = GenericDatumReader::builder(&writer_schema)
diff --git a/avro/tests/io.rs b/avro/tests/io.rs
index 0d95850..9947587 100644
--- a/avro/tests/io.rs
+++ b/avro/tests/io.rs
@@ -16,9 +16,9 @@
// under the License.
//! Port of <https://github.com/apache/avro/blob/release-1.9.1/lang/py/test/test_io.py>
-use apache_avro::{
- Error, Schema, error::Details, reader::datum::GenericDatumReader, to_avro_datum, types::Value,
-};
+use apache_avro::reader::datum::GenericDatumReader;
+use apache_avro::writer::datum::GenericDatumWriter;
+use apache_avro::{Error, Schema, error::Details, types::Value};
use apache_avro_test_helper::TestResult;
use pretty_assertions::assert_eq;
use std::{io::Cursor, sync::OnceLock};
@@ -233,7 +233,9 @@ fn test_validate() -> TestResult {
fn test_round_trip() -> TestResult {
for (raw_schema, value) in schemas_to_validate().iter() {
let schema = Schema::parse_str(raw_schema)?;
- let encoded = to_avro_datum(&schema, value.clone()).unwrap();
+ let encoded = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_value_to_vec(value.clone())?;
let decoded = GenericDatumReader::builder(&schema)
.build()?
.read_value(&mut Cursor::new(encoded))?;
@@ -246,7 +248,9 @@ fn test_round_trip() -> TestResult {
#[test]
fn test_binary_int_encoding() -> TestResult {
for (number, hex_encoding) in binary_encodings().iter() {
- let encoded = to_avro_datum(&Schema::Int, Value::Int(*number as i32))?;
+ let encoded = GenericDatumWriter::builder(&Schema::Int)
+ .build()?
+ .write_value_to_vec(Value::Int(*number as i32))?;
assert_eq!(&encoded, hex_encoding);
}
@@ -256,7 +260,9 @@ fn test_binary_int_encoding() -> TestResult {
#[test]
fn test_binary_long_encoding() -> TestResult {
for (number, hex_encoding) in binary_encodings().iter() {
- let encoded = to_avro_datum(&Schema::Long, Value::Long(*number))?;
+ let encoded = GenericDatumWriter::builder(&Schema::Long)
+ .build()?
+ .write_value_to_vec(Value::Long(*number))?;
assert_eq!(&encoded, hex_encoding);
}
@@ -279,7 +285,9 @@ fn test_schema_promotion() -> TestResult {
let original_value = &promotable_values[i];
for (j, reader_raw_schema) in promotable_schemas.iter().enumerate().skip(i + 1) {
let reader_schema = Schema::parse_str(reader_raw_schema)?;
- let encoded = to_avro_datum(&writer_schema, original_value.clone())?;
+ let encoded = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(original_value.clone())?;
let decoded = GenericDatumReader::builder(&writer_schema)
.reader_schema(&reader_schema)
.build()?
@@ -298,7 +306,9 @@ fn test_unknown_symbol() -> TestResult {
let reader_schema =
Schema::parse_str(r#"{"type": "enum", "name": "Test", "symbols": ["BAR", "BAZ"]}"#)?;
let original_value = Value::Enum(0, "FOO".to_string());
- let encoded = to_avro_datum(&writer_schema, original_value)?;
+ let encoded = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(original_value.clone())?;
let decoded = GenericDatumReader::builder(&writer_schema)
.reader_schema(&reader_schema)
.build()?
@@ -321,7 +331,9 @@ fn test_default_value() -> TestResult {
}}"#
))?;
let datum_to_read = Value::Record(vec![("H".to_string(), default_datum.clone())]);
- let encoded = to_avro_datum(long_record_schema(), long_record_datum().clone())?;
+ let encoded = GenericDatumWriter::builder(long_record_schema())
+ .build()?
+ .write_value_to_vec(long_record_datum().clone())?;
let datum_read = GenericDatumReader::builder(long_record_schema())
.reader_schema(&reader_schema)
.build()?
@@ -377,7 +389,9 @@ fn test_no_default_value() -> TestResult {
]
}"#,
)?;
- let encoded = to_avro_datum(long_record_schema(), long_record_datum().clone())?;
+ let encoded = GenericDatumWriter::builder(long_record_schema())
+ .build()?
+ .write_value_to_vec(long_record_datum().clone())?;
let result = GenericDatumReader::builder(long_record_schema())
.reader_schema(&reader_schema)
.build()?
@@ -405,7 +419,9 @@ fn test_projection() -> TestResult {
("E".to_string(), Value::Int(5)),
("F".to_string(), Value::Int(6)),
]);
- let encoded = to_avro_datum(long_record_schema(), long_record_datum().clone())?;
+ let encoded = GenericDatumWriter::builder(long_record_schema())
+ .build()?
+ .write_value_to_vec(long_record_datum().clone())?;
let datum_read = GenericDatumReader::builder(long_record_schema())
.reader_schema(&reader_schema)
.build()?
@@ -433,7 +449,9 @@ fn test_field_order() -> TestResult {
("F".to_string(), Value::Int(6)),
("E".to_string(), Value::Int(5)),
]);
- let encoded = to_avro_datum(long_record_schema(), long_record_datum().clone())?;
+ let encoded = GenericDatumWriter::builder(long_record_schema())
+ .build()?
+ .write_value_to_vec(long_record_datum().clone())?;
let datum_read = GenericDatumReader::builder(long_record_schema())
.reader_schema(&reader_schema)
.build()?
@@ -462,7 +480,11 @@ fn test_type_exception() -> Result<(), String> {
("E".to_string(), Value::Int(5)),
("F".to_string(), Value::String(String::from("Bad"))),
]);
- let encoded = to_avro_datum(&writer_schema, datum_to_write).map_err(Error::into_details);
+ let encoded = GenericDatumWriter::builder(&writer_schema)
+ .build()
+ .unwrap()
+ .write_value_to_vec(datum_to_write)
+ .map_err(Error::into_details);
match encoded {
Ok(_) => Err(String::from("Expected ValidationError, got Ok")),
Err(Details::Validation) => Ok(()),
diff --git a/avro/tests/schema.rs b/avro/tests/schema.rs
index 9ec8684..89e328a 100644
--- a/avro/tests/schema.rs
+++ b/avro/tests/schema.rs
@@ -15,18 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-use std::{
- collections::HashMap,
- io::{Cursor, Read},
-};
-
+use apache_avro::writer::datum::GenericDatumWriter;
use apache_avro::{
Codec, Error, Reader, Schema, Writer,
error::Details,
from_value,
reader::datum::GenericDatumReader,
schema::{EnumSchema, FixedSchema, Name, RecordField, RecordSchema},
- to_avro_datum, to_value,
+ to_value,
types::{Record, Value},
};
use apache_avro_test_helper::{
@@ -35,6 +31,10 @@ use apache_avro_test_helper::{
init,
};
use serde::{Deserialize, Serialize};
+use std::{
+ collections::HashMap,
+ io::{Cursor, Read},
+};
#[test]
fn test_correct_recursive_extraction() -> TestResult {
@@ -859,7 +859,9 @@ fn avro_old_issue_47() -> TestResult {
};
let ser_value = to_value(record.clone())?;
- let serialized_bytes = to_avro_datum(&schema, ser_value)?;
+ let serialized_bytes = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_value_to_vec(ser_value)?;
let de_value = GenericDatumReader::builder(&schema)
.build()?
@@ -979,12 +981,14 @@ fn test_avro_3785_deserialize_namespace_with_nullable_type_containing_reference_
bar_init: Bar::Bar0,
bar_use_parent: Some(BarUseParent { bar_use: Bar::Bar1 }),
};
- let avro_value = crate::to_value(foo1)?;
+ let avro_value = to_value(foo1)?;
assert!(
avro_value.validate(&writer_schema),
"value is valid for schema",
);
- let datum = to_avro_datum(&writer_schema, avro_value)?;
+ let datum = GenericDatumWriter::builder(&writer_schema)
+ .build()?
+ .write_value_to_vec(avro_value)?;
let mut x = &datum[..];
let reader_schema = Schema::parse_str(reader_schema)?;
let deser_value = GenericDatumReader::builder(&writer_schema)
diff --git a/avro/tests/to_from_avro_datum_schemata.rs b/avro/tests/to_from_avro_datum_schemata.rs
index 14feacd..0849367 100644
--- a/avro/tests/to_from_avro_datum_schemata.rs
+++ b/avro/tests/to_from_avro_datum_schemata.rs
@@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
-use apache_avro::{
- Codec, Reader, Schema, Writer, reader::datum::GenericDatumReader, to_avro_datum_schemata,
- types::Value,
-};
+use apache_avro::reader::datum::GenericDatumReader;
+use apache_avro::writer::datum::GenericDatumWriter;
+use apache_avro::{Codec, Reader, Schema, Writer, types::Value};
use apache_avro_test_helper::{TestResult, init};
static SCHEMA_A_STR: &str = r#"{
@@ -52,7 +51,10 @@ fn test_avro_3683_multiple_schemata_to_from_avro_datum() -> TestResult {
// this is the Schema we want to use for write/read
let schema_b = schemata[1];
let expected: Vec<u8> = vec![0, 0, 128, 63];
- let actual = to_avro_datum_schemata(schema_b, schemata.clone(), record.clone())?;
+ let actual = GenericDatumWriter::builder(schema_b)
+ .schemata(schemata.clone())?
+ .build()?
+ .write_value_to_vec(record.clone())?;
assert_eq!(actual, expected);
let value = GenericDatumReader::builder(schema_b)
@@ -79,7 +81,10 @@ fn avro_rs_106_test_multiple_schemata_to_from_avro_datum_with_resolution() -> Te
// this is the Schema we want to use for write/read
let schema_b = schemata[1];
let expected: Vec<u8> = vec![0, 0, 128, 63];
- let actual = to_avro_datum_schemata(schema_b, schemata.clone(), record.clone())?;
+ let actual = GenericDatumWriter::builder(schema_b)
+ .schemata(schemata.clone())?
+ .build()?
+ .write_value_to_vec(record.clone())?;
assert_eq!(actual, expected);
let value = GenericDatumReader::builder(schema_b)
(avro-rs) 01/08: chore: Make `writer` module a directory module (#497)
Posted by kr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a commit to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
commit 127aa7e5e543140933c6061d6df0587cf07e7f01
Author: Kriskras99 <gi...@kriskras99.nl>
AuthorDate: Tue Mar 3 15:10:58 2026 +0100
chore: Make `writer` module a directory module (#497)
---
avro/src/{writer.rs => writer/mod.rs} | 0
1 file changed, 0 insertions(+), 0 deletions(-)
diff --git a/avro/src/writer.rs b/avro/src/writer/mod.rs
similarity index 100%
rename from avro/src/writer.rs
rename to avro/src/writer/mod.rs
(avro-rs) 08/08: Add test suite for deser
Posted by kr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a commit to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
commit c1adfee11f728a56c04fefc30c4916bca3f77d24
Author: default <ad...@kriskras99.nl>
AuthorDate: Wed Mar 4 16:05:37 2026 +0000
Add test suite for deser
---
avro/src/reader/datum.rs | 6 +-
avro/src/serde/deser_schema/mod.rs | 954 ++++++++++++++++++++++++++++++++++++-
2 files changed, 954 insertions(+), 6 deletions(-)
diff --git a/avro/src/reader/datum.rs b/avro/src/reader/datum.rs
index f72b272..9eeb78e 100644
--- a/avro/src/reader/datum.rs
+++ b/avro/src/reader/datum.rs
@@ -135,7 +135,7 @@ impl<'s> GenericDatumReader<'s> {
}
}
- pub fn read_deser<R: Read, T: DeserializeOwned>(&self, reader: &mut R) -> AvroResult<T> {
+ pub fn read_deser<T: DeserializeOwned>(&self, reader: &mut impl Read) -> AvroResult<T> {
T::deserialize(SchemaAwareDeserializer::new(
reader,
self.writer,
@@ -352,11 +352,11 @@ mod tests {
}
let schema = Schema::parse_str(TEST_RECORD_SCHEMA_3240)?;
- let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111];
+ let mut encoded: &[u8] = &[54, 6, 102, 111, 111];
let error = GenericDatumReader::builder(&schema)
.build()?
- .read_deser::<_, TestRecord3240>(&mut encoded)
+ .read_deser::<TestRecord3240>(&mut encoded)
.unwrap_err();
assert_eq!(
diff --git a/avro/src/serde/deser_schema/mod.rs b/avro/src/serde/deser_schema/mod.rs
index fc6c560..7e61686 100644
--- a/avro/src/serde/deser_schema/mod.rs
+++ b/avro/src/serde/deser_schema/mod.rs
@@ -182,8 +182,8 @@ impl<'s, 'r, R: Read, S: Borrow<Schema>> SchemaAwareDeserializer<'s, 'r, R, S> {
}
}
-const DESERIALIZE_ANY: &str = "This value is compared by pointer value";
-const DESERIALIZE_ANY_FIELDS: &[&str] = &[];
+static DESERIALIZE_ANY: &str = "This value is compared by pointer value";
+static DESERIALIZE_ANY_FIELDS: &[&str] = &[];
impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
for SchemaAwareDeserializer<'s, 'r, R, S>
@@ -447,7 +447,7 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
V: serde::de::Visitor<'de>,
{
match self.schema {
- Schema::String => {
+ Schema::String | Schema::Uuid(UuidSchema::String) => {
let string = self.read_string()?;
visitor.visit_string(string)
}
@@ -727,3 +727,951 @@ impl<'de, 's, 'r, R: Read, S: Borrow<Schema>> Deserializer<'de>
self.deserialize_any(visitor)
}
}
+
+#[cfg(test)]
+mod tests {
+ use std::io::Cursor;
+
+ use num_bigint::BigInt;
+ use pretty_assertions::assert_eq;
+ use serde::de::{DeserializeOwned, Visitor};
+ use serde::{Deserialize, Serialize};
+ use uuid::Uuid;
+
+ use apache_avro_test_helper::TestResult;
+
+ use super::*;
+ use crate::reader::datum::GenericDatumReader;
+ use crate::writer::datum::GenericDatumWriter;
+ use crate::{AvroResult, Decimal};
+
+ #[track_caller]
+ fn assert_roundtrip<T>(value: T, schema: &Schema, schemata: Vec<&Schema>) -> AvroResult<()>
+ where
+ T: Serialize + DeserializeOwned + PartialEq + Debug + Clone,
+ {
+ let buf = GenericDatumWriter::builder(schema)
+ .schemata(schemata.clone())?
+ .build()?
+ .write_ser_to_vec(&value)?;
+
+ let decoded_value: T = GenericDatumReader::builder(schema)
+ .writer_schemata(schemata)?
+ .build()?
+ .read_deser(&mut Cursor::new(buf))?;
+
+ assert_eq!(decoded_value, value);
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_3955_decode_enum() -> TestResult {
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+ pub enum SourceType {
+ Sozu,
+ Haproxy,
+ HaproxyTcp,
+ }
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct AccessLog {
+ source: SourceType,
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "AccessLog",
+ "namespace": "com.clevercloud.accesslogs.common.avro",
+ "type": "record",
+ "fields": [{
+ "name": "source",
+ "type": {
+ "type": "enum",
+ "name": "SourceType",
+ "items": "string",
+ "symbols": ["SOZU", "HAPROXY", "HAPROXY_TCP"]
+ }
+ }]
+ }"#,
+ )?;
+
+ let data = AccessLog {
+ source: SourceType::Sozu,
+ };
+
+ assert_roundtrip(data, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_decode_enum_invalid_data() -> TestResult {
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
+ pub enum SourceType {
+ Sozu,
+ Haproxy,
+ HaproxyTcp,
+ }
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct AccessLog {
+ source: SourceType,
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "AccessLog",
+ "namespace": "com.clevercloud.accesslogs.common.avro",
+ "type": "record",
+ "fields": [{
+ "name": "source",
+ "type": {
+ "type": "enum",
+ "name": "SourceType",
+ "items": "string",
+ "symbols": ["SOZU", "HAPROXY", "HAPROXY_TCP"]
+ }
+ }]
+ }"#,
+ )?;
+
+ // Contains index 3 (4th symbol)
+ let data_with_unknown_index = vec![6u8];
+
+ let error = GenericDatumReader::builder(&schema)
+ .build()?
+ .read_deser::<AccessLog>(&mut Cursor::new(data_with_unknown_index))
+ .unwrap_err();
+
+ assert_eq!(error.to_string(), "Enum symbol index out of bounds: 3");
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_nested_struct() -> TestResult {
+ #[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
+ struct Test {
+ a: i64,
+ b: String,
+ c: Decimal,
+ }
+
+ #[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
+ struct TestInner {
+ a: Test,
+ b: i32,
+ }
+
+ let schemas = Schema::parse_list([
+ r#"{
+ "name": "Test",
+ "type": "record",
+ "fields": [
+ {
+ "name": "a",
+ "type": "long"
+ },
+ {
+ "name": "b",
+ "type": "string"
+ },
+ {
+ "name": "c",
+ "type": {
+ "type": "bytes",
+ "logicalType": "decimal",
+ "precision": 4,
+ "scale": 2
+ }
+ }
+ ]
+ }"#,
+ r#"{
+ "name": "TestInner",
+ "type": "record",
+ "fields": [
+ {
+ "name": "a",
+ "type": "Test"
+ },
+ {
+ "name": "b",
+ "type": "int"
+ }
+ ]
+ }"#,
+ ])?;
+
+ let test = Test {
+ a: 27,
+ b: "foo".to_string(),
+ c: Decimal::from(vec![1, 24]),
+ };
+
+ assert_roundtrip(test.clone(), &schemas[0], Vec::new())?;
+
+ let test_inner = TestInner { a: test, b: 35 };
+
+ assert_roundtrip(test_inner, &schemas[1], vec![&schemas[0]])?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_external_unit_enum() -> TestResult {
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ pub enum UnitExternalEnum {
+ Val1,
+ Val2,
+ }
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct TestUnitExternalEnum {
+ a: UnitExternalEnum,
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "TestUnitExternalEnum",
+ "type": "record",
+ "fields": [{
+ "name": "a",
+ "type": {
+ "type": "enum",
+ "name": "UnitExternalEnum",
+ "items": "string",
+ "symbols": ["Val1", "Val2"]
+ }
+ }]
+ }"#,
+ )?;
+
+ let alt_schema = Schema::parse_str(
+ r#"{
+ "name": "TestUnitExternalEnum",
+ "type": "record",
+ "fields": [{
+ "name": "a",
+ "type": [
+ {
+ "name": "Val1",
+ "type": "record",
+ "fields": []
+ },
+ {
+ "name": "Val2",
+ "type": "record",
+ "fields": []
+ }
+ ]
+ }]
+ }"#,
+ )?;
+
+ let value = TestUnitExternalEnum {
+ a: UnitExternalEnum::Val1,
+ };
+ assert_roundtrip(value.clone(), &schema, Vec::new())?;
+ assert_roundtrip(value, &alt_schema, Vec::new())?;
+
+ let value = TestUnitExternalEnum {
+ a: UnitExternalEnum::Val2,
+ };
+ assert_roundtrip(value.clone(), &alt_schema, Vec::new())?;
+ assert_roundtrip(value, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_internal_unit_enum() -> TestResult {
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ #[serde(tag = "t")]
+ pub enum UnitInternalEnum {
+ Val1,
+ Val2,
+ }
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct TestUnitInternalEnum {
+ a: UnitInternalEnum,
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "TestUnitInternalEnum",
+ "type": "record",
+ "fields": [{
+ "name": "a",
+ "type": {
+ "type": "record",
+ "name": "UnitInternalEnum",
+ "fields": [{
+ "name": "t",
+ "type": {
+ "type": "enum",
+ "name": "t",
+ "symbols": ["Val1", "Val2"]
+ }
+ }]
+ }
+ }]
+ }"#,
+ )?;
+
+ let value = TestUnitInternalEnum {
+ a: UnitInternalEnum::Val1,
+ };
+ assert_roundtrip(value, &schema, Vec::new())?;
+
+ let value = TestUnitInternalEnum {
+ a: UnitInternalEnum::Val2,
+ };
+ assert_roundtrip(value, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_adjacent_unit_enum() -> TestResult {
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ #[serde(tag = "t", content = "v")]
+ pub enum UnitAdjacentEnum {
+ Val1,
+ Val2,
+ }
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct TestUnitAdjacentEnum {
+ a: UnitAdjacentEnum,
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "TestUnitAdjacentEnum",
+ "type": "record",
+ "fields": [{
+ "name": "a",
+ "type": {
+ "type": "record",
+ "name": "UnitAdjacentEnum",
+ "fields": [
+ {
+ "name": "t",
+ "type": {
+ "type": "enum",
+ "name": "t",
+ "symbols": ["Val1", "Val2"]
+ }
+ },
+ {
+ "name": "v",
+ "default": null,
+ "type": ["null"]
+ }
+ ]
+ }
+ }]
+ }"#,
+ )?;
+
+ let value = TestUnitAdjacentEnum {
+ a: UnitAdjacentEnum::Val1,
+ };
+ assert_roundtrip(value, &schema, Vec::new())?;
+
+ let value = TestUnitAdjacentEnum {
+ a: UnitAdjacentEnum::Val2,
+ };
+ assert_roundtrip(value, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_untagged_unit_enum() -> TestResult {
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ #[serde(untagged)]
+ pub enum UnitUntaggedEnum {
+ Val1,
+ Val2,
+ }
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct TestUnitUntaggedEnum {
+ a: UnitUntaggedEnum,
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "TestUnitUntaggedEnum",
+ "type": "record",
+ "fields": [{
+ "name": "a",
+ "type": ["null"]
+ }]
+ }"#,
+ )?;
+
+ let value1 = TestUnitUntaggedEnum {
+ a: UnitUntaggedEnum::Val1,
+ };
+ assert_roundtrip(value1.clone(), &schema, Vec::new())?;
+
+ let value2 = TestUnitUntaggedEnum {
+ a: UnitUntaggedEnum::Val2,
+ };
+ let buf = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_ser_to_vec(&value1)?;
+
+ let decoded_value: TestUnitUntaggedEnum = GenericDatumReader::builder(&schema)
+ .build()?
+ .read_deser(&mut Cursor::new(buf))?;
+
+ // Val2 cannot troundtrip. All unit variants are serialized to the same null.
+ // This also doesn't roundtrip in serde_json.
+ assert_ne!(value2, decoded_value);
+ assert_eq!(decoded_value, value1);
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_mixed_enum() -> TestResult {
+ #[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
+ struct TestNullExternalEnum {
+ a: NullExternalEnum,
+ }
+
+ #[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
+ enum NullExternalEnum {
+ Val1,
+ Val2(),
+ Val3(()),
+ Val4(u64),
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "TestNullExternalEnum",
+ "type": "record",
+ "fields": [{
+ "name": "a",
+ "type": [
+ {
+ "name": "Val1",
+ "type": "record",
+ "fields": []
+ },
+ {
+ "name": "Val2",
+ "type": "record",
+ "fields": []
+ },
+ {
+ "name": "Val3",
+ "type": "record",
+ "fields": [{
+ "name": "field_0",
+ "type": "null"
+ }]
+ },
+ {
+ "name": "Val4",
+ "type": "record",
+ "fields": [{
+ "name": "field_0",
+ "type": {
+ "type": "fixed",
+ "name": "u64",
+ "size": 8
+ }
+ }]
+ }
+ ]
+ }]
+ }"#,
+ )?;
+
+ let data = [
+ TestNullExternalEnum {
+ a: NullExternalEnum::Val1,
+ },
+ TestNullExternalEnum {
+ a: NullExternalEnum::Val2(),
+ },
+ TestNullExternalEnum {
+ a: NullExternalEnum::Val2(),
+ },
+ TestNullExternalEnum {
+ a: NullExternalEnum::Val3(()),
+ },
+ TestNullExternalEnum {
+ a: NullExternalEnum::Val4(123),
+ },
+ ];
+
+ for value in data {
+ assert_roundtrip(value, &schema, Vec::new())?;
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_single_value_enum() -> TestResult {
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct TestSingleValueExternalEnum {
+ a: SingleValueExternalEnum,
+ }
+
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ enum SingleValueExternalEnum {
+ Double(f64),
+ String(String),
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "TestSingleValueExternalEnum",
+ "type": "record",
+ "fields": [{
+ "name": "a",
+ "type": [
+ {
+ "name": "Val1",
+ "type": "record",
+ "fields": [{
+ "name": "field_0",
+ "type": "double"
+ }]
+ },
+ {
+ "name": "Val2",
+ "type": "record",
+ "fields": [{
+ "name": "field_0",
+ "type": "string"
+ }]
+ }
+ ]
+ }]
+ }"#,
+ )?;
+
+ let alt_schema = Schema::parse_str(
+ r#"{
+ "name": "TestSingleValueExternalEnum",
+ "type": "record",
+ "fields": [{
+ "name": "a",
+ "type": ["double", "string"]
+ }]
+ }"#,
+ )?;
+
+ let double = TestSingleValueExternalEnum {
+ a: SingleValueExternalEnum::Double(64.0),
+ };
+ assert_roundtrip(double.clone(), &schema, Vec::new())?;
+ assert_roundtrip(double, &alt_schema, Vec::new())?;
+
+ let string = TestSingleValueExternalEnum {
+ a: SingleValueExternalEnum::String("test".to_string()),
+ };
+ assert_roundtrip(string.clone(), &schema, Vec::new())?;
+ assert_roundtrip(string, &alt_schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_struct_enum() -> TestResult {
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct TestStructExternalEnum {
+ a: StructExternalEnum,
+ }
+
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ enum StructExternalEnum {
+ Val1 { x: f32, y: f32 },
+ Val2 { x: f32, y: f32 },
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "TestStructExternalEnum",
+ "type": "record",
+ "fields": [{
+ "name": "a",
+ "type": [
+ {
+ "name": "Val1",
+ "type": "record",
+ "fields": [
+ {
+ "name": "x",
+ "type": "float"
+ },
+ {
+ "name": "y",
+ "type": "float"
+ }
+ ]
+ },
+ {
+ "name": "Val2",
+ "type": "record",
+ "fields": [
+ {
+ "name": "x",
+ "type": "float"
+ },
+ {
+ "name": "y",
+ "type": "float"
+ }
+ ]
+ }
+ ]
+ }]
+ }"#,
+ )?;
+
+ let value1 = TestStructExternalEnum {
+ a: StructExternalEnum::Val1 { x: 1.0, y: 2.0 },
+ };
+
+ assert_roundtrip(value1, &schema, Vec::new())?;
+
+ let value2 = TestStructExternalEnum {
+ a: StructExternalEnum::Val2 { x: 2.0, y: 1.0 },
+ };
+
+ assert_roundtrip(value2, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_struct_flatten() -> TestResult {
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct S1 {
+ f1: String,
+ #[serde(flatten)]
+ inner: S2,
+ }
+
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct S2 {
+ f2: String,
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "S1",
+ "type": "record",
+ "fields": [
+ {
+ "name": "f1",
+ "type": "string"
+ },
+ {
+ "name": "f2",
+ "type": "string"
+ }
+ ]
+ }"#,
+ )?;
+
+ let value = S1 {
+ f1: "Hello".to_owned(),
+ inner: S2 {
+ f2: "World".to_owned(),
+ },
+ };
+
+ assert_roundtrip(value, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_tuple_enum() -> TestResult {
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ struct TestTupleExternalEnum {
+ a: TupleExternalEnum,
+ }
+
+ #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+ enum TupleExternalEnum {
+ Val1(f32, f32),
+ Val2(f32, f32, f32),
+ }
+
+ let schema = Schema::parse_str(
+ r#"{
+ "name": "TestTupleExternalEnum",
+ "type": "record",
+ "fields": [{
+ "name": "a",
+ "type": [
+ {
+ "name": "Val1",
+ "type": "record",
+ "fields": [
+ {
+ "name": "field_0",
+ "type": "float"
+ },
+ {
+ "name": "field_1",
+ "type": "float"
+ }
+ ]
+ },
+ {
+ "name": "Val2",
+ "type": "record",
+ "fields": [
+ {
+ "name": "field_0",
+ "type": "float"
+ },
+ {
+ "name": "field_1",
+ "type": "float"
+ },
+ {
+ "name": "field_2",
+ "type": "float"
+ }
+ ]
+ }
+ ]
+ }]
+ }"#,
+ )?;
+
+ let value1 = TestTupleExternalEnum {
+ a: TupleExternalEnum::Val1(1.0, 2.0),
+ };
+
+ assert_roundtrip(value1, &schema, Vec::new())?;
+
+ let value2 = TestTupleExternalEnum {
+ a: TupleExternalEnum::Val1(2.0, 1.0),
+ };
+
+ assert_roundtrip(value2, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_date() -> TestResult {
+ let schema = Schema::Date;
+ assert_roundtrip(1i32, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_time_millis() -> TestResult {
+ let schema = Schema::TimeMillis;
+ assert_roundtrip(1i32, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_time_micros() -> TestResult {
+ let schema = Schema::TimeMicros;
+ assert_roundtrip(1i64, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_timestamp_millis() -> TestResult {
+ let schema = Schema::TimestampMillis;
+ assert_roundtrip(1i64, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_timestamp_micros() -> TestResult {
+ let schema = Schema::TimestampMicros;
+ assert_roundtrip(1i64, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_3916_timestamp_nanos() -> TestResult {
+ let schema = Schema::TimestampNanos;
+ assert_roundtrip(1i64, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_3853_local_timestamp_millis() -> TestResult {
+ let schema = Schema::LocalTimestampMillis;
+ assert_roundtrip(1i64, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_3853_local_timestamp_micros() -> TestResult {
+ let schema = Schema::LocalTimestampMicros;
+ assert_roundtrip(1i64, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_3916_local_timestamp_nanos() -> TestResult {
+ let schema = Schema::LocalTimestampNanos;
+ assert_roundtrip(1i64, &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_xxx_uuid() -> TestResult {
+ let schema = Schema::parse_str(
+ r#"{
+ "type": "fixed",
+ "logicalType": "uuid",
+ "size": 16,
+ "name": "uuid"
+ }"#,
+ )?;
+
+ let alt_schema = Schema::Uuid(UuidSchema::String);
+
+ let uuid = Uuid::parse_str("9ec535ff-3e2a-45bd-91d3-0a01321b5a49")?;
+
+ assert_roundtrip(uuid, &schema, Vec::new())?;
+
+ let buf = GenericDatumWriter::builder(&alt_schema)
+ .human_readable(true)
+ .build()?
+ .write_ser_to_vec(&uuid)?;
+
+ let decoded_value: Uuid = GenericDatumReader::builder(&alt_schema)
+ .human_readable(true)
+ .build()?
+ .read_deser(&mut Cursor::new(buf))?;
+
+ assert_eq!(decoded_value, uuid);
+
+ Ok(())
+ }
+
+ #[derive(Debug)]
+ struct Bytes(Vec<u8>);
+
+ impl<'de> Deserialize<'de> for Bytes {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ struct BytesVisitor;
+ impl Visitor<'_> for BytesVisitor {
+ type Value = Bytes;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+ formatter.write_str("a byte array")
+ }
+
+ fn visit_bytes<E>(self, v: &[u8]) -> Result<Self::Value, E>
+ where
+ E: serde::de::Error,
+ {
+ Ok(Bytes(v.to_vec()))
+ }
+ }
+ deserializer.deserialize_bytes(BytesVisitor)
+ }
+ }
+
+ #[test]
+ fn avro_3892_deserialize_bytes_from_decimal() -> TestResult {
+ let schema = Schema::parse_str(
+ r#"{
+ "type": "bytes",
+ "logicalType": "decimal",
+ "precision": 4,
+ "scale": 2
+ }"#,
+ )?;
+ let schema_union = Schema::parse_str(
+ r#"[
+ "null",
+ {
+ "type": "bytes",
+ "logicalType": "decimal",
+ "precision": 4,
+ "scale": 2
+ }
+ ]"#,
+ )?;
+
+ let expected_bytes = BigInt::from(123456789).to_signed_bytes_be();
+ let value = Decimal::from(&expected_bytes);
+ let buf = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_ser_to_vec(&value)?;
+
+ let decoded_value: Bytes = GenericDatumReader::builder(&schema)
+ .build()?
+ .read_deser(&mut Cursor::new(buf))?;
+
+ assert_eq!(decoded_value.0, expected_bytes);
+
+ let buf = GenericDatumWriter::builder(&schema_union)
+ .build()?
+ .write_ser_to_vec(&Some(value))?;
+
+ let decoded_value: Option<Bytes> = GenericDatumReader::builder(&schema_union)
+ .build()?
+ .read_deser(&mut Cursor::new(buf))?;
+
+ assert_eq!(decoded_value.unwrap().0, expected_bytes);
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_414_deserialize_char_from_string() -> TestResult {
+ let schema = Schema::String;
+
+ assert_roundtrip('a', &schema, Vec::new())?;
+ assert_roundtrip('👹', &schema, Vec::new())?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn avro_rs_414_deserialize_char_from_long_string() -> TestResult {
+ let schema = Schema::String;
+ let buf = GenericDatumWriter::builder(&schema)
+ .build()?
+ .write_ser_to_vec(&"avro")?;
+
+ let error = GenericDatumReader::builder(&schema)
+ .build()?
+ .read_deser::<char>(&mut Cursor::new(buf))
+ .unwrap_err();
+
+ assert_eq!(
+ error.to_string(),
+ "Failed to deserialize value of type char using schema String: String contains more than one character"
+ );
+
+ Ok(())
+ }
+}
(avro-rs) 05/08: fix: `Value::validate_schemata` should not set `enclosing_namespace`
Posted by kr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a commit to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
commit 3d3b1e90e314d51a5f09769915c28d55fa4f1599
Author: default <ad...@kriskras99.nl>
AuthorDate: Wed Mar 4 09:41:32 2026 +0000
fix: `Value::validate_schemata` should not set `enclosing_namespace`
`enclosing_namespace` is the namespace of the parent schema, but
there is no parent schema and should therefore not be set.
---
avro/src/types.rs | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/avro/src/types.rs b/avro/src/types.rs
index 90ea2b0..b6f658c 100644
--- a/avro/src/types.rs
+++ b/avro/src/types.rs
@@ -380,10 +380,8 @@ impl Value {
let rs = ResolvedSchema::try_from(schemata.clone())
.expect("Schemata didn't successfully resolve");
let schemata_len = schemata.len();
- schemata.iter().any(|schema| {
- let enclosing_namespace = schema.namespace();
-
- match self.validate_internal(schema, rs.get_names(), enclosing_namespace) {
+ schemata.iter().any(
+ |schema| match self.validate_internal(schema, rs.get_names(), None) {
Some(reason) => {
let log_message =
format!("Invalid value: {self:?} for schema: {schema:?}. Reason: {reason}");
@@ -395,8 +393,8 @@ impl Value {
false
}
None => true,
- }
- })
+ },
+ )
}
fn accumulate(accumulator: Option<String>, other: Option<String>) -> Option<String> {