You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by kr...@apache.org on 2026/03/01 21:42:12 UTC
(avro-rs) branch feat/enums updated (70fb997 -> c492f2b)
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a change to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
discard 70fb997 wip: Full enum support
omit 8a668c5 fix: Remove `PartialOrd,Ord` implementations from `Name`
omit 41bf7ac fix: Panic when `Option<()>::get_schema()` is called
omit 7915558 fix: Support type promotion and resolve edge cases
omit 7205975 feat: Add a `UnionSchemaBuilder`
add 1933c43 feat!: Add a `UnionSchemaBuilder` (#489)
add 0860320 fix!: Remove unused fields in `RecordField` (`order`, `position`)
add fb7513f feat: Rework `Name` to be more performant
add e07d7a8 fix: Enable `ref_option` lint and fix the warnings it generates
new c492f2b wip: Full enum support
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (70fb997)
\
N -- N -- N refs/heads/feat/enums (c492f2b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.clippy.toml | 1 +
Cargo.toml | 1 +
avro/src/bytes.rs | 3 +
avro/src/decode.rs | 17 +-
avro/src/encode.rs | 27 +-
avro/src/error.rs | 1 -
avro/src/reader/block.rs | 38 +-
avro/src/reader/mod.rs | 7 +-
avro/src/reader/single_object.rs | 2 +-
avro/src/schema/builders.rs | 1 -
avro/src/schema/mod.rs | 420 +++++---------
avro/src/schema/name.rs | 191 ++++---
avro/src/schema/parser.rs | 88 ++-
avro/src/schema/record/field.rs | 105 ++--
avro/src/schema/record/mod.rs | 2 +-
avro/src/schema/resolve.rs | 63 +-
avro/src/schema/union.rs | 34 +-
avro/src/schema_compatibility.rs | 8 +-
avro/src/schema_equality.rs | 39 +-
avro/src/serde/derive.rs | 157 ++---
avro/src/serde/deser_schema/array.rs | 21 +-
avro/src/serde/deser_schema/enums/mod.rs | 3 +
avro/src/serde/deser_schema/enums/plain.rs | 324 +++++++++++
avro/src/serde/deser_schema/map.rs | 19 +-
avro/src/serde/deser_schema/mod.rs | 110 ++--
avro/src/serde/deser_schema/record.rs | 37 +-
avro/src/serde/deser_schema/tuple.rs | 107 ++++
avro/src/serde/deser_schema/union.rs | 404 ++++++++++++-
avro/src/serde/ser_schema/mod.rs | 666 ++++++++++------------
avro/src/serde/ser_schema/record/mod.rs | 108 ++--
avro/src/serde/ser_schema/tuple.rs | 192 ++++++-
avro/src/serde/ser_schema/union.rs | 408 ++++++-------
avro/src/serde/with.rs | 53 +-
avro/src/types.rs | 172 +++---
avro/src/util.rs | 12 +-
avro/src/validator.rs | 78 +--
avro/src/writer.rs | 14 +-
avro/tests/avro_schema_component.rs | 4 +-
avro/tests/get_record_fields.rs | 58 +-
avro/tests/schema.rs | 40 +-
avro/tests/serde_human_readable_true.rs | 2 +-
avro/tests/validators.rs | 5 +-
avro_derive/src/attributes/mod.rs | 12 +-
avro_derive/src/attributes/serde.rs | 5 +-
avro_derive/src/case.rs | 8 +
avro_derive/src/enums/bare_union.rs | 84 ++-
avro_derive/src/enums/plain.rs | 3 +-
avro_derive/src/enums/record_internally_tagged.rs | 108 +---
avro_derive/src/enums/record_tag_content.rs | 55 +-
avro_derive/src/lib.rs | 70 +--
avro_derive/tests/derive.rs | 78 +--
avro_derive/tests/enum.rs | 66 ++-
52 files changed, 2639 insertions(+), 1892 deletions(-)
create mode 100644 .clippy.toml
create mode 100644 avro/src/serde/deser_schema/enums/mod.rs
create mode 100644 avro/src/serde/deser_schema/enums/plain.rs
create mode 100644 avro/src/serde/deser_schema/tuple.rs
(avro-rs) 01/01: wip: Full enum support
Posted by kr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kriskras99 pushed a commit to branch feat/enums
in repository https://gitbox.apache.org/repos/asf/avro-rs.git
commit c492f2bc22dc2e3b748b6040b767d8a93bcbbd2d
Author: default <ad...@kriskras99.nl>
AuthorDate: Fri Feb 27 12:26:38 2026 +0000
wip: Full enum support
---
avro/src/bigdecimal.rs | 4 +
avro/src/decode.rs | 2 +-
avro/src/error.rs | 11 +-
avro/src/reader/block.rs | 32 +
avro/src/reader/mod.rs | 5 +
avro/src/schema/builders.rs | 6 +-
avro/src/schema/mod.rs | 4 +
avro/src/schema/union.rs | 4 +
avro/src/serde/derive.rs | 1 +
avro/src/serde/deser_schema/array.rs | 91 +
avro/src/serde/deser_schema/enums/mod.rs | 3 +
avro/src/serde/deser_schema/enums/plain.rs | 324 ++
avro/src/serde/deser_schema/map.rs | 118 +
avro/src/serde/deser_schema/mod.rs | 637 ++++
avro/src/serde/deser_schema/record.rs | 359 +++
avro/src/serde/deser_schema/tuple.rs | 107 +
avro/src/serde/deser_schema/union.rs | 438 +++
avro/src/serde/mod.rs | 6 +-
avro/src/serde/ser_schema/array.rs | 186 ++
avro/src/serde/ser_schema/map.rs | 215 ++
avro/src/serde/ser_schema/mod.rs | 3475 +++++++--------------
avro/src/serde/ser_schema/record/field_default.rs | 197 ++
avro/src/serde/ser_schema/record/mod.rs | 250 ++
avro/src/serde/ser_schema/tuple.rs | 262 ++
avro/src/serde/ser_schema/union.rs | 544 ++++
avro/src/serde/with.rs | 307 ++
avro/src/writer.rs | 71 +-
avro/tests/avro-rs-285-bytes_deserialization.rs | 4 +-
avro/tests/serde_human_readable_false.rs | 256 +-
avro/tests/serde_human_readable_true.rs | 21 +-
avro_derive/src/attributes/avro.rs | 17 +
avro_derive/src/attributes/mod.rs | 184 +-
avro_derive/src/attributes/serde.rs | 9 +-
avro_derive/src/case.rs | 8 +
avro_derive/src/enums/bare_union.rs | 92 +
avro_derive/src/enums/mod.rs | 34 +-
avro_derive/src/enums/plain.rs | 5 +-
avro_derive/src/enums/record_internally_tagged.rs | 79 +
avro_derive/src/enums/record_tag_content.rs | 127 +
avro_derive/src/enums/union_of_records.rs | 14 +
avro_derive/src/lib.rs | 18 +-
avro_derive/tests/derive.proptest-regressions | 9 +
avro_derive/tests/derive.rs | 6 +-
avro_derive/tests/enum.rs | 158 +
44 files changed, 6209 insertions(+), 2491 deletions(-)
diff --git a/avro/src/bigdecimal.rs b/avro/src/bigdecimal.rs
index 7a6a379..c2f00b2 100644
--- a/avro/src/bigdecimal.rs
+++ b/avro/src/bigdecimal.rs
@@ -183,10 +183,14 @@ mod tests {
Ok(())
}
+ // TODO: Needs new schema aware deserializer
#[test]
+ #[ignore]
fn avro_rs_338_deserialize_serde_way() -> TestResult {
#[derive(Clone, PartialEq, Eq, Debug, Default, serde::Deserialize, serde::Serialize)]
+ #[serde(rename = "test")]
struct Test {
+ #[serde(with = "crate::serde::bigdecimal")]
big_decimal: BigDecimal,
}
diff --git a/avro/src/decode.rs b/avro/src/decode.rs
index cab6797..add154a 100644
--- a/avro/src/decode.rs
+++ b/avro/src/decode.rs
@@ -39,7 +39,7 @@ pub(crate) fn decode_long<R: Read>(reader: &mut R) -> AvroResult<Value> {
}
#[inline]
-fn decode_int<R: Read>(reader: &mut R) -> AvroResult<Value> {
+pub(crate) fn decode_int<R: Read>(reader: &mut R) -> AvroResult<Value> {
zag_i32(reader).map(Value::Int)
}
diff --git a/avro/src/error.rs b/avro/src/error.rs
index 12bee1e..bf389e4 100644
--- a/avro/src/error.rs
+++ b/avro/src/error.rs
@@ -156,7 +156,6 @@ pub enum Details {
#[error("Union index {index} out of bounds: {num_variants}")]
GetUnionVariant { index: i64, num_variants: usize },
- #[deprecated(since = "0.20.0", note = "This error variant is not generated anymore")]
#[error("Enum symbol index out of bounds: {num_variants}")]
EnumSymbolIndex { index: usize, num_variants: usize },
@@ -579,6 +578,16 @@ pub enum Details {
#[error("Failed to deserialize Avro value into value: {0}")]
DeserializeValue(String),
+ #[error("Failed to deserialize value of type {value_type} using schema {schema:?}: {value}")]
+ DeserializeValueWithSchema {
+ value_type: &'static str,
+ value: String,
+ schema: Schema,
+ },
+
+ #[error("Only expected `deserialize_identifier` to be called but `{0}` was called")]
+ DeserializeKey(String),
+
#[error("Failed to write buffer bytes during flush: {0}")]
WriteBytes(#[source] std::io::Error),
diff --git a/avro/src/reader/block.rs b/avro/src/reader/block.rs
index 06c07a2..2313281 100644
--- a/avro/src/reader/block.rs
+++ b/avro/src/reader/block.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use crate::serde::deser_schema::{Config, SchemaAwareDeserializer};
use crate::{
AvroResult, Codec, Error,
decode::{decode, decode_internal},
@@ -24,6 +25,7 @@ use crate::{
util,
};
use log::warn;
+use serde::de::DeserializeOwned;
use serde_json::from_slice;
use std::{
collections::HashMap,
@@ -205,6 +207,36 @@ impl<'r, R: Read> Block<'r, R> {
Ok(Some(item))
}
+ pub(super) fn read_next_deser<T: DeserializeOwned>(&mut self) -> AvroResult<Option<T>> {
+ if self.is_empty() {
+ self.read_block_next()?;
+ if self.is_empty() {
+ return Ok(None);
+ }
+ }
+
+ let mut block_bytes = &self.buf[self.buf_idx..];
+ let b_original = block_bytes.len();
+
+ let deserializer = SchemaAwareDeserializer::new(
+ &mut block_bytes,
+ &self.writer_schema,
+ Config {
+ names: &self.names_refs,
+ human_readable: false,
+ },
+ )?;
+ let item = T::deserialize(deserializer)?;
+
+ if b_original != 0 && b_original == block_bytes.len() {
+ // from_avro_datum did not consume any bytes, so return an error to avoid an infinite loop
+ return Err(Details::ReadBlock.into());
+ }
+ self.buf_idx += b_original - block_bytes.len();
+ self.message_count -= 1;
+ Ok(Some(item))
+ }
+
fn read_writer_schema(&mut self, metadata: &HashMap<String, Value>) -> AvroResult<()> {
let json: serde_json::Value = metadata
.get("avro.schema")
diff --git a/avro/src/reader/mod.rs b/avro/src/reader/mod.rs
index 0113e50..904600d 100644
--- a/avro/src/reader/mod.rs
+++ b/avro/src/reader/mod.rs
@@ -28,6 +28,7 @@ use crate::{
};
use block::Block;
use bon::bon;
+use serde::de::DeserializeOwned;
use std::{collections::HashMap, io::Read};
/// Main interface for reading Avro formatted values.
@@ -117,6 +118,10 @@ impl<'a, R: Read> Reader<'a, R> {
self.block.read_next(read_schema)
}
+
+ pub fn next_deser<T: DeserializeOwned>(&mut self) -> AvroResult<Option<T>> {
+ self.block.read_next_deser()
+ }
}
impl<R: Read> Iterator for Reader<'_, R> {
diff --git a/avro/src/schema/builders.rs b/avro/src/schema/builders.rs
index dd16b3e..466b244 100644
--- a/avro/src/schema/builders.rs
+++ b/avro/src/schema/builders.rs
@@ -107,13 +107,11 @@ impl Schema {
#[builder(finish_fn = build)]
pub fn record(
#[builder(start_fn)] name: Name,
- fields: Option<Vec<RecordField>>,
+ #[builder(default)] fields: Vec<RecordField>,
aliases: Option<Vec<Alias>>,
doc: Option<String>,
- attributes: Option<BTreeMap<String, JsonValue>>,
+ #[builder(default)] attributes: BTreeMap<String, JsonValue>,
) -> Self {
- let fields = fields.unwrap_or_default();
- let attributes = attributes.unwrap_or_default();
let record_schema = RecordSchema::builder()
.name(name)
.fields(fields)
diff --git a/avro/src/schema/mod.rs b/avro/src/schema/mod.rs
index 40fc0f9..dfb26d8 100644
--- a/avro/src/schema/mod.rs
+++ b/avro/src/schema/mod.rs
@@ -28,6 +28,10 @@ pub(crate) use crate::schema::resolve::{
ResolvedOwnedSchema, resolve_names, resolve_names_with_schemata,
};
pub use crate::schema::{
+ builders::{
+ SchemaArrayBuilder, SchemaEnumBuilder, SchemaFixedBuilder, SchemaMapBuilder,
+ SchemaRecordBuilder,
+ },
name::{Alias, Aliases, Name, Names, NamesRef, Namespace, NamespaceRef},
record::{RecordField, RecordFieldBuilder, RecordSchema, RecordSchemaBuilder},
resolve::ResolvedSchema,
diff --git a/avro/src/schema/union.rs b/avro/src/schema/union.rs
index 24edfee..6064e5e 100644
--- a/avro/src/schema/union.rs
+++ b/avro/src/schema/union.rs
@@ -79,6 +79,10 @@ impl UnionSchema {
self.variant_index.contains_key(&SchemaKind::Null)
}
+ pub fn index(&self) -> &BTreeMap<SchemaKind, usize> {
+ &self.variant_index
+ }
+
/// Optionally returns a reference to the schema matched by this value, as well as its position
/// within this union.
///
diff --git a/avro/src/serde/derive.rs b/avro/src/serde/derive.rs
index 770d11b..d615f70 100644
--- a/avro/src/serde/derive.rs
+++ b/avro/src/serde/derive.rs
@@ -629,6 +629,7 @@ where
}
}
+// TODO: This does not match the Serde implementation and therefore does not work
impl AvroSchemaComponent for core::time::Duration {
/// The schema is [`Schema::Duration`] with the name `duration`.
///
diff --git a/avro/src/serde/deser_schema/array.rs b/avro/src/serde/deser_schema/array.rs
new file mode 100644
index 0000000..61415c2
--- /dev/null
+++ b/avro/src/serde/deser_schema/array.rs
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io::Read;
+
+use serde::de::SeqAccess;
+
+use super::Config;
+use crate::serde::deser_schema::union::UnionDeserializer;
+use crate::{
+ Error, Schema, schema::ArraySchema, serde::deser_schema::SchemaAwareDeserializer, util::zag_i32,
+};
+
+pub struct ArrayDeserializer<'s, 'r, R: Read> {
+ reader: &'r mut R,
+ schema: &'s ArraySchema,
+ config: Config<'s>,
+ state: State,
+}
+
+impl<'s, 'r, R: Read> ArrayDeserializer<'s, 'r, R> {
+ pub fn new(reader: &'r mut R, schema: &'s ArraySchema, config: Config<'s>) -> Self {
+ Self {
+ reader,
+ schema,
+ config,
+ state: State::EndOfBlock,
+ }
+ }
+}
+
+enum State {
+ EndOfBlock,
+ ReadingValue(u32),
+ Finished,
+}
+
+impl<'de, 's, 'r, R: Read> SeqAccess<'de> for ArrayDeserializer<'s, 'r, R> {
+ type Error = Error;
+
+ fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>
+ where
+ T: serde::de::DeserializeSeed<'de>,
+ {
+ match self.state {
+ State::EndOfBlock => {
+ let remaining = zag_i32(self.reader)?;
+ if remaining < 0 {
+ let _bytes = zag_i32(self.reader)?;
+ }
+ if remaining == 0 {
+ self.state = State::Finished
+ } else {
+ self.state = State::ReadingValue(remaining.abs() as u32)
+ }
+ self.next_element_seed(seed)
+ }
+ State::ReadingValue(remaining) => {
+ let v = match self.schema.items.as_ref() {
+ Schema::Union(union) => {
+ seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
+ }
+ schema => seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?,
+ };
+
+ self.state = State::ReadingValue(remaining);
+
+ Ok(Some(v))
+ }
+ State::Finished => Ok(None),
+ }
+ }
+}
diff --git a/avro/src/serde/deser_schema/enums/mod.rs b/avro/src/serde/deser_schema/enums/mod.rs
new file mode 100644
index 0000000..65a8e66
--- /dev/null
+++ b/avro/src/serde/deser_schema/enums/mod.rs
@@ -0,0 +1,3 @@
+mod plain;
+
+pub use plain::PlainEnumAccess;
diff --git a/avro/src/serde/deser_schema/enums/plain.rs b/avro/src/serde/deser_schema/enums/plain.rs
new file mode 100644
index 0000000..5b4ca6e
--- /dev/null
+++ b/avro/src/serde/deser_schema/enums/plain.rs
@@ -0,0 +1,324 @@
+use crate::error::Details;
+use crate::schema::EnumSchema;
+use crate::util::zag_i32;
+use crate::{Error, Schema};
+use serde::Deserializer;
+use serde::de::{DeserializeSeed, EnumAccess, Unexpected, VariantAccess, Visitor};
+use std::io::Read;
+
+pub struct PlainEnumAccess<'s, 'r, R: Read> {
+ schema: &'s EnumSchema,
+ reader: &'r mut R,
+}
+
+impl<'s, 'r, R: Read> PlainEnumAccess<'s, 'r, R> {
+ pub fn new(reader: &'r mut R, schema: &'s EnumSchema) -> Self {
+ Self { schema, reader }
+ }
+}
+
+impl<'de, 's, 'r, R: Read> EnumAccess<'de> for PlainEnumAccess<'s, 'r, R> {
+ type Error = Error;
+ type Variant = Self;
+
+ fn variant_seed<V>(self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error>
+ where
+ V: DeserializeSeed<'de>,
+ {
+ let deserializer = EnumIdentifierDeserializer {
+ reader: self.reader,
+ schema: self.schema,
+ };
+ Ok((seed.deserialize(deserializer)?, self))
+ }
+}
+
+impl<'de, 's, 'r, R: Read> VariantAccess<'de> for PlainEnumAccess<'s, 'r, R> {
+ type Error = Error;
+
+ fn unit_variant(self) -> Result<(), Self::Error> {
+ Ok(())
+ }
+
+ fn newtype_variant_seed<T>(self, _seed: T) -> Result<T::Value, Self::Error>
+ where
+ T: DeserializeSeed<'de>,
+ {
+ let unexp = Unexpected::UnitVariant;
+ Err(serde::de::Error::invalid_type(unexp, &"newtype variant"))
+ }
+
+ fn tuple_variant<V>(self, _len: usize, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ let unexp = Unexpected::UnitVariant;
+ Err(serde::de::Error::invalid_type(unexp, &"newtype variant"))
+ }
+
+ fn struct_variant<V>(
+ self,
+ _fields: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ let unexp = Unexpected::UnitVariant;
+ Err(serde::de::Error::invalid_type(unexp, &"newtype variant"))
+ }
+}
+
+struct EnumIdentifierDeserializer<'s, 'r, R: Read> {
+ schema: &'s EnumSchema,
+ reader: &'r mut R,
+}
+
+impl<'s, 'r, R: Read> EnumIdentifierDeserializer<'s, 'r, R> {
+ fn error(&self, error: impl Into<String>) -> Error {
+ Error::new(Details::DeserializeValueWithSchema {
+ value_type: "enum",
+ value: error.into(),
+ schema: Schema::Enum(self.schema.clone()),
+ })
+ }
+}
+
+impl<'de, 's, 'r, R: Read> Deserializer<'de> for EnumIdentifierDeserializer<'s, 'r, R> {
+ type Error = Error;
+
+ fn deserialize_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_any, expected deserialize_identifier"))
+ }
+
+ fn deserialize_bool<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_bool, expected deserialize_identifier"))
+ }
+
+ fn deserialize_i8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_i8, expected deserialize_identifier"))
+ }
+
+ fn deserialize_i16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_i16, expected deserialize_identifier"))
+ }
+
+ fn deserialize_i32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_i32, expected deserialize_identifier"))
+ }
+
+ fn deserialize_i64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_i64, expected deserialize_identifier"))
+ }
+
+ fn deserialize_u8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_u8, expected deserialize_identifier"))
+ }
+
+ fn deserialize_u16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_u16, expected deserialize_identifier"))
+ }
+
+ fn deserialize_u32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_u32, expected deserialize_identifier"))
+ }
+
+ fn deserialize_u64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_u64, expected deserialize_identifier"))
+ }
+
+ fn deserialize_f32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_f32, expected deserialize_identifier"))
+ }
+
+ fn deserialize_f64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_f64, expected deserialize_identifier"))
+ }
+
+ fn deserialize_char<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_char, expected deserialize_identifier"))
+ }
+
+ fn deserialize_str<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_str, expected deserialize_identifier"))
+ }
+
+ fn deserialize_string<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_string, expected deserialize_identifier"))
+ }
+
+ fn deserialize_bytes<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_bytes, expected deserialize_identifier"))
+ }
+
+ fn deserialize_byte_buf<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_byte_buf, expected deserialize_identifier"))
+ }
+
+ fn deserialize_option<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_option, expected deserialize_identifier"))
+ }
+
+ fn deserialize_unit<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_unit, expected deserialize_identifier"))
+ }
+
+ fn deserialize_unit_struct<V>(
+ self,
+ _name: &'static str,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_unit_struct, expected deserialize_identifier"))
+ }
+
+ fn deserialize_newtype_struct<V>(
+ self,
+ _name: &'static str,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_newtype_struct, expected deserialize_identifier"))
+ }
+
+ fn deserialize_seq<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_seq, expected deserialize_identifier"))
+ }
+
+ fn deserialize_tuple<V>(self, _len: usize, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_tuple, expected deserialize_identifier"))
+ }
+
+ fn deserialize_tuple_struct<V>(
+ self,
+ _name: &'static str,
+ _len: usize,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_tuple_struct, expected deserialize_identifier"))
+ }
+
+ fn deserialize_map<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_map, expected deserialize_identifier"))
+ }
+
+ fn deserialize_struct<V>(
+ self,
+ _name: &'static str,
+ _fields: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_struct, expected deserialize_identifier"))
+ }
+
+ fn deserialize_enum<V>(
+ self,
+ _name: &'static str,
+ _variants: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ Err(self.error("Unexpected deserialize_enum, expected deserialize_identifier"))
+ }
+
+ fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ let index = zag_i32(self.reader)?;
+ let symbol =
+ self.schema
+ .symbols
+ .get(index as usize)
+ .ok_or_else(|| Details::EnumSymbolIndex {
+ index: index as usize,
+ num_variants: self.schema.symbols.len(),
+ })?;
+ visitor.visit_str(&symbol)
+ }
+
+ fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ self.deserialize_any(visitor)
+ }
+}
diff --git a/avro/src/serde/deser_schema/map.rs b/avro/src/serde/deser_schema/map.rs
new file mode 100644
index 0000000..6f06cf3
--- /dev/null
+++ b/avro/src/serde/deser_schema/map.rs
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io::Read;
+
+use serde::de::MapAccess;
+
+use super::Config;
+use crate::serde::deser_schema::union::UnionDeserializer;
+use crate::{
+ Error, Schema, schema::MapSchema, serde::deser_schema::SchemaAwareDeserializer, util::zag_i32,
+};
+
+pub struct MapDeserializer<'s, 'r, R: Read> {
+ reader: &'r mut R,
+ schema: &'s MapSchema,
+ config: Config<'s>,
+ state: State,
+}
+
+impl<'s, 'r, R: Read> MapDeserializer<'s, 'r, R> {
+ pub fn new(reader: &'r mut R, schema: &'s MapSchema, config: Config<'s>) -> Self {
+ Self {
+ reader,
+ schema,
+ config,
+ state: State::EndOfBlock,
+ }
+ }
+}
+
+enum State {
+ EndOfBlock,
+ ReadingKey(u32),
+ ReadingValue(u32),
+ Finished,
+}
+
+impl<'de, 's, 'r, R: Read> MapAccess<'de> for MapDeserializer<'s, 'r, R> {
+ type Error = Error;
+
+ fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Self::Error>
+ where
+ K: serde::de::DeserializeSeed<'de>,
+ {
+ match self.state {
+ State::EndOfBlock => {
+ let remaining = zag_i32(self.reader)?;
+ if remaining < 0 {
+ let _bytes = zag_i32(self.reader)?;
+ }
+ if remaining == 0 {
+ self.state = State::Finished
+ } else {
+ self.state = State::ReadingKey(remaining.abs() as u32)
+ }
+ self.next_key_seed(seed)
+ }
+ State::ReadingKey(remaining) => {
+ let v = seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ &Schema::String,
+ self.config,
+ )?)?;
+
+ self.state = State::ReadingValue(remaining);
+
+ Ok(Some(v))
+ }
+ State::Finished => Ok(None),
+ State::ReadingValue(_) => {
+ panic!("`next_key_seed` and `next_value_seed` where called in the wrong error")
+ }
+ }
+ }
+
+ fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::DeserializeSeed<'de>,
+ {
+ let State::ReadingValue(mut remaining) = self.state else {
+ panic!("`next_key_seed` and `next_value_seed` where called in the wrong error")
+ };
+ let v = match self.schema.types.as_ref() {
+ Schema::Union(union) => {
+ seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
+ }
+ schema => seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?,
+ };
+
+ remaining -= 1;
+ if remaining == 0 {
+ self.state = State::EndOfBlock;
+ } else {
+ self.state = State::ReadingKey(remaining);
+ }
+
+ Ok(v)
+ }
+}
diff --git a/avro/src/serde/deser_schema/mod.rs b/avro/src/serde/deser_schema/mod.rs
new file mode 100644
index 0000000..26efad1
--- /dev/null
+++ b/avro/src/serde/deser_schema/mod.rs
@@ -0,0 +1,637 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod array;
+mod enums;
+mod map;
+mod record;
+mod tuple;
+mod union;
+
+use std::fmt::{Debug, Formatter};
+use std::io::Read;
+
+use serde::Deserializer;
+
+use crate::serde::deser_schema::enums::PlainEnumAccess;
+use crate::serde::deser_schema::tuple::{
+ ManyTupleDeserializer, OneTupleDeserializer, UnitTupleDeserializer,
+};
+use crate::serde::deser_schema::union::UnionDeserializer;
+use crate::{
+ Error, Schema,
+ decode::decode_len,
+ error::Details,
+ schema::{DecimalSchema, InnerDecimalSchema, Names, SchemaKind, UuidSchema},
+ serde::deser_schema::{
+ array::ArrayDeserializer, map::MapDeserializer, record::RecordDeserializer,
+ },
+ util::{zag_i32, zag_i64},
+};
+
+#[derive(Debug, Clone, Copy)]
+pub struct Config<'s> {
+ /// All names that can be referenced in the schema being used for serialisation.
+ pub names: &'s Names,
+ /// Should `Serialize` implementations pick a human-readable format.
+ ///
+ /// It is recommended to set this to `false` as it results in compacter output.
+ pub human_readable: bool,
+}
+
+pub struct SchemaAwareDeserializer<'s, 'r, R: Read> {
+ reader: &'r mut R,
+ schema: &'s Schema,
+ config: Config<'s>,
+}
+
+impl<'s, 'r, R: Read> Debug for SchemaAwareDeserializer<'s, 'r, R> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("SchemaAwareDeserializer")
+ .field("schema", &self.schema)
+ .finish()
+ }
+}
+
+impl<'s, 'r, R: Read> SchemaAwareDeserializer<'s, 'r, R> {
+ pub fn new(reader: &'r mut R, schema: &'s Schema, config: Config<'s>) -> Result<Self, Error> {
+ if let Schema::Ref { name } = schema {
+ let schema = config
+ .names
+ .get(name)
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?;
+ Self::new(reader, schema, config)
+ } else {
+ Ok(Self {
+ reader,
+ schema,
+ config,
+ })
+ }
+ }
+
+ fn error(&self, ty: &'static str, error: impl Into<String>) -> Error {
+ Error::new(Details::DeserializeValueWithSchema {
+ value_type: ty,
+ value: error.into(),
+ schema: self.schema.clone(),
+ })
+ }
+
+ /// Create a new deserializer with the existing reader and config.
+ ///
+ /// This will resolve the schema if it is a reference.
+ fn with_different_schema(mut self, schema: &'s Schema) -> Result<Self, Error> {
+ let schema = if let Schema::Ref { name } = schema {
+ self.config
+ .names
+ .get(name)
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?
+ } else {
+ schema
+ };
+ self.schema = schema;
+ Ok(self)
+ }
+
+ fn read_int(&mut self, original_ty: &'static str) -> Result<i32, Error> {
+ match self.schema {
+ Schema::Int | Schema::Date | Schema::TimeMillis => zag_i32(self.reader),
+ _ => Err(self.error(
+ original_ty,
+ "Expected Schema::Int | Schema::Date | Schema::TimeMillis",
+ )),
+ }
+ }
+
+ fn read_long(&mut self, original_ty: &'static str) -> Result<i64, Error> {
+ match self.schema {
+ Schema::Long | Schema::TimeMicros | Schema::TimestampMillis | Schema::TimestampMicros
+ | Schema::TimestampNanos | Schema::LocalTimestampMillis | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos => zag_i64(self.reader),
+ _ => Err(self.error(
+ original_ty,
+ "Expected Schema::Long | Schema::TimeMicros | Schema::{,Local}Timestamp{Millis,Micros,Nanos}",
+ )),
+ }
+ }
+
+ fn read_string(&mut self) -> Result<String, Error> {
+ let bytes = self.read_bytes_with_len()?;
+ Ok(String::from_utf8(bytes).map_err(Details::ConvertToUtf8)?)
+ }
+
+ fn read_bytes_with_len(&mut self) -> Result<Vec<u8>, Error> {
+ let length = decode_len(self.reader)?;
+ self.read_bytes(length)
+ }
+
+ fn read_bytes(&mut self, length: usize) -> Result<Vec<u8>, Error> {
+ let mut buf = vec![0; length];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ Ok(buf)
+ }
+}
+
+const DESERIALIZE_ANY: &str = "This value is compared by pointer value";
+const DESERIALIZE_ANY_FIELDS: &[&str] = &[];
+
+impl<'de, 's, 'r, R: Read> Deserializer<'de> for SchemaAwareDeserializer<'s, 'r, R> {
+ type Error = Error;
+
+ fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ match self.schema {
+ Schema::Null => self.deserialize_unit(visitor),
+ Schema::Boolean => self.deserialize_bool(visitor),
+ Schema::Int | Schema::Date | Schema::TimeMillis => self.deserialize_i32(visitor),
+ Schema::Long
+ | Schema::TimeMicros
+ | Schema::TimestampMillis
+ | Schema::TimestampMicros
+ | Schema::TimestampNanos
+ | Schema::LocalTimestampMillis
+ | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos => self.deserialize_i64(visitor),
+ Schema::Float => self.deserialize_f32(visitor),
+ Schema::Double => self.deserialize_f64(visitor),
+ Schema::Bytes
+ | Schema::Fixed(_)
+ | Schema::Decimal(_)
+ | Schema::BigDecimal
+ | Schema::Uuid(UuidSchema::Fixed(_))
+ | Schema::Duration(_) => self.deserialize_byte_buf(visitor),
+ Schema::String | Schema::Uuid(UuidSchema::String) => self.deserialize_string(visitor),
+ Schema::Array(_) => self.deserialize_seq(visitor),
+ Schema::Map(_) => self.deserialize_map(visitor),
+ Schema::Union(union) => {
+ UnionDeserializer::new(self.reader, union, self.config)?.deserialize_any(visitor)
+ }
+ Schema::Record(_) => {
+ self.deserialize_struct(DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, visitor)
+ }
+ Schema::Enum(_) => {
+ self.deserialize_enum(DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, visitor)
+ }
+ Schema::Ref { .. } => unreachable!("References are resolved on deserializer creation"),
+ Schema::Uuid(UuidSchema::Bytes) => panic!("Unsupported"),
+ }
+ }
+
+ fn deserialize_bool<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ match self.schema {
+ Schema::Boolean => {
+ let mut buf = [0];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ // TODO: The TryFrom implementation wasn't working??
+ let boolean = match buf[0] {
+ 0 => false,
+ 1 => true,
+ _ => {
+ return Err(
+ self.error("bool", format!("{} is not a valid boolean", buf[0]))
+ );
+ }
+ };
+ visitor.visit_bool(boolean)
+ }
+ _ => Err(self.error("bool", "Expected a Schema::Boolean")),
+ }
+ }
+
+ fn deserialize_i8<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_int("i8")?;
+ let value = i8::try_from(int)
+ .map_err(|_| self.error("i8", format!("Could not convert int ({int}) to an i8")))?;
+ visitor.visit_i8(value)
+ }
+
+ fn deserialize_i16<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_int("i16")?;
+ let value = i16::try_from(int)
+ .map_err(|_| self.error("i16", format!("Could not convert int ({int}) to an i16")))?;
+ visitor.visit_i16(value)
+ }
+
+ fn deserialize_i32<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_int("i32")?;
+ let value = i32::try_from(int)
+ .map_err(|_| self.error("i32", format!("Could not convert int ({int}) to an i32")))?;
+ visitor.visit_i32(value)
+ }
+
+ fn deserialize_i64<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_long("i64")?;
+ let value = i64::try_from(int)
+ .map_err(|_| self.error("i64", format!("Could not convert int ({int}) to an i64")))?;
+ visitor.visit_i64(value)
+ }
+
+ fn deserialize_u8<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_int("u8")?;
+ let value = u8::try_from(int)
+ .map_err(|_| self.error("u8", format!("Could not convert int ({int}) to an u8")))?;
+ visitor.visit_u8(value)
+ }
+
+ fn deserialize_u16<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_int("u16")?;
+ let value = u16::try_from(int)
+ .map_err(|_| self.error("u16", format!("Could not convert int ({int}) to an u16")))?;
+ visitor.visit_u16(value)
+ }
+
+ fn deserialize_u32<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ let int = self.read_long("u32")?;
+ let value = u32::try_from(int)
+ .map_err(|_| self.error("u32", format!("Could not convert int ({int}) to an u32")))?;
+ visitor.visit_u32(value)
+ }
+
+ fn deserialize_u64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Fixed(fixed) = self.schema
+ && fixed.size == 8
+ && fixed.name.name() == "u64"
+ {
+ let mut buf = [0; 8];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_u64(u64::from_le_bytes(buf))
+ } else {
+ Err(self.error("u64", r#"Expected Schema::Fixed(name: "u64", size: 8)"#))
+ }
+ }
+
+ fn deserialize_f32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Float = self.schema {
+ let mut buf = [0; 4];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_f32(f32::from_le_bytes(buf))
+ } else {
+ Err(self.error("f32", r#"Expected Schema::Float)"#))
+ }
+ }
+
+ fn deserialize_f64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Double = self.schema {
+ let mut buf = [0; 8];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_f64(f64::from_le_bytes(buf))
+ } else {
+ Err(self.error("f64", r#"Expected Schema::Double)"#))
+ }
+ }
+
+ fn deserialize_char<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::String = self.schema {
+ let string = self.read_string()?;
+ let mut chars = string.chars();
+ if let Some(character) = chars.next() {
+ if chars.next().is_some() {
+ Err(self.error("char", "String contains more than one character"))
+ } else {
+ visitor.visit_char(character)
+ }
+ } else {
+ Err(self.error("char", "String is empty"))
+ }
+ } else {
+ Err(self.error("char", "Expected Schema::String"))
+ }
+ }
+
+ fn deserialize_str<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ self.deserialize_string(visitor)
+ }
+
+ fn deserialize_string<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::String = self.schema {
+ let string = self.read_string()?;
+ visitor.visit_string(string)
+ } else {
+ Err(self.error("string", "Expected Schema::String"))
+ }
+ }
+
+ fn deserialize_bytes<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ self.deserialize_byte_buf(visitor)
+ }
+
+ fn deserialize_byte_buf<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ match self.schema {
+ Schema::Bytes | Schema::BigDecimal | Schema::Decimal(DecimalSchema { inner: InnerDecimalSchema::Bytes, .. }) => {
+ let bytes = self.read_bytes_with_len()?;
+ visitor.visit_byte_buf(bytes)
+ }
+ Schema::Fixed(fixed) | Schema::Decimal(DecimalSchema { inner: InnerDecimalSchema::Fixed(fixed), ..}) | Schema::Uuid(UuidSchema::Fixed(fixed)) | Schema::Duration(fixed) => {
+ let bytes = self.read_bytes(fixed.size)?;
+ visitor.visit_byte_buf(bytes)
+ }
+ _ => Err(self.error("bytes", "Expected Schema::Bytes | Schema::Fixed | Schema::BigDecimal | Schema::Decimal | Schema::Uuid(Fixed) | Schema::Duration"))
+ }
+ }
+
+ fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Union(union) = self.schema
+ && union.variants().len() == 2
+ && let Some(null_index) = union.index().get(&SchemaKind::Null).copied()
+ {
+ let index = zag_i32(self.reader)?;
+ if index < 0 || index > 1 {
+ return Err(self.error("option", format!("Invalid union index {index}")));
+ }
+ let index = index as usize;
+ if index == null_index {
+ visitor.visit_none()
+ } else {
+ visitor.visit_some(self.with_different_schema(&union.variants()[index])?)
+ }
+ } else {
+ Err(self.error("option", "Expected Schema::Union([Null, _])"))
+ }
+ }
+
+ fn deserialize_unit<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Null = self.schema {
+ visitor.visit_unit()
+ } else {
+ Err(self.error("unit", "Expected Schema::Null"))
+ }
+ }
+
+ fn deserialize_unit_struct<V>(
+ self,
+ name: &'static str,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Record(record) = self.schema
+ && record.fields.len() == 0
+ && record.name.name() == name
+ {
+ visitor.visit_unit()
+ } else {
+ Err(self.error(
+ "unit struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == 0)"),
+ ))
+ }
+ }
+
+ fn deserialize_newtype_struct<V>(
+ self,
+ name: &'static str,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Record(record) = self.schema
+ && record.fields.len() == 1
+ && record.name.name() == name
+ {
+ visitor.visit_newtype_struct(self.with_different_schema(&record.fields[0].schema)?)
+ } else {
+ Err(self.error(
+ "unit struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == 1)"),
+ ))
+ }
+ }
+
+ fn deserialize_seq<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Array(array) = self.schema {
+ visitor.visit_seq(ArrayDeserializer::new(self.reader, array, self.config))
+ } else {
+ Err(self.error("array", "Expected Schema::Array"))
+ }
+ }
+
+ fn deserialize_tuple<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ println!("deserialize_tuple(len: {len}): {self:?}");
+ match self.schema {
+ Schema::Null if len == 0 => visitor.visit_seq(UnitTupleDeserializer),
+ schema if len == 1 => {
+ visitor.visit_seq(OneTupleDeserializer::new(self.reader, schema, self.config))
+ }
+ Schema::Record(record) if record.fields.len() == len => {
+ visitor.visit_seq(ManyTupleDeserializer::new(self.reader, record, self.config))
+ }
+ _ if len == 0 => Err(self.error("tuple", "Expected Schema::Null for unit tuple")),
+ _ => Err(self.error("tuple", format!("Expected Schema::Record for {len}-tuple"))),
+ }
+ }
+
+ fn deserialize_tuple_struct<V>(
+ self,
+ name: &'static str,
+ len: usize,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Record(record) = self.schema
+ && record.name.name() == name
+ && record.fields.len() == len
+ {
+ visitor.visit_map(RecordDeserializer::new(self.reader, record, self.config))
+ } else {
+ Err(self.error(
+ "tuple struct",
+ format!("Expected Schema::Record(fields.len() == {len})"),
+ ))
+ }
+ }
+
+ fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Map(map) = self.schema {
+ visitor.visit_map(MapDeserializer::new(self.reader, map, self.config))
+ } else {
+ Err(self.error("map", "Expected Schema::Map"))
+ }
+ }
+
+ fn deserialize_struct<V>(
+ self,
+ name: &'static str,
+ _fields: &'static [&'static str],
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ println!("deserialize_struct(name: {name}, fields = {_fields:?}): {self:?}");
+ if let Schema::Record(record) = self.schema
+ && record.name.name() == name
+ {
+ visitor.visit_map(RecordDeserializer::new(self.reader, record, self.config))
+ } else {
+ Err(self.error("struct", format!("Expected Schema::Record(name: {name})")))
+ }
+ }
+
+ fn deserialize_enum<V>(
+ self,
+ name: &'static str,
+ variants: &'static [&'static str],
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ println!("deserialize_enum(name: {name}, variants: {variants:?}): {self:?}");
+ match self.schema {
+ Schema::Enum(schema) => visitor.visit_enum(PlainEnumAccess::new(self.reader, schema)),
+ _ => panic!(
+ "deserializing enum, name: {name}, variants: {variants:#?}, {:?}",
+ self.schema
+ ),
+ }
+ }
+
+ fn deserialize_i128<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Fixed(fixed) = self.schema
+ && fixed.size == 16
+ && fixed.name.name() == "i128"
+ {
+ let mut buf = [0; 16];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_i128(i128::from_le_bytes(buf))
+ } else {
+ Err(self.error("i128", r#"Expected Schema::Fixed(name: "i128", size: 16)"#))
+ }
+ }
+
+ fn deserialize_u128<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ if let Schema::Fixed(fixed) = self.schema
+ && fixed.size == 16
+ && fixed.name.name() == "u128"
+ {
+ let mut buf = [0; 16];
+ self.reader
+ .read_exact(&mut buf)
+ .map_err(Details::ReadBytes)?;
+ visitor.visit_u128(u128::from_le_bytes(buf))
+ } else {
+ Err(self.error("u128", r#"Expected Schema::Fixed(name: "u128", size: 16)"#))
+ }
+ }
+
+ fn is_human_readable(&self) -> bool {
+ self.config.human_readable
+ }
+
+ fn deserialize_identifier<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ panic!("deserialize_identifier: {:?}", self.schema);
+ }
+
+ fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ // TODO: We can probably do something more efficient, but that might need the Seek trait bound
+ self.deserialize_any(visitor)
+ }
+}
diff --git a/avro/src/serde/deser_schema/record.rs b/avro/src/serde/deser_schema/record.rs
new file mode 100644
index 0000000..ad70999
--- /dev/null
+++ b/avro/src/serde/deser_schema/record.rs
@@ -0,0 +1,359 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::{Debug, Formatter};
+use std::io::Read;
+
+use serde::{Deserializer, de::MapAccess};
+
+use super::Config;
+use crate::serde::deser_schema::union::UnionDeserializer;
+use crate::{
+ Error, Schema, error::Details, schema::RecordSchema,
+ serde::deser_schema::SchemaAwareDeserializer,
+};
+
+pub struct RecordDeserializer<'s, 'r, R: Read> {
+ reader: &'r mut R,
+ schema: &'s RecordSchema,
+ config: Config<'s>,
+ current_field: State,
+}
+
+impl<'s, 'r, R: Read> Debug for RecordDeserializer<'s, 'r, R> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("RecordDeserializer")
+ .field("schema", &self.schema)
+ .field("config", &self.config)
+ .field("current_field", &self.current_field)
+ .finish()
+ }
+}
+
+impl<'s, 'r, R: Read> RecordDeserializer<'s, 'r, R> {
+ pub fn new(reader: &'r mut R, schema: &'s RecordSchema, config: Config<'s>) -> Self {
+ Self {
+ reader,
+ schema,
+ config,
+ current_field: State::Key(0),
+ }
+ }
+}
+
+#[derive(Debug)]
+enum State {
+ Key(usize),
+ Value(usize),
+}
+
+impl<'de, 's, 'r, R: Read> MapAccess<'de> for RecordDeserializer<'s, 'r, R> {
+ type Error = Error;
+
+ fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Self::Error>
+ where
+ K: serde::de::DeserializeSeed<'de>,
+ {
+ println!("next_key_seed: {self:?}");
+ let State::Key(index) = self.current_field else {
+ panic!("`next_key_seed` and `next_value_seed` where called in the wrong error")
+ };
+ if index >= self.schema.fields.len() {
+ // Finished reading this record
+ Ok(None)
+ } else {
+ let v = seed.deserialize(FieldName {
+ name: &self.schema.fields[index].name,
+ })?;
+ self.current_field = State::Value(index);
+ Ok(Some(v))
+ }
+ }
+
+ fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::DeserializeSeed<'de>,
+ {
+ println!("next_value_seed: {self:?}");
+ let State::Value(index) = self.current_field else {
+ panic!("`next_key_seed` and `next_value_seed` where called in the wrong error")
+ };
+ let v = match &self.schema.fields[index].schema {
+ Schema::Union(union) => {
+ seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
+ }
+ schema => seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?,
+ };
+ self.current_field = State::Key(index + 1);
+ Ok(v)
+ }
+
+ fn size_hint(&self) -> Option<usize> {
+ match self.current_field {
+ State::Key(index) | State::Value(index) => Some(self.schema.fields.len() - index),
+ }
+ }
+}
+
+/// "Deserializer" for the field name
+struct FieldName<'s> {
+ name: &'s str,
+}
+
+impl<'de, 's> Deserializer<'de> for FieldName<'s> {
+ type Error = Error;
+
+ fn deserialize_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_any".into()).into())
+ }
+
+ fn deserialize_bool<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_bool".into()).into())
+ }
+
+ fn deserialize_i8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_i8".into()).into())
+ }
+
+ fn deserialize_i16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_i16".into()).into())
+ }
+
+ fn deserialize_i32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_i32".into()).into())
+ }
+
+ fn deserialize_i64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_i64".into()).into())
+ }
+
+ fn deserialize_u8<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_u8".into()).into())
+ }
+
+ fn deserialize_u16<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_u16".into()).into())
+ }
+
+ fn deserialize_u32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_u32".into()).into())
+ }
+
+ fn deserialize_u64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_u64".into()).into())
+ }
+
+ fn deserialize_f32<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_f32".into()).into())
+ }
+
+ fn deserialize_f64<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_f64".into()).into())
+ }
+
+ fn deserialize_char<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_char".into()).into())
+ }
+
+ fn deserialize_str<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_str".into()).into())
+ }
+
+ fn deserialize_string<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_string".into()).into())
+ }
+
+ fn deserialize_bytes<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_bytes".into()).into())
+ }
+
+ fn deserialize_byte_buf<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_byte_buf".into()).into())
+ }
+
+ fn deserialize_option<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_option".into()).into())
+ }
+
+ fn deserialize_unit<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_unit".into()).into())
+ }
+
+ fn deserialize_unit_struct<V>(
+ self,
+ name: &'static str,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!("deserialize_unit_struct(name: {name})")).into())
+ }
+
+ fn deserialize_newtype_struct<V>(
+ self,
+ name: &'static str,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!("deserialize_newtype_struct(name: {name})")).into())
+ }
+
+ fn deserialize_seq<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_seq".into()).into())
+ }
+
+ fn deserialize_tuple<V>(self, len: usize, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!("deserialize_tuple(len: {len})")).into())
+ }
+
+ fn deserialize_tuple_struct<V>(
+ self,
+ name: &'static str,
+ len: usize,
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!(
+ "deserialize_tuple_struct(name: {name}, len: {len})"
+ ))
+ .into())
+ }
+
+ fn deserialize_map<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_map".into()).into())
+ }
+
+ fn deserialize_struct<V>(
+ self,
+ name: &'static str,
+ fields: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!(
+ "deserialize_struct(name: {name}, fields: {fields:?})"
+ ))
+ .into())
+ }
+
+ fn deserialize_enum<V>(
+ self,
+ name: &'static str,
+ variants: &'static [&'static str],
+ _visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey(format!(
+ "deserialize_enum(name: {name}, variants: {variants:?})"
+ ))
+ .into())
+ }
+
+ fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ println!("deserializing_identifier: {}", self.name);
+ visitor.visit_str(self.name)
+ }
+
+ fn deserialize_ignored_any<V>(self, _visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: serde::de::Visitor<'de>,
+ {
+ Err(Details::DeserializeKey("deserialize_ignored_any".into()).into())
+ }
+}
diff --git a/avro/src/serde/deser_schema/tuple.rs b/avro/src/serde/deser_schema/tuple.rs
new file mode 100644
index 0000000..c95e38a
--- /dev/null
+++ b/avro/src/serde/deser_schema/tuple.rs
@@ -0,0 +1,107 @@
+use crate::schema::RecordSchema;
+use crate::serde::deser_schema::union::UnionDeserializer;
+use crate::serde::deser_schema::{Config, SchemaAwareDeserializer};
+use crate::{Error, Schema};
+use serde::de::{DeserializeSeed, SeqAccess};
+use std::io::Read;
+
+pub struct ManyTupleDeserializer<'s, 'r, R: Read> {
+ reader: &'r mut R,
+ schema: &'s RecordSchema,
+ config: Config<'s>,
+ current_field: usize,
+}
+
+impl<'s, 'r, R: Read> ManyTupleDeserializer<'s, 'r, R> {
+ pub fn new(reader: &'r mut R, schema: &'s RecordSchema, config: Config<'s>) -> Self {
+ Self {
+ reader,
+ schema,
+ config,
+ current_field: 0,
+ }
+ }
+}
+
+impl<'de, 's, 'r, R: Read> SeqAccess<'de> for ManyTupleDeserializer<'s, 'r, R> {
+ type Error = Error;
+
+ fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>
+ where
+ T: DeserializeSeed<'de>,
+ {
+ if self.current_field < self.schema.fields.len() {
+ let schema = &self.schema.fields[self.current_field].schema;
+ let v = match schema {
+ Schema::Union(union) => {
+ seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
+ }
+ schema => seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?,
+ };
+ self.current_field += 1;
+ Ok(Some(v))
+ } else {
+ Ok(None)
+ }
+ }
+}
+
+pub struct UnitTupleDeserializer;
+impl<'de> SeqAccess<'de> for UnitTupleDeserializer {
+ type Error = Error;
+
+ fn next_element_seed<T>(&mut self, _seed: T) -> Result<Option<T::Value>, Self::Error>
+ where
+ T: DeserializeSeed<'de>,
+ {
+ Ok(None)
+ }
+}
+
+pub struct OneTupleDeserializer<'s, 'r, R: Read> {
+ reader: &'r mut R,
+ schema: &'s Schema,
+ config: Config<'s>,
+ field_read: bool,
+}
+
+impl<'s, 'r, R: Read> OneTupleDeserializer<'s, 'r, R> {
+ pub fn new(reader: &'r mut R, schema: &'s Schema, config: Config<'s>) -> Self {
+ Self {
+ reader,
+ schema,
+ config,
+ field_read: false,
+ }
+ }
+}
+
+impl<'de, 's, 'r, R: Read> SeqAccess<'de> for OneTupleDeserializer<'s, 'r, R> {
+ type Error = Error;
+
+ fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, Self::Error>
+ where
+ T: DeserializeSeed<'de>,
+ {
+ if self.field_read {
+ Ok(None)
+ } else {
+ let v = match self.schema {
+ Schema::Union(union) => {
+ seed.deserialize(UnionDeserializer::new(self.reader, union, self.config)?)?
+ }
+ schema => seed.deserialize(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?)?,
+ };
+ self.field_read = true;
+ Ok(Some(v))
+ }
+ }
+}
diff --git a/avro/src/serde/deser_schema/union.rs b/avro/src/serde/deser_schema/union.rs
new file mode 100644
index 0000000..480a474
--- /dev/null
+++ b/avro/src/serde/deser_schema/union.rs
@@ -0,0 +1,438 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::{Debug, Formatter};
+use std::io::Read;
+
+use serde::de::Visitor;
+
+use super::{Config, DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, SchemaAwareDeserializer};
+use crate::error::Details;
+use crate::schema::{SchemaKind, UuidSchema};
+use crate::util::zag_i32;
+use crate::{Error, Schema, schema::UnionSchema};
+
+pub struct UnionDeserializer<'s, 'r, R: Read> {
+ reader: &'r mut R,
+ schema: &'s UnionSchema,
+ config: Config<'s>,
+ variant: &'s Schema,
+}
+
+impl<'s, 'r, R: Read> Debug for UnionDeserializer<'s, 'r, R> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("UnionDeserializer")
+ .field("schema", self.schema)
+ .field("variant", &self.variant)
+ .finish()
+ }
+}
+
+impl<'s, 'r, R: Read> UnionDeserializer<'s, 'r, R> {
+ pub fn new(
+ reader: &'r mut R,
+ schema: &'s UnionSchema,
+ config: Config<'s>,
+ ) -> Result<Self, Error> {
+ let index = zag_i32(reader)?;
+ let variant =
+ schema
+ .variants()
+ .get(index as usize)
+ .ok_or_else(|| Details::GetUnionVariant {
+ index: index as i64,
+ num_variants: schema.variants().len(),
+ })?;
+ Ok(Self {
+ reader,
+ schema,
+ config,
+ variant,
+ })
+ }
+
+ fn error(&self, ty: &'static str, error: impl Into<String>) -> Error {
+ Error::new(Details::DeserializeValueWithSchema {
+ value_type: ty,
+ value: error.into(),
+ schema: Schema::Union(self.schema.clone()),
+ })
+ }
+}
+
+impl<'de, 's, 'r, R: Read> serde::Deserializer<'de> for UnionDeserializer<'s, 'r, R> {
+ type Error = Error;
+
+ fn deserialize_any<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_any: {self:?}");
+ match self.variant {
+ Schema::Null => visitor.visit_unit(),
+ Schema::Boolean => self.deserialize_bool(visitor),
+ Schema::Int | Schema::Date | Schema::TimeMillis => self.deserialize_i32(visitor),
+ Schema::Long
+ | Schema::TimeMicros
+ | Schema::TimestampMillis
+ | Schema::TimestampMicros
+ | Schema::TimestampNanos
+ | Schema::LocalTimestampMillis
+ | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos => self.deserialize_i64(visitor),
+ Schema::Float => self.deserialize_f32(visitor),
+ Schema::Double => self.deserialize_f64(visitor),
+ Schema::Bytes
+ | Schema::Fixed(_)
+ | Schema::Decimal(_)
+ | Schema::BigDecimal
+ | Schema::Uuid(UuidSchema::Fixed(_))
+ | Schema::Duration(_) => self.deserialize_byte_buf(visitor),
+ Schema::String | Schema::Uuid(UuidSchema::String) => self.deserialize_string(visitor),
+ Schema::Array(_) => self.deserialize_seq(visitor),
+ Schema::Map(_) => self.deserialize_map(visitor),
+ Schema::Record(_) => {
+ self.deserialize_struct(DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, visitor)
+ }
+ Schema::Enum(_) => {
+ self.deserialize_enum(DESERIALIZE_ANY, DESERIALIZE_ANY_FIELDS, visitor)
+ }
+ Schema::Ref { name } => {
+ let schema = self
+ .config
+ .names
+ .get(name)
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?;
+ self.variant = schema;
+ self.deserialize_any(visitor)
+ }
+ Schema::Union(_) => Err(self.error("any", "Nested unions are not supported")),
+ Schema::Uuid(UuidSchema::Bytes) => panic!("Unsupported"),
+ }
+ }
+
+ fn deserialize_bool<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_bool: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_bool(visitor)
+ }
+
+ fn deserialize_i8<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_i8: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_i8(visitor)
+ }
+
+ fn deserialize_i16<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_i16: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_i16(visitor)
+ }
+
+ fn deserialize_i32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_i32: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_i32(visitor)
+ }
+
+ fn deserialize_i64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_i64: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_i64(visitor)
+ }
+
+ fn deserialize_i128<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_i128: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_i128(visitor)
+ }
+
+ fn deserialize_u8<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_u8: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_u8(visitor)
+ }
+
+ fn deserialize_u16<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_u16: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_u16(visitor)
+ }
+
+ fn deserialize_u32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_u32: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_u32(visitor)
+ }
+
+ fn deserialize_u64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_u64: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_u64(visitor)
+ }
+
+ fn deserialize_u128<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_u128: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_u128(visitor)
+ }
+
+ fn deserialize_f32<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_f32: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_f32(visitor)
+ }
+
+ fn deserialize_f64<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_f64: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_f64(visitor)
+ }
+
+ fn deserialize_char<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_char: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_char(visitor)
+ }
+
+ fn deserialize_str<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_str: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_str(visitor)
+ }
+
+ fn deserialize_string<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_string: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_string(visitor)
+ }
+
+ fn deserialize_bytes<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_bytes: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_bytes(visitor)
+ }
+
+ fn deserialize_byte_buf<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_byte_buf: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_byte_buf(visitor)
+ }
+
+ fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_option: {self:?}");
+ if self.schema.variants().len() == 2 && self.schema.index().get(&SchemaKind::Null).is_some()
+ {
+ match self.variant {
+ Schema::Null => visitor.visit_none(),
+ schema => visitor.visit_some(SchemaAwareDeserializer::new(
+ self.reader,
+ schema,
+ self.config,
+ )?),
+ }
+ } else {
+ Err(self.error(
+ "option",
+ "Expected Schema::Union(variants.contains(Schema::Null), self.variants.len() == 2)",
+ ))
+ }
+ }
+
+ fn deserialize_unit<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_unit: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_unit(visitor)
+ }
+
+ fn deserialize_unit_struct<V>(
+ self,
+ name: &'static str,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_unit_struct: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_unit_struct(name, visitor)
+ }
+
+ fn deserialize_newtype_struct<V>(
+ self,
+ name: &'static str,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_newtype_struct: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_newtype_struct(name, visitor)
+ }
+
+ fn deserialize_seq<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_seq: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_seq(visitor)
+ }
+
+ fn deserialize_tuple<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_tuple: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_tuple(len, visitor)
+ }
+
+ fn deserialize_tuple_struct<V>(
+ self,
+ name: &'static str,
+ len: usize,
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_tuple_struct: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_tuple_struct(name, len, visitor)
+ }
+
+ fn deserialize_map<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_map: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_map(visitor)
+ }
+
+ fn deserialize_struct<V>(
+ self,
+ name: &'static str,
+ fields: &'static [&'static str],
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_struct: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_struct(name, fields, visitor)
+ }
+
+ fn deserialize_enum<V>(
+ self,
+ name: &'static str,
+ variants: &'static [&'static str],
+ visitor: V,
+ ) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_enum: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_enum(name, variants, visitor)
+ }
+
+ fn deserialize_identifier<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ println!("deserialize_identifier: {self:?}");
+ SchemaAwareDeserializer::new(self.reader, self.variant, self.config)?
+ .deserialize_identifier(visitor)
+ }
+
+ fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
+ where
+ V: Visitor<'de>,
+ {
+ // TODO: We can do something more efficient here
+ println!("deserialize_ignored_any: {self:?}");
+ self.deserialize_any(visitor)
+ }
+
+ fn is_human_readable(&self) -> bool {
+ self.config.human_readable
+ }
+}
diff --git a/avro/src/serde/mod.rs b/avro/src/serde/mod.rs
index b3bfd2a..96cf620 100644
--- a/avro/src/serde/mod.rs
+++ b/avro/src/serde/mod.rs
@@ -109,6 +109,7 @@
mod de;
mod derive;
+pub mod deser_schema;
mod ser;
pub(crate) mod ser_schema;
mod util;
@@ -117,7 +118,10 @@ mod with;
pub use de::from_value;
pub use derive::{AvroSchema, AvroSchemaComponent};
pub use ser::to_value;
-pub use with::{bytes, bytes_opt, fixed, fixed_opt, slice, slice_opt};
+pub use with::{
+ array, array_opt, bigdecimal, bigdecimal_opt, bytes, bytes_opt, fixed, fixed_opt, slice,
+ slice_opt,
+};
#[doc(hidden)]
pub use derive::get_record_fields_in_ctxt;
diff --git a/avro/src/serde/ser_schema/array.rs b/avro/src/serde/ser_schema/array.rs
new file mode 100644
index 0000000..1700409
--- /dev/null
+++ b/avro/src/serde/ser_schema/array.rs
@@ -0,0 +1,186 @@
+use std::io::Write;
+
+use serde::ser::SerializeSeq;
+
+use super::{Config, SchemaAwareSerializer};
+use crate::{Error, encode::encode_int, error::Details, schema::ArraySchema};
+
+#[expect(
+ private_interfaces,
+ reason = "Direct and Buffered should not be used directly"
+)]
+pub enum ArraySerializer<'s, 'w, W: Write> {
+ Direct(DirectArraySerializer<'s, 'w, W>),
+ Buffered(BufferedArraySerializer<'s, 'w, W>),
+}
+
+impl<'s, 'w, W: Write> ArraySerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ schema: &'s ArraySchema,
+ config: Config<'s>,
+ len: Option<usize>,
+ bytes_written: Option<usize>,
+ ) -> Result<Self, Error> {
+ if let Some(len) = len
+ && config.target_block_size.is_none()
+ {
+ Ok(Self::Direct(DirectArraySerializer::new(
+ writer,
+ schema,
+ config,
+ len,
+ bytes_written.unwrap_or(0),
+ )?))
+ } else {
+ let target_block_size = config.target_block_size.unwrap_or(1024);
+ Ok(Self::Buffered(BufferedArraySerializer::new(
+ writer,
+ schema,
+ config,
+ target_block_size,
+ bytes_written.unwrap_or(0),
+ )))
+ }
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeSeq for ArraySerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ match self {
+ ArraySerializer::Direct(direct) => direct.serialize_element(value),
+ ArraySerializer::Buffered(buffered) => buffered.serialize_element(value),
+ }
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ match self {
+ ArraySerializer::Direct(direct) => direct.end(),
+ ArraySerializer::Buffered(buffered) => buffered.end(),
+ }
+ }
+}
+
+struct DirectArraySerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ array: &'s ArraySchema,
+ config: Config<'s>,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> DirectArraySerializer<'s, 'w, W> {
+ pub fn new(
+ mut writer: &'w mut W,
+ array: &'s ArraySchema,
+ config: Config<'s>,
+ len: usize,
+ mut bytes_written: usize,
+ ) -> Result<Self, Error> {
+ bytes_written += encode_int(len as i32, &mut writer)?;
+ Ok(Self {
+ writer,
+ array,
+ config,
+ bytes_written,
+ })
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeSeq for DirectArraySerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(self.writer, &self.array.items, self.config)?;
+ self.bytes_written += value.serialize(ser)?;
+ Ok(())
+ }
+
+ fn end(mut self) -> Result<Self::Ok, Self::Error> {
+ self.writer.write_all(&[0]).map_err(Details::WriteBytes)?;
+ self.bytes_written += 1;
+
+ Ok(self.bytes_written)
+ }
+}
+struct BufferedArraySerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ buffer: Vec<u8>,
+ array: &'s ArraySchema,
+ config: Config<'s>,
+ bytes_written: usize,
+ items_written: usize,
+ target_block_size: usize,
+}
+
+impl<'s, 'w, W: Write> BufferedArraySerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ array: &'s ArraySchema,
+ config: Config<'s>,
+ target_block_size: usize,
+ bytes_written: usize,
+ ) -> Self {
+ Self {
+ writer,
+ buffer: Vec::new(),
+ array,
+ config,
+ bytes_written,
+ items_written: 0,
+ target_block_size,
+ }
+ }
+
+ fn write_block(&mut self) -> Result<(), Error> {
+ self.bytes_written += encode_int(0 - (self.items_written as i32), &mut *self.writer)?;
+ self.bytes_written += encode_int(self.buffer.len() as i32, &mut *self.writer)?;
+ self.writer
+ .write_all(&self.buffer)
+ .map_err(Details::WriteBytes)?;
+ self.bytes_written += self.buffer.len();
+ self.buffer.clear();
+ Ok(())
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeSeq for BufferedArraySerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(&mut self.buffer, &self.array.items, self.config)?;
+ value.serialize(ser)?;
+ self.items_written += 1;
+
+ if self.buffer.len() >= self.target_block_size {
+ self.write_block()?;
+ }
+
+ Ok(())
+ }
+
+ fn end(mut self) -> Result<Self::Ok, Self::Error> {
+ if self.items_written > 0 {
+ self.write_block()?;
+ }
+ assert_eq!(self.buffer.len(), 0, "Buffer must be empty at this point");
+
+ encode_int(0, self.writer)?;
+ self.bytes_written += 1;
+
+ Ok(self.bytes_written)
+ }
+}
diff --git a/avro/src/serde/ser_schema/map.rs b/avro/src/serde/ser_schema/map.rs
new file mode 100644
index 0000000..185b8a6
--- /dev/null
+++ b/avro/src/serde/ser_schema/map.rs
@@ -0,0 +1,215 @@
+use std::io::Write;
+
+use serde::ser::SerializeMap;
+
+use super::{Config, SchemaAwareSerializer};
+use crate::{Error, Schema, encode::encode_int, error::Details, schema::MapSchema};
+
+#[expect(
+ private_interfaces,
+ reason = "Direct and Buffered should not be used directly"
+)]
+pub enum MapSerializer<'s, 'w, W: Write> {
+ Direct(DirectMapSerializer<'s, 'w, W>),
+ Buffered(BufferedMapSerializer<'s, 'w, W>),
+}
+
+impl<'s, 'w, W: Write> MapSerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ schema: &'s MapSchema,
+ config: Config<'s>,
+ len: Option<usize>,
+ bytes_written: Option<usize>,
+ ) -> Result<Self, Error> {
+ if let Some(len) = len
+ && config.target_block_size.is_none()
+ {
+ Ok(Self::Direct(DirectMapSerializer::new(
+ writer,
+ schema,
+ config,
+ len,
+ bytes_written.unwrap_or(0),
+ )?))
+ } else {
+ let target_block_size = config.target_block_size.unwrap_or(1024);
+ Ok(Self::Buffered(BufferedMapSerializer::new(
+ writer,
+ schema,
+ config,
+ target_block_size,
+ bytes_written.unwrap_or(0),
+ )))
+ }
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeMap for MapSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ match self {
+ MapSerializer::Direct(direct) => direct.serialize_key(key),
+ MapSerializer::Buffered(buffered) => buffered.serialize_key(key),
+ }
+ }
+
+ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ match self {
+ MapSerializer::Direct(direct) => direct.serialize_value(value),
+ MapSerializer::Buffered(buffered) => buffered.serialize_value(value),
+ }
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ match self {
+ MapSerializer::Direct(direct) => direct.end(),
+ MapSerializer::Buffered(buffered) => buffered.end(),
+ }
+ }
+}
+
+struct DirectMapSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ map: &'s MapSchema,
+ config: Config<'s>,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> DirectMapSerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ map: &'s MapSchema,
+ config: Config<'s>,
+ len: usize,
+ mut bytes_written: usize,
+ ) -> Result<Self, Error> {
+ bytes_written += encode_int(len as i32, &mut *writer)?;
+ Ok(Self {
+ writer,
+ map,
+ config,
+ bytes_written,
+ })
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeMap for DirectMapSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(&mut *self.writer, &Schema::String, self.config)?;
+ self.bytes_written += key.serialize(ser)?;
+ Ok(())
+ }
+
+ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(&mut *self.writer, &self.map.types, self.config)?;
+ self.bytes_written += value.serialize(ser)?;
+ Ok(())
+ }
+
+ fn end(mut self) -> Result<Self::Ok, Self::Error> {
+ self.writer.write_all(&[0]).map_err(Details::WriteBytes)?;
+ self.bytes_written += 1;
+
+ Ok(self.bytes_written)
+ }
+}
+
+struct BufferedMapSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ buffer: Vec<u8>,
+ map: &'s MapSchema,
+ config: Config<'s>,
+ bytes_written: usize,
+ items_written: usize,
+ target_block_size: usize,
+}
+
+impl<'s, 'w, W: Write> BufferedMapSerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ map: &'s MapSchema,
+ config: Config<'s>,
+ target_block_size: usize,
+ bytes_written: usize,
+ ) -> Self {
+ Self {
+ writer,
+ buffer: Vec::new(),
+ map,
+ config,
+ bytes_written,
+ items_written: 0,
+ target_block_size,
+ }
+ }
+
+ fn write_block(&mut self) -> Result<(), Error> {
+ self.bytes_written += encode_int(0 - (self.items_written as i32), &mut *self.writer)?;
+ self.bytes_written += encode_int(self.buffer.len() as i32, &mut *self.writer)?;
+ self.writer
+ .write_all(&self.buffer)
+ .map_err(Details::WriteBytes)?;
+ self.bytes_written += self.buffer.len();
+ self.buffer.clear();
+ Ok(())
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeMap for BufferedMapSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(&mut self.buffer, &self.map.types, self.config)?;
+ self.bytes_written += key.serialize(ser)?;
+ Ok(())
+ }
+
+ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let ser = SchemaAwareSerializer::new(&mut self.buffer, &self.map.types, self.config)?;
+ self.bytes_written += value.serialize(ser)?;
+ self.items_written += 1;
+
+ if self.buffer.len() >= self.target_block_size {
+ self.write_block()?;
+ }
+
+ Ok(())
+ }
+
+ fn end(mut self) -> Result<Self::Ok, Self::Error> {
+ if self.items_written > 0 {
+ self.write_block()?;
+ }
+ assert_eq!(self.buffer.len(), 0, "Buffer must be empty at this point");
+
+ encode_int(0, self.writer)?;
+ self.bytes_written += 1;
+
+ Ok(self.bytes_written)
+ }
+}
diff --git a/avro/src/serde/ser_schema/mod.rs b/avro/src/serde/ser_schema/mod.rs
index 50db278..d355cbf 100644
--- a/avro/src/serde/ser_schema/mod.rs
+++ b/avro/src/serde/ser_schema/mod.rs
@@ -17,1925 +17,435 @@
//! Logic for serde-compatible schema-aware serialization which writes directly to a writer.
-use crate::schema::{DecimalSchema, InnerDecimalSchema, UuidSchema};
-use crate::{
- bigdecimal::big_decimal_as_bytes,
- encode::{encode_int, encode_long},
- error::{Details, Error},
- schema::{Name, NamesRef, Namespace, RecordField, RecordSchema, Schema},
- serde::util::StringSerializer,
-};
-use bigdecimal::BigDecimal;
-use serde::{Serialize, ser};
-use std::{cmp::Ordering, collections::HashMap, io::Write, str::FromStr};
-
-const COLLECTION_SERIALIZER_ITEM_LIMIT: usize = 1024;
-const COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY: usize = 32;
-const SINGLE_VALUE_INIT_BUFFER_SIZE: usize = 128;
-
-/// The sequence serializer for [`SchemaAwareWriteSerializer`].
+mod array;
+mod map;
+mod record;
+mod tuple;
+mod union;
+
+use crate::encode::{encode_int, encode_long};
+use crate::error::Details;
+use crate::schema::{DecimalSchema, InnerDecimalSchema, NamesRef, SchemaKind, UuidSchema};
+use crate::serde::with::{BytesType, SER_BYTES_TYPE};
+use crate::{Error, Schema};
+use serde::ser::SerializeMap;
+use serde::{Serialize, Serializer};
+use std::fmt::{Debug, Formatter};
+use std::io::Write;
+
+use crate::serde::ser_schema::tuple::{OneTupleSerializer, TupleSerializer, UnitTupleSerializer};
+use array::ArraySerializer;
+use map::MapSerializer;
+use record::RecordSerializer;
+use tuple::ManyTupleSerializer;
+use union::UnionAwareSerializer;
+
+/// Indicate to the serializer that a record field default is being serialized.
///
-/// This may break large arrays up into multiple blocks to avoid having
-/// to obtain the length of the entire array before being able to write any data to the underlying
-/// writer (see the [Data Serialization and Deserialization] for more info).
-/// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization)
-pub struct SchemaAwareWriteSerializeSeq<'a, 's, W: Write> {
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- item_schema: &'s Schema,
- item_buffer_size: usize,
- item_buffers: Vec<Vec<u8>>,
- bytes_written: usize,
-}
-
-impl<'a, 's, W: Write> SchemaAwareWriteSerializeSeq<'a, 's, W> {
- fn new(
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- item_schema: &'s Schema,
- len: Option<usize>,
- ) -> SchemaAwareWriteSerializeSeq<'a, 's, W> {
- SchemaAwareWriteSerializeSeq {
- ser,
- item_schema,
- item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE,
- item_buffers: Vec::with_capacity(
- len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY),
- ),
- bytes_written: 0,
- }
- }
-
- fn write_buffered_items(&mut self) -> Result<(), Error> {
- if !self.item_buffers.is_empty() {
- self.bytes_written +=
- encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?;
- for item in self.item_buffers.drain(..) {
- self.bytes_written += self
- .ser
- .writer
- .write(item.as_slice())
- .map_err(Details::WriteBytes)?;
- }
- }
-
- Ok(())
- }
-
- fn serialize_element<T: ser::Serialize>(&mut self, value: &T) -> Result<(), Error> {
- let mut item_buffer: Vec<u8> = Vec::with_capacity(self.item_buffer_size);
- let mut item_ser = SchemaAwareWriteSerializer::new(
- &mut item_buffer,
- self.item_schema,
- self.ser.names,
- self.ser.enclosing_namespace.clone(),
- );
- value.serialize(&mut item_ser)?;
-
- self.item_buffer_size = std::cmp::max(self.item_buffer_size, item_buffer.len() + 16);
-
- self.item_buffers.push(item_buffer);
-
- if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT {
- self.write_buffered_items()?;
- }
-
- Ok(())
- }
-
- fn end(mut self) -> Result<usize, Error> {
- self.write_buffered_items()?;
- self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?;
-
- Ok(self.bytes_written)
- }
-}
-
-impl<W: Write> ser::SerializeSeq for SchemaAwareWriteSerializeSeq<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- self.serialize_element(&value)
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- self.end()
- }
-}
-
-impl<W: Write> ser::SerializeTuple for SchemaAwareWriteSerializeSeq<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- ser::SerializeSeq::serialize_element(self, value)
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- ser::SerializeSeq::end(self)
- }
-}
-
-/// The map serializer for [`SchemaAwareWriteSerializer`].
-///
-/// This may break large maps up into multiple blocks to avoid having to obtain the size of the entire
-/// map before being able to write any data to the underlying writer
-/// (see [Data Serialization and Deserialization] for more info)
-/// [Data Serialization and Deserialization](https://avro.apache.org/docs/1.12.0/specification/#data-serialization-and-deserialization)
-pub struct SchemaAwareWriteSerializeMap<'a, 's, W: Write> {
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- item_schema: &'s Schema,
- item_buffer_size: usize,
- item_buffers: Vec<Vec<u8>>,
- bytes_written: usize,
-}
-
-impl<'a, 's, W: Write> SchemaAwareWriteSerializeMap<'a, 's, W> {
- fn new(
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- item_schema: &'s Schema,
- len: Option<usize>,
- ) -> SchemaAwareWriteSerializeMap<'a, 's, W> {
- SchemaAwareWriteSerializeMap {
- ser,
- item_schema,
- item_buffer_size: SINGLE_VALUE_INIT_BUFFER_SIZE,
- item_buffers: Vec::with_capacity(
- len.unwrap_or(COLLECTION_SERIALIZER_DEFAULT_INIT_ITEM_CAPACITY),
- ),
- bytes_written: 0,
- }
- }
-
- fn write_buffered_items(&mut self) -> Result<(), Error> {
- if !self.item_buffers.is_empty() {
- self.bytes_written +=
- encode_long(self.item_buffers.len() as i64, &mut self.ser.writer)?;
- for item in self.item_buffers.drain(..) {
- self.bytes_written += self
- .ser
- .writer
- .write(item.as_slice())
- .map_err(Details::WriteBytes)?;
- }
- }
-
- Ok(())
- }
-}
-
-impl<W: Write> ser::SerializeMap for SchemaAwareWriteSerializeMap<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- let mut element_buffer: Vec<u8> = Vec::with_capacity(self.item_buffer_size);
- let string_schema = Schema::String;
- let mut key_ser = SchemaAwareWriteSerializer::new(
- &mut element_buffer,
- &string_schema,
- self.ser.names,
- self.ser.enclosing_namespace.clone(),
- );
- key.serialize(&mut key_ser)?;
-
- self.item_buffers.push(element_buffer);
-
- Ok(())
- }
-
- fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- let last_index = self.item_buffers.len() - 1;
- let element_buffer = &mut self.item_buffers[last_index];
- let mut val_ser = SchemaAwareWriteSerializer::new(
- element_buffer,
- self.item_schema,
- self.ser.names,
- self.ser.enclosing_namespace.clone(),
- );
- value.serialize(&mut val_ser)?;
-
- self.item_buffer_size = std::cmp::max(self.item_buffer_size, element_buffer.len() + 16);
-
- if self.item_buffers.len() > COLLECTION_SERIALIZER_ITEM_LIMIT {
- self.write_buffered_items()?;
- }
-
- Ok(())
- }
-
- fn end(mut self) -> Result<Self::Ok, Self::Error> {
- self.write_buffered_items()?;
- self.bytes_written += self.ser.writer.write(&[0u8]).map_err(Details::WriteBytes)?;
-
- Ok(self.bytes_written)
- }
-}
-
-/// The struct serializer for [`SchemaAwareWriteSerializer`], which can serialize Avro records.
-///
-/// This can accept fields out of order, but doing so incurs a performance penalty, since it requires
-/// buffering serialized values in order to write them to the stream in order.
-pub struct SchemaAwareWriteSerializeStruct<'a, 's, W: Write> {
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- record_schema: &'s RecordSchema,
- /// Fields we received in the wrong order
- field_cache: HashMap<usize, Vec<u8>>,
- /// The current field name when serializing from a map (for `flatten` support).
- map_field_name: Option<String>,
- field_position: usize,
- bytes_written: usize,
-}
-
-impl<'a, 's, W: Write> SchemaAwareWriteSerializeStruct<'a, 's, W> {
- fn new(
- ser: &'a mut SchemaAwareWriteSerializer<'s, W>,
- record_schema: &'s RecordSchema,
- ) -> SchemaAwareWriteSerializeStruct<'a, 's, W> {
- SchemaAwareWriteSerializeStruct {
- ser,
- record_schema,
- field_cache: HashMap::new(),
- map_field_name: None,
- field_position: 0,
- bytes_written: 0,
- }
- }
-
- fn serialize_next_field<T>(
- &mut self,
- field: &RecordField,
- position: usize,
- value: &T,
- ) -> Result<(), Error>
- where
- T: ?Sized + ser::Serialize,
- {
- match self.field_position.cmp(&position) {
- Ordering::Equal => {
- // If we receive fields in order, write them directly to the main writer
- let mut value_ser = SchemaAwareWriteSerializer::new(
- &mut *self.ser.writer,
- &field.schema,
- self.ser.names,
- self.ser.enclosing_namespace.clone(),
- );
- self.bytes_written += value.serialize(&mut value_ser)?;
-
- self.field_position += 1;
- while let Some(bytes) = self.field_cache.remove(&self.field_position) {
- self.ser
- .writer
- .write_all(&bytes)
- .map_err(Details::WriteBytes)?;
- self.bytes_written += bytes.len();
- self.field_position += 1;
- }
- Ok(())
- }
- Ordering::Less => {
- // Current field position is smaller than this field position,
- // so we're still missing at least one field, save this field temporarily
- let mut bytes = Vec::new();
- let mut value_ser = SchemaAwareWriteSerializer::new(
- &mut bytes,
- &field.schema,
- self.ser.names,
- self.ser.enclosing_namespace.clone(),
- );
- value.serialize(&mut value_ser)?;
- if self.field_cache.insert(position, bytes).is_some() {
- Err(Details::FieldNameDuplicate(field.name.clone()).into())
- } else {
- Ok(())
- }
- }
- Ordering::Greater => {
- // Current field position is greater than this field position,
- // so we've already had this field
- Err(Details::FieldNameDuplicate(field.name.clone()).into())
- }
- }
- }
-
- fn end(mut self) -> Result<usize, Error> {
- // Write any fields that are `serde(skip)` or `serde(skip_serializing)`
- while self.field_position != self.record_schema.fields.len() {
- let field_info = &self.record_schema.fields[self.field_position];
- if let Some(bytes) = self.field_cache.remove(&self.field_position) {
- self.ser
- .writer
- .write_all(&bytes)
- .map_err(Details::WriteBytes)?;
- self.bytes_written += bytes.len();
- self.field_position += 1;
- } else if let Some(default) = &field_info.default {
- self.serialize_next_field(field_info, self.field_position, default)
- .map_err(|e| Details::SerializeRecordFieldWithSchema {
- field_name: field_info.name.clone(),
- record_schema: Schema::Record(self.record_schema.clone()),
- error: Box::new(e),
- })?;
- } else {
- return Err(Details::MissingDefaultForSkippedField {
- field_name: field_info.name.clone(),
- schema: Schema::Record(self.record_schema.clone()),
- }
- .into());
- }
- }
-
- debug_assert!(
- self.field_cache.is_empty(),
- "There should be no more unwritten fields at this point: {:?}",
- self.field_cache
- );
- debug_assert!(
- self.map_field_name.is_none(),
- "There should be no field name at this point: field {:?}",
- self.map_field_name
- );
- Ok(self.bytes_written)
- }
-}
-
-impl<W: Write> ser::SerializeStruct for SchemaAwareWriteSerializeStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- if let Some(position) = self.record_schema.lookup.get(key).copied() {
- let field = &self.record_schema.fields[position];
- self.serialize_next_field(field, position, value)
- .map_err(|e| {
- Details::SerializeRecordFieldWithSchema {
- field_name: key.to_string(),
- record_schema: Schema::Record(self.record_schema.clone()),
- error: Box::new(e),
- }
- .into()
- })
- } else {
- Err(Details::FieldName(String::from(key)).into())
- }
- }
-
- fn skip_field(&mut self, key: &'static str) -> Result<(), Self::Error> {
- if let Some(position) = self.record_schema.lookup.get(key).copied() {
- let field = &self.record_schema.fields[position];
- if let Some(default) = &field.default {
- self.serialize_next_field(field, position, default)
- .map_err(|e| {
- Details::SerializeRecordFieldWithSchema {
- field_name: key.to_string(),
- record_schema: Schema::Record(self.record_schema.clone()),
- error: Box::new(e),
- }
- .into()
- })
- } else {
- Err(Details::MissingDefaultForSkippedField {
- field_name: key.to_string(),
- schema: Schema::Record(self.record_schema.clone()),
- }
- .into())
- }
- } else {
- Err(Details::GetField(key.to_string()).into())
- }
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- self.end()
- }
-}
-
-/// This implementation is used to support `#[serde(flatten)]` as that uses [`SerializeMap`] instead of [`SerializeStruct`].
-///
-/// [`SerializeMap`](ser::SerializeMap)
-/// [`SerializeStruct`](ser::SerializeStruct)
-impl<W: Write> ser::SerializeMap for SchemaAwareWriteSerializeStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + Serialize,
- {
- let name = key.serialize(StringSerializer)?;
- let old = self.map_field_name.replace(name);
- debug_assert!(
- old.is_none(),
- "Expected a value instead of a key: old key: {old:?}, new key: {:?}",
- self.map_field_name
- );
- Ok(())
- }
-
- fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + Serialize,
- {
- let key = self.map_field_name.take().ok_or(Details::MapNoKey)?;
- if let Some(position) = self.record_schema.lookup.get(&key).copied() {
- let field = &self.record_schema.fields[position];
- self.serialize_next_field(field, position, value)
- .map_err(|e| {
- Details::SerializeRecordFieldWithSchema {
- field_name: key.to_string(),
- record_schema: Schema::Record(self.record_schema.clone()),
- error: Box::new(e),
- }
- .into()
- })
- } else {
- Err(Details::FieldName(key).into())
- }
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- self.end()
- }
-}
-
-impl<W: Write> ser::SerializeStructVariant for SchemaAwareWriteSerializeStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- ser::SerializeStruct::serialize_field(self, key, value)
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- ser::SerializeStruct::end(self)
- }
-}
-
-/// Map serializer that switches between Struct or Map.
-///
-/// This exists because when `#[serde(flatten)]` is used, struct fields are serialized as a map.
-pub enum SchemaAwareWriteSerializeMapOrStruct<'a, 's, W: Write> {
- Struct(SchemaAwareWriteSerializeStruct<'a, 's, W>),
- Map(SchemaAwareWriteSerializeMap<'a, 's, W>),
-}
-
-impl<W: Write> ser::SerializeMap for SchemaAwareWriteSerializeMapOrStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + Serialize,
- {
- match self {
- Self::Struct(s) => s.serialize_key(key),
- Self::Map(s) => s.serialize_key(key),
- }
- }
-
- fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + Serialize,
- {
- match self {
- Self::Struct(s) => s.serialize_value(value),
- Self::Map(s) => s.serialize_value(value),
- }
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- match self {
- Self::Struct(s) => s.end(),
- Self::Map(s) => s.end(),
- }
- }
-}
-
-/// The tuple struct serializer for [`SchemaAwareWriteSerializer`].
-///
-/// This can serialize to an Avro array, record, or big-decimal.
-/// When serializing to a record, fields must be provided in the correct order, since no names are provided.
-pub enum SchemaAwareWriteSerializeTupleStruct<'a, 's, W: Write> {
- Record(SchemaAwareWriteSerializeStruct<'a, 's, W>),
- Array(SchemaAwareWriteSerializeSeq<'a, 's, W>),
-}
-
-impl<W: Write> SchemaAwareWriteSerializeTupleStruct<'_, '_, W> {
- fn serialize_field<T>(&mut self, value: &T) -> Result<(), Error>
- where
- T: ?Sized + ser::Serialize,
- {
- use SchemaAwareWriteSerializeTupleStruct::*;
- match self {
- Record(_record_ser) => {
- unimplemented!("Tuple struct serialization to record is not supported!");
- }
- Array(array_ser) => array_ser.serialize_element(&value),
- }
- }
-
- fn end(self) -> Result<usize, Error> {
- use SchemaAwareWriteSerializeTupleStruct::*;
- match self {
- Record(record_ser) => record_ser.end(),
- Array(array_ser) => array_ser.end(),
- }
- }
-}
-
-impl<W: Write> ser::SerializeTupleStruct for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- self.serialize_field(&value)
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- self.end()
- }
-}
-
-impl<W: Write> ser::SerializeTupleVariant for SchemaAwareWriteSerializeTupleStruct<'_, '_, W> {
- type Ok = usize;
- type Error = Error;
-
- fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>
- where
- T: ?Sized + ser::Serialize,
- {
- self.serialize_field(&value)
- }
-
- fn end(self) -> Result<Self::Ok, Self::Error> {
- self.end()
- }
-}
-
-/// A [`Serializer`](ser::Serializer) implementation that serializes directly to raw Avro data.
-///
-/// If data does not match with the schema it will return an error.
-///
-/// This does not keep state and can therefore be reused to write to the same writer.
-pub struct SchemaAwareWriteSerializer<'s, W: Write> {
- writer: &'s mut W,
- root_schema: &'s Schema,
- names: &'s NamesRef<'s>,
- enclosing_namespace: Namespace,
-}
-
-impl<'s, W: Write> SchemaAwareWriteSerializer<'s, W> {
- /// Create a new [`SchemaAwareWriteSerializer`].
- ///
- /// `writer` is the [`std::fmt::Write`] stream to be written to.
- ///
- /// `schema` is the schema of the value to be written.
- ///
- /// `names` is the mapping of schema names to schemas, to be used for type reference lookups
- ///
- /// `enclosing_namespace` is the enclosing namespace to be used for type reference lookups
- pub fn new(
- writer: &'s mut W,
- schema: &'s Schema,
- names: &'s NamesRef<'s>,
- enclosing_namespace: Namespace,
- ) -> SchemaAwareWriteSerializer<'s, W> {
- SchemaAwareWriteSerializer {
- writer,
- root_schema: schema,
- names,
- enclosing_namespace,
- }
- }
-
- fn get_ref_schema(&self, name: &'s Name) -> Result<&'s Schema, Error> {
- let full_name = name.fully_qualified_name(self.enclosing_namespace.as_deref());
-
- let ref_schema = self.names.get(full_name.as_ref()).copied();
-
- ref_schema.ok_or_else(|| Details::SchemaResolutionError(full_name.as_ref().clone()).into())
- }
-
- fn write_bytes(&mut self, bytes: &[u8]) -> Result<usize, Error> {
- let mut bytes_written: usize = 0;
-
- bytes_written += encode_long(bytes.len() as i64, &mut self.writer)?;
- // write_all() will retry when the error is ErrorKind::Interrupted (happens mostly on network storage)
- self.writer.write_all(bytes).map_err(Details::WriteBytes)?;
- bytes_written += bytes.len();
-
- Ok(bytes_written)
- }
-
- fn serialize_bool_with_schema(&mut self, value: bool, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "bool",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Boolean => self
- .writer
- .write(&[u8::from(value)])
- .map_err(|e| Details::WriteBytes(e).into()),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Boolean => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bool_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "No matching Schema::Bool found in {:?}",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: Bool"))),
- }
- }
-
- fn serialize_i32_with_schema(&mut self, value: i32, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "int (i8 | i16 | i32)",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Int | Schema::TimeMillis | Schema::Date => encode_int(value, &mut self.writer),
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Int
- | Schema::TimeMillis
- | Schema::Date
- | Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_i32_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching int-like schema in {union_schema:?}"
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: Int/Long"))),
- }
- }
-
- fn serialize_i64_with_schema(&mut self, value: i64, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "i64",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Int | Schema::TimeMillis | Schema::Date => {
- let int_value =
- i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?;
- encode_int(int_value, &mut self.writer)
- }
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => encode_long(value, &mut self.writer),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Int
- | Schema::TimeMillis
- | Schema::Date
- | Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_i64_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching int/long-like schema in {:?}",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))),
- }
- }
-
- fn serialize_i128_with_schema(&mut self, value: i128, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "i128",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Fixed(fixed) if fixed.size == 16 && fixed.name.name() == "i128" => {
- self.writer
- .write_all(&value.to_le_bytes())
- .map_err(Details::WriteBytes)?;
- Ok(16)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Fixed(fixed) if fixed.size == 16 && fixed.name.name() == "i128" => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_i128_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- r#"Cannot find a Fixed(size = 16, name = "i128") schema in {:?}"#,
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: i128"))),
- }
- }
-
- fn serialize_u8_with_schema(&mut self, value: u8, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "u8",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Int | Schema::TimeMillis | Schema::Date => {
- encode_int(value as i32, &mut self.writer)
- }
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer),
- Schema::Bytes => self.write_bytes(&[value]),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Int
- | Schema::TimeMillis
- | Schema::Date
- | Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos
- | Schema::Bytes => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_u8_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching Int-like, Long-like or Bytes schema in {union_schema:?}"
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Int"))),
- }
- }
-
- fn serialize_u32_with_schema(&mut self, value: u32, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "unsigned int (u16 | u32)",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Int | Schema::TimeMillis | Schema::Date => {
- let int_value =
- i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?;
- encode_int(int_value, &mut self.writer)
- }
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => encode_long(value as i64, &mut self.writer),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Int
- | Schema::TimeMillis
- | Schema::Date
- | Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_u32_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching Int-like or Long-like schema in {union_schema:?}"
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Int/Long"))),
- }
- }
-
- fn serialize_u64_with_schema(&mut self, value: u64, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "u64",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Int | Schema::TimeMillis | Schema::Date => {
- let int_value =
- i32::try_from(value).map_err(|cause| create_error(cause.to_string()))?;
- encode_int(int_value, &mut self.writer)
- }
- Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => {
- let long_value =
- i64::try_from(value).map_err(|cause| create_error(cause.to_string()))?;
- encode_long(long_value, &mut self.writer)
- }
- Schema::Fixed(fixed) if fixed.size == 8 && fixed.name.name() == "u64" => {
- self.writer
- .write_all(&value.to_le_bytes())
- .map_err(Details::WriteBytes)?;
- Ok(8)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Int
- | Schema::TimeMillis
- | Schema::Date
- | Schema::Long
- | Schema::TimeMicros
- | Schema::TimestampMillis
- | Schema::TimestampMicros
- | Schema::TimestampNanos
- | Schema::LocalTimestampMillis
- | Schema::LocalTimestampMicros
- | Schema::LocalTimestampNanos => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_u64_with_schema(value, variant_schema);
- }
- Schema::Fixed(fixed) if fixed.size == 8 && fixed.name.name() == "u64" => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_u64_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- r#"Cannot find a matching Int-like, Long-like or Fixed(size = 8, name "u64") schema in {:?}"#,
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: u64"))),
- }
- }
-
- fn serialize_u128_with_schema(&mut self, value: u128, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "u128",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Fixed(fixed) if fixed.size == 16 && fixed.name.name() == "u128" => {
- self.writer
- .write_all(&value.to_le_bytes())
- .map_err(Details::WriteBytes)?;
- Ok(16)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Fixed(fixed) if fixed.size == 16 && fixed.name.name() == "u128" => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_u128_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- r#"Cannot find a Fixed(size = 16, name = "u128") schema in {:?}"#,
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: u128"))),
- }
- }
-
- fn serialize_f32_with_schema(&mut self, value: f32, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "f32",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Float => self
- .writer
- .write(&value.to_le_bytes())
- .map_err(|e| Details::WriteBytes(e).into()),
- Schema::Double => self
- .writer
- .write(&(value as f64).to_le_bytes())
- .map_err(|e| Details::WriteBytes(e).into()),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Float | Schema::Double => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_f32_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a Float schema in {:?}",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Float"))),
- }
- }
-
- fn serialize_f64_with_schema(&mut self, value: f64, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "f64",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Float => self
- .writer
- .write(&(value as f32).to_le_bytes())
- .map_err(|e| Details::WriteBytes(e).into()),
- Schema::Double => self
- .writer
- .write(&value.to_le_bytes())
- .map_err(|e| Details::WriteBytes(e).into()),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Float | Schema::Double => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_f64_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a Double schema in {:?}",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Double"))),
- }
- }
-
- fn serialize_char_with_schema(&mut self, value: char, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "char",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::String | Schema::Bytes => self.write_bytes(String::from(value).as_bytes()),
- Schema::Fixed(fixed) if fixed.size == 4 && fixed.name.name() == "char" => {
- self.writer
- .write_all(&u32::from(value).to_le_bytes())
- .map_err(Details::WriteBytes)?;
- Ok(4)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::String | Schema::Bytes => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_char_with_schema(value, variant_schema);
- }
- Schema::Fixed(fixed) if fixed.size == 4 && fixed.name.name() == "char" => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_char_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- r#"Cannot find a matching String, Bytes or Fixed(size = 4, name = "char") schema in {union_schema:?}"#
- )))
- }
- expected => Err(create_error(format!("Expected {expected}. Got: char"))),
- }
- }
-
- fn serialize_str_with_schema(&mut self, value: &str, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "string",
- value: format!("{value}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::String | Schema::Bytes | Schema::Uuid(UuidSchema::String) => {
- self.write_bytes(value.as_bytes())
- }
- Schema::Uuid(UuidSchema::Bytes | UuidSchema::Fixed(_)) => {
- Err(create_error("Expected bytes but got a string. Did you mean to use `Schema::Uuid(UuidSchema::String)` or `utils::serde_set_human_readable(false)`?".to_string()))
- }
- Schema::BigDecimal => {
- // If we get a string for a `BigDecimal` type, expect a display string representation, such as "12.75"
- let decimal_val =
- BigDecimal::from_str(value).map_err(|e| create_error(e.to_string()))?;
- let decimal_bytes = big_decimal_as_bytes(&decimal_val)?;
- self.write_bytes(decimal_bytes.as_slice())
- }
- Schema::Fixed(fixed_schema) => {
- if value.len() == fixed_schema.size {
- self.writer
- .write(value.as_bytes())
- .map_err(|e| Details::WriteBytes(e).into())
- } else {
- Err(create_error(format!(
- "Fixed schema size ({}) does not match the value length ({})",
- fixed_schema.size,
- value.len()
- )))
- }
- }
- Schema::Ref { name } => {
- let ref_schema = self.get_ref_schema(name)?;
- self.serialize_str_with_schema(value, ref_schema)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::String
- | Schema::Bytes
- | Schema::Uuid(UuidSchema::String)
- | Schema::Fixed(_)
- | Schema::Ref { name: _ } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_str_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected one of the union variants {:?}. Got: String",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: String"))),
- }
- }
-
- fn serialize_bytes_with_schema(
- &mut self,
- value: &[u8],
- schema: &Schema,
- ) -> Result<usize, Error> {
- let create_error = |cause: String| {
- use std::fmt::Write;
- let mut v_str = String::with_capacity(value.len());
- for b in value {
- if write!(&mut v_str, "{b:x}").is_err() {
- v_str.push_str("??");
- }
- }
- Error::new(Details::SerializeValueWithSchema {
- value_type: "bytes",
- value: format!("{v_str}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::String | Schema::Bytes | Schema::BigDecimal => self.write_bytes(value),
- Schema::Uuid(UuidSchema::Bytes) => {
- if value.len() == 16 {
- self.write_bytes(value)
- } else {
- Err(create_error(format!(
- "Expected 16 bytes for `Schema::Uuid(Bytes)` but got {} bytes",
- value.len()
- )))
- }
- }
- Schema::Uuid(UuidSchema::String) if value.len() == 16 => {
- Err(create_error("Expected a string, but got 16 bytes. Did you mean to use `Schema::Uuid(UuidSchema::Fixed)` or `utils::serde_set_human_readable(true)`?".to_string()))
- }
- Schema::Fixed(fixed_schema) | Schema::Uuid(UuidSchema::Fixed(fixed_schema)) => {
- if value.len() == fixed_schema.size {
- self.writer
- .write(value)
- .map_err(|e| Details::WriteBytes(e).into())
- } else {
- Err(create_error(format!(
- "Fixed schema size ({}) does not match the value length ({})",
- fixed_schema.size,
- value.len()
- )))
- }
- }
- Schema::Duration(_) => {
- if value.len() == 12 {
- self.writer
- .write(value)
- .map_err(|e| Details::WriteBytes(e).into())
- } else {
- Err(create_error(format!(
- "Duration length must be 12! Got ({})",
- value.len()
- )))
- }
- }
- Schema::Decimal(decimal_schema) => match &decimal_schema.inner {
- InnerDecimalSchema::Bytes => self.write_bytes(value),
- InnerDecimalSchema::Fixed(fixed_schema) => {
- match fixed_schema.size.checked_sub(value.len()) {
- Some(pad) => {
- let pad_val = match value.len() {
- 0 => 0,
- _ => value[0],
- };
- let padding = vec![pad_val; pad];
- let mut bytes_written = self
- .writer
- .write(padding.as_slice())
- .map_err(Details::WriteBytes)?;
- bytes_written +=
- self.writer.write(value).map_err(Details::WriteBytes)?;
- Ok(bytes_written)
- }
- None => Err(Details::CompareFixedSizes {
- size: fixed_schema.size,
- n: value.len(),
- }
- .into()),
- }
- }
- },
- Schema::Ref { name } => {
- let ref_schema = self.get_ref_schema(name)?;
- self.serialize_bytes_with_schema(value, ref_schema)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::String
- | Schema::Bytes
- | Schema::BigDecimal
- | Schema::Decimal(DecimalSchema {
- inner: InnerDecimalSchema::Bytes,
- ..
- })
- | Schema::Ref { name: _ } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bytes_with_schema(value, variant_schema);
- }
- Schema::Uuid(UuidSchema::Bytes) if value.len() == 16 => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bytes_with_schema(value, variant_schema);
- }
- Schema::Fixed(fixed) | Schema::Uuid(UuidSchema::Fixed(fixed))
- if fixed.size == value.len() =>
- {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bytes_with_schema(value, variant_schema);
- }
- Schema::Decimal(DecimalSchema {
- inner: InnerDecimalSchema::Fixed(fixed),
- ..
- }) if fixed.size >= value.len() => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bytes_with_schema(value, variant_schema);
- }
- Schema::Duration(_) if value.len() == 12 => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_bytes_with_schema(value, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal or Ref schema in {union_schema:?}"
- )))
- }
- unsupported => Err(create_error(format!(
- "Expected String, Bytes, Uuid, BigDecimal, Fixed, Duration, Decimal, Ref or Union schema. Got: {unsupported}"
- ))),
- }
- }
+/// This is needed because the serializer takes a `&'static str` for the enum name and variant name.
+/// When this value is encountered, the serializer will blindly trust the variant index.
+///
+/// To prevent users from abusing this fact, the string is compared by pointer value. Because the static
+/// is not public, there is no way for a user to obtain that value.
+static SERIALIZING_SCHEMA_DEFAULT: &str = "The pointer value is used, not the string itself";
- fn serialize_none_with_schema(&mut self, schema: &Schema) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "none",
- value: format!("None. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
+pub enum MapOrRecordSerializer<'s, 'w, W: Write> {
+ Map(MapSerializer<'s, 'w, W>),
+ Record(RecordSerializer<'s, 'w, W>),
+}
- match schema {
- Schema::Null => Ok(0),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Null => {
- return encode_int(i as i32, &mut *self.writer);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching Null schema in {:?}",
- union_schema.schemas
- )))
- }
- expected => Err(create_error(format!("Expected: {expected}. Got: Null"))),
- }
- }
+impl<'s, 'w, W: Write> SerializeMap for MapOrRecordSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
- fn serialize_some_with_schema<T>(&mut self, value: &T, schema: &Schema) -> Result<usize, Error>
+ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
where
- T: ?Sized + ser::Serialize,
+ T: ?Sized + Serialize,
{
- let mut inner_ser = SchemaAwareWriteSerializer::new(
- &mut *self.writer,
- schema,
- self.names,
- self.enclosing_namespace.clone(),
- );
- value.serialize(&mut inner_ser)
- }
-
- fn serialize_unit_struct_with_schema(
- &mut self,
- name: &'static str,
- schema: &Schema,
- ) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "unit struct",
- value: format!("{name}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Record(sch) => match sch.fields.len() {
- 0 => Ok(0),
- too_many => Err(create_error(format!(
- "Too many fields: {too_many}. Expected: 0"
- ))),
- },
- Schema::Null => Ok(0),
- Schema::Ref { name: ref_name } => {
- let ref_schema = self.get_ref_schema(ref_name)?;
- self.serialize_unit_struct_with_schema(name, ref_schema)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Record(record_schema) if record_schema.fields.is_empty() => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_unit_struct_with_schema(name, variant_schema);
- }
- Schema::Null | Schema::Ref { name: _ } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_unit_struct_with_schema(name, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Cannot find a matching Null schema in {union_schema:?}"
- )))
- }
- unsupported => Err(create_error(format!(
- "Expected Null or Union schema. Got: {unsupported}"
- ))),
- }
- }
-
- fn serialize_unit_variant_with_schema(
- &mut self,
- name: &'static str,
- variant_index: u32,
- variant: &'static str,
- schema: &Schema,
- ) -> Result<usize, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "unit variant",
- value: format!("{name}::{variant} (index={variant_index}). Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Enum(enum_schema) => {
- if variant_index as usize >= enum_schema.symbols.len() {
- return Err(create_error(format!(
- "Variant index out of bounds: {}. The Enum schema has '{}' symbols",
- variant_index,
- enum_schema.symbols.len()
- )));
- }
-
- encode_int(variant_index as i32, &mut self.writer)
- }
- Schema::Union(union_schema) => {
- if variant_index as usize >= union_schema.schemas.len() {
- return Err(create_error(format!(
- "Variant index out of bounds: {}. The union schema has '{}' schemas",
- variant_index,
- union_schema.schemas.len()
- )));
- }
-
- encode_int(variant_index as i32, &mut self.writer)?;
- self.serialize_unit_struct_with_schema(
- name,
- &union_schema.schemas[variant_index as usize],
- )
- }
- Schema::Ref { name: ref_name } => {
- let ref_schema = self.get_ref_schema(ref_name)?;
- self.serialize_unit_variant_with_schema(name, variant_index, variant, ref_schema)
- }
- unsupported => Err(create_error(format!(
- "Unsupported schema: {unsupported:?}. Expected: Enum, Union or Ref"
- ))),
+ match self {
+ MapOrRecordSerializer::Map(map) => map.serialize_key(key),
+ MapOrRecordSerializer::Record(record) => record.serialize_key(key),
}
}
- fn serialize_newtype_struct_with_schema<T>(
- &mut self,
- _name: &'static str,
- value: &T,
- schema: &Schema,
- ) -> Result<usize, Error>
+ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
where
- T: ?Sized + ser::Serialize,
+ T: ?Sized + Serialize,
{
- let mut inner_ser = SchemaAwareWriteSerializer::new(
- &mut *self.writer,
- schema,
- self.names,
- self.enclosing_namespace.clone(),
- );
- // Treat any newtype struct as a transparent wrapper around the contained type
- value.serialize(&mut inner_ser)
+ match self {
+ MapOrRecordSerializer::Map(map) => map.serialize_value(value),
+ MapOrRecordSerializer::Record(record) => record.serialize_value(value),
+ }
}
- fn serialize_newtype_variant_with_schema<T>(
- &mut self,
- name: &'static str,
- variant_index: u32,
- variant: &'static str,
- value: &T,
- schema: &Schema,
- ) -> Result<usize, Error>
- where
- T: ?Sized + ser::Serialize,
- {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "newtype variant",
- value: format!("{name}::{variant}(?) (index={variant_index}). Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Union(union_schema) => {
- let variant_schema = union_schema
- .schemas
- .get(variant_index as usize)
- .ok_or_else(|| {
- create_error(format!(
- "No variant schema at position {variant_index} for {union_schema:?}"
- ))
- })?;
-
- encode_int(variant_index as i32, &mut self.writer)?;
- self.serialize_newtype_struct_with_schema(variant, value, variant_schema)
- }
- _ => Err(create_error(format!(
- "Expected Union schema. Got: {schema}"
- ))),
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ match self {
+ MapOrRecordSerializer::Map(map) => map.end(),
+ MapOrRecordSerializer::Record(record) => record.end(),
}
}
+}
- fn serialize_seq_with_schema<'a>(
- &'a mut self,
- len: Option<usize>,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeSeq<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- let len_str = len
- .map(|l| format!("{l}"))
- .unwrap_or_else(|| String::from("?"));
+#[derive(Clone, Copy)]
+pub struct Config<'s> {
+ /// All names that can be referenced in the schema being used for serialisation.
+ pub names: &'s NamesRef<'s>,
+ /// At what block size to start a new block.
+ ///
+ /// This is a minimum value, the block size will always be larger than this except for the last
+ /// block.
+ ///
+ /// When set to `None` all values will be written in a single block. This can be faster as no
+ /// intermediate buffer is used, but deserialization can be slower as the block size is not written.
+ pub target_block_size: Option<usize>,
+ /// Should `Serialize` implementations pick a human-readable format.
+ ///
+ /// It is recommended to set this to `false` as it results in compacter output.
+ pub human_readable: bool,
+}
- Error::new(Details::SerializeValueWithSchema {
- value_type: "sequence",
- value: format!("sequence (len={len_str}). Cause: {cause}"),
- schema: schema.clone(),
- })
- };
+pub struct SchemaAwareSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ schema: &'s Schema,
+ config: Config<'s>,
+}
- match schema {
- Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new(
- self,
- array_schema.items.as_ref(),
- len,
- )),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Array(_) => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_seq_with_schema(len, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected Array schema in {union_schema:?}"
- )))
- }
- _ => Err(create_error(format!("Expected: {schema}. Got: Array"))),
- }
+impl<'s, 'w, W: Write> Debug for SchemaAwareSerializer<'s, 'w, W> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("SchemaAwareSerializer")
+ .field("schema", &self.schema)
+ .finish()
}
+}
- fn serialize_tuple_with_schema<'a>(
- &'a mut self,
- len: usize,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeSeq<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "tuple",
- value: format!("tuple (len={len}). Cause: {cause}"),
- schema: schema.clone(),
+impl<'s, 'w, W: Write> SchemaAwareSerializer<'s, 'w, W> {
+ pub fn new(writer: &'w mut W, schema: &'s Schema, config: Config<'s>) -> Result<Self, Error> {
+ if let Schema::Ref { name } = schema {
+ let schema = config
+ .names
+ .get(name)
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?;
+ Self::new(writer, schema, config)
+ } else {
+ Ok(Self {
+ writer,
+ schema,
+ config,
})
- };
-
- match schema {
- Schema::Array(array_schema) => Ok(SchemaAwareWriteSerializeSeq::new(
- self,
- array_schema.items.as_ref(),
- Some(len),
- )),
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Array(_) => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_tuple_with_schema(len, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected Array schema in {union_schema:?}"
- )))
- }
- _ => Err(create_error(format!("Expected: {schema}. Got: Array"))),
}
}
- fn serialize_tuple_struct_with_schema<'a>(
- &'a mut self,
- name: &'static str,
- len: usize,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeTupleStruct<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "tuple struct",
- value: format!(
- "{name}({}). Cause: {cause}",
- vec!["?"; len].as_slice().join(",")
- ),
- schema: schema.clone(),
- })
+ /// Create a new serializer with the existing writer and config.
+ ///
+ /// This will resolve the schema if it is a reference.
+ fn with_different_schema(mut self, schema: &'s Schema) -> Result<Self, Error> {
+ let schema = if let Schema::Ref { name } = schema {
+ self.config
+ .names
+ .get(name)
+ .copied()
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()))?
+ } else {
+ schema
};
-
- match schema {
- Schema::Array(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Array(
- SchemaAwareWriteSerializeSeq::new(self, &sch.items, Some(len)),
- )),
- Schema::Record(sch) => Ok(SchemaAwareWriteSerializeTupleStruct::Record(
- SchemaAwareWriteSerializeStruct::new(self, sch),
+ self.schema = schema;
+ Ok(self)
+ }
+
+ fn error(&self, ty: &'static str, error: impl Into<String>) -> Error {
+ Error::new(Details::SerializeValueWithSchema {
+ value_type: ty,
+ value: error.into(),
+ schema: self.schema.clone(),
+ })
+ }
+
+ fn serialize_int(self, original_ty: &'static str, v: i32) -> Result<usize, Error> {
+ match self.schema {
+ Schema::Int | Schema::Date | Schema::TimeMillis => encode_int(v, &mut *self.writer),
+ Schema::Union(union) => UnionAwareSerializer::new(self.writer, union, self.config)
+ .serialize_int(original_ty, v),
+ _ => Err(self.error(
+ original_ty,
+ "Expected Schema::Int | Schema::Date | Schema::TimeMillis",
)),
- Schema::Ref { name: ref_name } => {
- let ref_schema = self.get_ref_schema(ref_name)?;
- self.serialize_tuple_struct_with_schema(name, len, ref_schema)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Record(inner) => {
- if inner.fields.len() == len {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_tuple_struct_with_schema(
- name,
- len,
- variant_schema,
- );
- }
- }
- Schema::Array(_) | Schema::Ref { name: _ } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_tuple_struct_with_schema(
- name,
- len,
- variant_schema,
- );
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected Record, Array or Ref schema in {union_schema:?}"
- )))
- }
- _ => Err(create_error(format!(
- "Expected Record, Array, Ref or Union schema. Got: {schema}"
- ))),
}
}
- fn serialize_tuple_variant_with_schema<'a>(
- &'a mut self,
- name: &'static str,
- variant_index: u32,
- variant: &'static str,
- len: usize,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeTupleStruct<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "tuple variant",
- value: format!(
- "{name}::{variant}({}) (index={variant_index}). Cause: {cause}",
- vec!["?"; len].as_slice().join(",")
- ),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Union(union_schema) => {
- let variant_schema = union_schema
- .schemas
- .get(variant_index as usize)
- .ok_or_else(|| {
- create_error(format!(
- "Cannot find a variant at position {variant_index} in {union_schema:?}"
- ))
- })?;
-
- encode_int(variant_index as i32, &mut self.writer)?;
- self.serialize_tuple_struct_with_schema(variant, len, variant_schema)
+ fn serialize_long(self, original_ty: &'static str, v: i64) -> Result<usize, Error> {
+ match self.schema {
+ Schema::Long | Schema::TimeMicros | Schema::TimestampMillis | Schema::TimestampMicros
+ | Schema::TimestampNanos | Schema::LocalTimestampMillis | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos => {
+ encode_long(v, &mut *self.writer)
}
- _ => Err(create_error(format!(
- "Expected Union schema. Got: {schema}"
- ))),
- }
- }
-
- fn serialize_map_with_schema<'a>(
- &'a mut self,
- len: Option<usize>,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeMapOrStruct<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- let len_str = len
- .map(|l| format!("{l}"))
- .unwrap_or_else(|| String::from("?"));
-
- Error::new(Details::SerializeValueWithSchema {
- value_type: "map",
- value: format!("map (size={len_str}). Cause: {cause}"),
- schema: schema.clone(),
- })
- };
-
- match schema {
- Schema::Map(map_schema) => Ok(SchemaAwareWriteSerializeMapOrStruct::Map(
- SchemaAwareWriteSerializeMap::new(self, map_schema.types.as_ref(), len),
- )),
- Schema::Ref { name: ref_name } => {
- let ref_schema = self.get_ref_schema(ref_name)?;
- self.serialize_map_with_schema(len, ref_schema)
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_long(original_ty, v)
}
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Map(_) | Schema::Record(_) | Schema::Ref { .. } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_map_with_schema(len, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected a Map schema in {union_schema:?}"
- )))
+ _ => {
+ Err(self.error(original_ty, "Expected Schema::Long | Schema::TimeMicros | Schema::{,Local}Timestamp{Millis,Micros,Nanos}"))
}
- Schema::Record(record_schema) => Ok(SchemaAwareWriteSerializeMapOrStruct::Struct(
- SchemaAwareWriteSerializeStruct::new(self, record_schema),
- )),
- _ => Err(create_error(format!(
- "Expected Map or Union schema. Got: {schema}"
- ))),
}
}
- fn serialize_struct_with_schema<'a>(
- &'a mut self,
- name: &'static str,
- len: usize,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeStruct<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "struct",
- value: format!("{name}{{ ... }}. Cause: {cause}"),
- schema: schema.clone(),
- })
- };
+ fn write_bytes(&mut self, bytes: &[u8]) -> Result<usize, Error> {
+ self.writer.write_all(bytes).map_err(Details::WriteBytes)?;
- match schema {
- Schema::Record(record_schema) => {
- Ok(SchemaAwareWriteSerializeStruct::new(self, record_schema))
- }
- Schema::Ref { name: ref_name } => {
- let ref_schema = self.get_ref_schema(ref_name)?;
- self.serialize_struct_with_schema(name, len, ref_schema)
- }
- Schema::Union(union_schema) => {
- for (i, variant_schema) in union_schema.schemas.iter().enumerate() {
- match variant_schema {
- Schema::Record(inner)
- if inner.fields.len() == len && inner.name.name() == name =>
- {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_struct_with_schema(name, len, variant_schema);
- }
- Schema::Ref { name: _ } => {
- encode_int(i as i32, &mut *self.writer)?;
- return self.serialize_struct_with_schema(name, len, variant_schema);
- }
- _ => { /* skip */ }
- }
- }
- Err(create_error(format!(
- "Expected Record or Ref schema in {union_schema:?}"
- )))
- }
- _ => Err(create_error(format!(
- "Expected Record, Ref or Union schema. Got: {schema}"
- ))),
- }
+ Ok(bytes.len())
}
- fn serialize_struct_variant_with_schema<'a>(
- &'a mut self,
- name: &'static str,
- variant_index: u32,
- variant: &'static str,
- len: usize,
- schema: &'s Schema,
- ) -> Result<SchemaAwareWriteSerializeStruct<'a, 's, W>, Error> {
- let create_error = |cause: String| {
- Error::new(Details::SerializeValueWithSchema {
- value_type: "struct variant",
- value: format!("{name}::{variant}{{ ... }} (size={len}. Cause: {cause})"),
- schema: schema.clone(),
- })
- };
+ fn write_bytes_with_len(&mut self, bytes: &[u8]) -> Result<usize, Error> {
+ let mut bytes_written: usize = 0;
- match schema {
- Schema::Union(union_schema) => {
- let variant_schema = union_schema
- .schemas
- .get(variant_index as usize)
- .ok_or_else(|| {
- create_error(format!(
- "Cannot find variant at position {variant_index} in {union_schema:?}"
- ))
- })?;
-
- encode_int(variant_index as i32, &mut self.writer)?;
- self.serialize_struct_with_schema(variant, len, variant_schema)
- }
- _ => Err(create_error(format!(
- "Expected Union schema. Got: {schema}"
- ))),
- }
+ bytes_written += encode_long(bytes.len() as i64, &mut *self.writer)?;
+ bytes_written += self.write_bytes(bytes)?;
+
+ Ok(bytes_written)
}
}
-impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s, W> {
+impl<'s, 'w, W: Write> Serializer for SchemaAwareSerializer<'s, 'w, W> {
+ /// Amount of bytes written
type Ok = usize;
type Error = Error;
- type SerializeSeq = SchemaAwareWriteSerializeSeq<'a, 's, W>;
- type SerializeTuple = SchemaAwareWriteSerializeSeq<'a, 's, W>;
- type SerializeTupleStruct = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>;
- type SerializeTupleVariant = SchemaAwareWriteSerializeTupleStruct<'a, 's, W>;
- type SerializeMap = SchemaAwareWriteSerializeMapOrStruct<'a, 's, W>;
- type SerializeStruct = SchemaAwareWriteSerializeStruct<'a, 's, W>;
- type SerializeStructVariant = SchemaAwareWriteSerializeStruct<'a, 's, W>;
+ type SerializeSeq = ArraySerializer<'s, 'w, W>;
+ type SerializeTuple = TupleSerializer<'s, 'w, W>;
+ type SerializeTupleStruct = ManyTupleSerializer<'s, 'w, W>;
+ type SerializeTupleVariant = ManyTupleSerializer<'s, 'w, W>;
+ type SerializeMap = MapOrRecordSerializer<'s, 'w, W>;
+ type SerializeStruct = RecordSerializer<'s, 'w, W>;
+ type SerializeStructVariant = RecordSerializer<'s, 'w, W>;
fn serialize_bool(self, v: bool) -> Result<Self::Ok, Self::Error> {
- self.serialize_bool_with_schema(v, self.root_schema)
+ println!("serialize_bool({v}): {self:?}");
+ match self.schema {
+ Schema::Boolean => {
+ self.writer
+ .write_all(&[v as u8])
+ .map_err(Details::WriteBytes)?;
+ Ok(1)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_bool(v)
+ }
+ _ => Err(self.error("bool", "Expected Schema::Boolean")),
+ }
}
fn serialize_i8(self, v: i8) -> Result<Self::Ok, Self::Error> {
- self.serialize_i32(v as i32)
+ println!("serialize_i8({v}): {self:?}");
+ self.serialize_int("i8", i32::from(v))
}
fn serialize_i16(self, v: i16) -> Result<Self::Ok, Self::Error> {
- self.serialize_i32(v as i32)
+ println!("serialize_i16({v}): {self:?}");
+ self.serialize_int("i16", i32::from(v))
}
fn serialize_i32(self, v: i32) -> Result<Self::Ok, Self::Error> {
- self.serialize_i32_with_schema(v, self.root_schema)
+ println!("serialize_i32({v}): {self:?}");
+ self.serialize_int("i32", v)
}
fn serialize_i64(self, v: i64) -> Result<Self::Ok, Self::Error> {
- self.serialize_i64_with_schema(v, self.root_schema)
+ println!("serialize_i64({v}): {self:?}");
+ self.serialize_long("i64", v)
}
- fn serialize_i128(self, v: i128) -> Result<Self::Ok, Self::Error> {
- self.serialize_i128_with_schema(v, self.root_schema)
+ fn serialize_i128(mut self, v: i128) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_i128({v}): {self:?}");
+ match self.schema {
+ Schema::Fixed(fixed) if fixed.name.name() == "i128" && fixed.size == 16 => {
+ let bytes = v.to_le_bytes();
+ self.write_bytes(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_i128(v)
+ }
+ _ => Err(self.error("i128", r#"Expected Schema::Fixed(name: "i128", size: 16)"#)),
+ }
}
fn serialize_u8(self, v: u8) -> Result<Self::Ok, Self::Error> {
- self.serialize_u8_with_schema(v, self.root_schema)
+ println!("serialize_u8({v}): {self:?}");
+ self.serialize_int("u8", i32::from(v))
}
fn serialize_u16(self, v: u16) -> Result<Self::Ok, Self::Error> {
- self.serialize_u32(v as u32)
+ println!("serialize_u16({v}): {self:?}");
+ self.serialize_int("u16", i32::from(v))
}
fn serialize_u32(self, v: u32) -> Result<Self::Ok, Self::Error> {
- self.serialize_u32_with_schema(v, self.root_schema)
+ println!("serialize_u32({v}): {self:?}");
+ self.serialize_long("u32", i64::from(v))
}
- fn serialize_u64(self, v: u64) -> Result<Self::Ok, Self::Error> {
- self.serialize_u64_with_schema(v, self.root_schema)
+ fn serialize_u64(mut self, v: u64) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_u64({v}): {self:?}");
+ match self.schema {
+ Schema::Fixed(fixed) if fixed.name.name() == "u64" && fixed.size == 8 => {
+ let bytes = v.to_le_bytes();
+ self.write_bytes(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_u64(v)
+ }
+ _ => Err(self.error("u64", r#"Expected Schema::Fixed(name: "u64", size: 8)"#)),
+ }
}
- fn serialize_u128(self, v: u128) -> Result<Self::Ok, Self::Error> {
- self.serialize_u128_with_schema(v, self.root_schema)
+ fn serialize_u128(mut self, v: u128) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_u128({v}): {self:?}");
+ match self.schema {
+ Schema::Fixed(fixed) if fixed.name.name() == "u128" && fixed.size == 16 => {
+ let bytes = v.to_le_bytes();
+ self.write_bytes(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_u128(v)
+ }
+ _ => Err(self.error("u128", r#"Expected Schema::Fixed(name: "u128", size: 16)"#)),
+ }
}
- fn serialize_f32(self, v: f32) -> Result<Self::Ok, Self::Error> {
- self.serialize_f32_with_schema(v, self.root_schema)
+ fn serialize_f32(mut self, v: f32) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_f32({v}): {self:?}");
+ match self.schema {
+ Schema::Float => {
+ let bytes = v.to_le_bytes();
+ self.write_bytes(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_f32(v)
+ }
+ _ => Err(self.error("f32", r"Expected Schema::Float")),
+ }
}
- fn serialize_f64(self, v: f64) -> Result<Self::Ok, Self::Error> {
- self.serialize_f64_with_schema(v, self.root_schema)
+ fn serialize_f64(mut self, v: f64) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_f64({v}): {self:?}");
+ match self.schema {
+ Schema::Float => {
+ let bytes = v.to_le_bytes();
+ self.write_bytes(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_f64(v)
+ }
+ _ => Err(self.error("f64", r"Expected Schema::Double")),
+ }
}
- fn serialize_char(self, v: char) -> Result<Self::Ok, Self::Error> {
- self.serialize_char_with_schema(v, self.root_schema)
+ fn serialize_char(mut self, v: char) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_char({v}): {self:?}");
+ match self.schema {
+ Schema::String => {
+ let bytes = v.to_string().into_bytes();
+ self.write_bytes_with_len(&bytes)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_char(v)
+ }
+ _ => Err(self.error("char", r"Expected Schema::String")),
+ }
}
- fn serialize_str(self, v: &str) -> Result<Self::Ok, Self::Error> {
- self.serialize_str_with_schema(v, self.root_schema)
+ fn serialize_str(mut self, v: &str) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_str({v}): {self:?}");
+ match self.schema {
+ Schema::String | Schema::Uuid(UuidSchema::String) => {
+ self.write_bytes_with_len(v.as_bytes())
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_str(v)
+ }
+ _ => Err(self.error("str", "Expected Schema::String | Schema::Uuid(String)")),
+ }
}
- fn serialize_bytes(self, v: &[u8]) -> Result<Self::Ok, Self::Error> {
- self.serialize_bytes_with_schema(v, self.root_schema)
+ fn serialize_bytes(mut self, v: &[u8]) -> Result<Self::Ok, Self::Error> {
+ println!("serialize_bytes({v:?}): {self:?}");
+ match (SER_BYTES_TYPE.get(), self.schema) {
+ (BytesType::Unset | BytesType::Bytes, Schema::Bytes | Schema::BigDecimal | Schema::Decimal(DecimalSchema { inner: InnerDecimalSchema::Bytes, .. })) => {
+ self.write_bytes_with_len(v)
+ }
+ (BytesType::Unset | BytesType::Fixed, Schema::Fixed(fixed) | Schema::Decimal(DecimalSchema { inner: InnerDecimalSchema::Fixed(fixed), ..}) | Schema::Uuid(UuidSchema::Fixed(fixed)) | Schema::Duration(fixed)) => {
+ if fixed.size != v.len() {
+ Err(self.error("bytes", format!("Fixed size ({}) does not match value length ({})", fixed.size, v.len())))
+ } else {
+ self.write_bytes(v)
+ }
+ }
+ (_, Schema::Union(union)) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_bytes(v)
+ }
+ _ => Err(self.error("bytes", "Expected Schema::Bytes | Schema::BigDecimal | Schema::Decimal(Bytes | Fixed) | Schema::Fixed | Schema::Uuid(Fixed) | Schema::Duration")),
+ }
}
fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
- self.serialize_none_with_schema(self.root_schema)
+ println!("serialize_none: {self:?}");
+ if let Schema::Union(union) = self.schema
+ && union.variants().len() == 2
+ && let Some(index) = union.index().get(&SchemaKind::Null).copied()
+ {
+ encode_int(index as i32, &mut *self.writer)
+ } else {
+ Err(self.error(
+ "None",
+ "Expected Schema::Union(variants.len() == 2 && variants.contains(Schema::Null))",
+ ))
+ }
}
fn serialize_some<T>(self, value: &T) -> Result<Self::Ok, Self::Error>
where
- T: ?Sized + ser::Serialize,
+ T: ?Sized + Serialize,
{
- self.serialize_some_with_schema(value, self.root_schema)
+ println!("serialize_some: {self:?}");
+ if let Schema::Union(union) = self.schema
+ && union.variants().len() == 2
+ && let Some(index) = union.index().get(&SchemaKind::Null).copied()
+ {
+ // Convert the index of null to the other index
+ let index = (index + 1) & 1;
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let ser = self.with_different_schema(&union.variants()[index])?;
+ bytes_written += value.serialize(ser)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "None",
+ "Expected Schema::Union(variants.len() == 2 && variants.contains(Schema::Null))",
+ ))
+ }
}
fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
- self.serialize_none()
+ println!("serialize_unit: {self:?}");
+ match self.schema {
+ Schema::Null => Ok(0),
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_unit()
+ }
+ _ => Err(self.error("unit", "Expected Schema::Null")),
+ }
}
fn serialize_unit_struct(self, name: &'static str) -> Result<Self::Ok, Self::Error> {
- self.serialize_unit_struct_with_schema(name, self.root_schema)
+ println!("serialize_unit_struct(name: {name}): {self:?}");
+ match self.schema {
+ Schema::Record(record) if record.name.name() == name && record.fields.is_empty() => {
+ Ok(0)
+ }
+ Schema::Union(union) => UnionAwareSerializer::new(self.writer, union, self.config)
+ .serialize_unit_struct(name),
+ _ => Err(self.error(
+ "unit struct",
+ format!("Expected Schema::Record(name: {name}, fields: [])"),
+ )),
+ }
}
fn serialize_unit_variant(
@@ -1944,7 +454,46 @@ impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s
variant_index: u32,
variant: &'static str,
) -> Result<Self::Ok, Self::Error> {
- self.serialize_unit_variant_with_schema(name, variant_index, variant, self.root_schema)
+ println!(
+ "serialize_struct_variant(name: {name}, index: {variant_index}, variant: {variant}): {self:?}"
+ );
+ match self.schema {
+ Schema::Enum(enum_schema) => {
+ if name.as_ptr() == SERIALIZING_SCHEMA_DEFAULT.as_ptr() {
+ encode_int(variant_index as i32, &mut *self.writer)
+ } else {
+ if enum_schema.symbols[variant_index as usize] != variant {
+ return Err(self.error(
+ "unit variant",
+ format!(
+ "Enum variant ({variant}) is not at index {variant_index} in symbols"
+ ),
+ ));
+ }
+ encode_int(variant_index as i32, &mut *self.writer)
+ }
+ }
+ Schema::Union(union) => {
+ if let Some(index) = union.index().get(&SchemaKind::Null).copied() {
+ // Bare union
+ encode_int(index as i32, &mut *self.writer)
+ } else {
+ // Union of records
+ let Some(Schema::Record(record)) = union.variants().get(variant_index as usize)
+ else {
+ return Err(self.error("unit variant", format!("Union does not contain null and variant at index {variant_index} is not a record")));
+ };
+ if record.name.name() != variant {
+ return Err(self.error("unit variant", format!("Union does not contain null and record at index {variant_index} is not named {variant}")));
+ }
+ if !record.fields.is_empty() {
+ return Err(self.error("unit variant", format!("Union does not contain null and record at index {variant_index} is not empty")));
+ }
+ encode_int(variant_index as i32, &mut *self.writer)
+ }
+ }
+ _ => Err(self.error("unit variant", "Expected Schema::Enum | Schema::Union")),
+ }
}
fn serialize_newtype_struct<T>(
@@ -1953,36 +502,93 @@ impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s
value: &T,
) -> Result<Self::Ok, Self::Error>
where
- T: ?Sized + ser::Serialize,
+ T: ?Sized + Serialize,
{
- self.serialize_newtype_struct_with_schema(name, value, self.root_schema)
+ println!("serialize_newtype_struct(name: {name}): {self:?}");
+ match self.schema {
+ Schema::Record(record) if record.name.name() == name && record.fields.len() == 1 => {
+ let schema = &record.fields[0].schema;
+ let ser = self.with_different_schema(schema)?;
+ value.serialize(ser)
+ }
+ Schema::Union(union) => UnionAwareSerializer::new(self.writer, union, self.config)
+ .serialize_newtype_struct(name, value),
+ _ => Err(self.error(
+ "newtype struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == 1)"),
+ )),
+ }
}
fn serialize_newtype_variant<T>(
self,
- name: &'static str,
+ _name: &'static str,
variant_index: u32,
variant: &'static str,
value: &T,
) -> Result<Self::Ok, Self::Error>
where
- T: ?Sized + ser::Serialize,
+ T: ?Sized + Serialize,
{
- self.serialize_newtype_variant_with_schema(
- name,
- variant_index,
- variant,
- value,
- self.root_schema,
- )
+ println!(
+ "serialize_newtype_variant(name: {_name}, index: {variant_index}, variant: {variant}): {self:?}"
+ );
+ match self.schema {
+ Schema::Union(union) => {
+ if let Some(Schema::Record(record)) = union.variants().get(variant_index as usize)
+ && record.name.name() == variant
+ && record.fields.len() == 1
+ {
+ // Union of records
+ let mut bytes_written = encode_int(variant_index as i32, &mut *self.writer)?;
+
+ let schema = &record.fields[0].schema;
+ let ser = self.with_different_schema(schema)?;
+ bytes_written += value.serialize(ser)?;
+ Ok(bytes_written)
+ } else {
+ // Bare union, UnionAwareSerializer will write the correct index
+ let ser = UnionAwareSerializer::new(self.writer, union, self.config);
+ value.serialize(ser)
+ }
+ }
+ _ => Err(self.error("newtype variant", "Expected Schema::Union")),
+ }
}
fn serialize_seq(self, len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
- self.serialize_seq_with_schema(len, self.root_schema)
+ println!("serialize_seq(len: {len:?}): {self:?}");
+ match self.schema {
+ Schema::Array(array) => {
+ ArraySerializer::new(self.writer, array, self.config, len, None)
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_seq(len)
+ }
+ _ => Err(self.error("seq", "Expected Schema::Array")),
+ }
}
fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, Self::Error> {
- self.serialize_tuple_with_schema(len, self.root_schema)
+ println!("serialize_tuple(len: {len}): {self:?}");
+ match self.schema {
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_tuple(len)
+ }
+ Schema::Null if len == 0 => Ok(TupleSerializer::Unit(UnitTupleSerializer::new(None))),
+ schema if len == 1 => Ok(TupleSerializer::One(OneTupleSerializer::new(
+ self.writer,
+ schema,
+ self.config,
+ ))),
+ Schema::Record(record) if record.fields.len() == len => Ok(TupleSerializer::Many(
+ ManyTupleSerializer::new(self.writer, record, self.config, None),
+ )),
+ _ => Err(self.error(
+ "tuple",
+ format!("Expected Schema::Record(fields.len() == {len})"),
+ )),
+ }
}
fn serialize_tuple_struct(
@@ -1990,27 +596,101 @@ impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s
name: &'static str,
len: usize,
) -> Result<Self::SerializeTupleStruct, Self::Error> {
- self.serialize_tuple_struct_with_schema(name, len, self.root_schema)
+ println!("serialize_tuple_struct(name: {name}, len: {len}): {self:?}");
+ match self.schema {
+ Schema::Record(record) if record.name.name() == name && record.fields.len() == len => {
+ Ok(ManyTupleSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ None,
+ ))
+ }
+ Schema::Union(union) => UnionAwareSerializer::new(self.writer, union, self.config)
+ .serialize_tuple_struct(name, len),
+ _ => Err(self.error(
+ "tuple",
+ format!("Expected Schema::Record(name: {name}, fields.len() == {len})"),
+ )),
+ }
}
fn serialize_tuple_variant(
self,
- name: &'static str,
+ _name: &'static str,
variant_index: u32,
variant: &'static str,
len: usize,
) -> Result<Self::SerializeTupleVariant, Self::Error> {
- self.serialize_tuple_variant_with_schema(
- name,
- variant_index,
- variant,
- len,
- self.root_schema,
- )
+ println!(
+ "serialize_tuple_variant(name: {_name}, index: {variant_index}, variant: {variant}, len: {len}): {self:?}"
+ );
+ match self.schema {
+ Schema::Union(union) => {
+ if let Some(Schema::Record(record)) = union.variants().get(variant_index as usize)
+ && record.name.name() == variant
+ && record.fields.len() == len
+ {
+ // Union of records
+ let bytes_written = encode_int(variant_index as i32, &mut *self.writer)?;
+
+ Ok(ManyTupleSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else if let Some((index, Schema::Record(record))) = union
+ .variants()
+ .iter()
+ .enumerate()
+ .find(|(_i, s)| s.name().is_some_and(|n| n.name() == variant))
+ && record.fields.len() == len
+ {
+ // Bare union
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+
+ Ok(ManyTupleSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else {
+ Err(self.error(
+ "tuple variant",
+ format!("Expected Schema::Union(variants.contains(Schema::Record(name: {variant}, fields.len() == {len})))"),
+ ))
+ }
+ }
+ _ => Err(self.error("tuple variant", "Expected Schema::Union")),
+ }
}
fn serialize_map(self, len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
- self.serialize_map_with_schema(len, self.root_schema)
+ println!("serialize_map(len: {len:?}): {self:?}");
+ match self.schema {
+ Schema::Map(map) => Ok(MapOrRecordSerializer::Map(MapSerializer::new(
+ self.writer,
+ map,
+ self.config,
+ len,
+ None,
+ )?)),
+ Schema::Record(record) => {
+ // Structs with flattened fields will be serialized as a map
+ Ok(MapOrRecordSerializer::Record(RecordSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ len,
+ )))
+ }
+ Schema::Union(union) => {
+ UnionAwareSerializer::new(self.writer, union, self.config).serialize_map(len)
+ }
+ _ => Err(self.error("map", "Expected Schema::Map")),
+ }
}
fn serialize_struct(
@@ -2018,64 +698,140 @@ impl<'a, 's, W: Write> ser::Serializer for &'a mut SchemaAwareWriteSerializer<'s
name: &'static str,
len: usize,
) -> Result<Self::SerializeStruct, Self::Error> {
- self.serialize_struct_with_schema(name, len, self.root_schema)
+ println!("serialize_struct(name: {name}, len: {len}): {self:?}");
+ match self.schema {
+ // For unit variants with tag,content `len` will be 1 but we expect 2.
+ Schema::Record(record)
+ if record.name.name() == name && record.fields.len() == len
+ || record.fields.len() == 2 =>
+ {
+ Ok(RecordSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ None,
+ ))
+ }
+ Schema::Union(union) => UnionAwareSerializer::new(self.writer, union, self.config)
+ .serialize_struct(name, len),
+ _ => Err(self.error(
+ "struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == {len})"),
+ )),
+ }
}
fn serialize_struct_variant(
self,
- name: &'static str,
+ _name: &'static str,
variant_index: u32,
variant: &'static str,
len: usize,
) -> Result<Self::SerializeStructVariant, Self::Error> {
- self.serialize_struct_variant_with_schema(
- name,
- variant_index,
- variant,
- len,
- self.root_schema,
- )
+ println!(
+ "serialize_struct_variant(name: {_name}, index: {variant_index}, variant: {variant}, len: {len}): {self:?}"
+ );
+ match self.schema {
+ Schema::Union(union) => {
+ if let Some(Schema::Record(record)) = union.variants().get(variant_index as usize)
+ && record.name.name() == variant
+ && record.fields.len() == len
+ {
+ // Union of records
+ let bytes_written = encode_int(variant_index as i32, &mut *self.writer)?;
+
+ Ok(RecordSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else if let Some((index, Schema::Record(record))) = union
+ .variants()
+ .iter()
+ .enumerate()
+ .find(|(_i, s)| s.name().is_some_and(|n| n.name() == variant))
+ && record.fields.len() == len
+ {
+ // Bare union
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+
+ Ok(RecordSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else {
+ Err(self.error(
+ "struct variant",
+ format!("Expected Schema::Union(variants.contains(Schema::Record(name: {variant}, fields.len() == {len})))"),
+ ))
+ }
+ }
+ _ => Err(self.error("struct variant", "Expected Schema::Union")),
+ }
}
fn is_human_readable(&self) -> bool {
- crate::util::is_human_readable()
+ self.config.human_readable
}
}
#[cfg(test)]
mod tests {
- use super::*;
- use crate::schema::FixedSchema;
+ use super::{Config, SchemaAwareSerializer};
+ use crate::schema::{FixedSchema, Name};
use crate::{
- Days, Duration, Millis, Months, Reader, Writer, decimal::Decimal, error::Details,
- from_value, schema::ResolvedSchema,
+ Days, Duration, Error, Millis, Months, Reader, Schema, Writer, decimal::Decimal,
+ error::Details, from_value, schema::ResolvedSchema,
};
use apache_avro_test_helper::TestResult;
use bigdecimal::BigDecimal;
use num_bigint::{BigInt, Sign};
use serde::{Deserialize, Serialize};
use serde_bytes::{ByteArray, Bytes};
- use std::{
- collections::{BTreeMap, HashMap},
- marker::PhantomData,
- };
+ use std::collections::HashMap;
+ use std::{collections::BTreeMap, marker::PhantomData};
use uuid::Uuid;
#[test]
fn test_serialize_null() -> TestResult {
let schema = Schema::Null;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- ().serialize(&mut serializer)?;
- None::<()>.serialize(&mut serializer)?;
- None::<i32>.serialize(&mut serializer)?;
- None::<String>.serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ None::<()>
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ None::<i32>
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ None::<String>
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
- assert_eq!(buffer.as_slice(), Vec::<u8>::new().as_slice());
+ assert!(buffer.is_empty());
Ok(())
}
@@ -2084,13 +840,23 @@ mod tests {
fn test_serialize_bool() -> TestResult {
let schema = Schema::Boolean;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- true.serialize(&mut serializer)?;
- false.serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ true.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ false.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
assert_eq!(buffer.as_slice(), &[1, 0]);
@@ -2101,19 +867,33 @@ mod tests {
fn test_serialize_int() -> TestResult {
let schema = Schema::Int;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 4u8.serialize(&mut serializer)?;
- 31u16.serialize(&mut serializer)?;
- 13u32.serialize(&mut serializer)?;
- 7i8.serialize(&mut serializer)?;
- (-57i16).serialize(&mut serializer)?;
- 129i32.serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ 4u8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 31u16.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 13u32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 7i8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ (-57i16).serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 129i32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
- assert_eq!(buffer.as_slice(), &[8, 62, 26, 14, 113, 130, 2]);
+ assert_eq!(buffer.as_slice(), &[8, 62, 14, 113, 130, 2]);
Ok(())
}
@@ -2122,24 +902,53 @@ mod tests {
fn test_serialize_long() -> TestResult {
let schema = Schema::Long;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 4u8.serialize(&mut serializer)?;
- 31u16.serialize(&mut serializer)?;
- 13u32.serialize(&mut serializer)?;
- 291u64.serialize(&mut serializer)?;
- 7i8.serialize(&mut serializer)?;
- (-57i16).serialize(&mut serializer)?;
- 129i32.serialize(&mut serializer)?;
- (-432i64).serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- assert_eq!(
- buffer.as_slice(),
- &[8, 62, 26, 198, 4, 14, 113, 130, 2, 223, 6]
+ assert!(
+ 4u8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
);
+ assert!(
+ 31u16
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 13u32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 291u64
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 7i8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ (-57i16)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 129i32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ (-432i64).serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+
+ assert_eq!(buffer.as_slice(), &[26, 223, 6]);
Ok(())
}
@@ -2148,15 +957,29 @@ mod tests {
fn test_serialize_float() -> TestResult {
let schema = Schema::Float;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 4.7f32.serialize(&mut serializer)?;
- (-14.1f64).serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ 4.7f32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ (-14.1f64)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
- assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]);
+ assert_eq!(buffer.as_slice(), &[102, 102, 150, 64]);
Ok(())
}
@@ -2165,13 +988,28 @@ mod tests {
fn test_serialize_double() -> TestResult {
let schema = Schema::Float;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 4.7f32.serialize(&mut serializer)?;
- (-14.1f64).serialize(&mut serializer)?;
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ 4.7f32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ (-14.1f32).serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ (-14.1f64)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
assert_eq!(buffer.as_slice(), &[102, 102, 150, 64, 154, 153, 97, 193]);
@@ -2182,20 +1020,38 @@ mod tests {
fn test_serialize_bytes() -> TestResult {
let schema = Schema::Bytes;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 'a'.serialize(&mut serializer)?;
- "test".serialize(&mut serializer)?;
- Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?;
- assert!(().serialize(&mut serializer).is_err());
- assert!(PhantomData::<String>.serialize(&mut serializer).is_err());
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- assert_eq!(
- buffer.as_slice(),
- &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4]
+ assert!(
+ 'a'.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ "test"
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ Bytes::new(&[12, 3, 7, 91, 4]).serialize(SchemaAwareSerializer::new(
+ &mut buffer,
+ &schema,
+ config,
+ )?)?;
+ assert!(
+ ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ PhantomData::<String>
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
);
+ assert_eq!(buffer.as_slice(), &[10, 12, 3, 7, 91, 4]);
+
Ok(())
}
@@ -2203,19 +1059,30 @@ mod tests {
fn test_serialize_string() -> TestResult {
let schema = Schema::String;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 'a'.serialize(&mut serializer)?;
- "test".serialize(&mut serializer)?;
- Bytes::new(&[12, 3, 7, 91, 4]).serialize(&mut serializer)?;
- assert!(().serialize(&mut serializer).is_err());
- assert!(PhantomData::<String>.serialize(&mut serializer).is_err());
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- assert_eq!(
- buffer.as_slice(),
- &[2, b'a', 8, b't', b'e', b's', b't', 10, 12, 3, 7, 91, 4]
+ 'a'.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ "test".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ Bytes::new(&[12, 3, 7, 91, 4])
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config,)?)
+ .is_err()
+ );
+ assert!(
+ ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
);
+ assert!(
+ PhantomData::<String>
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+
+ assert_eq!(buffer.as_slice(), &[2, b'a', 8, b't', b'e', b's', b't']);
Ok(())
}
@@ -2234,37 +1101,51 @@ mod tests {
)?;
#[derive(Serialize)]
- #[serde(rename_all = "camelCase")]
+ #[serde(rename_all = "camelCase", rename = "TestRecord")]
struct GoodTestRecord {
string_field: String,
int_field: i32,
}
#[derive(Serialize)]
- #[serde(rename_all = "camelCase")]
+ #[serde(rename_all = "camelCase", rename = "TestRecord")]
struct BadTestRecord {
foo_string_field: String,
bar_int_field: i32,
}
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let good_record = GoodTestRecord {
string_field: String::from("test"),
int_field: 10,
};
- good_record.serialize(&mut serializer)?;
+ good_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let bad_record = BadTestRecord {
foo_string_field: String::from("test"),
bar_int_field: 10,
};
- assert!(bad_record.serialize(&mut serializer).is_err());
+ assert!(
+ bad_record
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
- assert!("".serialize(&mut serializer).is_err());
- assert!(Some("").serialize(&mut serializer).is_err());
+ assert!(
+ "".serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ Some("")
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
assert_eq!(buffer.as_slice(), &[8, b't', b'e', b's', b't', 20]);
@@ -2282,14 +1163,18 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
#[derive(Serialize)]
struct EmptyRecord;
- EmptyRecord.serialize(&mut serializer)?;
+ EmptyRecord.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
#[derive(Serialize)]
+ #[serde(rename = "EmptyRecord")]
struct NonEmptyRecord {
foo: String,
}
@@ -2297,21 +1182,21 @@ mod tests {
foo: "bar".to_string(),
};
match record
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::FieldName(field_name)) if field_name == "foo" => (),
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- match ().serialize(&mut serializer).map_err(Error::into_details) {
+ match ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?).map_err(Error::into_details) {
Err(Details::SerializeValueWithSchema {
value_type,
value,
schema,
}) => {
- assert_eq!(value_type, "none"); // serialize_unit() delegates to serialize_none()
- assert_eq!(value, "None. Cause: Expected: Record. Got: Null");
+ assert_eq!(value_type, "unit");
+ assert_eq!(value, "Expected Schema::Null");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2333,6 +1218,7 @@ mod tests {
)?;
#[derive(Serialize)]
+ #[serde(rename_all = "UPPERCASE")]
enum Suit {
Spades,
Hearts,
@@ -2341,15 +1227,18 @@ mod tests {
}
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- Suit::Spades.serialize(&mut serializer)?;
- Suit::Hearts.serialize(&mut serializer)?;
- Suit::Diamonds.serialize(&mut serializer)?;
- Suit::Clubs.serialize(&mut serializer)?;
+ Suit::Spades.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ Suit::Hearts.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ Suit::Diamonds.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ Suit::Clubs.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match None::<()>
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2357,8 +1246,8 @@ mod tests {
value,
schema,
}) => {
- assert_eq!(value_type, "none");
- assert_eq!(value, "None. Cause: Expected: Enum. Got: Null");
+ assert_eq!(value_type, "None");
+ assert_eq!(value, "Expected Schema::Union([null, _])");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2379,14 +1268,17 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let arr: Vec<i64> = vec![10, 5, 400];
- arr.serialize(&mut serializer)?;
+ arr.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match vec![1_f32]
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2395,13 +1287,15 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "f32");
- assert_eq!(value, "1. Cause: Expected: Long. Got: Float");
+ assert_eq!(value, "Expected Schema::Float");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- assert_eq!(buffer.as_slice(), &[6, 20, 10, 160, 6, 0]);
+ // The 2 at the end is because the DirectArraySerializer immediately writes the length and doesn't
+ // know the values will be invalid yet
+ assert_eq!(buffer.as_slice(), &[6, 20, 10, 160, 6, 0, 2]);
Ok(())
}
@@ -2416,35 +1310,43 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let mut map: BTreeMap<String, i64> = BTreeMap::new();
map.insert(String::from("item1"), 10);
map.insert(String::from("item2"), 400);
- map.serialize(&mut serializer)?;
+ map.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let mut map: BTreeMap<String, &str> = BTreeMap::new();
map.insert(String::from("item1"), "value1");
- match map.serialize(&mut serializer).map_err(Error::into_details) {
+ match map
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .map_err(Error::into_details)
+ {
Err(Details::SerializeValueWithSchema {
value_type,
value,
schema,
}) => {
- assert_eq!(value_type, "string");
- assert_eq!(value, "value1. Cause: Expected: Long. Got: String");
+ assert_eq!(value_type, "str");
+ assert_eq!(value, "Expected Schema::String | Schema::Uuid(String)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
+ // The last 7 bytes are written because the DirectMapSerializer will write immediately and doesn't
+ // know yet that the value is not a long
assert_eq!(
buffer.as_slice(),
&[
4, 10, b'i', b't', b'e', b'm', b'1', 20, 10, b'i', b't', b'e', b'm', b'2', 160, 6,
- 0
+ 0, 2, 10, 105, 116, 101, 109, 49
]
);
@@ -2466,16 +1368,23 @@ mod tests {
}
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- Some(10i64).serialize(&mut serializer)?;
- None::<i64>.serialize(&mut serializer)?;
- NullableLong::Long(400).serialize(&mut serializer)?;
- NullableLong::Null.serialize(&mut serializer)?;
+ Some(10i64).serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ None::<i64>.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ NullableLong::Long(400).serialize(SchemaAwareSerializer::new(
+ &mut buffer,
+ &schema,
+ config,
+ )?)?;
+ NullableLong::Null.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match "invalid"
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2483,11 +1392,8 @@ mod tests {
value,
schema,
}) => {
- assert_eq!(value_type, "string");
- assert_eq!(
- value,
- "invalid. Cause: Expected one of the union variants [Null, Long]. Got: String"
- );
+ assert_eq!(value_type, "str");
+ assert_eq!(value, "Expected Schema::String | Schema::Uuid(String)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2514,15 +1420,26 @@ mod tests {
}
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- LongOrString::Null.serialize(&mut serializer)?;
- LongOrString::Long(400).serialize(&mut serializer)?;
- LongOrString::Str(String::from("test")).serialize(&mut serializer)?;
+ LongOrString::Null.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ LongOrString::Long(400).serialize(SchemaAwareSerializer::new(
+ &mut buffer,
+ &schema,
+ config,
+ )?)?;
+ LongOrString::Str(String::from("test")).serialize(SchemaAwareSerializer::new(
+ &mut buffer,
+ &schema,
+ config,
+ )?)?;
match 1_f64
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2531,10 +1448,7 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "f64");
- assert_eq!(
- value,
- "1. Cause: Cannot find a Double schema in [Null, Long, String]"
- );
+ assert_eq!(value, "Expected Schema::Double");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2559,14 +1473,21 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- Bytes::new(&[10, 124, 31, 97, 14, 201, 3, 88]).serialize(&mut serializer)?;
+ Bytes::new(&[10, 124, 31, 97, 14, 201, 3, 88]).serialize(SchemaAwareSerializer::new(
+ &mut buffer,
+ &schema,
+ config,
+ )?)?;
// non-8 size
match Bytes::new(&[123])
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2575,10 +1496,7 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "bytes");
- assert_eq!(
- value,
- "7b. Cause: Fixed schema size (8) does not match the value length (1)"
- ); // Bytes represents its values as hexadecimals: '7b' is 123
+ assert_eq!(value, "Fixed size (8) does not match value length (1)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2586,7 +1504,7 @@ mod tests {
// array
match [1; 8]
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2594,8 +1512,9 @@ mod tests {
value,
schema,
}) => {
- assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?!
- assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array");
+ // Arrays are serialized as a tuple by Serde
+ assert_eq!(value_type, "tuple");
+ assert_eq!(value, "Expected Schema::Record(fields.len() == 8)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2603,7 +1522,7 @@ mod tests {
// slice
match &[1, 2, 3, 4, 5, 6, 7, 8]
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2611,8 +1530,9 @@ mod tests {
value,
schema,
}) => {
- assert_eq!(*value_type, "tuple"); // TODO: why is this 'tuple' ?!
- assert_eq!(value, "tuple (len=8). Cause: Expected: Fixed. Got: Array");
+ // This is a tuple as Serde serializes array as a tuple
+ assert_eq!(*value_type, "tuple");
+ assert_eq!(value, "Expected Schema::Record(fields.len() == 8)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2635,20 +1555,23 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let val = Decimal::from(&[251, 155]);
- val.serialize(&mut serializer)?;
+ val.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
- match ().serialize(&mut serializer).map_err(Error::into_details) {
+ match ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?).map_err(Error::into_details) {
Err(Details::SerializeValueWithSchema {
value_type,
value,
schema,
}) => {
- assert_eq!(value_type, "none");
- assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null");
+ assert_eq!(value_type, "unit");
+ assert_eq!(value, "Expected Schema::Null");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2673,20 +1596,23 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let val = Decimal::from(&[0, 0, 0, 0, 0, 0, 251, 155]);
- val.serialize(&mut serializer)?;
+ val.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
- match ().serialize(&mut serializer).map_err(Error::into_details) {
+ match ().serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?).map_err(Error::into_details) {
Err(Details::SerializeValueWithSchema {
value_type,
value,
schema,
}) => {
- assert_eq!(value_type, "none");
- assert_eq!(value, "None. Cause: Expected: Decimal. Got: Null");
+ assert_eq!(value_type, "unit");
+ assert_eq!(value, "Expected Schema::Null");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2707,11 +1633,17 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let val = BigDecimal::new(BigInt::new(Sign::Plus, vec![50024]), 2);
- val.serialize(&mut serializer)?;
+ crate::serde::bigdecimal::serialize(
+ &val,
+ SchemaAwareSerializer::new(&mut buffer, &schema, config)?,
+ )?;
assert_eq!(buffer.as_slice(), &[10, 6, 0, 195, 104, 4]);
@@ -2731,21 +1663,30 @@ mod tests {
assert!(!crate::util::is_human_readable());
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
"8c28da81-238c-4326-bddd-4e3d00cc5099"
.parse::<Uuid>()?
- .serialize(&mut serializer)?;
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
- match 1_u8.serialize(&mut serializer).map_err(Error::into_details) {
+ match 1_u8
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .map_err(Error::into_details)
+ {
Err(Details::SerializeValueWithSchema {
value_type,
value,
schema,
}) => {
assert_eq!(value_type, "u8");
- assert_eq!(value, "1. Cause: Expected: Uuid. Got: Int");
+ assert_eq!(
+ value,
+ "Expected Schema::Int | Schema::Date | Schema::TimeMillis"
+ );
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -2771,17 +1712,24 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 100_u8.serialize(&mut serializer)?;
- 1000_u16.serialize(&mut serializer)?;
- 10000_u32.serialize(&mut serializer)?;
- 1000_i16.serialize(&mut serializer)?;
- 10000_i32.serialize(&mut serializer)?;
+ 100_u8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 1000_u16.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 10000_u32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 1000_i16.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 10000_i32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match 10000_f32
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2790,16 +1738,13 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "f32");
- assert_eq!(value, "10000. Cause: Expected: Date. Got: Float");
+ assert_eq!(value, "Expected Schema::Float");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- assert_eq!(
- buffer.as_slice(),
- &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1]
- );
+ assert_eq!(buffer.as_slice(), &[200, 1, 208, 15, 208, 15, 160, 156, 1]);
Ok(())
}
@@ -2814,17 +1759,24 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 100_u8.serialize(&mut serializer)?;
- 1000_u16.serialize(&mut serializer)?;
- 10000_u32.serialize(&mut serializer)?;
- 1000_i16.serialize(&mut serializer)?;
- 10000_i32.serialize(&mut serializer)?;
+ 100_u8.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 1000_u16.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 10000_u32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 1000_i16.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ 10000_i32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match 10000_f32
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2833,16 +1785,13 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "f32");
- assert_eq!(value, "10000. Cause: Expected: TimeMillis. Got: Float");
+ assert_eq!(value, "Expected Schema::Float");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- assert_eq!(
- buffer.as_slice(),
- &[200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1]
- );
+ assert_eq!(buffer.as_slice(), &[200, 1, 208, 15, 208, 15, 160, 156, 1]);
Ok(())
}
@@ -2857,18 +1806,37 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 100_u8.serialize(&mut serializer)?;
- 1000_u16.serialize(&mut serializer)?;
- 10000_u32.serialize(&mut serializer)?;
- 1000_i16.serialize(&mut serializer)?;
- 10000_i32.serialize(&mut serializer)?;
- 10000_i64.serialize(&mut serializer)?;
+ assert!(
+ 100_u8
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 1000_u16
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 10000_u32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 1000_i16
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 10000_i32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 10000_i64.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match 10000_f32
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2877,18 +1845,13 @@ mod tests {
schema,
}) => {
assert_eq!(value_type, "f32");
- assert_eq!(value, "10000. Cause: Expected: TimeMicros. Got: Float");
+ assert_eq!(value, "Expected Schema::Float");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- assert_eq!(
- buffer.as_slice(),
- &[
- 200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1, 160, 156, 1
- ]
- );
+ assert_eq!(buffer.as_slice(), &[160, 156, 1, 160, 156, 1,]);
Ok(())
}
@@ -2904,19 +1867,37 @@ mod tests {
))?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer =
- SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 100_u8.serialize(&mut serializer)?;
- 1000_u16.serialize(&mut serializer)?;
- 10000_u32.serialize(&mut serializer)?;
- 1000_i16.serialize(&mut serializer)?;
- 10000_i32.serialize(&mut serializer)?;
- 10000_i64.serialize(&mut serializer)?;
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
+
+ assert!(
+ 100_u8
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 1000_u16
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 10000_u32.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert!(
+ 1000_i16
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ assert!(
+ 10000_i32
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .is_err()
+ );
+ 10000_i64.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match 10000_f64
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2929,23 +1910,13 @@ mod tests {
capital_precision.replace_range(..1, &c.to_uppercase().to_string());
}
assert_eq!(value_type, "f64");
- assert_eq!(
- value,
- format!(
- "10000. Cause: Expected: Timestamp{capital_precision}. Got: Double"
- )
- );
+ assert_eq!(value, format!("Expected Schema::Double"));
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
}
- assert_eq!(
- buffer.as_slice(),
- &[
- 200, 1, 208, 15, 160, 156, 1, 208, 15, 160, 156, 1, 160, 156, 1
- ]
- );
+ assert_eq!(buffer.as_slice(), &[160, 156, 1, 160, 156, 1,]);
}
Ok(())
@@ -2963,15 +1934,18 @@ mod tests {
)?;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
let duration_bytes =
ByteArray::new(Duration::new(Months::new(3), Days::new(2), Millis::new(1200)).into());
- duration_bytes.serialize(&mut serializer)?;
+ duration_bytes.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
match [1; 12]
- .serialize(&mut serializer)
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
.map_err(Error::into_details)
{
Err(Details::SerializeValueWithSchema {
@@ -2979,11 +1953,9 @@ mod tests {
value,
schema,
}) => {
- assert_eq!(value_type, "tuple"); // TODO: why is this 'tuple' ?!
- assert_eq!(
- value,
- "tuple (len=12). Cause: Expected: Duration. Got: Array"
- );
+ // This is a tuple because Serde serializes arrays [T; N] as tuples
+ assert_eq!(value_type, "tuple");
+ assert_eq!(value, "Expected Schema::Record(fields.len() == 12)");
assert_eq!(schema, schema);
}
unexpected => panic!("Expected an error. Got: {unexpected:?}"),
@@ -3015,6 +1987,7 @@ mod tests {
struct TestRecord {
string_field: String,
int_field: i32,
+ #[serde(with = "crate::serde::bigdecimal")]
big_decimal_field: BigDecimal,
uuid_field: Uuid,
// #[serde(skip_serializing_if = "Option::is_none")] => Never ignore None!
@@ -3024,8 +1997,11 @@ mod tests {
assert!(!crate::util::is_human_readable());
let mut buffer: Vec<u8> = Vec::new();
let rs = ResolvedSchema::try_from(&schema)?;
- let mut serializer =
- SchemaAwareWriteSerializer::new(&mut buffer, &schema, rs.get_names(), None);
+ let config = Config {
+ names: rs.get_names(),
+ target_block_size: None,
+ human_readable: false,
+ };
let good_record = TestRecord {
string_field: String::from("test"),
@@ -3040,7 +2016,7 @@ mod tests {
inner_record: None,
})),
};
- good_record.serialize(&mut serializer)?;
+ good_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
assert_eq!(
buffer.as_slice(),
@@ -3055,7 +2031,9 @@ mod tests {
Ok(())
}
+ // TODO: Figure out what to do with Option<Enum> mapping to Union([Null, ..])
#[test]
+ #[ignore]
fn avro_rs_337_serialize_union_record_variant() -> TestResult {
let schema = Schema::parse_str(
r#"{
@@ -3105,33 +2083,38 @@ mod tests {
let mut buffer: Vec<u8> = Vec::new();
let rs = ResolvedSchema::try_from(&schema)?;
- let mut serializer =
- SchemaAwareWriteSerializer::new(&mut buffer, &schema, rs.get_names(), None);
+ let config = Config {
+ names: rs.get_names(),
+ target_block_size: None,
+ human_readable: false,
+ };
let foo_record = TestRecord {
inner_union: InnerUnion::InnerVariantFoo(InnerRecordFoo {
foo: String::from("foo"),
}),
};
- foo_record.serialize(&mut serializer)?;
+ foo_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let bar_record = TestRecord {
inner_union: InnerUnion::InnerVariantBar(InnerRecordBar {
bar: String::from("bar"),
}),
};
- bar_record.serialize(&mut serializer)?;
+ bar_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let int_record = TestRecord {
inner_union: InnerUnion::IntField(1),
};
- int_record.serialize(&mut serializer)?;
+ int_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let string_record = TestRecord {
inner_union: InnerUnion::StringField(String::from("string")),
};
- string_record.serialize(&mut serializer)?;
+ string_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
Ok(())
}
+ // TODO: Figure out what to do with Option<Enum> mapping to Union([Null, ..])
#[test]
+ #[ignore]
fn avro_rs_337_serialize_option_union_record_variant() -> TestResult {
let schema = Schema::parse_str(
r#"{
@@ -3148,7 +2131,7 @@ mod tests {
]},
{"name": "intField", "type": "int"},
{"name": "stringField", "type": "string"}
- ]
+ ],
}]
}"#,
)?;
@@ -3182,31 +2165,34 @@ mod tests {
let mut buffer: Vec<u8> = Vec::new();
let rs = ResolvedSchema::try_from(&schema)?;
- let mut serializer =
- SchemaAwareWriteSerializer::new(&mut buffer, &schema, rs.get_names(), None);
+ let config = Config {
+ names: rs.get_names(),
+ target_block_size: None,
+ human_readable: false,
+ };
let null_record = TestRecord { inner_union: None };
- null_record.serialize(&mut serializer)?;
+ null_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let foo_record = TestRecord {
inner_union: Some(InnerUnion::InnerVariantFoo(InnerRecordFoo {
foo: String::from("foo"),
})),
};
- foo_record.serialize(&mut serializer)?;
+ foo_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let bar_record = TestRecord {
inner_union: Some(InnerUnion::InnerVariantBar(InnerRecordBar {
bar: String::from("bar"),
})),
};
- bar_record.serialize(&mut serializer)?;
+ bar_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let int_record = TestRecord {
inner_union: Some(InnerUnion::IntField(1)),
};
- int_record.serialize(&mut serializer)?;
+ int_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
let string_record = TestRecord {
inner_union: Some(InnerUnion::StringField(String::from("string"))),
};
- string_record.serialize(&mut serializer)?;
+ string_record.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
Ok(())
}
@@ -3216,9 +2202,9 @@ mod tests {
struct Foo {
a: String,
b: String,
- c: usize,
+ c: i64,
d: f64,
- e: usize,
+ e: i64,
}
let schema = Schema::parse_str(
r#"
@@ -3282,149 +2268,37 @@ mod tests {
let schema = Schema::String;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 'a'.serialize(&mut serializer)?;
-
- assert_eq!(buffer.as_slice(), &[2, b'a']);
-
- Ok(())
- }
-
- #[test]
- fn avro_rs_414_serialize_char_as_bytes() -> TestResult {
- let schema = Schema::Bytes;
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- 'a'.serialize(&mut serializer)?;
+ 'a'.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
assert_eq!(buffer.as_slice(), &[2, b'a']);
Ok(())
}
- #[test]
- fn avro_rs_414_serialize_char_as_fixed() -> TestResult {
- let schema = Schema::Fixed(FixedSchema {
- name: Name::new("char")?,
- aliases: None,
- doc: None,
- size: 4,
- attributes: Default::default(),
- });
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- 'a'.serialize(&mut serializer)?;
-
- assert_eq!(buffer.as_slice(), &[b'a', 0, 0, 0]);
-
- Ok(())
- }
-
#[test]
fn avro_rs_414_serialize_emoji_char_as_string() -> TestResult {
let schema = Schema::String;
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- '👹'.serialize(&mut serializer)?;
-
- assert_eq!(buffer.as_slice(), &[8, 240, 159, 145, 185]);
-
- Ok(())
- }
-
- #[test]
- fn avro_rs_414_serialize_emoji_char_as_bytes() -> TestResult {
- let schema = Schema::Bytes;
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- '👹'.serialize(&mut serializer)?;
+ '👹'.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
assert_eq!(buffer.as_slice(), &[8, 240, 159, 145, 185]);
Ok(())
}
- #[test]
- fn avro_rs_414_serialize_emoji_char_as_fixed() -> TestResult {
- let schema = Schema::Fixed(FixedSchema {
- name: Name::new("char")?,
- aliases: None,
- doc: None,
- size: 4,
- attributes: Default::default(),
- });
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- '👹'.serialize(&mut serializer)?;
-
- // This is a different byte value than the tests above. This is because by creating a String
- // the unicode value is normalized by Rust
- assert_eq!(buffer.as_slice(), &[121, 244, 1, 0]);
-
- Ok(())
- }
-
- #[test]
- fn avro_rs_414_serialize_char_as_fixed_wrong_name() -> TestResult {
- let schema = Schema::Fixed(FixedSchema {
- name: Name::new("characters")?,
- aliases: None,
- doc: None,
- size: 4,
- attributes: Default::default(),
- });
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- assert!(matches!(
- 'a'.serialize(&mut serializer).unwrap_err().details(),
- Details::SerializeValueWithSchema { .. }
- ));
-
- Ok(())
- }
-
- #[test]
- fn avro_rs_414_serialize_char_as_fixed_wrong_size() -> TestResult {
- let schema = Schema::Fixed(FixedSchema {
- name: Name::new("char")?,
- aliases: None,
- doc: None,
- size: 1,
- attributes: Default::default(),
- });
-
- let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
-
- assert!(matches!(
- 'a'.serialize(&mut serializer).unwrap_err().details(),
- Details::SerializeValueWithSchema { .. }
- ));
-
- Ok(())
- }
-
#[test]
fn avro_rs_414_serialize_i128_as_fixed() -> TestResult {
let schema = Schema::Fixed(FixedSchema {
@@ -3436,10 +2310,14 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- let bytes_written = i128::MAX.serialize(&mut serializer)?;
+ let bytes_written =
+ i128::MAX.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
assert_eq!(bytes_written, 16);
assert_eq!(
@@ -3464,11 +2342,17 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
assert!(matches!(
- i128::MAX.serialize(&mut serializer).unwrap_err().details(),
+ i128::MAX
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .unwrap_err()
+ .details(),
Details::SerializeValueWithSchema { .. }
));
@@ -3486,11 +2370,17 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
assert!(matches!(
- i128::MAX.serialize(&mut serializer).unwrap_err().details(),
+ i128::MAX
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .unwrap_err()
+ .details(),
Details::SerializeValueWithSchema { .. }
));
@@ -3508,10 +2398,14 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
- let bytes_written = u128::MAX.serialize(&mut serializer)?;
+ let bytes_written =
+ u128::MAX.serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
assert_eq!(bytes_written, 16);
assert_eq!(
@@ -3536,11 +2430,17 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
assert!(matches!(
- u128::MAX.serialize(&mut serializer).unwrap_err().details(),
+ u128::MAX
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .unwrap_err()
+ .details(),
Details::SerializeValueWithSchema { .. }
));
@@ -3558,11 +2458,17 @@ mod tests {
});
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
assert!(matches!(
- u128::MAX.serialize(&mut serializer).unwrap_err().details(),
+ u128::MAX
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)
+ .unwrap_err()
+ .details(),
Details::SerializeValueWithSchema { .. }
));
@@ -3576,17 +2482,30 @@ mod tests {
{ "name": "fixed4", "type": "fixed", "size": 4 },
{ "name": "fixed8", "type": "fixed", "size": 8 }
]"#,
- )
- .unwrap();
+ )?;
+
+ #[derive(Serialize)]
+ enum UnionOfFixed {
+ #[serde(with = "crate::serde::fixed")]
+ Four([u8; 4]),
+ #[serde(with = "crate::serde::fixed")]
+ Eight([u8; 8]),
+ }
let mut buffer: Vec<u8> = Vec::new();
- let names = HashMap::new();
- let mut serializer = SchemaAwareWriteSerializer::new(&mut buffer, &schema, &names, None);
- let bytes_written = crate::serde::fixed::serialize(&[0, 1, 2, 3], &mut serializer)?;
- assert_eq!(bytes_written, 4);
- let bytes_written =
- crate::serde::fixed::serialize(&[4, 5, 6, 7, 8, 9, 10, 11], &mut serializer)?;
- assert_eq!(bytes_written, 8);
+ let config = Config {
+ names: &HashMap::new(),
+ target_block_size: None,
+ human_readable: false,
+ };
+
+ let bytes_written = UnionOfFixed::Four([0, 1, 2, 3])
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert_eq!(bytes_written, 5);
+
+ let bytes_written = UnionOfFixed::Eight([4, 5, 6, 7, 8, 9, 10, 11])
+ .serialize(SchemaAwareSerializer::new(&mut buffer, &schema, config)?)?;
+ assert_eq!(bytes_written, 9);
assert_eq!(buffer, &[0, 0, 1, 2, 3, 2, 4, 5, 6, 7, 8, 9, 10, 11][..]);
diff --git a/avro/src/serde/ser_schema/record/field_default.rs b/avro/src/serde/ser_schema/record/field_default.rs
new file mode 100644
index 0000000..89e48e1
--- /dev/null
+++ b/avro/src/serde/ser_schema/record/field_default.rs
@@ -0,0 +1,197 @@
+use crate::schema::SchemaKind;
+use crate::{Schema, serde::ser_schema::SERIALIZING_SCHEMA_DEFAULT};
+use serde::Serialize;
+use serde_json::Value;
+
+#[derive(Debug)]
+pub struct SchemaAwareDefault<'v, 's> {
+ value: &'v Value,
+ schema: &'s Schema,
+}
+
+impl<'v, 's> SchemaAwareDefault<'v, 's> {
+ pub fn new(value: &'v Value, schema: &'s Schema) -> Self {
+ Self { value, schema }
+ }
+
+ fn serialize_as_newtype_variant<S>(
+ &self,
+ serializer: S,
+ index: usize,
+ ) -> Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ let Schema::Union(union) = self.schema else {
+ panic!("This function should only be called if self.schema is a Union");
+ };
+ let value = Self::new(self.value, &union.variants()[index]);
+ let index = index as u32;
+ serializer.serialize_newtype_variant(
+ SERIALIZING_SCHEMA_DEFAULT,
+ index,
+ SERIALIZING_SCHEMA_DEFAULT,
+ &value,
+ )
+ }
+
+ fn recursive_type_check(value: &Value, schema: &Schema) -> bool {
+ match (value, schema) {
+ (Value::Null, Schema::Null)
+ | (Value::Bool(_), Schema::Boolean)
+ | (Value::String(_), Schema::Bytes | Schema::String) => true,
+ (Value::Number(n), Schema::Int | Schema::Date | Schema::TimeMillis) if n.is_i64() => {
+ let long = n.as_i64().unwrap();
+ i32::try_from(long).is_ok()
+ }
+ (
+ Value::Number(n),
+ Schema::Long
+ | Schema::TimeMicros
+ | Schema::TimestampMillis
+ | Schema::TimestampMicros
+ | Schema::TimestampNanos
+ | Schema::LocalTimestampMillis
+ | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos,
+ ) if n.is_i64() => true,
+ (Value::Number(n), Schema::Float | Schema::Double) if n.is_f64() => true,
+ (Value::String(s), Schema::Fixed(fixed)) => s.len() == fixed.size,
+ (Value::String(s), Schema::Enum(enum_schema)) => enum_schema.symbols.contains(s),
+ (Value::Object(o), Schema::Record(record)) => record.fields.iter().all(|field| {
+ if let Some(value) = o.get(&field.name) {
+ Self::recursive_type_check(value, &field.schema)
+ } else {
+ field.default.is_some()
+ }
+ }),
+ (Value::Object(o), Schema::Map(map)) => o
+ .values()
+ .all(|value| Self::recursive_type_check(value, &map.types)),
+ (Value::Array(a), Schema::Array(array)) => a
+ .iter()
+ .all(|value| Self::recursive_type_check(value, &array.items)),
+ (_, Schema::Union(union)) => union
+ .variants()
+ .iter()
+ .any(|variant| Self::recursive_type_check(value, variant)),
+ _ => false,
+ }
+ }
+}
+
+impl<'v, 's> Serialize for SchemaAwareDefault<'v, 's> {
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ use serde::ser::Error;
+ match (&self.value, self.schema) {
+ (Value::Null, Schema::Null) => serializer.serialize_unit(),
+ (Value::Bool(boolean), Schema::Boolean) => serializer.serialize_bool(*boolean),
+ (Value::Number(n), Schema::Int | Schema::Date | Schema::TimeMillis) if n.is_i64() => {
+ let long = n.as_i64().unwrap();
+ let int = i32::try_from(long).map_err(|_| {
+ S::Error::custom(format!("Default {long} is too large for {:?}", self.schema))
+ })?;
+ serializer.serialize_i32(int)
+ }
+ (
+ Value::Number(n),
+ Schema::Long
+ | Schema::TimeMicros
+ | Schema::TimestampMillis
+ | Schema::TimestampMicros
+ | Schema::TimestampNanos
+ | Schema::LocalTimestampMillis
+ | Schema::LocalTimestampMicros
+ | Schema::LocalTimestampNanos,
+ ) if n.is_i64() => {
+ let long = n.as_i64().unwrap();
+ serializer.serialize_i64(long)
+ }
+ (Value::Number(n), Schema::Float) if n.is_f64() => {
+ let float = n.as_f64().unwrap();
+ serializer.serialize_f32(float as f32)
+ }
+ (Value::Number(n), Schema::Double) if n.is_f64() => {
+ let double = n.as_f64().unwrap();
+ serializer.serialize_f64(double)
+ }
+ (Value::String(s), Schema::Bytes | Schema::Fixed(_)) => {
+ serializer.serialize_bytes(s.as_bytes())
+ }
+ (Value::String(s), Schema::String) => serializer.serialize_str(s),
+ (Value::String(s), Schema::Enum(enum_schema)) => {
+ let Some((variant_index, _)) = enum_schema
+ .symbols
+ .iter()
+ .enumerate()
+ .find(|(_i, symbol)| *symbol == s)
+ else {
+ return Err(S::Error::custom(format!(
+ "Could not find `{s}` in enum: {enum_schema:?}"
+ )));
+ };
+
+ serializer.serialize_unit_variant(
+ SERIALIZING_SCHEMA_DEFAULT,
+ variant_index as u32,
+ SERIALIZING_SCHEMA_DEFAULT,
+ )
+ }
+ // This abuses the support for flattened fields, which are also serialized as a map.
+ (Value::Object(o), Schema::Record(record)) => serializer.collect_map(
+ o.iter()
+ .enumerate()
+ .map(|(i, (k, v))| (k, Self::new(v, &record.fields[i].schema))),
+ ),
+ (Value::Object(o), Schema::Map(map)) => {
+ serializer.collect_map(o.iter().map(|(k, v)| (k, Self::new(v, &map.types))))
+ }
+ (Value::Array(a), Schema::Array(array)) => {
+ serializer.collect_seq(a.iter().map(|v| Self::new(v, &array.items)))
+ }
+ (_, Schema::Union(union)) => {
+ if union.variants().len() == 2
+ && let Some(index) = union.index().get(&SchemaKind::Null)
+ {
+ // Fast path for options
+ if self.value == &Value::Null {
+ serializer.serialize_none()
+ } else {
+ let other_index = (index + 1) & 1;
+ let value = Self::new(self.value, &union.variants()[other_index]);
+ serializer.serialize_some(&value)
+ }
+ } else {
+ // Find the first variant that can match this value
+ for (index, variant) in union.variants().iter().enumerate() {
+ match (self.value, variant) {
+ (Value::Null, Schema::Null) => {
+ let index = index as u32;
+ return serializer.serialize_unit_variant(
+ SERIALIZING_SCHEMA_DEFAULT,
+ index,
+ SERIALIZING_SCHEMA_DEFAULT,
+ );
+ }
+ _ if Self::recursive_type_check(self.value, variant) => {
+ return self.serialize_as_newtype_variant(serializer, index);
+ }
+ _ => {}
+ }
+ }
+ Err(S::Error::custom(format!(
+ "Could not match default to any variant of {:?}, default: {:?}",
+ self.schema, self.value
+ )))
+ }
+ }
+ _ => Err(S::Error::custom(format!(
+ "Unexpected default for {:?}, default: {:?}",
+ self.schema, self.value
+ ))),
+ }
+ }
+}
diff --git a/avro/src/serde/ser_schema/record/mod.rs b/avro/src/serde/ser_schema/record/mod.rs
new file mode 100644
index 0000000..12e268f
--- /dev/null
+++ b/avro/src/serde/ser_schema/record/mod.rs
@@ -0,0 +1,250 @@
+use std::{cmp::Ordering, collections::HashMap, io::Write};
+
+use serde::ser::{Serialize, SerializeMap, SerializeStruct, SerializeStructVariant};
+
+use super::{Config, SchemaAwareSerializer};
+use crate::serde::ser_schema::record::field_default::SchemaAwareDefault;
+use crate::{
+ Error, Schema,
+ error::Details,
+ schema::{RecordField, RecordSchema},
+ serde::util::StringSerializer,
+};
+
+mod field_default;
+
+pub struct RecordSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ schema: &'s RecordSchema,
+ config: Config<'s>,
+ /// Fields we received in the wrong order
+ field_cache: HashMap<usize, Vec<u8>>,
+ /// The current field name when serializing from a map (for `flatten` support).
+ map_field_name: Option<String>,
+ field_position: usize,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> RecordSerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ schema: &'s RecordSchema,
+ config: Config<'s>,
+ bytes_written: Option<usize>,
+ ) -> Self {
+ Self {
+ writer,
+ schema,
+ config,
+ field_cache: HashMap::new(),
+ map_field_name: None,
+ field_position: 0,
+ bytes_written: bytes_written.unwrap_or(0),
+ }
+ }
+
+ fn serialize_next_field<T>(
+ &mut self,
+ field: &RecordField,
+ position: usize,
+ value: &T,
+ ) -> Result<(), Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ match self.field_position.cmp(&position) {
+ Ordering::Equal => {
+ // If we receive fields in order, write them directly to the main writer
+ let value_ser =
+ SchemaAwareSerializer::new(&mut *self.writer, &field.schema, self.config)?;
+ self.bytes_written += value.serialize(value_ser)?;
+
+ self.field_position += 1;
+ while let Some(bytes) = self.field_cache.remove(&self.field_position) {
+ self.writer.write_all(&bytes).map_err(Details::WriteBytes)?;
+ self.bytes_written += bytes.len();
+ self.field_position += 1;
+ }
+ Ok(())
+ }
+ Ordering::Less => {
+ // Current field position is smaller than this field position,
+ // so we're still missing at least one field, save this field temporarily
+ let mut bytes = Vec::new();
+ let value_ser = SchemaAwareSerializer::new(&mut bytes, &field.schema, self.config)?;
+ value.serialize(value_ser)?;
+ if self.field_cache.insert(position, bytes).is_some() {
+ Err(Details::FieldNameDuplicate(field.name.clone()).into())
+ } else {
+ Ok(())
+ }
+ }
+ Ordering::Greater => {
+ // Current field position is greater than this field position,
+ // so we've already had this field
+ Err(Details::FieldNameDuplicate(field.name.clone()).into())
+ }
+ }
+ }
+
+ fn end(mut self) -> Result<usize, Error> {
+ // Write any fields that are `serde(skip)` or `serde(skip_serializing)`
+ while self.field_position != self.schema.fields.len() {
+ let field_info = &self.schema.fields[self.field_position];
+ if let Some(bytes) = self.field_cache.remove(&self.field_position) {
+ self.writer.write_all(&bytes).map_err(Details::WriteBytes)?;
+ self.bytes_written += bytes.len();
+ self.field_position += 1;
+ } else if let Some(default) = &field_info.default {
+ self.serialize_next_field(
+ field_info,
+ self.field_position,
+ &SchemaAwareDefault::new(default, &field_info.schema),
+ )
+ .map_err(|e| Details::SerializeRecordFieldWithSchema {
+ field_name: field_info.name.clone(),
+ record_schema: Schema::Record(self.schema.clone()),
+ error: Box::new(e),
+ })?;
+ } else {
+ return Err(Details::MissingDefaultForSkippedField {
+ field_name: field_info.name.clone(),
+ schema: Schema::Record(self.schema.clone()),
+ }
+ .into());
+ }
+ }
+
+ debug_assert!(
+ self.field_cache.is_empty(),
+ "There should be no more unwritten fields at this point: {:?}",
+ self.field_cache
+ );
+ debug_assert!(
+ self.map_field_name.is_none(),
+ "There should be no field name at this point: field {:?}",
+ self.map_field_name
+ );
+ Ok(self.bytes_written)
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeStruct for RecordSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ if let Some(position) = self.schema.lookup.get(key).copied() {
+ let field = &self.schema.fields[position];
+ self.serialize_next_field(field, position, value)
+ .map_err(|e| {
+ Details::SerializeRecordFieldWithSchema {
+ field_name: key.to_string(),
+ record_schema: Schema::Record(self.schema.clone()),
+ error: Box::new(e),
+ }
+ .into()
+ })
+ } else {
+ Err(Details::FieldName(String::from(key)).into())
+ }
+ }
+
+ fn skip_field(&mut self, key: &'static str) -> Result<(), Self::Error> {
+ if let Some(position) = self.schema.lookup.get(key).copied() {
+ let field = &self.schema.fields[position];
+ if let Some(default) = &field.default {
+ self.serialize_next_field(field, position, default)
+ .map_err(|e| {
+ Details::SerializeRecordFieldWithSchema {
+ field_name: key.to_string(),
+ record_schema: Schema::Record(self.schema.clone()),
+ error: Box::new(e),
+ }
+ .into()
+ })
+ } else {
+ Err(Details::MissingDefaultForSkippedField {
+ field_name: key.to_string(),
+ schema: Schema::Record(self.schema.clone()),
+ }
+ .into())
+ }
+ } else {
+ Err(Details::GetField(key.to_string()).into())
+ }
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ self.end()
+ }
+}
+
+/// This implementation is used to support `#[serde(flatten)]` as that uses [`SerializeMap`] instead of [`SerializeStruct`].
+///
+/// [`SerializeMap`](ser::SerializeMap)
+/// [`SerializeStruct`](ser::SerializeStruct)
+impl<'s, 'w, W: Write> SerializeMap for RecordSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ let name = key.serialize(StringSerializer)?;
+ let old = self.map_field_name.replace(name);
+ debug_assert!(
+ old.is_none(),
+ "Expected a value instead of a key: old key: {old:?}, new key: {:?}",
+ self.map_field_name
+ );
+ Ok(())
+ }
+
+ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ let key = self.map_field_name.take().ok_or(Details::MapNoKey)?;
+ if let Some(position) = self.schema.lookup.get(&key).copied() {
+ let field = &self.schema.fields[position];
+ self.serialize_next_field(field, position, value)
+ .map_err(|e| {
+ Details::SerializeRecordFieldWithSchema {
+ field_name: key.to_string(),
+ record_schema: Schema::Record(self.schema.clone()),
+ error: Box::new(e),
+ }
+ .into()
+ })
+ } else {
+ Err(Details::FieldName(key).into())
+ }
+ }
+
+ // TODO: Implement serialize_entry, which allows us to skip storing the key
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ self.end()
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeStructVariant for RecordSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_field<T>(&mut self, key: &'static str, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ SerializeStruct::serialize_field(self, key, value)
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ self.end()
+ }
+}
diff --git a/avro/src/serde/ser_schema/tuple.rs b/avro/src/serde/ser_schema/tuple.rs
new file mode 100644
index 0000000..712294a
--- /dev/null
+++ b/avro/src/serde/ser_schema/tuple.rs
@@ -0,0 +1,262 @@
+use std::io::Write;
+
+use super::{Config, SchemaAwareSerializer};
+use crate::error::Details;
+use crate::schema::UnionSchema;
+use crate::serde::ser_schema::union::UnionAwareSerializer;
+use crate::{Error, Schema, schema::RecordSchema};
+use serde::Serialize;
+use serde::ser::{SerializeTuple, SerializeTupleStruct, SerializeTupleVariant};
+
+pub enum TupleSerializer<'s, 'w, W: Write> {
+ Unit(UnitTupleSerializer),
+ One(OneTupleSerializer<'s, 'w, W>),
+ OneUnion(OneUnionTupleSerializer<'s, 'w, W>),
+ Many(ManyTupleSerializer<'s, 'w, W>),
+}
+
+impl<'s, 'w, W: Write> SerializeTuple for TupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ match self {
+ TupleSerializer::Unit(tuple) => tuple.serialize_element(value),
+ TupleSerializer::One(tuple) => tuple.serialize_element(value),
+ TupleSerializer::OneUnion(tuple) => tuple.serialize_element(value),
+ TupleSerializer::Many(tuple) => tuple.serialize_element(value),
+ }
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ match self {
+ TupleSerializer::Unit(tuple) => tuple.end(),
+ TupleSerializer::One(tuple) => tuple.end(),
+ TupleSerializer::OneUnion(tuple) => tuple.end(),
+ TupleSerializer::Many(tuple) => SerializeTuple::end(tuple),
+ }
+ }
+}
+
+pub struct ManyTupleSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ schema: &'s RecordSchema,
+ config: Config<'s>,
+ field_position: usize,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> ManyTupleSerializer<'s, 'w, W> {
+ pub fn new(
+ writer: &'w mut W,
+ schema: &'s RecordSchema,
+ config: Config<'s>,
+ bytes_written: Option<usize>,
+ ) -> Self {
+ Self {
+ writer,
+ schema,
+ config,
+ field_position: 0,
+ bytes_written: bytes_written.unwrap_or(0),
+ }
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeTuple for ManyTupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ let schema = &self.schema.fields[self.field_position].schema;
+ let ser = SchemaAwareSerializer::new(&mut *self.writer, schema, self.config)?;
+ self.bytes_written += value.serialize(ser)?;
+ self.field_position += 1;
+ Ok(())
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ assert_eq!(self.field_position, self.schema.fields.len());
+ Ok(self.bytes_written)
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeTupleStruct for ManyTupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ self.serialize_element(value)
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ SerializeTuple::end(self)
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeTupleVariant for ManyTupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + serde::Serialize,
+ {
+ self.serialize_element(value)
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ SerializeTuple::end(self)
+ }
+}
+
+pub struct OneTupleSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ schema: &'s Schema,
+ config: Config<'s>,
+ written_field: bool,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> OneTupleSerializer<'s, 'w, W> {
+ pub fn new(writer: &'w mut W, schema: &'s Schema, config: Config<'s>) -> Self {
+ Self {
+ writer,
+ schema,
+ config,
+ written_field: false,
+ bytes_written: 0,
+ }
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeTuple for OneTupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ if self.written_field {
+ panic!(
+ "Invalid serialize implementation, serialize_element was called more than once for tuple with one element"
+ )
+ }
+ match self.schema {
+ Schema::Union(union) => {
+ self.bytes_written +=
+ value.serialize(UnionAwareSerializer::new(self.writer, union, self.config))?;
+ }
+ schema => {
+ self.bytes_written += value.serialize(SchemaAwareSerializer::new(
+ self.writer,
+ schema,
+ self.config,
+ )?)?;
+ }
+ }
+ Ok(())
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ if !self.written_field {
+ panic!(
+ "Invalid serialize implementation, serialize_element was never called for tuple with one element"
+ )
+ }
+ Ok(self.bytes_written)
+ }
+}
+
+pub struct OneUnionTupleSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ schema: &'s UnionSchema,
+ config: Config<'s>,
+ written_field: bool,
+ bytes_written: usize,
+}
+
+impl<'s, 'w, W: Write> OneUnionTupleSerializer<'s, 'w, W> {
+ pub fn new(writer: &'w mut W, schema: &'s UnionSchema, config: Config<'s>) -> Self {
+ Self {
+ writer,
+ schema,
+ config,
+ written_field: false,
+ bytes_written: 0,
+ }
+ }
+}
+
+impl<'s, 'w, W: Write> SerializeTuple for OneUnionTupleSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ if self.written_field {
+ panic!(
+ "Invalid serialize implementation, serialize_element was called more than once for tuple with one element"
+ )
+ }
+ self.bytes_written += value.serialize(UnionAwareSerializer::new(
+ self.writer,
+ self.schema,
+ self.config,
+ ))?;
+ Ok(())
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ if !self.written_field {
+ panic!(
+ "Invalid serialize implementation, serialize_element was never called for tuple with one element"
+ )
+ }
+ Ok(self.bytes_written)
+ }
+}
+
+pub struct UnitTupleSerializer {
+ bytes_written: usize,
+}
+
+impl UnitTupleSerializer {
+ pub fn new(bytes_written: Option<usize>) -> Self {
+ Self {
+ bytes_written: bytes_written.unwrap_or(0),
+ }
+ }
+}
+
+impl SerializeTuple for UnitTupleSerializer {
+ type Ok = usize;
+ type Error = Error;
+
+ fn serialize_element<T>(&mut self, _value: &T) -> Result<(), Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ Err(Error::new(Details::SerializeValueWithSchema {
+ value_type: "tuple",
+ value: "Expected no elements for the unit tuple".into(),
+ schema: Schema::Null,
+ }))
+ }
+
+ fn end(self) -> Result<Self::Ok, Self::Error> {
+ Ok(self.bytes_written)
+ }
+}
diff --git a/avro/src/serde/ser_schema/union.rs b/avro/src/serde/ser_schema/union.rs
new file mode 100644
index 0000000..09562c6
--- /dev/null
+++ b/avro/src/serde/ser_schema/union.rs
@@ -0,0 +1,544 @@
+use super::array::ArraySerializer;
+use super::map::MapSerializer;
+use super::record::RecordSerializer;
+use super::tuple::{
+ ManyTupleSerializer, OneUnionTupleSerializer, TupleSerializer, UnitTupleSerializer,
+};
+use super::{Config, MapOrRecordSerializer, SchemaAwareSerializer};
+use crate::encode::{encode_int, encode_long};
+use crate::error::Details;
+use crate::schema::{RecordSchema, SchemaKind, UnionSchema};
+use crate::serde::with::{BytesType, SER_BYTES_TYPE};
+use crate::{Error, Schema};
+use serde::{Serialize, Serializer};
+use std::io::Write;
+
+pub struct UnionAwareSerializer<'s, 'w, W: Write> {
+ writer: &'w mut W,
+ union_schema: &'s UnionSchema,
+ config: Config<'s>,
+}
+
+impl<'s, 'w, W: Write> UnionAwareSerializer<'s, 'w, W> {
+ pub fn new(writer: &'w mut W, union_schema: &'s UnionSchema, config: Config<'s>) -> Self {
+ Self {
+ writer,
+ union_schema,
+ config,
+ }
+ }
+
+ fn error(&self, ty: &'static str, error: impl Into<String>) -> Error {
+ Error::new(Details::SerializeValueWithSchema {
+ value_type: ty,
+ value: error.into(),
+ schema: Schema::Union(self.union_schema.clone()),
+ })
+ }
+
+ pub(super) fn serialize_int(self, original_ty: &'static str, v: i32) -> Result<usize, Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Int).copied() else {
+ return Err(self.error(
+ original_ty,
+ "Expected Schema::Int | Schema::Date | Schema::TimeMillis in variants",
+ ));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ bytes_written += encode_int(v, &mut *self.writer)?;
+ Ok(bytes_written)
+ }
+
+ pub(super) fn serialize_long(self, original_ty: &'static str, v: i64) -> Result<usize, Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Long).copied() else {
+ return Err(self.error(original_ty, "Expected Schema::Long | Schema::TimeMicros | Schema::{,Local}Timestamp{Millis,Micros,Nanos} in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ bytes_written += encode_long(v, &mut *self.writer)?;
+ Ok(bytes_written)
+ }
+
+ fn find_named_schema(&self, name: &str) -> Result<Option<(usize, &'s Schema)>, Error> {
+ self.union_schema
+ .variants()
+ .iter()
+ .enumerate()
+ .find(|(_i, s)| s.name().is_some_and(|n| n.name() == name))
+ .map(|(i, s)| self.resolve_schema(s).map(|s| (i, s)))
+ .transpose()
+ }
+
+ fn find_fixed_n(&self, len: usize) -> Result<Option<(usize, &'s Schema)>, Error> {
+ for (index, schema) in self.union_schema.variants().iter().enumerate() {
+ if let Schema::Fixed(fixed) = self.resolve_schema(schema)?
+ && fixed.size == len
+ {
+ return Ok(Some((index, schema)));
+ }
+ }
+ Ok(None)
+ }
+
+ fn find_record_n_fields(&self, len: usize) -> Result<Option<(usize, &'s RecordSchema)>, Error> {
+ for (index, schema) in self.union_schema.variants().iter().enumerate() {
+ if let Schema::Record(record) = self.resolve_schema(schema)?
+ && record.fields.len() == len
+ {
+ return Ok(Some((index, record)));
+ }
+ }
+ Ok(None)
+ }
+
+ fn resolve_schema(&self, schema: &'s Schema) -> Result<&'s Schema, Error> {
+ if let Schema::Ref { name } = schema {
+ self.config
+ .names
+ .get(name)
+ .copied()
+ .ok_or_else(|| Details::SchemaResolutionError(name.clone()).into())
+ } else {
+ Ok(schema)
+ }
+ }
+
+ fn write_bytes(&mut self, bytes: &[u8]) -> Result<usize, Error> {
+ self.writer.write_all(bytes).map_err(Details::WriteBytes)?;
+
+ Ok(bytes.len())
+ }
+
+ fn write_bytes_with_len(&mut self, bytes: &[u8]) -> Result<usize, Error> {
+ let mut bytes_written: usize = 0;
+
+ bytes_written += encode_long(bytes.len() as i64, &mut *self.writer)?;
+ bytes_written += self.write_bytes(bytes)?;
+
+ Ok(bytes_written)
+ }
+}
+
+impl<'s, 'w, W: Write> Serializer for UnionAwareSerializer<'s, 'w, W> {
+ type Ok = usize;
+ type Error = Error;
+ type SerializeSeq = ArraySerializer<'s, 'w, W>;
+ type SerializeTuple = TupleSerializer<'s, 'w, W>;
+ type SerializeTupleStruct = ManyTupleSerializer<'s, 'w, W>;
+ type SerializeTupleVariant = ManyTupleSerializer<'s, 'w, W>;
+ type SerializeMap = MapOrRecordSerializer<'s, 'w, W>;
+ type SerializeStruct = RecordSerializer<'s, 'w, W>;
+ type SerializeStructVariant = RecordSerializer<'s, 'w, W>;
+
+ fn serialize_bool(self, v: bool) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Boolean).copied() else {
+ return Err(self.error("bool", "Expected Schema::Boolean in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ self.writer
+ .write_all(&[v as u8])
+ .map_err(Details::WriteBytes)?;
+ bytes_written += 1;
+ Ok(bytes_written)
+ }
+
+ fn serialize_i8(self, v: i8) -> Result<Self::Ok, Self::Error> {
+ self.serialize_int("i8", i32::from(v))
+ }
+
+ fn serialize_i16(self, v: i16) -> Result<Self::Ok, Self::Error> {
+ self.serialize_int("i16", i32::from(v))
+ }
+
+ fn serialize_i32(self, v: i32) -> Result<Self::Ok, Self::Error> {
+ self.serialize_int("i32", v)
+ }
+
+ fn serialize_i64(self, v: i64) -> Result<Self::Ok, Self::Error> {
+ self.serialize_long("i64", v)
+ }
+
+ fn serialize_i128(mut self, v: i128) -> Result<Self::Ok, Self::Error> {
+ if let Some((index, Schema::Fixed(fixed))) = self.find_named_schema("i128")?
+ && fixed.size == 16
+ {
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_le_bytes();
+ bytes_written += self.write_bytes(&bytes)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "i128",
+ r#"Expected Schema::Fixed(name: "i128", size: 16) in variants"#,
+ ))
+ }
+ }
+
+ fn serialize_u8(self, v: u8) -> Result<Self::Ok, Self::Error> {
+ self.serialize_int("u8", i32::from(v))
+ }
+
+ fn serialize_u16(self, v: u16) -> Result<Self::Ok, Self::Error> {
+ self.serialize_int("u16", i32::from(v))
+ }
+
+ fn serialize_u32(self, v: u32) -> Result<Self::Ok, Self::Error> {
+ self.serialize_long("u32", i64::from(v))
+ }
+
+ fn serialize_u64(mut self, v: u64) -> Result<Self::Ok, Self::Error> {
+ if let Some((index, Schema::Fixed(fixed))) = self.find_named_schema("u64")?
+ && fixed.size == 8
+ {
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_le_bytes();
+ bytes_written += self.write_bytes(&bytes)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "u64",
+ r#"Expected Schema::Fixed(name: "u64", size: 8) in variants"#,
+ ))
+ }
+ }
+
+ fn serialize_u128(mut self, v: u128) -> Result<Self::Ok, Self::Error> {
+ if let Some((index, Schema::Fixed(fixed))) = self.find_named_schema("u128")?
+ && fixed.size == 16
+ {
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_le_bytes();
+ bytes_written += self.write_bytes(&bytes)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "u64",
+ r#"Expected Schema::Fixed(name: "u128", size: 16) in variants"#,
+ ))
+ }
+ }
+
+ fn serialize_f32(mut self, v: f32) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Float).copied() else {
+ return Err(self.error("f32", "Expected Schema::Float in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_le_bytes();
+ bytes_written += self.write_bytes(&bytes)?;
+ Ok(bytes_written)
+ }
+
+ fn serialize_f64(mut self, v: f64) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Double).copied() else {
+ return Err(self.error("f64", "Expected Schema::Double in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_le_bytes();
+ bytes_written += self.write_bytes(&bytes)?;
+ Ok(bytes_written)
+ }
+
+ fn serialize_char(mut self, v: char) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::String).copied() else {
+ return Err(self.error("char", "Expected Schema::String in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.to_string().into_bytes();
+ bytes_written += self.write_bytes_with_len(&bytes)?;
+ Ok(bytes_written)
+ }
+
+ fn serialize_str(mut self, v: &str) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::String).copied() else {
+ return Err(self.error("str", "Expected Schema::String in variants"));
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let bytes = v.as_bytes();
+ bytes_written += self.write_bytes_with_len(&bytes)?;
+ Ok(bytes_written)
+ }
+
+ fn serialize_bytes(mut self, v: &[u8]) -> Result<Self::Ok, Self::Error> {
+ let (index, schema) = match SER_BYTES_TYPE.get() {
+ BytesType::Bytes => {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Bytes).copied() else {
+ return Err(self.error("bytes", "Expected Schema::Bytes in variants"));
+ };
+ (index, &Schema::Bytes)
+ }
+ BytesType::Fixed => {
+ let Some((index, schema)) = self.find_fixed_n(v.len())? else {
+ return Err(self.error(
+ "bytes",
+ format!("Expected Schema::Fixed(size = {}) in variants", v.len()),
+ ));
+ };
+ (index, schema)
+ }
+ BytesType::Unset => {
+ let potential_bytes_index =
+ self.union_schema.index().get(&SchemaKind::Bytes).copied();
+ let potential_fixed_index = self.find_fixed_n(v.len())?;
+ match (potential_bytes_index, potential_fixed_index) {
+ (Some(bytes_index), Some((fixed_index, fixed_schema))) => {
+ if bytes_index < fixed_index {
+ (bytes_index, &Schema::Bytes)
+ } else {
+ (fixed_index, fixed_schema)
+ }
+ }
+ (Some(bytes_index), None) => (bytes_index, &Schema::Bytes),
+ (None, Some((fixed_index, fixed_schema))) => (fixed_index, fixed_schema),
+ (None, None) => {
+ return Err(self.error(
+ "bytes",
+ format!(
+ "Expected Schema::Bytes or Schema::Fixed(size: {}) in variants",
+ v.len()
+ ),
+ ));
+ }
+ }
+ }
+ };
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ match schema {
+ Schema::Bytes => {
+ bytes_written += self.write_bytes_with_len(v)?;
+ }
+ Schema::Fixed(_) => {
+ bytes_written += self.write_bytes(v)?;
+ }
+ _ => unreachable!(),
+ }
+ Ok(bytes_written)
+ }
+
+ fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
+ Err(self.error("None", "Nested unions are not supported"))
+ }
+
+ fn serialize_some<T>(self, _value: &T) -> Result<Self::Ok, Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ Err(self.error("Some", "Nested unions are not supported"))
+ }
+
+ fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Null).copied() else {
+ return Err(self.error("()", "Expected Schema::Null in variants"));
+ };
+ encode_int(index as i32, self.writer)
+ }
+
+ fn serialize_unit_struct(self, name: &'static str) -> Result<Self::Ok, Self::Error> {
+ let Some((index, Schema::Record(schema))) = self.find_named_schema(name)? else {
+ return Err(self.error(
+ "unit struct",
+ format!("Expected Schema::Record(name: {name}, fields: []) in variants"),
+ ));
+ };
+ if !schema.fields.is_empty() {
+ return Err(self.error(
+ "unit struct",
+ format!("Expected Schema::Record(name: {name}, fields: []) in variants"),
+ ));
+ }
+ encode_int(index as i32, self.writer)
+ }
+
+ fn serialize_unit_variant(
+ self,
+ name: &'static str,
+ variant_index: u32,
+ variant: &'static str,
+ ) -> Result<Self::Ok, Self::Error> {
+ if let Some((index, Schema::Enum(schema))) = self.find_named_schema(name)?
+ && schema.symbols[variant_index as usize] == variant
+ {
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ bytes_written += encode_int(variant_index as i32, &mut *self.writer)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "unit variant",
+ format!(
+ "Expected Enum(name: {name}, symbols[{variant_index}] == {variant}) in variants"
+ ),
+ ))
+ }
+ }
+
+ fn serialize_newtype_struct<T>(
+ self,
+ name: &'static str,
+ value: &T,
+ ) -> Result<Self::Ok, Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ if let Some((index, Schema::Record(record))) = self.find_named_schema(name)?
+ && record.fields.len() == 1
+ {
+ let mut bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ let ser =
+ SchemaAwareSerializer::new(self.writer, &record.fields[0].schema, self.config)?;
+ bytes_written += value.serialize(ser)?;
+ Ok(bytes_written)
+ } else {
+ Err(self.error(
+ "newtype struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == 1) in variants"),
+ ))
+ }
+ }
+
+ fn serialize_newtype_variant<T>(
+ self,
+ _name: &'static str,
+ _variant_index: u32,
+ _variant: &'static str,
+ _value: &T,
+ ) -> Result<Self::Ok, Self::Error>
+ where
+ T: ?Sized + Serialize,
+ {
+ Err(self.error("newtype struct", "Nested unions are not supported"))
+ }
+
+ fn serialize_seq(self, len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {
+ let Some(index) = self.union_schema.index().get(&SchemaKind::Array).copied() else {
+ return Err(self.error("seq", "Expected Schema::Array in variants"));
+ };
+ let Some(Schema::Array(schema)) = self.union_schema.variants().get(index) else {
+ unreachable!("SchemaKind::Array at this index, so there must be a Schema:Array");
+ };
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ ArraySerializer::new(self.writer, schema, self.config, len, Some(bytes_written))
+ }
+
+ fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, Self::Error> {
+ if len == 0 {
+ let index = self
+ .union_schema
+ .index()
+ .get(&SchemaKind::Null)
+ .copied()
+ .ok_or_else(|| self.error("tuple", "Expected Schema::Null in variants"))?;
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(TupleSerializer::Unit(UnitTupleSerializer::new(Some(
+ bytes_written,
+ ))))
+ } else if len == 1 {
+ Ok(TupleSerializer::OneUnion(OneUnionTupleSerializer::new(
+ self.writer,
+ self.union_schema,
+ self.config,
+ )))
+ } else if let Some((index, record)) = self.find_record_n_fields(len)? {
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(TupleSerializer::Many(ManyTupleSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ )))
+ } else {
+ Err(self.error(
+ "tuple",
+ format!(
+ "Expected Schema::Null | Schema::Record(fields.len() == {len}) in variants"
+ ),
+ ))
+ }
+ }
+
+ fn serialize_tuple_struct(
+ self,
+ name: &'static str,
+ len: usize,
+ ) -> Result<Self::SerializeTupleStruct, Self::Error> {
+ if let Some((index, Schema::Record(schema))) = self.find_named_schema(name)?
+ && schema.fields.len() == len
+ {
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(ManyTupleSerializer::new(
+ self.writer,
+ schema,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else {
+ Err(self.error(
+ "newtype struct",
+ format!("Expected Schema::Record(name: {name}, fields.len() == {len}) in variants"),
+ ))
+ }
+ }
+
+ fn serialize_tuple_variant(
+ self,
+ _name: &'static str,
+ _variant_index: u32,
+ _variant: &'static str,
+ _len: usize,
+ ) -> Result<Self::SerializeTupleVariant, Self::Error> {
+ Err(self.error("tuple variant", "Nested unions are not supported"))
+ }
+
+ fn serialize_map(self, len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {
+ if let Some(index) = self.union_schema.index().get(&SchemaKind::Map).copied() {
+ let Some(Schema::Map(map)) = self.union_schema.variants().get(index) else {
+ unreachable!("SchemaKind::Map at this index, so there must be a Schema:Map");
+ };
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(MapOrRecordSerializer::Map(MapSerializer::new(
+ self.writer,
+ map,
+ self.config,
+ len,
+ Some(bytes_written),
+ )?))
+ } else if let Some((index, record)) = self.find_record_n_fields(len.unwrap())? {
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(MapOrRecordSerializer::Record(RecordSerializer::new(
+ self.writer,
+ record,
+ self.config,
+ Some(bytes_written),
+ )))
+ } else {
+ Err(self.error("map", "Expected Schema::Map in variants"))
+ }
+ }
+
+ fn serialize_struct(
+ self,
+ name: &'static str,
+ _len: usize,
+ ) -> Result<Self::SerializeStruct, Self::Error> {
+ if let Some((index, Schema::Record(schema))) = self.find_named_schema(name)? {
+ let bytes_written = encode_int(index as i32, &mut *self.writer)?;
+ Ok(RecordSerializer::new(
+ self.writer,
+ schema,
+ self.config,
+ Some(bytes_written),
+ ))
+ } else {
+ Err(self.error(
+ "struct",
+ format!("Expect Schema::Record(name: {name}) in variants"),
+ ))
+ }
+ }
+
+ fn serialize_struct_variant(
+ self,
+ _name: &'static str,
+ _variant_index: u32,
+ _variant: &'static str,
+ _len: usize,
+ ) -> Result<Self::SerializeStructVariant, Self::Error> {
+ Err(self.error("struct variant", "Nested unions are not supported"))
+ }
+}
diff --git a/avro/src/serde/with.rs b/avro/src/serde/with.rs
index d6a2a8b..fc3b5bc 100644
--- a/avro/src/serde/with.rs
+++ b/avro/src/serde/with.rs
@@ -506,6 +506,313 @@ pub mod slice_opt {
}
}
+/// Efficient (de)serialization of [`BigDecimal`] values.
+///
+/// This module is intended to be used through the Serde `with` attribute.
+///
+/// Use [`apache_avro::serde::bigdecimal_opt`] for optional decimal values.
+///
+/// When used with different serialization formats, this is equivalent to [`serde_bytes`].
+///
+/// See usage with below example:
+/// ```
+/// # use apache_avro::AvroSchema;
+/// # use serde::{Deserialize, Serialize};
+/// # use bigdecimal::BigDecimal;
+/// #[derive(AvroSchema, Serialize, Deserialize)]
+/// struct StructWithBytes<'a> {
+/// #[avro(with)]
+/// #[serde(with = "apache_avro::serde::bigdecimal")]
+/// decimal: BigDecimal,
+/// }
+/// ```
+///
+/// [`BigDecimal`]: bigdecimal::BigDecimal
+/// [`apache_avro::serde::bigdecimal_opt`]: bigdecimal_opt
+pub mod bigdecimal {
+ use std::collections::HashSet;
+
+ use bigdecimal::BigDecimal;
+ use serde::{Deserializer, Serializer, de::Error as _, ser::Error as _};
+
+ use crate::{
+ Schema,
+ bigdecimal::{big_decimal_as_bytes, deserialize_big_decimal},
+ schema::{Name, Namespace, RecordField},
+ serde::with::BytesType,
+ };
+
+ /// Returns [`Schema::BigDecimal`]
+ pub fn get_schema_in_ctxt(_: &mut HashSet<Name>, _: &Namespace) -> Schema {
+ Schema::BigDecimal
+ }
+
+ /// Returns `None`
+ pub fn get_record_fields_in_ctxt(
+ _: usize,
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Option<Vec<RecordField>> {
+ None
+ }
+
+ pub fn serialize<S>(decimal: &BigDecimal, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ let _guard = super::BytesTypeGuard::set(BytesType::Bytes);
+ let decimal_bytes = big_decimal_as_bytes(decimal).map_err(S::Error::custom)?;
+ serde_bytes::serialize(&decimal_bytes, serializer)
+ }
+
+ pub fn deserialize<'de, D>(deserializer: D) -> Result<BigDecimal, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let _bytes_guard = super::BytesTypeGuard::set(BytesType::Bytes);
+ let _guard = super::BorrowedGuard::set(true);
+ let bytes: &'de [u8] = serde_bytes::deserialize(deserializer)?;
+
+ deserialize_big_decimal(bytes).map_err(D::Error::custom)
+ }
+}
+
+/// Efficient (de)serialization of optional [`BigDecimal`] values.
+///
+/// This module is intended to be used through the Serde `with` attribute.
+///
+/// Use [`apache_avro::serde::bigdecimal`] for non-optional decimal values.
+///
+/// When used with different serialization formats, this is equivalent to [`serde_bytes`].
+///
+/// See usage with below example:
+/// ```
+/// # use apache_avro::AvroSchema;
+/// # use serde::{Deserialize, Serialize};
+/// # use bigdecimal::BigDecimal;
+/// #[derive(AvroSchema, Serialize, Deserialize)]
+/// struct StructWithBytes<'a> {
+/// #[avro(with)]
+/// #[serde(with = "apache_avro::serde::bigdecimal_opt")]
+/// decimal: Option<BigDecimal>,
+/// }
+/// ```
+///
+/// [`BigDecimal`]: bigdecimal::BigDecimal
+/// [`apache_avro::serde::bigdecimal`]: bigdecimal
+pub mod bigdecimal_opt {
+ use bigdecimal::BigDecimal;
+ use serde::{Deserializer, Serializer, de::Error as _, ser::Error as _};
+ use std::collections::HashSet;
+
+ use crate::{
+ Schema,
+ bigdecimal::{big_decimal_as_bytes, deserialize_big_decimal},
+ schema::{Name, Namespace, RecordField, UnionSchema},
+ serde::with::BytesType,
+ };
+
+ /// Returns `Schema::Union(Schema::Null, Schema::BigDecimal)`
+ pub fn get_schema_in_ctxt(_: &mut HashSet<Name>, _: &Namespace) -> Schema {
+ Schema::Union(
+ UnionSchema::new(vec![Schema::Null, Schema::BigDecimal])
+ .expect("This is a valid union"),
+ )
+ }
+
+ /// Returns `None`
+ pub fn get_record_fields_in_ctxt(
+ _: usize,
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Option<Vec<RecordField>> {
+ None
+ }
+
+ pub fn serialize<S, B>(decimal: &Option<BigDecimal>, serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ {
+ let _guard = super::BytesTypeGuard::set(BytesType::Bytes);
+ if let Some(decimal) = decimal {
+ let decimal_bytes = big_decimal_as_bytes(decimal).map_err(S::Error::custom)?;
+ serde_bytes::serialize(&Some(decimal_bytes), serializer)
+ } else {
+ serde_bytes::serialize(&None::<Vec<u8>>, serializer)
+ }
+ }
+
+ pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<BigDecimal>, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let _bytes_guard = super::BytesTypeGuard::set(BytesType::Bytes);
+ let _guard = super::BorrowedGuard::set(true);
+ let bytes: Option<&'de [u8]> = serde_bytes::deserialize(deserializer)?;
+ if let Some(bytes) = bytes {
+ deserialize_big_decimal(bytes)
+ .map(Some)
+ .map_err(D::Error::custom)
+ } else {
+ Ok(None)
+ }
+ }
+}
+
+/// (De)serialize an Rust array (`[T; N]`) as an Avro [`Schema::Array`].
+///
+/// This module is intended to be used through the Serde `with` attribute.
+///
+/// Use [`apache_avro::serde::array_opt`] for optional array values.
+///
+/// See usage with below example:
+/// ```
+/// # use apache_avro::AvroSchema;
+/// # use serde::{Deserialize, Serialize};
+/// #[derive(AvroSchema, Serialize, Deserialize)]
+/// struct StructWithBytes<'a> {
+/// #[avro(with)]
+/// #[serde(with = "apache_avro::serde::array")]
+/// array: [i32; 10],
+/// }
+/// ```
+///
+/// [`apache_avro::serde::array_opt`]: array_opt
+pub mod array {
+ use crate::{
+ AvroSchema, AvroSchemaComponent, Schema,
+ schema::{Name, Namespace, RecordField},
+ };
+ use serde::de::DeserializeOwned;
+ use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
+ use std::collections::HashSet;
+
+ /// Returns `Schema::Array(T::get_schema())`
+ pub fn get_schema_in_ctxt<T: AvroSchemaComponent>(
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Schema {
+ Schema::array(T::get_schema()).build()
+ }
+
+ /// Returns `None`
+ pub fn get_record_fields_in_ctxt(
+ _: usize,
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Option<Vec<RecordField>> {
+ None
+ }
+
+ pub fn serialize<const N: usize, S, T>(value: &[T; N], serializer: S) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ T: Serialize,
+ {
+ value.as_slice().serialize(serializer)
+ }
+
+ pub fn deserialize<'de, const N: usize, D, T>(deserializer: D) -> Result<[T; N], D::Error>
+ where
+ D: Deserializer<'de>,
+ T: DeserializeOwned,
+ {
+ let bytes = <Vec<T> as Deserialize>::deserialize(deserializer)?;
+ bytes.try_into().map_err(|v: Vec<T>| {
+ D::Error::custom(format!(
+ "Deserialized array has length {} which does not match array length of {N}",
+ v.len()
+ ))
+ })
+ }
+}
+
+/// (De)serialize an optional Rust array (`Option<[T; N]>`) as an Avro `Schema::Union([Schema::Null, Schema::Array])`.
+///
+/// This module is intended to be used through the Serde `with` attribute.
+///
+/// Use [`apache_avro::serde::array`] for non-optional array values.
+///
+/// When used with different serialization formats, this is equivalent to [`serde_bytes`].
+///
+/// See usage with below example:
+/// ```
+/// # use apache_avro::AvroSchema;
+/// # use serde::{Deserialize, Serialize};
+/// #[derive(AvroSchema, Serialize, Deserialize)]
+/// struct StructWithBytes<'a> {
+/// #[avro(with)]
+/// #[serde(with = "apache_avro::serde::array_opt")]
+/// array: Option<[i32; 10]>,
+/// }
+/// ```
+///
+/// [`apache_avro::serde::array`]: array
+pub mod array_opt {
+ use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
+ use std::collections::HashSet;
+
+ use crate::{
+ AvroSchema, AvroSchemaComponent, Schema,
+ schema::{Name, Namespace, RecordField, UnionSchema},
+ };
+
+ /// Returns `Schema::Union(Schema::Null, Schema::Array(T::get_schema()))`
+ pub fn get_schema_in_ctxt<T: AvroSchemaComponent>(
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Schema {
+ Schema::Union(
+ UnionSchema::new(vec![Schema::Null, Schema::array(T::get_schema()).build()])
+ .expect("This is a valid union"),
+ )
+ }
+
+ /// Returns `None`
+ pub fn get_record_fields_in_ctxt(
+ _: usize,
+ _: &mut HashSet<Name>,
+ _: &Namespace,
+ ) -> Option<Vec<RecordField>> {
+ None
+ }
+
+ pub fn serialize<const N: usize, S, T>(
+ value: &Option<[T; N]>,
+ serializer: S,
+ ) -> Result<S::Ok, S::Error>
+ where
+ S: Serializer,
+ T: Serialize,
+ {
+ if let Some(array) = value {
+ Some(array.as_slice()).serialize(serializer)
+ } else {
+ None::<Vec<T>>.serialize(serializer)
+ }
+ }
+
+ pub fn deserialize<'de, const N: usize, D, T>(
+ deserializer: D,
+ ) -> Result<Option<[T; N]>, D::Error>
+ where
+ D: Deserializer<'de>,
+ T: Deserialize<'de>,
+ {
+ let bytes = <Option<Vec<T>> as Deserialize>::deserialize(deserializer)?;
+ if let Some(bytes) = bytes {
+ Ok(Some(bytes.try_into().map_err(|v: Vec<T>| {
+ D::Error::custom(format!(
+ "Deserialized array has length {} which does not match array length of {N}",
+ v.len()
+ ))
+ })?))
+ } else {
+ Ok(None)
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
use crate::{Schema, from_value, to_value, types::Value};
diff --git a/avro/src/writer.rs b/avro/src/writer.rs
index fe264a1..22b9aa1 100644
--- a/avro/src/writer.rs
+++ b/avro/src/writer.rs
@@ -22,9 +22,13 @@ use crate::{
error::Details,
headers::{HeaderBuilder, RabinFingerprintHeader},
schema::{NamesRef, ResolvedOwnedSchema, ResolvedSchema, Schema},
- serde::{AvroSchema, ser_schema::SchemaAwareWriteSerializer},
+ serde::{
+ AvroSchema,
+ ser_schema::{Config, SchemaAwareSerializer},
+ },
types::Value,
};
+use bon::Builder;
use serde::Serialize;
use std::{
collections::HashMap, io::Write, marker::PhantomData, mem::ManuallyDrop, ops::RangeInclusive,
@@ -49,6 +53,8 @@ pub struct Writer<'a, W: Write> {
marker: [u8; 16],
has_header: bool,
user_metadata: HashMap<String, Value>,
+ human_readable: bool,
+ map_array_target_block_size: Option<usize>,
}
#[bon::bon]
@@ -67,6 +73,8 @@ impl<'a, W: Write> Writer<'a, W> {
#[builder(default = false)]
has_header: bool,
#[builder(default)] user_metadata: HashMap<String, Value>,
+ #[builder(default = false)] human_readable: bool,
+ map_array_target_block_size: Option<usize>,
) -> AvroResult<Self> {
let resolved_schema = if let Some(schemata) = schemata {
ResolvedSchema::try_from(schemata)?
@@ -84,6 +92,8 @@ impl<'a, W: Write> Writer<'a, W> {
marker,
has_header,
user_metadata,
+ human_readable,
+ map_array_target_block_size,
})
}
}
@@ -271,13 +281,16 @@ impl<'a, W: Write> Writer<'a, W> {
pub fn append_ser<S: Serialize>(&mut self, value: S) -> AvroResult<usize> {
let n = self.maybe_write_header()?;
- let mut serializer = SchemaAwareWriteSerializer::new(
+ let serializer = SchemaAwareSerializer::new(
&mut self.buffer,
self.schema,
- self.resolved_schema.get_names(),
- None,
- );
- value.serialize(&mut serializer)?;
+ Config {
+ names: self.resolved_schema.get_names(),
+ target_block_size: self.map_array_target_block_size,
+ human_readable: self.human_readable,
+ },
+ )?;
+ value.serialize(serializer)?;
self.num_values += 1;
if self.buffer.len() >= self.block_size {
@@ -704,12 +717,29 @@ impl GenericSingleObjectWriter {
}
/// Writer that encodes messages according to the single object encoding v1 spec
+#[derive(Builder)]
pub struct SpecificSingleObjectWriter<T>
where
T: AvroSchema,
{
+ // TODO: Make default fallibe
+ #[builder(
+ with = |schema: Schema| -> Result<_, Error> { ResolvedOwnedSchema::new(schema) },
+ default = ResolvedOwnedSchema::new(T::get_schema()).expect("Invalid schema")
+ )]
resolved: ResolvedOwnedSchema,
+ #[builder(
+ default = RabinFingerprintHeader::from_schema(resolved.get_root_schema()).build_header(),
+ with = |header_builder: impl HeaderBuilder| header_builder.build_header(),
+ )]
header: Vec<u8>,
+ /// Should data be encoded as human-readable where possible.
+ human_readable: bool,
+ /// How many bytes should blocks of array and map values be.
+ ///
+ /// Every block except the last will have at least this size.
+ map_array_target_block_size: Option<usize>,
+ #[builder(skip)]
_model: PhantomData<T>,
}
@@ -725,6 +755,8 @@ where
Ok(Self {
resolved,
header,
+ human_readable: false,
+ map_array_target_block_size: None,
_model: PhantomData,
})
}
@@ -736,6 +768,8 @@ where
resolved,
header,
_model: PhantomData,
+ human_readable: false,
+ map_array_target_block_size: None,
})
}
@@ -786,6 +820,8 @@ where
self.resolved.get_names(),
data,
writer,
+ self.human_readable,
+ self.map_array_target_block_size,
)?;
Ok(bytes + self.header.len())
@@ -852,9 +888,19 @@ pub fn write_avro_datum_ref<T: Serialize, W: Write>(
names: &NamesRef,
data: &T,
writer: &mut W,
+ human_readable: bool,
+ map_array_target_block_size: Option<usize>,
) -> AvroResult<usize> {
- let mut serializer = SchemaAwareWriteSerializer::new(writer, schema, names, None);
- data.serialize(&mut serializer)
+ let serializer = SchemaAwareSerializer::new(
+ writer,
+ schema,
+ Config {
+ names,
+ target_block_size: map_array_target_block_size,
+ human_readable,
+ },
+ )?;
+ data.serialize(serializer)
}
/// Encode a value into raw Avro data, also performs schema validation.
@@ -952,6 +998,7 @@ mod tests {
#[test]
fn avro_rs_193_write_avro_datum_ref() -> TestResult {
#[derive(Serialize)]
+ #[serde(rename = "test")]
struct TestStruct {
a: i64,
b: String,
@@ -969,7 +1016,8 @@ mod tests {
zig_i64(3, &mut expected)?;
expected.extend([b'f', b'o', b'o']);
- let bytes = write_avro_datum_ref(&schema, &HashMap::new(), &data, &mut writer)?;
+ let bytes =
+ write_avro_datum_ref(&schema, &HashMap::new(), &data, &mut writer, false, None)?;
assert_eq!(bytes, expected.len());
assert_eq!(writer, expected);
@@ -1245,6 +1293,7 @@ mod tests {
}
#[derive(Debug, Clone, Deserialize, Serialize)]
+ #[serde(rename = "test")]
struct TestSerdeSerialize {
a: i64,
b: String,
@@ -1576,7 +1625,7 @@ mod tests {
let schema = r#"
{
"type":"record",
- "name":"TestSingleObjectWrtierSerialize",
+ "name":"TestSingleObjectWriter",
"fields":[
{
"name":"a",
@@ -1785,7 +1834,7 @@ mod tests {
Err(e) => {
assert_eq!(
e.to_string(),
- r#"Failed to serialize field 'time' for record Record(RecordSchema { name: Name { name: "Conference", .. }, fields: [RecordField { name: "name", schema: String, .. }, RecordField { name: "date", aliases: ["time2", "time"], schema: Union(UnionSchema { schemas: [Null, Long] }), .. }], .. }): Failed to serialize value of type f64 using schema Union(UnionSchema { schemas: [Null, Long] }): 12345678.9. Cause: Cannot find a Double schema in [Null, Long]"#
+ r#"Failed to serialize field 'time' for record Record(RecordSchema { name: Name { name: "Conference", .. }, fields: [RecordField { name: "name", schema: String, .. }, RecordField { name: "date", aliases: ["time2", "time"], schema: Union(UnionSchema { schemas: [Null, Long] }), .. }], .. }): Failed to serialize value of type f64 using schema Long: Expected Schema::Double"#
);
}
}
diff --git a/avro/tests/avro-rs-285-bytes_deserialization.rs b/avro/tests/avro-rs-285-bytes_deserialization.rs
index b9ebe80..cf02567 100644
--- a/avro/tests/avro-rs-285-bytes_deserialization.rs
+++ b/avro/tests/avro-rs-285-bytes_deserialization.rs
@@ -36,7 +36,7 @@ fn avro_rs_285_bytes_deserialization_round_trip() -> TestResult {
let raw_schema = r#"
{
"type": "record",
- "name": "SimpleRecord",
+ "name": "ExampleByteArray",
"fields": [
{"name": "data_bytes", "type": ["null", "bytes"], "default": null},
{"name": "description", "type": ["null", "string"], "default": null}
@@ -84,7 +84,7 @@ fn avro_rs_285_bytes_deserialization_filtered_round_trip() -> TestResult {
let raw_schema = r#"
{
"type": "record",
- "name": "SimpleRecord",
+ "name": "ExampleByteArray",
"fields": [
{"name": "data_bytes", "type": ["null", "bytes"], "default": null},
{"name": "description", "type": ["null", "string"], "default": null}
diff --git a/avro/tests/serde_human_readable_false.rs b/avro/tests/serde_human_readable_false.rs
index 2f8be3f..8289914 100644
--- a/avro/tests/serde_human_readable_false.rs
+++ b/avro/tests/serde_human_readable_false.rs
@@ -15,131 +15,131 @@
// specific language governing permissions and limitations
// under the License.
-use apache_avro::{AvroSchema, Schema, SpecificSingleObjectWriter, schema::UuidSchema};
-use apache_avro_test_helper::TestResult;
-use pretty_assertions::assert_eq;
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-#[test]
-fn avro_rs_53_uuid_with_fixed() -> TestResult {
- #[derive(Debug, Serialize, Deserialize)]
- struct Comment {
- id: apache_avro::Uuid,
- }
-
- impl AvroSchema for Comment {
- fn get_schema() -> Schema {
- Schema::parse_str(
- r#"{
- "type" : "record",
- "name" : "Comment",
- "fields" : [ {
- "name" : "id",
- "type" : {
- "type" : "fixed",
- "size" : 16,
- "logicalType" : "uuid",
- "name": "FixedUUID"
- }
- } ]
- }"#,
- )
- .expect("Invalid Comment Avro schema")
- }
- }
-
- let payload = Comment {
- id: "de2df598-9948-4988-b00a-a41c0e287398".parse()?,
- };
- let mut buffer = Vec::new();
-
- // serialize the Uuid as Fixed
- assert!(!apache_avro::util::set_serde_human_readable(false));
- let bytes = SpecificSingleObjectWriter::<Comment>::new()?.write_ref(&payload, &mut buffer)?;
- assert_eq!(bytes, 26);
-
- Ok(())
-}
-
-#[test]
-fn avro_rs_440_uuid_string() -> TestResult {
- #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
- #[serde(transparent)]
- struct CustomUuid {
- #[avro(with = || Schema::Uuid(UuidSchema::String))]
- inner: Uuid,
- }
- let uuid = CustomUuid {
- inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
- };
- let mut buffer = Vec::new();
-
- assert!(!apache_avro::util::set_serde_human_readable(false));
- let writer = SpecificSingleObjectWriter::new()?;
- assert_eq!(
- writer.write(uuid, &mut buffer).unwrap_err().to_string(),
- "Failed to serialize value of type bytes using schema Uuid(String): 55e840e29b41d4a7164466554400. Cause: Expected a string, but got 16 bytes. Did you mean to use `Schema::Uuid(UuidSchema::Fixed)` or `utils::serde_set_human_readable(true)`?"
- );
-
- Ok(())
-}
-
-#[test]
-fn avro_rs_440_uuid_bytes() -> TestResult {
- #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
- #[serde(transparent)]
- struct CustomUuid {
- #[avro(with = || Schema::Uuid(UuidSchema::Bytes))]
- inner: Uuid,
- }
- let uuid = CustomUuid {
- inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
- };
- let mut buffer = Vec::new();
-
- assert!(!apache_avro::util::set_serde_human_readable(false));
- let writer = SpecificSingleObjectWriter::new()?;
- writer.write(uuid, &mut buffer)?;
-
- assert_eq!(
- buffer.as_slice(),
- &[
- 195, 1, 46, 208, 56, 148, 57, 0, 104, 249, 32, 85, 14, 132, 0, 226, 155, 65, 212, 167,
- 22, 68, 102, 85, 68, 0, 0
- ][..]
- );
-
- Ok(())
-}
-
-#[test]
-fn avro_rs_440_uuid_fixed() -> TestResult {
- #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
- #[serde(transparent)]
- struct CustomUuid {
- inner: Uuid,
- }
- assert!(matches!(
- CustomUuid::get_schema(),
- Schema::Uuid(UuidSchema::Fixed(_))
- ));
- let uuid = CustomUuid {
- inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
- };
- let mut buffer = Vec::new();
-
- assert!(!apache_avro::util::set_serde_human_readable(false));
- let writer = SpecificSingleObjectWriter::new()?;
- writer.write(uuid, &mut buffer)?;
-
- assert_eq!(
- buffer.as_slice(),
- &[
- 195, 1, 22, 19, 155, 41, 216, 175, 73, 144, 85, 14, 132, 0, 226, 155, 65, 212, 167, 22,
- 68, 102, 85, 68, 0, 0
- ][..]
- );
-
- Ok(())
-}
+// use apache_avro::{AvroSchema, Schema, SpecificSingleObjectWriter, schema::UuidSchema};
+// use apache_avro_test_helper::TestResult;
+// use pretty_assertions::assert_eq;
+// use serde::{Deserialize, Serialize};
+// use uuid::Uuid;
+//
+// #[test]
+// fn avro_rs_53_uuid_with_fixed() -> TestResult {
+// #[derive(Debug, Serialize, Deserialize)]
+// struct Comment {
+// id: apache_avro::Uuid,
+// }
+//
+// impl AvroSchema for Comment {
+// fn get_schema() -> Schema {
+// Schema::parse_str(
+// r#"{
+// "type" : "record",
+// "name" : "Comment",
+// "fields" : [ {
+// "name" : "id",
+// "type" : {
+// "type" : "fixed",
+// "size" : 16,
+// "logicalType" : "uuid",
+// "name": "FixedUUID"
+// }
+// } ]
+// }"#,
+// )
+// .expect("Invalid Comment Avro schema")
+// }
+// }
+//
+// let payload = Comment {
+// id: "de2df598-9948-4988-b00a-a41c0e287398".parse()?,
+// };
+// let mut buffer = Vec::new();
+//
+// // serialize the Uuid as Fixed
+// assert!(!apache_avro::util::set_serde_human_readable(false));
+// let bytes = SpecificSingleObjectWriter::<Comment>::new()?.write_ref(&payload, &mut buffer)?;
+// assert_eq!(bytes, 26);
+//
+// Ok(())
+// }
+//
+// #[test]
+// fn avro_rs_440_uuid_string() -> TestResult {
+// #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
+// #[serde(transparent)]
+// struct CustomUuid {
+// #[avro(with = || Schema::Uuid(UuidSchema::String))]
+// inner: Uuid,
+// }
+// let uuid = CustomUuid {
+// inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
+// };
+// let mut buffer = Vec::new();
+//
+// assert!(!apache_avro::util::set_serde_human_readable(false));
+// let writer = SpecificSingleObjectWriter::new()?;
+// assert_eq!(
+// writer.write(uuid, &mut buffer).unwrap_err().to_string(),
+// "Failed to serialize value of type bytes using schema Uuid(String): 55e840e29b41d4a7164466554400. Cause: Expected a string, but got 16 bytes. Did you mean to use `Schema::Uuid(UuidSchema::Fixed)` or `utils::serde_set_human_readable(true)`?"
+// );
+//
+// Ok(())
+// }
+//
+// #[test]
+// fn avro_rs_440_uuid_bytes() -> TestResult {
+// #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
+// #[serde(transparent)]
+// struct CustomUuid {
+// #[avro(with = || Schema::Uuid(UuidSchema::Bytes))]
+// inner: Uuid,
+// }
+// let uuid = CustomUuid {
+// inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
+// };
+// let mut buffer = Vec::new();
+//
+// assert!(!apache_avro::util::set_serde_human_readable(false));
+// let writer = SpecificSingleObjectWriter::new()?;
+// writer.write(uuid, &mut buffer)?;
+//
+// assert_eq!(
+// buffer.as_slice(),
+// &[
+// 195, 1, 46, 208, 56, 148, 57, 0, 104, 249, 32, 85, 14, 132, 0, 226, 155, 65, 212, 167,
+// 22, 68, 102, 85, 68, 0, 0
+// ][..]
+// );
+//
+// Ok(())
+// }
+//
+// #[test]
+// fn avro_rs_440_uuid_fixed() -> TestResult {
+// #[derive(apache_avro_derive::AvroSchema, Serialize, Deserialize)]
+// #[serde(transparent)]
+// struct CustomUuid {
+// inner: Uuid,
+// }
+// assert!(matches!(
+// CustomUuid::get_schema(),
+// Schema::Uuid(UuidSchema::Fixed(_))
+// ));
+// let uuid = CustomUuid {
+// inner: Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000")?,
+// };
+// let mut buffer = Vec::new();
+//
+// assert!(!apache_avro::util::set_serde_human_readable(false));
+// let writer = SpecificSingleObjectWriter::new()?;
+// writer.write(uuid, &mut buffer)?;
+//
+// assert_eq!(
+// buffer.as_slice(),
+// &[
+// 195, 1, 22, 19, 155, 41, 216, 175, 73, 144, 85, 14, 132, 0, 226, 155, 65, 212, 167, 22,
+// 68, 102, 85, 68, 0, 0
+// ][..]
+// );
+//
+// Ok(())
+// }
diff --git a/avro/tests/serde_human_readable_true.rs b/avro/tests/serde_human_readable_true.rs
index d1c71d7..fd61d84 100644
--- a/avro/tests/serde_human_readable_true.rs
+++ b/avro/tests/serde_human_readable_true.rs
@@ -55,7 +55,10 @@ fn avro_rs_53_uuid_with_string_true() -> TestResult {
// serialize the Uuid as String
assert!(apache_avro::util::set_serde_human_readable(true));
- let bytes = SpecificSingleObjectWriter::<Comment>::new()?.write_ref(&payload, &mut buffer)?;
+ let bytes = SpecificSingleObjectWriter::<Comment>::builder()
+ .human_readable(true)
+ .build()
+ .write_ref(&payload, &mut buffer)?;
assert_eq!(bytes, 47);
Ok(())
@@ -75,7 +78,9 @@ fn avro_rs_440_uuid_string() -> TestResult {
let mut buffer = Vec::new();
assert!(apache_avro::util::set_serde_human_readable(true));
- let writer = SpecificSingleObjectWriter::new()?;
+ let writer = SpecificSingleObjectWriter::builder()
+ .human_readable(true)
+ .build();
writer.write(uuid, &mut buffer)?;
assert_eq!(
@@ -100,10 +105,12 @@ fn avro_rs_440_uuid_bytes() -> TestResult {
let mut buffer = Vec::new();
assert!(apache_avro::util::set_serde_human_readable(true));
- let writer = SpecificSingleObjectWriter::new()?;
+ let writer = SpecificSingleObjectWriter::builder()
+ .human_readable(true)
+ .build();
assert_eq!(
writer.write(uuid, &mut buffer).unwrap_err().to_string(),
- r#"Failed to serialize value of type string using schema Uuid(Bytes): 550e8400-e29b-41d4-a716-446655440000. Cause: Expected bytes but got a string. Did you mean to use `Schema::Uuid(UuidSchema::String)` or `utils::serde_set_human_readable(false)`?"#
+ r#"Failed to serialize value of type str using schema Uuid(Bytes): Expected Schema::String | Schema::Uuid(String)"#
);
Ok(())
@@ -126,10 +133,12 @@ fn avro_rs_440_uuid_fixed() -> TestResult {
let mut buffer = Vec::new();
assert!(apache_avro::util::set_serde_human_readable(true));
- let writer = SpecificSingleObjectWriter::new()?;
+ let writer = SpecificSingleObjectWriter::builder()
+ .human_readable(true)
+ .build();
assert_eq!(
writer.write(uuid, &mut buffer).unwrap_err().to_string(),
- r#"Failed to serialize value of type string using schema Uuid(Fixed(FixedSchema { name: Name { name: "uuid", .. }, size: 16, .. })): 550e8400-e29b-41d4-a716-446655440000. Cause: Expected bytes but got a string. Did you mean to use `Schema::Uuid(UuidSchema::String)` or `utils::serde_set_human_readable(false)`?"#
+ r#"Failed to serialize value of type str using schema Uuid(Fixed(FixedSchema { name: Name { name: "uuid", .. }, size: 16, .. })): Expected Schema::String | Schema::Uuid(String)"#
);
Ok(())
diff --git a/avro_derive/src/attributes/avro.rs b/avro_derive/src/attributes/avro.rs
index eb11b9e..a633a32 100644
--- a/avro_derive/src/attributes/avro.rs
+++ b/avro_derive/src/attributes/avro.rs
@@ -57,6 +57,23 @@ pub struct ContainerAttributes {
/// Set the default value if this schema is used as a field
#[darling(default)]
pub default: Option<String>,
+ #[darling(default)]
+ pub repr: Option<EnumRepr>,
+}
+
+/// What kind of schema to use for an enum.
+#[derive(Debug, FromMeta, PartialEq, Clone, Copy)]
+pub enum EnumRepr {
+ /// Create a `Schema::Enum`, only works for plain enums.
+ Enum,
+ /// Untagged
+ BareUnion,
+ /// Externally tagged
+ UnionOfRecords,
+ /// Adjacently tagged (`#[serde(tag = "type", content = "value")]`)
+ RecordTagContent,
+ /// Internally tagged (`#[serde(tag = "type")]`)
+ RecordInternallyTagged,
}
impl ContainerAttributes {
diff --git a/avro_derive/src/attributes/mod.rs b/avro_derive/src/attributes/mod.rs
index cfdae4f..15a2561 100644
--- a/avro_derive/src/attributes/mod.rs
+++ b/avro_derive/src/attributes/mod.rs
@@ -24,14 +24,31 @@ use syn::{AttrStyle, Attribute, Expr, Ident, Path, spanned::Spanned};
mod avro;
mod serde;
+/// What kind of schema to use for an enum.
+#[derive(Debug, PartialEq)]
+pub enum EnumRepr {
+ /// Create a `Schema::Enum`, only works for plain enums.
+ Enum,
+ /// Untagged
+ BareUnion,
+ /// Externally tagged
+ UnionOfRecords,
+ /// Adjacently tagged (`#[serde(tag = "type", content = "value")]`)
+ RecordTagContent { tag: String, content: String },
+ /// Internally tagged (`#[serde(tag = "type")]`)
+ RecordInternallyTagged { tag: String },
+}
+
#[derive(Default)]
pub struct NamedTypeOptions {
pub name: String,
pub doc: Option<String>,
pub aliases: Vec<String>,
pub rename_all: RenameRule,
+ pub rename_all_fields: RenameRule,
pub transparent: bool,
pub default: TokenStream,
+ pub repr: Option<EnumRepr>,
}
impl NamedTypeOptions {
@@ -52,27 +69,28 @@ impl NamedTypeOptions {
let mut errors = Vec::new();
// Check for any Serde attributes that are hard errors
- if serde.tag.is_some()
- || serde.content.is_some()
- || serde.untagged
- || serde.variant_identifier
- || serde.field_identifier
- {
+ if serde.variant_identifier || serde.field_identifier {
errors.push(syn::Error::new(
span,
- "AvroSchema derive does not support changing the tagging Serde generates (`tag`, `content`, `untagged`, `variant_identifier`, `field_identifier`)",
+ r#"AvroSchema: `#[serde(variant_identifier)]` and `#[serde(field_identifier)]` are not supported"#,
));
}
if serde.remote.is_some() {
errors.push(syn::Error::new(
span,
- "AvroSchema derive does not support the Serde `remote` attribute",
+ r#"AvroSchema: `#[serde(remote = "..")]` is not supported"#,
));
}
if serde.rename_all.deserialize != serde.rename_all.serialize {
errors.push(syn::Error::new(
span,
- r#"AvroSchema derive does not support different rename rules for serializing and deserializing (`rename_all(serialize = "..", deserialize = "..")`)"#
+ r#"AvroSchema: rename rules for serializing and deserializing must match (`rename_all(serialize = "..", deserialize = "..")`)"#
+ ));
+ }
+ if serde.rename_all_fields.deserialize != serde.rename_all_fields.serialize {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: rename rules for serializing and deserializing must match (`rename_all_fields(serialize = "..", deserialize = "..")`)"#
));
}
@@ -80,13 +98,13 @@ impl NamedTypeOptions {
if avro.name.is_some() && avro.name != serde.rename {
errors.push(syn::Error::new(
span,
- r#"#[avro(name = "..")] must match #[serde(rename = "..")], it's also deprecated. Please use only `#[serde(rename = "..")]`"#,
+ r#"AvroSchema: #[avro(name = "..")] must match #[serde(rename = "..")], it's also deprecated. Please use only `#[serde(rename = "..")]`"#,
));
}
if avro.rename_all != RenameRule::None && serde.rename_all.serialize != avro.rename_all {
errors.push(syn::Error::new(
span,
- r#"#[avro(rename_all = "..")] must match #[serde(rename_all = "..")], it's also deprecated. Please use only `#[serde(rename_all = "..")]`"#,
+ r#"AvroSchema: #[avro(rename_all = "..")] must match #[serde(rename_all = "..")], it's also deprecated. Please use only `#[serde(rename_all = "..")]`"#,
));
}
if serde.transparent
@@ -97,14 +115,110 @@ impl NamedTypeOptions {
|| !avro.alias.is_empty()
|| avro.rename_all != RenameRule::None
|| serde.rename_all.serialize != RenameRule::None
- || serde.rename_all.deserialize != RenameRule::None)
+ || serde.rename_all.deserialize != RenameRule::None
+ || serde.untagged
+ || serde.tag.is_some()
+ || serde.content.is_some())
{
errors.push(syn::Error::new(
span,
- "AvroSchema: #[serde(transparent)] is incompatible with all other attributes",
+ "AvroSchema: `#[serde(transparent)]` is incompatible with all other attributes",
));
}
+ let repr = if let Some(repr) = avro.repr {
+ match repr {
+ avro::EnumRepr::Enum => {
+ if serde.tag.is_some() || serde.content.is_some() || serde.untagged {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[avro(repr = "enum")]` is incompatible with `#[serde(tag = "..")]`, `#[serde(content = "..")]`, and `#[serde(untagged)]`"#,
+ ));
+ None
+ } else {
+ Some(EnumRepr::Enum)
+ }
+ }
+ avro::EnumRepr::BareUnion => {
+ if serde.untagged {
+ Some(EnumRepr::BareUnion)
+ } else {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[avro(repr = "bare_union")]` requires `#[serde(untagged)]`"#,
+ ));
+ None
+ }
+ }
+ avro::EnumRepr::UnionOfRecords => {
+ if serde.tag.is_some() || serde.content.is_some() || serde.untagged {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[avro(repr = "union_of_records")]` is incompatible with `#[serde(tag = "..")]`, `#[serde(content = "..")]`, and `#[serde(untagged)]`"#,
+ ));
+ None
+ } else {
+ Some(EnumRepr::UnionOfRecords)
+ }
+ }
+ avro::EnumRepr::RecordTagContent => {
+ if let Some(tag) = serde.tag
+ && let Some(content) = serde.content
+ {
+ Some(EnumRepr::RecordTagContent { tag, content })
+ } else {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[avro(repr = "record_tag_content")]` requires `#[serde(tag = "..", content = "..")]`"#,
+ ));
+ None
+ }
+ }
+ avro::EnumRepr::RecordInternallyTagged => {
+ if let Some(tag) = serde.tag
+ && serde.content.is_none()
+ {
+ Some(EnumRepr::RecordInternallyTagged { tag })
+ } else {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: `#[avro(repr = "discriminator_value")]` requires `#[serde(tag = "..")]` and is incompatible with `#[serde(content = "..")]`"#,
+ ));
+ None
+ }
+ }
+ }
+ } else {
+ if let Some(content) = serde.content
+ && let Some(tag) = serde.tag
+ {
+ Some(EnumRepr::RecordTagContent { tag, content })
+ } else if serde.untagged {
+ Some(EnumRepr::BareUnion)
+ } else if let Some(tag) = serde.tag {
+ Some(EnumRepr::RecordInternallyTagged { tag })
+ } else {
+ None
+ }
+ };
+
+ let default = match avro.default {
+ None => quote! { None },
+ Some(default_value) => {
+ if let Err(err) = serde_json::from_str::<serde_json::Value>(&default_value[..]) {
+ errors.push(syn::Error::new(
+ ident.span(),
+ format!("Invalid Avro `default` JSON: \n{err}"),
+ ));
+ quote! { None }
+ } else {
+ quote! {
+ Some(serde_json::from_str(#default_value).expect("Unreachable! This was checked at compile time"))
+ }
+ }
+ }
+ };
+
if !errors.is_empty() {
return Err(errors);
}
@@ -118,35 +232,22 @@ impl NamedTypeOptions {
let doc = avro.doc.or_else(|| extract_rustdoc(attributes));
- let default = match avro.default {
- None => quote! { None },
- Some(default_value) => {
- let _: serde_json::Value =
- serde_json::from_str(&default_value[..]).map_err(|e| {
- vec![syn::Error::new(
- ident.span(),
- format!("Invalid Avro `default` JSON: \n{e}"),
- )]
- })?;
- quote! {
- Some(::serde_json::from_str(#default_value).expect(format!("Invalid JSON: {:?}", #default_value).as_str()))
- }
- }
- };
-
Ok(Self {
name: full_schema_name,
doc,
aliases: avro.alias,
rename_all: serde.rename_all.serialize,
+ rename_all_fields: serde.rename_all_fields.serialize,
transparent: serde.transparent,
default,
+ repr,
})
}
}
pub struct VariantOptions {
pub rename: Option<String>,
+ pub rename_all: RenameRule,
}
impl VariantOptions {
@@ -165,7 +266,13 @@ impl VariantOptions {
if serde.other || serde.untagged {
errors.push(syn::Error::new(
span,
- "AvroSchema derive does not support changing the tagging Serde generates (`other`, `untagged`)",
+ r#"AvroSchema: `#[serde(other)]` and `#[serde(untagged)]` are not supported"#,
+ ));
+ }
+ if serde.rename_all.deserialize != serde.rename_all.serialize {
+ errors.push(syn::Error::new(
+ span,
+ r#"AvroSchema: rename rules for serializing and deserializing must match (`rename_all(serialize = "..", deserialize = "..")`)"#
));
}
@@ -173,7 +280,7 @@ impl VariantOptions {
if avro.rename.is_some() && serde.rename != avro.rename {
errors.push(syn::Error::new(
span,
- r#"`#[avro(rename = "..")]` must match `#[serde(rename = "..")]`, it's also deprecated. Please use only `#[serde(rename = "..")]`"#
+ r#"AvroSchema: `#[avro(rename = "..")]` must match `#[serde(rename = "..")]`, it's also deprecated. Please use only `#[serde(rename = "..")]`"#
));
}
@@ -183,6 +290,7 @@ impl VariantOptions {
Ok(Self {
rename: serde.rename,
+ rename_all: serde.rename_all.serialize,
})
}
}
@@ -221,7 +329,7 @@ impl With {
} else {
Err(syn::Error::new(
span,
- r#"`#[avro(with)]` requires `#[serde(with = "some_module")]` or provide a function to call `#[avro(with = some_fn)]`"#,
+ r#"AvroSchema: `#[avro(with)]` requires `#[serde(with = "some_module")]` or provide a function to call `#[avro(with = some_fn)]`"#,
))
}
}
@@ -288,7 +396,7 @@ impl FieldOptions {
if serde.getter.is_some() {
errors.push(syn::Error::new(
span,
- "AvroSchema derive does not support the Serde `getter` attribute",
+ r#"AvroSchema: `#[serde(getter = "..")]` is not supported"#,
));
}
@@ -296,26 +404,26 @@ impl FieldOptions {
if avro.skip && !(serde.skip || (serde.skip_serializing && serde.skip_deserializing)) {
errors.push(syn::Error::new(
span,
- "`#[avro(skip)]` requires `#[serde(skip)]`, it's also deprecated. Please use only `#[serde(skip)]`"
+ "AvroSchema: `#[avro(skip)]` requires `#[serde(skip)]`, it's also deprecated. Please use only `#[serde(skip)]`"
));
}
if avro.flatten && !serde.flatten {
errors.push(syn::Error::new(
span,
- "`#[avro(flatten)]` requires `#[serde(flatten)]`, it's also deprecated. Please use only `#[serde(flatten)]`"
+ "AvroSchema: `#[avro(flatten)]` requires `#[serde(flatten)]`, it's also deprecated. Please use only `#[serde(flatten)]`"
));
}
// TODO: rename and alias checking can be relaxed with a more complex check, would require the field name
if avro.rename.is_some() && serde.rename != avro.rename {
errors.push(syn::Error::new(
span,
- r#"`#[avro(rename = "..")]` must match `#[serde(rename = "..")]`, it's also deprecated. Please use only `#[serde(rename = "..")]`"#
+ r#"AvroSchema: `#[avro(rename = "..")]` must match `#[serde(rename = "..")]`, it's also deprecated. Please use only `#[serde(rename = "..")]`"#
));
}
if !avro.alias.is_empty() && serde.alias != avro.alias {
errors.push(syn::Error::new(
span,
- r#"`#[avro(alias = "..")]` must match `#[serde(alias = "..")]`, it's also deprecated. Please use only `#[serde(alias = "..")]`"#
+ r#"AvroSchema: `#[avro(alias = "..")]` must match `#[serde(alias = "..")]`, it's also deprecated. Please use only `#[serde(alias = "..")]`"#
));
}
if ((serde.skip_serializing && !serde.skip_deserializing)
@@ -324,7 +432,7 @@ impl FieldOptions {
{
errors.push(syn::Error::new(
span,
- "`#[serde(skip_serializing)]` and `#[serde(skip_serializing_if)]` are incompatible with `#[avro(default = false)]`"
+ "AvroSchema: `#[serde(skip_serializing)]` and `#[serde(skip_serializing_if)]` are incompatible with `#[avro(default = false)]`"
));
}
let with = match With::from_avro_and_serde(&avro.with, serde.with.as_deref(), span) {
diff --git a/avro_derive/src/attributes/serde.rs b/avro_derive/src/attributes/serde.rs
index b1ca6fa..6d7f55c 100644
--- a/avro_derive/src/attributes/serde.rs
+++ b/avro_derive/src/attributes/serde.rs
@@ -92,8 +92,8 @@ pub struct ContainerAttributes {
#[darling(default)]
pub rename_all: RenameAll,
/// Rename all the fields of the struct variants in this enum.
- #[darling(default, rename = "rename_all_fields")]
- pub _rename_all_fields: RenameAll,
+ #[darling(default)]
+ pub rename_all_fields: RenameAll,
/// Error when encountering unknown fields when deserialising.
#[darling(default, rename = "deny_unknown_fields")]
pub _deny_unknown_fields: bool,
@@ -155,8 +155,9 @@ pub struct VariantAttributes {
/// Aliases for this variant, only used during deserialisation.
#[darling(multiple, rename = "alias")]
pub _alias: Vec<String>,
- #[darling(default, rename = "rename_all")]
- pub _rename_all: RenameAll,
+ /// Rename struct fields of this variant
+ #[darling(default)]
+ pub rename_all: RenameAll,
/// Do not serialize or deserialize this variant.
#[darling(default, rename = "skip")]
pub _skip: bool,
diff --git a/avro_derive/src/case.rs b/avro_derive/src/case.rs
index c958562..ad02a9a 100644
--- a/avro_derive/src/case.rs
+++ b/avro_derive/src/case.rs
@@ -136,6 +136,14 @@ impl RenameRule {
ScreamingKebabCase => ScreamingSnakeCase.apply_to_field(field).replace('_', "-"),
}
}
+
+ /// Returns the `RenameRule` if it is not `None`, `rule_b` otherwise.
+ pub fn or(self, rule_b: Self) -> Self {
+ match self {
+ None => rule_b,
+ _ => self,
+ }
+ }
}
pub struct ParseError {
diff --git a/avro_derive/src/enums/bare_union.rs b/avro_derive/src/enums/bare_union.rs
new file mode 100644
index 0000000..26d9ee7
--- /dev/null
+++ b/avro_derive/src/enums/bare_union.rs
@@ -0,0 +1,92 @@
+use crate::attributes::{NamedTypeOptions, VariantOptions};
+use crate::type_to_schema_expr;
+use proc_macro2::{Span, TokenStream};
+use quote::quote;
+use syn::spanned::Spanned;
+use syn::{DataEnum, Fields};
+
+pub fn get_data_enum_schema_def(
+ container_attrs: &NamedTypeOptions,
+ data_enum: DataEnum,
+ ident_span: Span,
+) -> Result<TokenStream, Vec<syn::Error>> {
+ let mut variant_expr = Vec::new();
+ let mut have_null = false;
+ for variant in data_enum.variants {
+ let field_attrs = VariantOptions::new(&variant.attrs, variant.span())?;
+ let name = field_attrs.rename.unwrap_or_else(|| {
+ container_attrs
+ .rename_all
+ .apply_to_variant(&variant.ident.to_string())
+ });
+ match variant.fields {
+ Fields::Named(named) => {
+ for field in named.named {
+ let ident = field_attrs
+ .rename_all
+ .or(container_attrs.rename_all_fields)
+ .apply_to_field(&field.ident.unwrap().to_string());
+ let schema_expr = type_to_schema_expr(&field.ty)?;
+ variant_expr.push(quote! {
+ fields.push(::apache_avro::schema::RecordField::builder()
+ .name(#ident.to_string())
+ .schema(#schema_expr)
+ .build())
+ });
+ }
+ }
+ Fields::Unnamed(unnamed) => {
+ if unnamed.unnamed.len() == 0 {
+ return Err(vec![syn::Error::new(
+ unnamed.span(),
+ "AvroSchema: Empty tuple variants are not supported for bare unions",
+ )]);
+ } else if unnamed.unnamed.len() == 1 {
+ let only_one = unnamed.unnamed.iter().next().expect("There is one");
+ let schema_expr = type_to_schema_expr(&only_one.ty)?;
+ variant_expr.push(schema_expr);
+ } else if unnamed.unnamed.len() > 1 {
+ let mut fields = Vec::with_capacity(unnamed.unnamed.len());
+ for (index, field) in unnamed.unnamed.iter().enumerate() {
+ let field_schema_expr = type_to_schema_expr(&field.ty)?;
+ fields.push(quote! {
+ ::apache_avro::schema::RecordField::builder()
+ .name(format!("field_{}", #index))
+ .schema(#field_schema_expr)
+ .build()
+ });
+ }
+
+ let schema_expr = quote! {
+ ::apache_avro::schema::Schema::Record(
+ ::apache_avro::schema::RecordSchema::builder()
+ .name(::apache_avro::schema::Name::new_with_enclosing_namespace(#name, enclosing_namespace).expect(&format!("Unable to parse variant record name for schema {}", #name)[..]))
+ .fields(vec![
+ #(#fields,)*
+ ])
+ .build()
+ )
+ };
+ variant_expr.push(schema_expr);
+ }
+ }
+ Fields::Unit => {
+ if have_null {
+ return Err(vec![syn::Error::new(
+ ident_span,
+ "More than one variant maps to Schema::Null, this is not supported for bare unions",
+ )]);
+ }
+ variant_expr.push(quote! { ::apache_avro::schema::Schema::Null });
+ have_null = true;
+ }
+ }
+ }
+ Ok(quote! {
+ let mut builder = ::apache_avro::schema::UnionSchema::builder();
+
+ #(builder.variant(#variant_expr).expect("Duplicate Schema found");)*
+
+ ::apache_avro::schema::Schema::Union(builder.build())
+ })
+}
diff --git a/avro_derive/src/enums/mod.rs b/avro_derive/src/enums/mod.rs
index e44f4a1..83af7a4 100644
--- a/avro_derive/src/enums/mod.rs
+++ b/avro_derive/src/enums/mod.rs
@@ -15,9 +15,13 @@
// specific language governing permissions and limitations
// under the License.
+mod bare_union;
mod plain;
+mod record_internally_tagged;
+mod record_tag_content;
+mod union_of_records;
-use crate::attributes::NamedTypeOptions;
+use crate::attributes::{EnumRepr, NamedTypeOptions};
use proc_macro2::{Ident, Span, TokenStream};
use syn::{Attribute, DataEnum, Fields, Meta};
@@ -27,13 +31,27 @@ pub fn get_data_enum_schema_def(
data_enum: DataEnum,
ident_span: Span,
) -> Result<TokenStream, Vec<syn::Error>> {
- if data_enum.variants.iter().all(|v| Fields::Unit == v.fields) {
- plain::schema_def(container_attrs, data_enum, ident_span)
- } else {
- Err(vec![syn::Error::new(
- ident_span,
- "AvroSchema: derive does not work for enums with non unit structs",
- )])
+ match &container_attrs.repr {
+ None => {
+ if data_enum.variants.iter().all(|v| Fields::Unit == v.fields) {
+ plain::schema_def(container_attrs, data_enum, ident_span)
+ } else {
+ union_of_records::get_data_enum_schema_def(container_attrs, data_enum, ident_span)
+ }
+ }
+ Some(EnumRepr::Enum) => plain::schema_def(container_attrs, data_enum, ident_span),
+ Some(EnumRepr::BareUnion) => {
+ bare_union::get_data_enum_schema_def(container_attrs, data_enum, ident_span)
+ }
+ Some(EnumRepr::UnionOfRecords) => {
+ union_of_records::get_data_enum_schema_def(container_attrs, data_enum, ident_span)
+ }
+ Some(EnumRepr::RecordTagContent { tag, content }) => {
+ record_tag_content::get_data_enum_schema_def(container_attrs, data_enum, tag, content)
+ }
+ Some(EnumRepr::RecordInternallyTagged { tag }) => {
+ record_internally_tagged::get_data_enum_schema_def(container_attrs, data_enum, tag)
+ }
}
}
diff --git a/avro_derive/src/enums/plain.rs b/avro_derive/src/enums/plain.rs
index 89de59c..f1ebdaa 100644
--- a/avro_derive/src/enums/plain.rs
+++ b/avro_derive/src/enums/plain.rs
@@ -46,10 +46,9 @@ pub fn schema_def(
};
symbols.push(name);
}
- let full_schema_name = &container_attrs.name;
Ok(quote! {
::apache_avro::schema::Schema::Enum(apache_avro::schema::EnumSchema {
- name: ::apache_avro::schema::Name::new(#full_schema_name).expect(&format!("Unable to parse enum name for schema {}", #full_schema_name)[..]),
+ name,
aliases: #enum_aliases,
doc: #doc,
symbols: vec![#(#symbols.to_owned()),*],
@@ -60,7 +59,7 @@ pub fn schema_def(
} else {
Err(vec![syn::Error::new(
ident_span,
- "AvroSchema: derive does not work for enums with non unit structs",
+ r#"AvroSchema: `#[avro(repr = "enum")]` does not work for enums with non-unit variants"#,
)])
}
}
diff --git a/avro_derive/src/enums/record_internally_tagged.rs b/avro_derive/src/enums/record_internally_tagged.rs
new file mode 100644
index 0000000..f3331dc
--- /dev/null
+++ b/avro_derive/src/enums/record_internally_tagged.rs
@@ -0,0 +1,79 @@
+use crate::attributes::{NamedTypeOptions, VariantOptions};
+use crate::{aliases, preserve_optional, type_to_schema_expr};
+use proc_macro2::TokenStream;
+use quote::quote;
+use syn::spanned::Spanned;
+use syn::{DataEnum, Fields};
+
+pub fn get_data_enum_schema_def(
+ container_attrs: &NamedTypeOptions,
+ data_enum: DataEnum,
+ tag: &str,
+) -> Result<TokenStream, Vec<syn::Error>> {
+ let doc = preserve_optional(container_attrs.doc.as_ref());
+ let enum_aliases = aliases(&container_attrs.aliases);
+ let mut symbols = Vec::new();
+ let mut field_additions = Vec::new();
+ for variant in data_enum.variants {
+ let field_attrs = VariantOptions::new(&variant.attrs, variant.span())?;
+ let name = field_attrs.rename.unwrap_or_else(|| {
+ container_attrs
+ .rename_all
+ .apply_to_variant(&variant.ident.to_string())
+ });
+ match variant.fields {
+ Fields::Named(named) => {
+ for field in named.named {
+ let ident = field_attrs
+ .rename_all
+ .or(container_attrs.rename_all_fields)
+ .apply_to_field(&field.ident.unwrap().to_string());
+ let schema_expr = type_to_schema_expr(&field.ty)?;
+ field_additions.push(quote! {
+ fields.push(::apache_avro::schema::RecordField::builder()
+ .name(#ident.to_string())
+ .schema(#schema_expr)
+ .build())
+ });
+ }
+ }
+ Fields::Unnamed(unnamed) => {
+ if unnamed.unnamed.len() == 1 {
+ let only_one = unnamed.unnamed.iter().next().expect("There is one");
+ let schema_expr = type_to_schema_expr(&only_one.ty)?;
+ field_additions.push(schema_expr);
+ } else if unnamed.unnamed.len() > 1 {
+ return Err(vec![syn::Error::new(
+ unnamed.span(),
+ "Tuple variants are not supported for internally tagged enums",
+ )]);
+ }
+ }
+ Fields::Unit => {}
+ }
+ symbols.push(name);
+ }
+ Ok(quote! {
+ let mut builder = ::apache_avro::schema::UnionSchema::builder();
+
+ let tag_name = ::apache_avro::schema::Name::new_with_enclosing_namespace(#tag, enclosing_namespace).expect(&format!("Unable to parse name for schema tag {}", #tag)[..]);
+ let tag_schema = ::apache_avro::schema::Schema::r#enum(tag_name, vec![#(#symbols.to_owned()),*]).build();
+
+ let mut fields = ::std::vec::Vec::new();
+ fields.push(::apache_avro::schema::RecordField::builder()
+ .name(#tag)
+ .schema(tag_schema)
+ .build()
+ );
+ #(
+ #field_additions
+ )*
+ ::apache_avro::schema::Schema::Record(::apache_avro::schema::RecordSchema::builder()
+ .name(name)
+ .maybe_aliases(#enum_aliases)
+ .maybe_doc(#doc)
+ .fields(fields)
+ .build()
+ )
+ })
+}
diff --git a/avro_derive/src/enums/record_tag_content.rs b/avro_derive/src/enums/record_tag_content.rs
new file mode 100644
index 0000000..5227de2
--- /dev/null
+++ b/avro_derive/src/enums/record_tag_content.rs
@@ -0,0 +1,127 @@
+use crate::attributes::{NamedTypeOptions, VariantOptions};
+use crate::{aliases, preserve_optional, type_to_schema_expr};
+use proc_macro2::TokenStream;
+use quote::quote;
+use syn::spanned::Spanned;
+use syn::{DataEnum, Fields};
+
+pub fn get_data_enum_schema_def(
+ container_attrs: &NamedTypeOptions,
+ data_enum: DataEnum,
+ tag: &str,
+ content: &str,
+) -> Result<TokenStream, Vec<syn::Error>> {
+ let doc = preserve_optional(container_attrs.doc.as_ref());
+ let enum_aliases = aliases(&container_attrs.aliases);
+ let mut symbols = Vec::new();
+ let mut schema_definitions = Vec::new();
+ for variant in data_enum.variants {
+ let field_attrs = VariantOptions::new(&variant.attrs, variant.span())?;
+ let name = field_attrs.rename.unwrap_or_else(|| {
+ container_attrs
+ .rename_all
+ .apply_to_variant(&variant.ident.to_string())
+ });
+ match variant.fields {
+ Fields::Named(named) => {
+ let mut fields = Vec::with_capacity(named.named.len());
+ for field in named.named {
+ let ident = field_attrs
+ .rename_all
+ .or(container_attrs.rename_all_fields)
+ .apply_to_field(&field.ident.unwrap().to_string());
+ let schema_expr = type_to_schema_expr(&field.ty)?;
+ fields.push(quote! {
+ ::apache_avro::schema::RecordField::builder()
+ .name(#ident.to_string())
+ .schema(#schema_expr)
+ .build()
+ });
+ }
+
+ let schema_expr = quote! {
+ ::apache_avro::schema::Schema::Record(
+ ::apache_avro::schema::RecordSchema::builder()
+ .name(::apache_avro::schema::Name::new_with_enclosing_namespace(#name, enclosing_namespace).expect(&format!("Unable to parse variant record name for schema {}", #name)[..]))
+ .fields(vec![
+ #(#fields,)*
+ ])
+ .build()
+ )
+ };
+ schema_definitions.push(schema_expr);
+ }
+ Fields::Unnamed(unnamed) => {
+ if unnamed.unnamed.is_empty() {
+ let schema_expr = quote! {
+ ::apache_avro::schema::Schema::Null
+ };
+ schema_definitions.push(schema_expr);
+ } else if unnamed.unnamed.len() == 1 {
+ let only_one = unnamed.unnamed.iter().next().expect("There is one");
+ let field_schema_expr = type_to_schema_expr(&only_one.ty)?;
+ schema_definitions.push(field_schema_expr);
+ } else if unnamed.unnamed.len() > 1 {
+ let mut fields = Vec::with_capacity(unnamed.unnamed.len());
+ for (index, field) in unnamed.unnamed.iter().enumerate() {
+ let field_schema_expr = type_to_schema_expr(&field.ty)?;
+ fields.push(quote! {
+ ::apache_avro::schema::RecordField::builder()
+ .name(format!("field_{}", #index))
+ .schema(#field_schema_expr)
+ .build()
+ });
+ }
+
+ let schema_expr = quote! {
+ ::apache_avro::schema::Schema::Record(
+ ::apache_avro::schema::RecordSchema::builder()
+ .name(::apache_avro::schema::Name::new_with_enclosing_namespace(#name, enclosing_namespace).expect(&format!("Unable to parse variant record name for schema {}", #name)[..]))
+ .fields(vec![
+ #(#fields,)*
+ ])
+ .build()
+ )
+ };
+ schema_definitions.push(schema_expr);
+ }
... 325 lines suppressed ...