You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/05/13 13:03:42 UTC
[avro] branch avro-3512-rust created (now acd7f0260)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a change to branch avro-3512-rust
in repository https://gitbox.apache.org/repos/asf/avro.git
at acd7f0260 AVRO-3512: Fix the aliases' namespace when parsing named schemata
This branch includes the following new commits:
new acd7f0260 AVRO-3512: Fix the aliases' namespace when parsing named schemata
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[avro] 01/01: AVRO-3512: Fix the aliases' namespace when parsing named schemata
Posted by mg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch avro-3512-rust
in repository https://gitbox.apache.org/repos/asf/avro.git
commit acd7f02600fba57f3551f317081860fa6bcfbe49
Author: Martin Tzvetanov Grigorov <mg...@apache.org>
AuthorDate: Fri May 13 16:02:25 2022 +0300
AVRO-3512: Fix the aliases' namespace when parsing named schemata
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
---
doc-deprecated/src/content/xdocs/spec.xml | 2 +-
doc/content/en/docs/next/Specification/_index.md | 2 +-
lang/rust/avro/src/schema.rs | 143 +++++++++++++++++++++--
3 files changed, 137 insertions(+), 10 deletions(-)
diff --git a/doc-deprecated/src/content/xdocs/spec.xml b/doc-deprecated/src/content/xdocs/spec.xml
index 46aa71126..9bc07f0f6 100644
--- a/doc-deprecated/src/content/xdocs/spec.xml
+++ b/doc-deprecated/src/content/xdocs/spec.xml
@@ -398,7 +398,7 @@
<title>Aliases</title>
<p>Named types and fields may have aliases. An implementation
may optionally use aliases to map a writer's schema to the
- reader's. This faciliates both schema evolution as well as
+ reader's. This facilitates both schema evolution as well as
processing disparate datasets.</p>
<p>Aliases function by re-writing the writer's schema using
aliases from the reader's schema. For example, if the
diff --git a/doc/content/en/docs/next/Specification/_index.md b/doc/content/en/docs/next/Specification/_index.md
index 49e030dbe..cf5914089 100755
--- a/doc/content/en/docs/next/Specification/_index.md
+++ b/doc/content/en/docs/next/Specification/_index.md
@@ -257,7 +257,7 @@ Complex types (`record`, `enum`, `array`, `map`, `fixed`) have no namespace, but
A schema or protocol may not contain multiple definitions of a fullname. Further, a name must be defined before it is used ("before" in the depth-first, left-to-right traversal of the JSON parse tree, where the types attribute of a protocol is always deemed to come "before" the messages attribute.)
### Aliases
-Named types and fields may have aliases. An implementation may optionally use aliases to map a writer's schema to the reader's. This faciliates both schema evolution as well as processing disparate datasets.
+Named types and fields may have aliases. An implementation may optionally use aliases to map a writer's schema to the reader's. This facilitates both schema evolution as well as processing disparate datasets.
Aliases function by re-writing the writer's schema using aliases from the reader's schema. For example, if the writer's schema was named "Foo" and the reader's schema is named "Bar" and has an alias of "Foo", then the implementation would act as though "Foo" were named "Bar" when reading. Similarly, if data was written as a record with a field named "x" and is read as a record with a field named "y" with alias "x", then the implementation would act as though "x" were named "y" when reading.
diff --git a/lang/rust/avro/src/schema.rs b/lang/rust/avro/src/schema.rs
index ecbdad441..320b36a80 100644
--- a/lang/rust/avro/src/schema.rs
+++ b/lang/rust/avro/src/schema.rs
@@ -1115,6 +1115,7 @@ impl Parser {
Some(ref ns) => format!("{}.{}", ns, alias),
None => alias.clone(),
};
+
let alias_name = Name::new(alias_fullname.as_str()).unwrap();
self.resolving_schemas.remove(&alias_name);
self.parsed_schemas.insert(alias_name, schema.clone());
@@ -1148,8 +1149,6 @@ impl Parser {
complex: &Map<String, Value>,
enclosing_namespace: &Namespace,
) -> AvroResult<Schema> {
- let name = Name::parse(complex)?;
- let aliases = complex.aliases();
let fields_opt = complex.get("fields");
if fields_opt.is_none() {
@@ -1158,6 +1157,9 @@ impl Parser {
}
}
+ let name = Name::parse(complex)?;
+ let aliases = fix_aliases_namespace(complex.aliases(), &name.namespace);
+
let mut lookup = BTreeMap::new();
let fully_qualified_name = name.fully_qualified_name(enclosing_namespace);
self.register_resolving_schema(&fully_qualified_name, &aliases);
@@ -1199,9 +1201,6 @@ impl Parser {
complex: &Map<String, Value>,
enclosing_namespace: &Namespace,
) -> AvroResult<Schema> {
- let name = Name::parse(complex)?;
- let aliases = complex.aliases();
- let fully_qualified_name = name.fully_qualified_name(enclosing_namespace);
let symbols_opt = complex.get("symbols");
if symbols_opt.is_none() {
@@ -1210,6 +1209,10 @@ impl Parser {
}
}
+ let name = Name::parse(complex)?;
+ let fully_qualified_name = name.fully_qualified_name(enclosing_namespace);
+ let aliases = fix_aliases_namespace(complex.aliases(), &name.namespace);
+
let symbols: Vec<String> = symbols_opt
.and_then(|v| v.as_array())
.ok_or(Error::GetEnumSymbolsField)
@@ -1297,9 +1300,6 @@ impl Parser {
complex: &Map<String, Value>,
enclosing_namespace: &Namespace,
) -> AvroResult<Schema> {
- let name = Name::parse(complex)?;
- let fully_qualified_name = name.fully_qualified_name(enclosing_namespace);
- let aliases = complex.aliases();
let size_opt = complex.get("size");
if size_opt.is_none() {
if let Some(seen) = self.get_already_seen_schema(complex, enclosing_namespace) {
@@ -1316,6 +1316,10 @@ impl Parser {
.and_then(|v| v.as_i64())
.ok_or(Error::GetFixedSizeField)?;
+ let name = Name::parse(complex)?;
+ let fully_qualified_name = name.fully_qualified_name(enclosing_namespace);
+ let aliases = fix_aliases_namespace(complex.aliases(), &name.namespace);
+
let schema = Schema::Fixed {
name,
aliases: aliases.clone(),
@@ -1329,6 +1333,29 @@ impl Parser {
}
}
+// A type alias may be specified either as a fully namespace-qualified, or relative
+// to the namespace of the name it is an alias for. For example, if a type named "a.b"
+// has aliases of "c" and "x.y", then the fully qualified names of its aliases are "a.c"
+// and "x.y".
+// https://avro.apache.org/docs/current/spec.html#Aliases
+fn fix_aliases_namespace(aliases: Aliases, namespace: &Namespace) -> Aliases {
+ aliases.map(|aliases| {
+ aliases
+ .iter()
+ .map(|alias| {
+ if alias.find('.').is_none() {
+ match namespace {
+ Some(ref ns) => format!("{}.{}", ns, alias),
+ None => alias.clone(),
+ }
+ } else {
+ alias.clone()
+ }
+ })
+ .collect()
+ })
+}
+
fn get_schema_type_name(name: Name, value: Value) -> Name {
match value.get("type") {
Some(Value::Object(complex_type)) => match complex_type.name() {
@@ -3505,4 +3532,104 @@ mod tests {
let _schema = Schema::parse_str(&schema_str).expect("test failed");
assert_eq!(schema, _schema);
}
+
+ #[test]
+ fn avro_3512_alias_with_null_namespace_record() {
+ let schema = Schema::parse_str(
+ r#"
+ {
+ "type": "record",
+ "name": "a",
+ "namespace": "space",
+ "aliases": ["b", "x.y", ".c"],
+ "fields" : [
+ {"name": "time", "type": "long"}
+ ]
+ }
+ "#,
+ )
+ .unwrap();
+
+ if let Schema::Record { ref aliases, .. } = schema {
+ match aliases {
+ Some(aliases) => {
+ assert_eq!(aliases.len(), 3);
+ assert_eq!(aliases[0], "space.b");
+ assert_eq!(aliases[1], "x.y");
+ assert_eq!(aliases[2], ".c");
+ }
+ None => {
+ panic!("'aliases' must be Some");
+ }
+ }
+ } else {
+ panic!("The Schema should be a record: {:?}", schema);
+ }
+ }
+
+ #[test]
+ fn avro_3512_alias_with_null_namespace_enum() {
+ let schema = Schema::parse_str(
+ r#"
+ {
+ "type": "enum",
+ "name": "a",
+ "namespace": "space",
+ "aliases": ["b", "x.y", ".c"],
+ "symbols" : [
+ "symbol1", "symbol2"
+ ]
+ }
+ "#,
+ )
+ .unwrap();
+
+ if let Schema::Enum { ref aliases, .. } = schema {
+ match aliases {
+ Some(aliases) => {
+ assert_eq!(aliases.len(), 3);
+ assert_eq!(aliases[0], "space.b");
+ assert_eq!(aliases[1], "x.y");
+ assert_eq!(aliases[2], ".c");
+ }
+ None => {
+ panic!("'aliases' must be Some");
+ }
+ }
+ } else {
+ panic!("The Schema should be an enum: {:?}", schema);
+ }
+ }
+
+ #[test]
+ fn avro_3512_alias_with_null_namespace_fixed() {
+ let schema = Schema::parse_str(
+ r#"
+ {
+ "type": "fixed",
+ "name": "a",
+ "namespace": "space",
+ "aliases": ["b", "x.y", ".c"],
+ "size" : 12
+ }
+ "#,
+ )
+ .unwrap();
+
+ if let Schema::Fixed { ref aliases, .. } = schema {
+ match aliases {
+ Some(aliases) => {
+ assert_eq!(aliases.len(), 3);
+ assert_eq!(aliases[0], "space.b");
+ assert_eq!(aliases[1], "x.y");
+ assert_eq!(aliases[2], ".c");
+ }
+ None => {
+ panic!("'aliases' must be Some");
+ }
+ }
+ } else {
+ panic!("The Schema should be a fixed: {:?}", schema);
+ }
+ }
}