You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/04/19 18:24:08 UTC
[avro] branch master updated: AVRO-2883: Fix namespace mapping (#1610)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new cf0cb14de AVRO-2883: Fix namespace mapping (#1610)
cf0cb14de is described below
commit cf0cb14de5498cf420265217e8f4d3a84dd2f4a4
Author: Zoltan Csizmadia <zc...@gmail.com>
AuthorDate: Tue Apr 19 13:24:05 2022 -0500
AVRO-2883: Fix namespace mapping (#1610)
* Remove unused package references
* Replace namespace in text schema
* Remove namespace mapping
* Add unit tests
* Match namespace mapping used in ticket
* Make ReplaceMappedNamespacesInSchema private
* Mark NamespaceMapping obsolete
Co-authored-by: Zoltan Csizmadia <Cs...@valassis.com>
---
lang/csharp/src/apache/codegen/Avro.codegen.csproj | 9 --
lang/csharp/src/apache/codegen/AvroGen.cs | 13 +--
lang/csharp/src/apache/main/CodeGen/CodeGen.cs | 76 ++++++++++++-
.../csharp/src/apache/test/AvroGen/AvroGenTests.cs | 123 ++++++++++++++++-----
4 files changed, 170 insertions(+), 51 deletions(-)
diff --git a/lang/csharp/src/apache/codegen/Avro.codegen.csproj b/lang/csharp/src/apache/codegen/Avro.codegen.csproj
index e0e2b8e68..dfb438d37 100644
--- a/lang/csharp/src/apache/codegen/Avro.codegen.csproj
+++ b/lang/csharp/src/apache/codegen/Avro.codegen.csproj
@@ -54,15 +54,6 @@
<WarningsAsErrors />
</PropertyGroup>
- <!-- See lang/csharp/README.md for tool and library dependency update strategy -->
- <ItemGroup>
- <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
- <PackageReference Include="System.CodeDom" Version="$(SystemCodeDomVersion)" />
- <PackageReference Include="System.Reflection" Version="$(SystemReflectionVersion)" />
- <PackageReference Include="System.Reflection.Emit.ILGeneration" Version="$(SystemReflectionEmitILGenerationVersion)" />
- <PackageReference Include="System.Reflection.Emit.Lightweight" Version="$(SystemReflectionEmitLightweightVersion)" />
- </ItemGroup>
-
<ItemGroup>
<ProjectReference Include="..\main\Avro.main.csproj" />
</ItemGroup>
diff --git a/lang/csharp/src/apache/codegen/AvroGen.cs b/lang/csharp/src/apache/codegen/AvroGen.cs
index f1572c32f..d11690272 100644
--- a/lang/csharp/src/apache/codegen/AvroGen.cs
+++ b/lang/csharp/src/apache/codegen/AvroGen.cs
@@ -159,13 +159,9 @@ namespace Avro
try
{
string text = System.IO.File.ReadAllText(infile);
- Protocol protocol = Protocol.Parse(text);
CodeGen codegen = new CodeGen();
- codegen.AddProtocol(protocol);
-
- foreach (var entry in namespaceMapping)
- codegen.NamespaceMapping[entry.Key] = entry.Value;
+ codegen.AddProtocol(text, namespaceMapping);
codegen.GenerateCode();
codegen.WriteTypes(outdir);
@@ -185,13 +181,8 @@ namespace Avro
try
{
string text = System.IO.File.ReadAllText(infile);
- Schema schema = Schema.Parse(text);
-
CodeGen codegen = new CodeGen();
- codegen.AddSchema(schema);
-
- foreach (var entry in namespaceMapping)
- codegen.NamespaceMapping[entry.Key] = entry.Value;
+ codegen.AddSchema(text, namespaceMapping);
codegen.GenerateCode();
codegen.WriteTypes(outdir);
diff --git a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs
index 85ba07da6..9176f4f21 100644
--- a/lang/csharp/src/apache/main/CodeGen/CodeGen.cs
+++ b/lang/csharp/src/apache/main/CodeGen/CodeGen.cs
@@ -24,6 +24,7 @@ using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
+using System.Text.RegularExpressions;
using Microsoft.CSharp;
namespace Avro
@@ -63,6 +64,7 @@ namespace Avro
/// <value>
/// The namespace mapping.
/// </value>
+ [Obsolete("NamespaceMapping is not used, use AddProtocol(string ...) or AddSchema(string ...) instead!")]
public IDictionary<string, string> NamespaceMapping { get; private set; }
/// <summary>
@@ -80,7 +82,6 @@ namespace Avro
{
Schemas = new List<Schema>();
Protocols = new List<Protocol>();
- NamespaceMapping = new Dictionary<string, string>();
NamespaceLookup = new Dictionary<string, CodeNamespace>(StringComparer.Ordinal);
}
@@ -103,6 +104,19 @@ namespace Avro
Protocols.Add(protocol);
}
+ /// <summary>
+ /// Parses and adds a protocol object to generate code for.
+ /// </summary>
+ /// <param name="protocolText">The protocol.</param>
+ /// <param name="namespaceMapping">namespace mapping key value pairs.</param>
+ public virtual void AddProtocol(string protocolText, IEnumerable<KeyValuePair<string, string>> namespaceMapping = null)
+ {
+ // Map namespaces
+ protocolText = ReplaceMappedNamespacesInSchema(protocolText, namespaceMapping);
+ Protocol protocol = Protocol.Parse(protocolText);
+ Protocols.Add(protocol);
+ }
+
/// <summary>
/// Adds a schema object to generate code for.
/// </summary>
@@ -112,6 +126,19 @@ namespace Avro
Schemas.Add(schema);
}
+ /// <summary>
+ /// Parses and adds a schema object to generate code for.
+ /// </summary>
+ /// <param name="schemaText">schema object.</param>
+ /// <param name="namespaceMapping">namespace mapping key value pairs.</param>
+ public virtual void AddSchema(string schemaText, IEnumerable<KeyValuePair<string, string>> namespaceMapping = null)
+ {
+ // Map namespaces
+ schemaText = ReplaceMappedNamespacesInSchema(schemaText, namespaceMapping);
+ Schema schema = Schema.Parse(schemaText);
+ Schemas.Add(schema);
+ }
+
/// <summary>
/// Adds a namespace object for the given name into the dictionary if it doesn't exist yet.
/// </summary>
@@ -129,9 +156,7 @@ namespace Avro
if (!NamespaceLookup.TryGetValue(name, out CodeNamespace ns))
{
- ns = NamespaceMapping.TryGetValue(name, out string csharpNamespace)
- ? new CodeNamespace(csharpNamespace)
- : new CodeNamespace(CodeGenUtil.Instance.Mangle(name));
+ ns = new CodeNamespace(CodeGenUtil.Instance.Mangle(name));
foreach (CodeNamespaceImport nci in CodeGenUtil.Instance.NamespaceImports)
{
@@ -1158,5 +1183,48 @@ namespace Avro
}
}
}
+
+ /// <summary>
+ /// Replace namespace(s) in schema or protocol definition.
+ /// </summary>
+ /// <param name="input">input schema or protocol definition.</param>
+ /// <param name="namespaceMapping">namespace mappings object.</param>
+ private static string ReplaceMappedNamespacesInSchema(string input, IEnumerable<KeyValuePair<string, string>> namespaceMapping)
+ {
+ if (namespaceMapping == null || input == null)
+ return input;
+
+ // Replace namespace in "namespace" definitions:
+ // "namespace": "originalnamespace" -> "namespace": "mappednamespace"
+ // "namespace": "originalnamespace.whatever" -> "namespace": "mappednamespace.whatever"
+ // Note: It keeps the original whitespaces
+ return Regex.Replace(input, @"""namespace""(\s*):(\s*)""([^""]*)""", m =>
+ {
+ // m.Groups[1]: whitespaces before ':'
+ // m.Groups[2]: whitespaces after ':'
+ // m.Groups[3]: the namespace
+
+ string ns = m.Groups[3].Value;
+
+ foreach (var mapping in namespaceMapping)
+ {
+ // Full match
+ if (mapping.Key == ns)
+ {
+ ns = mapping.Value;
+ break;
+ }
+ else
+ // Partial match
+ if (ns.StartsWith($"{mapping.Key}."))
+ {
+ ns = $"{mapping.Value}.{ns.Substring(mapping.Key.Length + 1)}";
+ break;
+ }
+ }
+
+ return $@"""namespace""{m.Groups[1].Value}:{m.Groups[2].Value}""{ns}""";
+ });
+ }
}
}
diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs
index 93e453190..2426bc275 100644
--- a/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs
+++ b/lang/csharp/src/apache/test/AvroGen/AvroGenTests.cs
@@ -261,6 +261,51 @@ namespace Avro.Test.AvroGen
]
}";
+ // https://issues.apache.org/jira/browse/AVRO-2883
+ private const string _schema_avro_2883 = @"
+{
+ ""type"" : ""record"",
+ ""name"" : ""TestModel"",
+ ""namespace"" : ""my.avro.ns"",
+ ""fields"" : [ {
+ ""name"" : ""eventType"",
+ ""type"" : {
+ ""type"" : ""enum"",
+ ""name"" : ""EventType"",
+ ""symbols"" : [ ""CREATE"", ""UPDATE"", ""DELETE"" ]
+ }
+} ]
+}";
+
+ // https://issues.apache.org/jira/browse/AVRO-3046
+ private const string _schema_avro_3046 = @"
+{
+ ""type"": ""record"",
+ ""name"": ""ExampleRecord"",
+ ""namespace"": ""com.example"",
+ ""fields"": [
+ {
+ ""name"": ""Id"",
+ ""type"": ""string"",
+ ""logicalType"": ""UUID""
+ },
+ {
+ ""name"": ""InnerRecord"",
+ ""type"": {
+ ""type"": ""record"",
+ ""name"": ""InnerRecord"",
+ ""fields"": [
+ {
+ ""name"": ""Id"",
+ ""type"": ""string"",
+ ""logicalType"": ""UUID""
+ }
+ ]
+ }
+ }
+ ]
+}";
+
private Assembly TestSchema(
string schema,
IEnumerable<string> typeNamesToCheck = null,
@@ -417,6 +462,18 @@ namespace Avro.Test.AvroGen
{
"org/apache/avro/codegentest/testdata/NullableLogicalTypesArray.cs"
})]
+ [TestCase(
+ _schema_avro_2883,
+ new string[]
+ {
+ "my.avro.ns.TestModel",
+ "my.avro.ns.EventType",
+ },
+ new string[]
+ {
+ "my/avro/ns/TestModel.cs",
+ "my/avro/ns/EventType.cs"
+ })]
public void GenerateSchema(string schema, IEnumerable<string> typeNamesToCheck, IEnumerable<string> generatedFilesToCheck)
{
TestSchema(schema, typeNamesToCheck, generatedFilesToCheck: generatedFilesToCheck);
@@ -433,6 +490,45 @@ namespace Avro.Test.AvroGen
{
"org/apache/csharp/codegentest/testdata/NullableLogicalTypesArray.cs"
})]
+ [TestCase(
+ _nestedLogicalTypesUnion,
+ "org.apache.avro.codegentest.testdata", "org.apache.csharp.codegentest.testdata",
+ new string[]
+ {
+ "org.apache.csharp.codegentest.testdata.NestedLogicalTypesUnion",
+ "org.apache.csharp.codegentest.testdata.RecordInUnion"
+ },
+ new string[]
+ {
+ "org/apache/csharp/codegentest/testdata/NestedLogicalTypesUnion.cs",
+ "org/apache/csharp/codegentest/testdata/RecordInUnion.cs"
+ })]
+ [TestCase(
+ _schema_avro_2883,
+ "my.avro.ns", "my.csharp.ns",
+ new string[]
+ {
+ "my.csharp.ns.TestModel",
+ "my.csharp.ns.EventType",
+ },
+ new string[]
+ {
+ "my/csharp/ns/TestModel.cs",
+ "my/csharp/ns/EventType.cs"
+ })]
+ [TestCase(
+ _schema_avro_3046,
+ "com.example", "Example",
+ new string[]
+ {
+ "Example.ExampleRecord",
+ "Example.InnerRecord",
+ },
+ new string[]
+ {
+ "Example/ExampleRecord.cs",
+ "Example/InnerRecord.cs"
+ })]
[TestCase(
_nullableLogicalTypesArray,
"org.apache.avro.codegentest.testdata", "org.apache.@return.@int", // Reserved keywords in namespace
@@ -497,33 +593,6 @@ namespace Avro.Test.AvroGen
TestSchema(schema, typeNamesToCheck, new Dictionary<string, string> { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck);
}
- [TestCase(
- _nestedLogicalTypesUnion,
- "org.apache.avro.codegentest.testdata", "org.apache.csharp.codegentest.testdata",
- new string[]
- {
- "org.apache.avro.codegentest.testdata.NestedLogicalTypesUnion",
- "org.apache.avro.codegentest.testdata.RecordInUnion"
- },
- new string[]
- {
- "org/apache/csharp/codegentest/testdata/NestedLogicalTypesUnion.cs",
- "org/apache/csharp/codegentest/testdata/RecordInUnion.cs"
- })]
- public void GenerateSchemaWithNamespaceMapping_Bug_AVRO_2883(
- string schema,
- string namespaceMappingFrom,
- string namespaceMappingTo,
- IEnumerable<string> typeNamesToCheck,
- IEnumerable<string> generatedFilesToCheck)
- {
- // !!! This is a bug which must be fixed
- // !!! Once it is fixed, this test will fail and this test can be removed
- // https://issues.apache.org/jira/browse/AVRO-2883
- // https://issues.apache.org/jira/browse/AVRO-3046
- Assert.Throws<AssertionException>(() => TestSchema(schema, typeNamesToCheck, new Dictionary<string, string> { { namespaceMappingFrom, namespaceMappingTo } }, generatedFilesToCheck));
- }
-
[TestCase(_logicalTypesWithCustomConversion, typeof(AvroTypeException))]
[TestCase(_customConversionWithLogicalTypes, typeof(SchemaParseException))]
[TestCase(_nestedLogicalTypesUnionFixedDecimal, typeof(SchemaParseException))]