You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/03/29 10:21:12 UTC
[avro] branch branch-1.11 updated: AVRO-3468: Handle default values for logical types (#1622)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch branch-1.11
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/branch-1.11 by this push:
new ff45ec9 AVRO-3468: Handle default values for logical types (#1622)
ff45ec9 is described below
commit ff45ec9b7192be52abf7e57fdf8142112679ee9d
Author: Lucas Heimberg <20...@users.noreply.github.com>
AuthorDate: Tue Mar 29 10:31:52 2022 +0200
AVRO-3468: Handle default values for logical types (#1622)
* AVRO-3468: Add handling of default values for logical types
This extends the resolver to handle default values for logical types
according to their base schema.
* AVRO-3468: Fix tests for conversion of default values to decimal logical type
Because of a possible bug in the handling of TestCaseData in NUnit,
the tests for the decimal logical type are separate from the tests
for other logical types.
* AVRO-3468: Add tests for decimal binary encoding
* AVRO-3468: Fix tests by parsing decimals culture-invariant
* AVRO-3468: Removes redundant using statement
* AVRO-3468: Add comment about CultureInfo to tests
Co-authored-by: l.heimberg <l....@cid.com>
(cherry picked from commit 65eb24a6275632d02eed8bca54c71314bfdb42e2)
---
lang/csharp/src/apache/main/IO/Resolver.cs | 4 +
lang/csharp/src/apache/test/AvroDecimalTest.cs | 1 +
.../csharp/src/apache/test/Generic/GenericTests.cs | 105 ++++++++++++++++++---
.../test/Specific/RecordWithOptionalLogicalType.cs | 74 +++++++++++++++
.../src/apache/test/Specific/SpecificTests.cs | 39 ++++++++
.../src/apache/test/Util/LogicalTypeTests.cs | 35 ++++++-
6 files changed, 245 insertions(+), 13 deletions(-)
diff --git a/lang/csharp/src/apache/main/IO/Resolver.cs b/lang/csharp/src/apache/main/IO/Resolver.cs
index c77aca7..60d7966 100644
--- a/lang/csharp/src/apache/main/IO/Resolver.cs
+++ b/lang/csharp/src/apache/main/IO/Resolver.cs
@@ -158,6 +158,10 @@ namespace Avro.IO
EncodeDefaultValue(enc, (schema as UnionSchema).Schemas[0], jtok);
break;
+ case Schema.Type.Logical:
+ EncodeDefaultValue(enc, (schema as LogicalSchema).BaseSchema, jtok);
+ break;
+
default:
throw new AvroException("Unsupported schema type " + schema.Tag);
}
diff --git a/lang/csharp/src/apache/test/AvroDecimalTest.cs b/lang/csharp/src/apache/test/AvroDecimalTest.cs
index e10210b..4a8654a 100644
--- a/lang/csharp/src/apache/test/AvroDecimalTest.cs
+++ b/lang/csharp/src/apache/test/AvroDecimalTest.cs
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
using NUnit.Framework;
namespace Avro.test
diff --git a/lang/csharp/src/apache/test/Generic/GenericTests.cs b/lang/csharp/src/apache/test/Generic/GenericTests.cs
index 05aa5bc..8e0a86c 100644
--- a/lang/csharp/src/apache/test/Generic/GenericTests.cs
+++ b/lang/csharp/src/apache/test/Generic/GenericTests.cs
@@ -17,16 +17,97 @@
*/
using System;
using System.IO;
-using System.Linq;
using Avro.IO;
using System.Collections.Generic;
+using System.Text;
using Avro.Generic;
using NUnit.Framework;
+using Decoder = Avro.IO.Decoder;
+using Encoder = Avro.IO.Encoder;
namespace Avro.Test.Generic
{
class GenericTests
{
+ private static string intToUtf8(int value)
+ {
+ var decimalLogicalType = new Avro.Util.Decimal();
+ var logicalSchema = (LogicalSchema)
+ Schema.Parse(@"{ ""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4 }");
+
+ byte[] byteArray = (byte[])decimalLogicalType.ConvertToBaseValue(new AvroDecimal(value), logicalSchema);
+
+ return Encoding.GetEncoding("iso-8859-1").GetString(byteArray);
+ }
+
+ [Test]
+ public void ConvertsDecimalZeroToLogicalType() => ConvertsDefaultToLogicalType(
+ @"{""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4}",
+ @$"""{intToUtf8(0)}""", new AvroDecimal(0));
+
+ [Test]
+ public void ConvertsDecimalIntegerToLogicalType() => ConvertsDefaultToLogicalType(
+ @"{""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4}",
+ @$"""{intToUtf8(1234)}""", new AvroDecimal(1234));
+
+ [Test]
+ public void ConvertsDecimalScaledToLogicalType() => ConvertsDefaultToLogicalType(
+ @"{""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 3}",
+ @$"""{intToUtf8(1234)}""", new AvroDecimal(1.234));
+
+ private static IEnumerable<TestCaseData> ConvertsDefaultToLogicalTypeSource = new List<TestCaseData>()
+ {
+ new TestCaseData(@"{""type"": ""string"", ""logicalType"": ""uuid""}", @"""00000000-0000-0000-0000-000000000000""", new Guid()),
+ new TestCaseData(@"{""type"": ""string"", ""logicalType"": ""uuid""}", @"""00000000000000000000000000000000""", new Guid()),
+ new TestCaseData(@"{""type"": ""string"", ""logicalType"": ""uuid""}", @"""12345678-1234-5678-1234-123456789012""", new Guid("12345678-1234-5678-1234-123456789012")),
+ new TestCaseData(@"{""type"": ""string"", ""logicalType"": ""uuid""}", @"""12345678123456781234123456789012""", new Guid("12345678-1234-5678-1234-123456789012")),
+ new TestCaseData(@"{""type"": ""int"", ""logicalType"": ""date""}", "0", DateTime.UnixEpoch),
+ new TestCaseData(@"{""type"": ""int"", ""logicalType"": ""date""}", "123456", DateTime.UnixEpoch.AddDays(123456)),
+ new TestCaseData(@"{""type"": ""long"", ""logicalType"": ""time-micros""}", "0", new TimeSpan()),
+ new TestCaseData(@"{""type"": ""long"", ""logicalType"": ""time-micros""}", "123456", new TimeSpan(123456*TimeSpan.TicksPerMillisecond/1000)),
+ new TestCaseData(@"{""type"": ""int"", ""logicalType"": ""time-millis""}", "0", new TimeSpan()),
+ new TestCaseData(@"{""type"": ""int"", ""logicalType"": ""time-millis""}", "123456", new TimeSpan(0, 0, 0, 0, 123456)),
+ new TestCaseData(@"{""type"": ""long"", ""logicalType"": ""timestamp-micros""}", "0", DateTime.UnixEpoch),
+ new TestCaseData(@"{""type"": ""long"", ""logicalType"": ""timestamp-micros""}", "123456", DateTime.UnixEpoch.AddTicks(123456*TimeSpan.TicksPerMillisecond/1000)),
+ new TestCaseData(@"{""type"": ""long"", ""logicalType"": ""timestamp-millis""}", "0", DateTime.UnixEpoch),
+ new TestCaseData(@"{""type"": ""long"", ""logicalType"": ""timestamp-millis""}", "123456", DateTime.UnixEpoch.AddMilliseconds(123456))
+ };
+
+ [TestCaseSource(nameof(ConvertsDefaultToLogicalTypeSource))]
+ public void ConvertsDefaultToLogicalType(string typeDefinition, string defaultDefinition, object expected)
+ {
+ var writerSchemaString = @"{
+ ""type"": ""record"",
+ ""name"": ""Foo"",
+ ""fields"": [
+ ]
+}";
+
+ var readerSchemaString = $@"{{
+ ""type"": ""record"",
+ ""name"": ""Foo"",
+ ""fields"": [
+ {{
+ ""name"": ""x"",
+ ""type"": {typeDefinition},
+ ""default"": {defaultDefinition}
+ }}
+ ]
+}}";
+ var writerSchema = Schema.Parse(writerSchemaString);
+
+ Stream stream;
+
+ serialize(writerSchemaString,
+ MkRecord(new object[] { }, (RecordSchema)writerSchema),
+ out stream,
+ out _);
+
+ var output = deserialize<GenericRecord>(stream, writerSchema, Schema.Parse(readerSchemaString)).GetValue(0);
+
+ Assert.AreEqual(expected, output);
+ }
+
private static void test<T>(string s, T value)
{
Stream ms;
@@ -98,7 +179,7 @@ namespace Avro.Test.Generic
new object[] { "f1", new byte[] { 1, 2 } })]
public void TestRecord(string schema, object[] kv)
{
- test(schema, mkRecord(kv, Schema.Parse(schema) as RecordSchema));
+ test(schema, MkRecord(kv, Schema.Parse(schema) as RecordSchema));
}
[TestCase("{\"type\": \"map\", \"values\": \"string\"}",
@@ -166,7 +247,7 @@ namespace Avro.Test.Generic
new object[] { "f1", "v1" })]
public void TestUnion_record(string unionSchema, string recordSchema, object[] value)
{
- test(unionSchema, mkRecord(value, Schema.Parse(recordSchema) as RecordSchema));
+ test(unionSchema, MkRecord(value, Schema.Parse(recordSchema) as RecordSchema));
}
[TestCase("[{\"type\": \"enum\", \"symbols\": [\"s1\", \"s2\"], \"name\": \"e\"}, \"string\"]",
@@ -344,8 +425,8 @@ namespace Avro.Test.Generic
new object[] { "f1", true, "f2", "d" }, Description = "Default field")]
public void TestResolution_record(string ws, object[] actual, string rs, object[] expected)
{
- TestResolution(ws, mkRecord(actual, Schema.Parse(ws) as RecordSchema), rs,
- mkRecord(expected, Schema.Parse(rs) as RecordSchema));
+ TestResolution(ws, MkRecord(actual, Schema.Parse(ws) as RecordSchema), rs,
+ MkRecord(expected, Schema.Parse(rs) as RecordSchema));
}
[TestCase("{\"type\":\"map\",\"values\":\"int\"}", new object[] { "a", 100, "b", -202 },
@@ -419,11 +500,11 @@ namespace Avro.Test.Generic
{
if (expectedExceptionType != null)
{
- Assert.Throws(expectedExceptionType, () => { testResolutionMismatch(ws, mkRecord(actual, Schema.Parse(ws) as RecordSchema), rs); });
+ Assert.Throws(expectedExceptionType, () => { testResolutionMismatch(ws, MkRecord(actual, Schema.Parse(ws) as RecordSchema), rs); });
}
else
{
- testResolutionMismatch(ws, mkRecord(actual, Schema.Parse(ws) as RecordSchema), rs);
+ testResolutionMismatch(ws, MkRecord(actual, Schema.Parse(ws) as RecordSchema), rs);
}
}
@@ -491,7 +572,7 @@ namespace Avro.Test.Generic
"{\"type\":\"record\",\"name\":\"r\",\"fields\":" +
"[{\"name\":\"a\",\"type\":{\"type\":\"array\",\"items\":\"int\"}}]}");
- Func<int[], GenericRecord> makeRec = arr => mkRecord(new object[] { "a", arr }, schema);
+ Func<int[], GenericRecord> makeRec = arr => MkRecord(new object[] { "a", arr }, schema);
var rec1 = makeRec(new[] { 69, 23 });
var rec2 = makeRec(new[] { 42, 11 });
@@ -506,7 +587,7 @@ namespace Avro.Test.Generic
"{\"type\":\"record\",\"name\":\"r\",\"fields\":" +
"[{\"name\":\"a\",\"type\":{\"type\":\"array\",\"items\":\"int\"}}]}");
- Func<int[], GenericRecord> makeRec = arr => mkRecord(new object[] { "a", arr }, schema);
+ Func<int[], GenericRecord> makeRec = arr => MkRecord(new object[] { "a", arr }, schema);
// Intentionally duplicated so reference equality doesn't apply
var rec1 = makeRec(new[] { 89, 12, 66 });
@@ -522,7 +603,7 @@ namespace Avro.Test.Generic
"{\"type\":\"record\",\"name\":\"r\",\"fields\":" +
"[{\"name\":\"a\",\"type\":{\"type\":\"map\",\"values\":\"int\"}}]}");
- Func<int, GenericRecord> makeRec = value => mkRecord(
+ Func<int, GenericRecord> makeRec = value => MkRecord(
new object[] { "a", new Dictionary<string, int> { { "key", value } } }, schema);
var rec1 = makeRec(52);
@@ -538,7 +619,7 @@ namespace Avro.Test.Generic
"{\"type\":\"record\",\"name\":\"r\",\"fields\":" +
"[{\"name\":\"a\",\"type\":{\"type\":\"map\",\"values\":\"int\"}}]}");
- Func<int, GenericRecord> makeRec = value => mkRecord(
+ Func<int, GenericRecord> makeRec = value => MkRecord(
new object[] { "a", new Dictionary<string, int> { { "key", value } } }, schema);
var rec1 = makeRec(69);
@@ -547,7 +628,7 @@ namespace Avro.Test.Generic
Assert.AreNotEqual(rec1, rec2);
}
- private static GenericRecord mkRecord(object[] kv, RecordSchema s)
+ public static GenericRecord MkRecord(object[] kv, RecordSchema s)
{
GenericRecord input = new GenericRecord(s);
for (int i = 0; i < kv.Length; i += 2)
diff --git a/lang/csharp/src/apache/test/Specific/RecordWithOptionalLogicalType.cs b/lang/csharp/src/apache/test/Specific/RecordWithOptionalLogicalType.cs
new file mode 100644
index 0000000..585032e
--- /dev/null
+++ b/lang/csharp/src/apache/test/Specific/RecordWithOptionalLogicalType.cs
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// ------------------------------------------------------------------------------
+// <auto-generated>
+// Generated by avrogen, version 1.11.0.0
+// Changes to this file may cause incorrect behavior and will be lost if code
+// is regenerated
+// </auto-generated>
+// ------------------------------------------------------------------------------
+namespace Avro.Test.Specific.@return
+{
+ using System;
+ using System.Collections.Generic;
+ using System.Text;
+ using Avro;
+ using Avro.Specific;
+
+ public partial class RecordWithOptionalLogicalType : ISpecificRecord
+ {
+ public static Schema _SCHEMA = Avro.Schema.Parse("{\"type\":\"record\",\"name\":\"RecordWithOptionalLogicalType\",\"namespace\":\"Avro.Test.Sp" +
+ "ecific.return\",\"fields\":[{\"name\":\"x\",\"default\":10,\"type\":{\"type\":\"int\",\"logicalT" +
+ "ype\":\"date\"}}]}");
+ private System.DateTime _x;
+ public virtual Schema Schema
+ {
+ get
+ {
+ return RecordWithOptionalLogicalType._SCHEMA;
+ }
+ }
+ public System.DateTime x
+ {
+ get
+ {
+ return this._x;
+ }
+ set
+ {
+ this._x = value;
+ }
+ }
+ public virtual object Get(int fieldPos)
+ {
+ switch (fieldPos)
+ {
+ case 0: return this.x;
+ default: throw new AvroRuntimeException("Bad index " + fieldPos + " in Get()");
+ };
+ }
+ public virtual void Put(int fieldPos, object fieldValue)
+ {
+ switch (fieldPos)
+ {
+ case 0: this.x = (System.DateTime)fieldValue; break;
+ default: throw new AvroRuntimeException("Bad index " + fieldPos + " in Put()");
+ };
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/test/Specific/SpecificTests.cs b/lang/csharp/src/apache/test/Specific/SpecificTests.cs
index 46b93e5..1ba6840 100644
--- a/lang/csharp/src/apache/test/Specific/SpecificTests.cs
+++ b/lang/csharp/src/apache/test/Specific/SpecificTests.cs
@@ -24,6 +24,8 @@ using Avro.IO;
using Avro.Specific;
using Avro.Test.Specific;
using System.Collections.Generic;
+using Avro.Generic;
+using Avro.Test.Generic;
using Avro.Test.Specific.@return;
#if !NETCOREAPP
@@ -440,6 +442,43 @@ namespace Avro.Test
Assert.AreEqual(0, dstRecord.UserMatrix[2].Count);
}
+ private static void serializeGeneric<T>(string writerSchema, T actual, out Stream stream, out Schema ws)
+ {
+ var ms = new MemoryStream();
+ Encoder e = new BinaryEncoder(ms);
+ ws = Schema.Parse(writerSchema);
+ GenericWriter<T> w = new GenericWriter<T>(ws);
+ w.Write(actual, e);
+ ms.Flush();
+ ms.Position = 0;
+ stream = ms;
+ }
+
+ [Test]
+ public void DeserializeToLogicalTypeWithDefault()
+ {
+ var writerSchemaString = @"{
+ ""type"": ""record"",
+ ""name"": ""RecordWithOptionalLogicalType"",
+ ""namespace"": ""Avro.Test.Specific.return"",
+ ""fields"": [
+ ]}";
+
+ var writerSchema = Schema.Parse(writerSchemaString);
+
+ Stream stream;
+
+ serializeGeneric(writerSchemaString,
+ GenericTests.MkRecord(new object[] { }, (RecordSchema)writerSchema),
+ out stream,
+ out _);
+
+ RecordWithOptionalLogicalType output = deserialize<RecordWithOptionalLogicalType>(stream, writerSchema, RecordWithOptionalLogicalType._SCHEMA);
+
+ Assert.AreEqual(output.x, new DateTime(1970, 1, 11));
+
+ }
+
private static S deserialize<S>(Stream ms, Schema ws, Schema rs) where S : class, ISpecificRecord
{
long initialPos = ms.Position;
diff --git a/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs b/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs
index 6b2762a..32ce7c3 100644
--- a/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs
+++ b/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs
@@ -18,6 +18,7 @@
using System;
using System.Globalization;
+using System.Numerics;
using Avro.Util;
using NUnit.Framework;
@@ -26,6 +27,37 @@ namespace Avro.Test
[TestFixture]
class LogicalTypeTests
{
+ [TestCase("0", 0, new byte[] { 0 })]
+ [TestCase("1.01", 2, new byte[] { 101 })]
+ [TestCase("123456789123456789.56", 2, new byte[] { 0, 171, 84, 169, 143, 129, 101, 36, 108 })]
+ [TestCase("1234", 0, new byte[] { 4, 210 })]
+ [TestCase("1234.5", 1, new byte[] { 48, 57 })]
+ [TestCase("1234.56", 2, new byte[] { 1, 226, 64 })]
+ [TestCase("-0", 0, new byte[] { 0 })]
+ [TestCase("-1.01", 2, new byte[] { 155 })]
+ [TestCase("-123456789123456789.56", 2, new byte[] { 255, 84, 171, 86, 112, 126, 154, 219, 148 })]
+ [TestCase("-1234", 0, new byte[] { 251, 46 })]
+ [TestCase("-1234.5", 1, new byte[] { 207, 199 })]
+ [TestCase("-1234.56", 2, new byte[] { 254, 29, 192 })]
+ // This tests ensures that changes to Decimal.ConvertToBaseValue and ConvertToLogicalValue can be validated (bytes)
+ public void TestDecimalConvert(string s, int scale, byte[] converted)
+ {
+ var schema = (LogicalSchema)Schema.Parse(@$"{{""type"": ""bytes"", ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": {scale}}}");
+
+ var avroDecimal = new Avro.Util.Decimal();
+ // CultureInfo.InvariantCulture ensures that "." is always accepted as the decimal point
+ var decimalVal = (AvroDecimal)decimal.Parse(s, CultureInfo.InvariantCulture);
+
+ // TestDecimal tests ConvertToLogicalValue(ConvertToBaseValue(...)) which might hide symmetrical breaking changes in both functions
+ // The following 2 tests are checking the conversions seperately
+
+ // Validate Decimal.ConvertToBaseValue
+ Assert.AreEqual(converted, avroDecimal.ConvertToBaseValue(decimalVal, schema));
+
+ // Validate Decimal.ConvertToLogicalValue
+ Assert.AreEqual(decimalVal, (AvroDecimal)avroDecimal.ConvertToLogicalValue(converted, schema));
+ }
+
[TestCase("1234.56")]
[TestCase("-1234.56")]
[TestCase("123456789123456789.56")]
@@ -37,7 +69,8 @@ namespace Avro.Test
var schema = (LogicalSchema)Schema.Parse("{\"type\": \"bytes\", \"logicalType\": \"decimal\", \"precision\": 4, \"scale\": 2 }");
var avroDecimal = new Avro.Util.Decimal();
- var decimalVal = (AvroDecimal)decimal.Parse(s);
+ // CultureInfo.InvariantCulture ensures that "." is always accepted as the decimal point
+ var decimalVal = (AvroDecimal)decimal.Parse(s, CultureInfo.InvariantCulture);
var convertedDecimalVal = (AvroDecimal)avroDecimal.ConvertToLogicalValue(avroDecimal.ConvertToBaseValue(decimalVal, schema), schema);