You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/05/03 21:21:45 UTC
[avro] branch master updated: AVRO-3078: Add local-timestamp-millis and local-timestamp-micros logical types to C# (#1628)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new c2bd724f8 AVRO-3078: Add local-timestamp-millis and local-timestamp-micros logical types to C# (#1628)
c2bd724f8 is described below
commit c2bd724f8af1f67ef540f6b8f80fab88663a92bf
Author: Zoltan Csizmadia <zc...@gmail.com>
AuthorDate: Tue May 3 16:21:39 2022 -0500
AVRO-3078: Add local-timestamp-millis and local-timestamp-micros logical types to C# (#1628)
* Add local-timestamp-millis and micros
* Add more UTC based local-timestamp tests
* Fix whitespace
Co-authored-by: Zoltan Csizmadia <Cs...@valassis.com>
---
.../apache/main/Util/LocalTimestampMicrosecond.cs | 63 ++++++++++++++++
.../apache/main/Util/LocalTimestampMillisecond.cs | 62 ++++++++++++++++
.../src/apache/main/Util/LogicalTypeFactory.cs | 2 +
.../src/apache/test/AvroGen/AvroGenSchemaTests.cs | 6 +-
.../src/apache/test/Util/LogicalTypeTests.cs | 86 ++++++++++++++++++++++
5 files changed, 218 insertions(+), 1 deletion(-)
diff --git a/lang/csharp/src/apache/main/Util/LocalTimestampMicrosecond.cs b/lang/csharp/src/apache/main/Util/LocalTimestampMicrosecond.cs
new file mode 100644
index 000000000..36014c97a
--- /dev/null
+++ b/lang/csharp/src/apache/main/Util/LocalTimestampMicrosecond.cs
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+namespace Avro.Util
+{
+ /// <summary>
+ /// The 'local-timestamp-micros' logical type.
+ /// </summary>
+ public class LocalTimestampMicrosecond : LogicalUnixEpochType<DateTime>
+ {
+ /// <summary>
+ /// The logical type name for LocalTimestampMicrosecond.
+ /// </summary>
+ public static readonly string LogicalTypeName = "local-timestamp-micros";
+
+ /// <summary>
+ /// Initializes a new LocalTimestampMicrosecond logical type.
+ /// </summary>
+ public LocalTimestampMicrosecond()
+ : base(LogicalTypeName)
+ {
+ }
+
+ /// <inheritdoc/>
+ public override void ValidateSchema(LogicalSchema schema)
+ {
+ if (Schema.Type.Long != schema.BaseSchema.Tag)
+ {
+ throw new AvroTypeException("'local-timestamp-micros' can only be used with an underlying long type");
+ }
+ }
+
+ /// <inheritdoc/>
+ public override object ConvertToBaseValue(object logicalValue, LogicalSchema schema)
+ {
+ DateTime date = ((DateTime)logicalValue).ToUniversalTime();
+ return (date - UnixEpochDateTime).Ticks / TicksPerMicrosecond;
+ }
+
+ /// <inheritdoc/>
+ public override object ConvertToLogicalValue(object baseValue, LogicalSchema schema)
+ {
+ return UnixEpochDateTime.AddTicks((long)baseValue * TicksPerMicrosecond).ToLocalTime();
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/Util/LocalTimestampMillisecond.cs b/lang/csharp/src/apache/main/Util/LocalTimestampMillisecond.cs
new file mode 100644
index 000000000..4ae86fd08
--- /dev/null
+++ b/lang/csharp/src/apache/main/Util/LocalTimestampMillisecond.cs
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+
+namespace Avro.Util
+{
+ /// <summary>
+ /// The 'local-timestamp-millis' logical type.
+ /// </summary>
+ public class LocalTimestampMillisecond : LogicalUnixEpochType<DateTime>
+ {
+ /// <summary>
+ /// The logical type name for LocalTimestampMillisecond.
+ /// </summary>
+ public static readonly string LogicalTypeName = "local-timestamp-millis";
+
+ /// <summary>
+ /// Initializes a new LocalTimestampMillisecond logical type.
+ /// </summary>
+ public LocalTimestampMillisecond()
+ : base(LogicalTypeName)
+ {
+ }
+
+ /// <inheritdoc/>
+ public override void ValidateSchema(LogicalSchema schema)
+ {
+ if (Schema.Type.Long != schema.BaseSchema.Tag)
+ {
+ throw new AvroTypeException("'local-timestamp-millis' can only be used with an underlying long type");
+ }
+ }
+
+ /// <inheritdoc/>
+ public override object ConvertToBaseValue(object logicalValue, LogicalSchema schema)
+ {
+ DateTime date = ((DateTime)logicalValue).ToUniversalTime();
+ return (long)(date - UnixEpochDateTime).TotalMilliseconds;
+ }
+
+ /// <inheritdoc/>
+ public override object ConvertToLogicalValue(object baseValue, LogicalSchema schema)
+ {
+ return UnixEpochDateTime.AddMilliseconds((long)baseValue).ToLocalTime();
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/Util/LogicalTypeFactory.cs b/lang/csharp/src/apache/main/Util/LogicalTypeFactory.cs
index e7fa31556..f4086ab5a 100644
--- a/lang/csharp/src/apache/main/Util/LogicalTypeFactory.cs
+++ b/lang/csharp/src/apache/main/Util/LogicalTypeFactory.cs
@@ -39,6 +39,8 @@ namespace Avro.Util
{
{ Decimal.LogicalTypeName, new Decimal() },
{ Date.LogicalTypeName, new Date() },
+ { LocalTimestampMillisecond.LogicalTypeName, new LocalTimestampMillisecond() },
+ { LocalTimestampMicrosecond.LogicalTypeName, new LocalTimestampMicrosecond() },
{ TimeMillisecond.LogicalTypeName, new TimeMillisecond() },
{ TimeMicrosecond.LogicalTypeName, new TimeMicrosecond() },
{ TimestampMillisecond.LogicalTypeName, new TimestampMillisecond() },
diff --git a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs
index e31dc383b..807acbda9 100644
--- a/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs
+++ b/lang/csharp/src/apache/test/AvroGen/AvroGenSchemaTests.cs
@@ -693,6 +693,10 @@ namespace Avro.Test.AvroGen
{ ""name"" : ""timestampmillis"", ""type"" : {""type"": ""long"", ""logicalType"": ""timestamp-millis""} },
{ ""name"" : ""nullibiletimestampmicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""timestamp-micros""}] },
{ ""name"" : ""timestampmicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""timestamp-micros""} },
+ { ""name"" : ""nulliblelocaltimestampmillis"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""local-timestamp-millis""}] },
+ { ""name"" : ""localtimestampmillis"", ""type"" : {""type"": ""long"", ""logicalType"": ""local-timestamp-millis""} },
+ { ""name"" : ""nullibilelocaltimestampmicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""local-timestamp-micros""}] },
+ { ""name"" : ""locallocaltimestampmicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""local-timestamp-micros""} },
{ ""name"" : ""nullibiletimemicros"", ""type"" : [""null"", {""type"": ""long"", ""logicalType"": ""time-micros""}] },
{ ""name"" : ""timemicros"", ""type"" : {""type"": ""long"", ""logicalType"": ""time-micros""} },
{ ""name"" : ""nullibiletimemillis"", ""type"" : [""null"", {""type"": ""int"", ""logicalType"": ""time-millis""}] },
@@ -703,7 +707,7 @@ namespace Avro.Test.AvroGen
{ ""name"" : ""decimalfixed"", ""type"" : {""type"": {""type"" : ""fixed"", ""size"": 16, ""name"": ""df""}, ""logicalType"": ""decimal"", ""precision"": 4, ""scale"": 2} }
]
}",
- new object[] { "schematest.LogicalTypes", typeof(Guid?), typeof(Guid), typeof(DateTime?), typeof(DateTime), typeof(DateTime?), typeof(DateTime), typeof(TimeSpan?), typeof(TimeSpan), typeof(TimeSpan?), typeof(TimeSpan), typeof(AvroDecimal?), typeof(AvroDecimal), typeof(AvroDecimal?), typeof(AvroDecimal) })]
+ new object[] { "schematest.LogicalTypes", typeof(Guid?), typeof(Guid), typeof(DateTime?), typeof(DateTime), typeof(DateTime?), typeof(DateTime), typeof(DateTime?), typeof(DateTime), typeof(DateTime?), typeof(DateTime), typeof(TimeSpan?), typeof(TimeSpan), typeof(TimeSpan?), typeof(TimeSpan), typeof(AvroDecimal?), typeof(AvroDecimal), typeof(AvroDecimal?), typeof(AvroDecimal) })]
[TestCase(@"
{
""namespace"": ""enum.base"",
diff --git a/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs b/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs
index 45903d551..d7443391c 100644
--- a/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs
+++ b/lang/csharp/src/apache/test/Util/LogicalTypeTests.cs
@@ -199,6 +199,7 @@ namespace Avro.Test
var avroTimestampMilli = new TimestampMillisecond();
var convertedDate = (DateTime)avroTimestampMilli.ConvertToLogicalValue(avroTimestampMilli.ConvertToBaseValue(date, schema), schema);
Assert.AreEqual(expectedDate, convertedDate);
+ Assert.AreEqual(DateTimeKind.Utc, convertedDate.Kind);
}
[TestCase("01/01/2019 14:20:00Z", "01/01/2019 14:20:00Z")]
@@ -232,6 +233,91 @@ namespace Avro.Test
var avroTimestampMicro = new TimestampMicrosecond();
var convertedDate = (DateTime)avroTimestampMicro.ConvertToLogicalValue(avroTimestampMicro.ConvertToBaseValue(date, schema), schema);
Assert.AreEqual(expectedDate, convertedDate);
+ Assert.AreEqual(DateTimeKind.Utc, convertedDate.Kind);
+ }
+
+ [TestCase("01/01/2019 14:20:00", "01/01/2019 14:20:00")]
+ [TestCase("05/05/2019 14:20:00", "05/05/2019 14:20:00")]
+ [TestCase("05/05/2019 00:00:00", "05/05/2019 00:00:00")]
+ [TestCase("01/01/2019 14:20:00.1", "01/01/2019 14:20:00.1")]
+ [TestCase("01/01/2019 14:20:00.01", "01/01/2019 14:20:00.01")]
+ [TestCase("01/01/2019 14:20:00.001", "01/01/2019 14:20:00.001")]
+ [TestCase("01/01/2019 14:20:00.0001", "01/01/2019 14:20:00")]
+ [TestCase("01/01/2019 14:20:00.0009", "01/01/2019 14:20:00")] // there is no rounding up
+ [TestCase("01/01/2019 14:20:00.0019", "01/01/2019 14:20:00.001")] // there is no rounding up
+ [TestCase("01/01/2019 14:20:00Z", "01/01/2019 14:20:00Z")] // UTC timestamps, but will check will in local TZ
+ [TestCase("01/01/2019 14:20:00.1Z", "01/01/2019 14:20:00.1Z")]
+ [TestCase("01/01/2019 14:20:00.01Z", "01/01/2019 14:20:00.01Z")]
+ [TestCase("01/01/2019 14:20:00.001Z", "01/01/2019 14:20:00.001Z")]
+ public void TestLocalTimestampMillisecond(string s, string e)
+ {
+ var schema = (LogicalSchema)Schema.Parse("{\"type\": \"long\", \"logicalType\": \"local-timestamp-millis\"}");
+
+ var date = DateTime.Parse(s, CultureInfo.GetCultureInfo("en-US").DateTimeFormat, DateTimeStyles.RoundtripKind);
+
+ if (date.Kind != DateTimeKind.Utc)
+ {
+ date = DateTime.Parse(s, CultureInfo.GetCultureInfo("en-US").DateTimeFormat, DateTimeStyles.AssumeLocal);
+ }
+
+ var expectedDate = DateTime.Parse(e, CultureInfo.GetCultureInfo("en-US").DateTimeFormat, DateTimeStyles.RoundtripKind);
+
+ if (expectedDate.Kind != DateTimeKind.Utc)
+ {
+ expectedDate = DateTime.Parse(e, CultureInfo.GetCultureInfo("en-US").DateTimeFormat, DateTimeStyles.AssumeLocal);
+ }
+
+ expectedDate = expectedDate.ToLocalTime();
+
+ var avroLocalTimestampMilli = new LocalTimestampMillisecond();
+ var convertedDate = (DateTime)avroLocalTimestampMilli.ConvertToLogicalValue(avroLocalTimestampMilli.ConvertToBaseValue(date, schema), schema);
+ Assert.AreEqual(expectedDate, convertedDate);
+ Assert.AreEqual(DateTimeKind.Local, convertedDate.Kind);
+ }
+
+ [TestCase("01/01/2019 14:20:00", "01/01/2019 14:20:00")]
+ [TestCase("05/05/2019 14:20:00", "05/05/2019 14:20:00")]
+ [TestCase("05/05/2019 00:00:00", "05/05/2019 00:00:00")]
+ [TestCase("01/01/2019 14:20:00.1", "01/01/2019 14:20:00.1")]
+ [TestCase("01/01/2019 14:20:00.01", "01/01/2019 14:20:00.01")]
+ [TestCase("01/01/2019 14:20:00.001", "01/01/2019 14:20:00.001")]
+ [TestCase("01/01/2019 14:20:00.0001", "01/01/2019 14:20:00.0001")]
+ [TestCase("01/01/2019 14:20:00.00001", "01/01/2019 14:20:00.00001")]
+ [TestCase("01/01/2019 14:20:00.000001", "01/01/2019 14:20:00.000001")]
+ [TestCase("01/01/2019 14:20:00.0000001", "01/01/2019 14:20:00")]
+ [TestCase("01/01/2019 14:20:00.0000009", "01/01/2019 14:20:00")] // there is no rounding up
+ [TestCase("01/01/2019 14:20:00.0000019", "01/01/2019 14:20:00.000001")] // there is no rounding up
+ [TestCase("01/01/2019 14:20:00Z", "01/01/2019 14:20:00Z")] // UTC timestamps, but will check will in local TZ
+ [TestCase("01/01/2019 14:20:00.1Z", "01/01/2019 14:20:00.1Z")]
+ [TestCase("01/01/2019 14:20:00.01Z", "01/01/2019 14:20:00.01Z")]
+ [TestCase("01/01/2019 14:20:00.001Z", "01/01/2019 14:20:00.001Z")]
+ [TestCase("01/01/2019 14:20:00.0001Z", "01/01/2019 14:20:00.0001Z")]
+ [TestCase("01/01/2019 14:20:00.00001Z", "01/01/2019 14:20:00.00001Z")]
+ [TestCase("01/01/2019 14:20:00.000001Z", "01/01/2019 14:20:00.000001Z")]
+ public void TestLocalTimestampMicrosecond(string s, string e)
+ {
+ var schema = (LogicalSchema)Schema.Parse("{\"type\": \"long\", \"logicalType\": \"local-timestamp-micros\"}");
+
+ var date = DateTime.Parse(s, CultureInfo.GetCultureInfo("en-US").DateTimeFormat, DateTimeStyles.RoundtripKind);
+
+ if (date.Kind != DateTimeKind.Utc)
+ {
+ date = DateTime.Parse(s, CultureInfo.GetCultureInfo("en-US").DateTimeFormat, DateTimeStyles.AssumeLocal);
+ }
+
+ var expectedDate = DateTime.Parse(e, CultureInfo.GetCultureInfo("en-US").DateTimeFormat, DateTimeStyles.RoundtripKind);
+
+ if (expectedDate.Kind != DateTimeKind.Utc)
+ {
+ expectedDate = DateTime.Parse(e, CultureInfo.GetCultureInfo("en-US").DateTimeFormat, DateTimeStyles.AssumeLocal);
+ }
+
+ expectedDate = expectedDate.ToLocalTime();
+
+ var avroLocalTimestampMicro = new LocalTimestampMicrosecond();
+ var convertedDate = (DateTime)avroLocalTimestampMicro.ConvertToLogicalValue(avroLocalTimestampMicro.ConvertToBaseValue(date, schema), schema);
+ Assert.AreEqual(expectedDate, convertedDate);
+ Assert.AreEqual(DateTimeKind.Local, convertedDate.Kind);
}
[TestCase("01:20:10", "01:20:10", false)]