You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by dk...@apache.org on 2020/05/22 12:26:23 UTC
[avro] branch master updated: AVRO-2714: [C#] Optimize reading on
.NET Core and Standard 2.1+ (#792)
This is an automated email from the ASF dual-hosted git repository.
dkulp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new 61f162d AVRO-2714: [C#] Optimize reading on .NET Core and Standard 2.1+ (#792)
61f162d is described below
commit 61f162de6f441f65297cf8b57c0e8c36474be7d6
Author: Eric Erhardt <er...@microsoft.com>
AuthorDate: Fri May 22 07:26:16 2020 -0500
AVRO-2714: [C#] Optimize reading on .NET Core and Standard 2.1+ (#792)
* AVRO-2714: [C#] Optimize reading on .NET Core 2.1+
We can use stackallocs and ArrayPool to save on allocations when running on .NET Core.
* Add netstandard2.1 target back.
---
lang/csharp/src/apache/main/Avro.main.csproj | 13 ++-
lang/csharp/src/apache/main/Generic/GenericEnum.cs | 2 +
lang/csharp/src/apache/main/IO/BinaryDecoder.cs | 77 +---------------
.../apache/main/IO/BinaryDecoder.netstandard2.0.cs | 92 +++++++++++++++++++
.../main/IO/BinaryDecoder.notnetstandard2.0.cs | 100 +++++++++++++++++++++
lang/csharp/src/apache/main/Protocol/Message.cs | 2 +
lang/csharp/src/apache/main/Protocol/Protocol.cs | 4 +
lang/csharp/src/apache/main/Schema/EnumSchema.cs | 2 +
lang/csharp/src/apache/main/Schema/Field.cs | 2 +
lang/csharp/src/apache/main/Schema/Property.cs | 2 +
lang/csharp/src/apache/main/Schema/SchemaName.cs | 6 +-
.../src/apache/main/Specific/ObjectCreator.cs | 2 +
12 files changed, 224 insertions(+), 80 deletions(-)
diff --git a/lang/csharp/src/apache/main/Avro.main.csproj b/lang/csharp/src/apache/main/Avro.main.csproj
index d8d03e7..539143f 100644
--- a/lang/csharp/src/apache/main/Avro.main.csproj
+++ b/lang/csharp/src/apache/main/Avro.main.csproj
@@ -1,4 +1,4 @@
-<!--
+<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
@@ -19,7 +19,7 @@
<Import Project="../../../common.props" />
<PropertyGroup>
- <TargetFrameworks>netstandard2.0</TargetFrameworks>
+ <TargetFrameworks>netstandard2.0;netstandard2.1;netcoreapp2.1</TargetFrameworks>
<AssemblyName>Avro</AssemblyName>
<RootNamespace>Avro</RootNamespace>
<SignAssembly>true</SignAssembly>
@@ -52,7 +52,7 @@
</ItemGroup>
<ItemGroup>
- <PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="2.9.4">
+ <PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="2.9.8">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
@@ -67,4 +67,11 @@
<PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.3.0" />
</ItemGroup>
+ <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
+ <Compile Remove="**/*.notnetstandard2.0.cs" />
+ </ItemGroup>
+ <ItemGroup Condition="'$(TargetFramework)' != 'netstandard2.0'">
+ <Compile Remove="**/*.netstandard2.0.cs" />
+ </ItemGroup>
+
</Project>
diff --git a/lang/csharp/src/apache/main/Generic/GenericEnum.cs b/lang/csharp/src/apache/main/Generic/GenericEnum.cs
index 1e81304..00357f2 100644
--- a/lang/csharp/src/apache/main/Generic/GenericEnum.cs
+++ b/lang/csharp/src/apache/main/Generic/GenericEnum.cs
@@ -65,7 +65,9 @@ namespace Avro.Generic
/// <inheritdoc/>
public override int GetHashCode()
{
+#pragma warning disable CA1307 // Specify StringComparison
return 17 * Value.GetHashCode();
+#pragma warning restore CA1307 // Specify StringComparison
}
/// <inheritdoc/>
diff --git a/lang/csharp/src/apache/main/IO/BinaryDecoder.cs b/lang/csharp/src/apache/main/IO/BinaryDecoder.cs
index afa59ca..56aaa6e 100644
--- a/lang/csharp/src/apache/main/IO/BinaryDecoder.cs
+++ b/lang/csharp/src/apache/main/IO/BinaryDecoder.cs
@@ -23,7 +23,7 @@ namespace Avro.IO
/// <summary>
/// Decoder for Avro binary format
/// </summary>
- public class BinaryDecoder : Decoder
+ public partial class BinaryDecoder : Decoder
{
private readonly Stream stream;
@@ -85,47 +85,6 @@ namespace Avro.IO
}
/// <summary>
- /// A float is written as 4 bytes.
- /// The float is converted into a 32-bit integer using a method equivalent to
- /// Java's floatToIntBits and then encoded in little-endian format.
- /// </summary>
- /// <returns></returns>
- public float ReadFloat()
- {
- byte[] buffer = read(4);
-
- if (!BitConverter.IsLittleEndian)
- Array.Reverse(buffer);
-
- return BitConverter.ToSingle(buffer, 0);
-
- //int bits = (Stream.ReadByte() & 0xff |
- //(Stream.ReadByte()) & 0xff << 8 |
- //(Stream.ReadByte()) & 0xff << 16 |
- //(Stream.ReadByte()) & 0xff << 24);
- //return intBitsToFloat(bits);
- }
-
- /// <summary>
- /// A double is written as 8 bytes.
- /// The double is converted into a 64-bit integer using a method equivalent to
- /// Java's doubleToLongBits and then encoded in little-endian format.
- /// </summary>
- /// <returns>A double value.</returns>
- public double ReadDouble()
- {
- long bits = (stream.ReadByte() & 0xffL) |
- (stream.ReadByte() & 0xffL) << 8 |
- (stream.ReadByte() & 0xffL) << 16 |
- (stream.ReadByte() & 0xffL) << 24 |
- (stream.ReadByte() & 0xffL) << 32 |
- (stream.ReadByte() & 0xffL) << 40 |
- (stream.ReadByte() & 0xffL) << 48 |
- (stream.ReadByte() & 0xffL) << 56;
- return BitConverter.Int64BitsToDouble(bits);
- }
-
- /// <summary>
/// Bytes are encoded as a long followed by that many bytes of data.
/// </summary>
/// <returns></returns>
@@ -135,19 +94,6 @@ namespace Avro.IO
}
/// <summary>
- /// Reads a string written by <see cref="BinaryEncoder.WriteString(string)"/>.
- /// </summary>
- /// <returns>String read from the stream.</returns>
- public string ReadString()
- {
- int length = ReadInt();
- byte[] buffer = new byte[length];
- //TODO: Fix this because it's lame;
- ReadFixed(buffer);
- return System.Text.Encoding.UTF8.GetString(buffer);
- }
-
- /// <summary>
/// Reads an enumeration.
/// </summary>
/// <returns>Ordinal value of the enum.</returns>
@@ -323,11 +269,6 @@ namespace Avro.IO
return buffer;
}
- private static float intBitsToFloat(int value)
- {
- return BitConverter.ToSingle(BitConverter.GetBytes(value), 0);
- }
-
private byte read()
{
int n = stream.ReadByte();
@@ -335,17 +276,6 @@ namespace Avro.IO
throw new AvroException("End of stream reached");
}
- private void Read(byte[] buffer, int start, int len)
- {
- while (len > 0)
- {
- int n = stream.Read(buffer, start, len);
- if (n <= 0) throw new AvroException("End of stream reached");
- start += n;
- len -= n;
- }
- }
-
private long doReadItemCount()
{
long result = ReadLong();
@@ -366,10 +296,5 @@ namespace Avro.IO
{
stream.Seek(p, SeekOrigin.Current);
}
-
- internal void skip(long block_size)
- {
- throw new NotImplementedException();
- }
}
}
diff --git a/lang/csharp/src/apache/main/IO/BinaryDecoder.netstandard2.0.cs b/lang/csharp/src/apache/main/IO/BinaryDecoder.netstandard2.0.cs
new file mode 100644
index 0000000..91afeb5
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/BinaryDecoder.netstandard2.0.cs
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+
+namespace Avro.IO
+{
+ /// <content>
+ /// Contains the netstandard2.0 specific functionality for BinaryDecoder.
+ /// </content>
+ public partial class BinaryDecoder
+ {
+ /// <summary>
+ /// A float is written as 4 bytes.
+ /// The float is converted into a 32-bit integer using a method equivalent to
+ /// Java's floatToIntBits and then encoded in little-endian format.
+ /// </summary>
+ /// <returns></returns>
+ public float ReadFloat()
+ {
+ byte[] buffer = read(4);
+
+ if (!BitConverter.IsLittleEndian)
+ Array.Reverse(buffer);
+
+ return BitConverter.ToSingle(buffer, 0);
+
+ //int bits = (Stream.ReadByte() & 0xff |
+ //(Stream.ReadByte()) & 0xff << 8 |
+ //(Stream.ReadByte()) & 0xff << 16 |
+ //(Stream.ReadByte()) & 0xff << 24);
+ //return intBitsToFloat(bits);
+ }
+
+ /// <summary>
+ /// A double is written as 8 bytes.
+ /// The double is converted into a 64-bit integer using a method equivalent to
+ /// Java's doubleToLongBits and then encoded in little-endian format.
+ /// </summary>
+ /// <returns>A double value.</returns>
+ public double ReadDouble()
+ {
+ long bits = (stream.ReadByte() & 0xffL) |
+ (stream.ReadByte() & 0xffL) << 8 |
+ (stream.ReadByte() & 0xffL) << 16 |
+ (stream.ReadByte() & 0xffL) << 24 |
+ (stream.ReadByte() & 0xffL) << 32 |
+ (stream.ReadByte() & 0xffL) << 40 |
+ (stream.ReadByte() & 0xffL) << 48 |
+ (stream.ReadByte() & 0xffL) << 56;
+ return BitConverter.Int64BitsToDouble(bits);
+ }
+
+ /// <summary>
+ /// Reads a string written by <see cref="BinaryEncoder.WriteString(string)"/>.
+ /// </summary>
+ /// <returns>String read from the stream.</returns>
+ public string ReadString()
+ {
+ int length = ReadInt();
+ byte[] buffer = new byte[length];
+ //TODO: Fix this because it's lame;
+ ReadFixed(buffer);
+ return System.Text.Encoding.UTF8.GetString(buffer);
+ }
+
+ private void Read(byte[] buffer, int start, int len)
+ {
+ while (len > 0)
+ {
+ int n = stream.Read(buffer, start, len);
+ if (n <= 0) throw new AvroException("End of stream reached");
+ start += n;
+ len -= n;
+ }
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/IO/BinaryDecoder.notnetstandard2.0.cs b/lang/csharp/src/apache/main/IO/BinaryDecoder.notnetstandard2.0.cs
new file mode 100644
index 0000000..206fe86
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/BinaryDecoder.notnetstandard2.0.cs
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+using System.Buffers;
+using System.Buffers.Binary;
+using System.Text;
+
+namespace Avro.IO
+{
+ /// <content>
+ /// Contains the netstandard2.1 and netcoreapp2.1 specific functionality for BinaryDecoder.
+ /// </content>
+ public partial class BinaryDecoder
+ {
+ private const int StackallocThreshold = 256;
+
+ /// <summary>
+ /// A float is written as 4 bytes.
+ /// The float is converted into a 32-bit integer using a method equivalent to
+ /// Java's floatToIntBits and then encoded in little-endian format.
+ /// </summary>
+ /// <returns></returns>
+ public float ReadFloat()
+ {
+ Span<byte> buffer = stackalloc byte[4];
+ Read(buffer);
+
+ return BitConverter.Int32BitsToSingle(BinaryPrimitives.ReadInt32LittleEndian(buffer));
+ }
+
+ /// <summary>
+ /// A double is written as 8 bytes.
+ /// The double is converted into a 64-bit integer using a method equivalent to
+ /// Java's doubleToLongBits and then encoded in little-endian format.
+ /// </summary>
+ /// <returns>A double value.</returns>
+ public double ReadDouble()
+ {
+ Span<byte> buffer = stackalloc byte[8];
+ Read(buffer);
+
+ return BitConverter.Int64BitsToDouble(BinaryPrimitives.ReadInt64LittleEndian(buffer));
+ }
+
+ /// <summary>
+ /// Reads a string written by <see cref="BinaryEncoder.WriteString(string)"/>.
+ /// </summary>
+ /// <returns>String read from the stream.</returns>
+ public string ReadString()
+ {
+ byte[] bufferArray = null;
+
+ int length = ReadInt();
+ Span<byte> buffer = length <= StackallocThreshold ?
+ stackalloc byte[length] :
+ (bufferArray = ArrayPool<byte>.Shared.Rent(length));
+
+ Read(buffer);
+
+ string result = Encoding.UTF8.GetString(buffer);
+
+ if (bufferArray != null)
+ {
+ ArrayPool<byte>.Shared.Return(bufferArray);
+ }
+
+ return result;
+ }
+
+ private void Read(byte[] buffer, int start, int len)
+ {
+ Read(buffer.AsSpan(start, len));
+ }
+
+ private void Read(Span<byte> buffer)
+ {
+ while (!buffer.IsEmpty)
+ {
+ int n = stream.Read(buffer);
+ if (n <= 0) throw new AvroException("End of stream reached");
+ buffer = buffer.Slice(n);
+ }
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/Protocol/Message.cs b/lang/csharp/src/apache/main/Protocol/Message.cs
index 5e455ef..732438c 100644
--- a/lang/csharp/src/apache/main/Protocol/Message.cs
+++ b/lang/csharp/src/apache/main/Protocol/Message.cs
@@ -198,7 +198,9 @@ namespace Avro
/// <returns></returns>
public override int GetHashCode()
{
+#pragma warning disable CA1307 // Specify StringComparison
return Name.GetHashCode() +
+#pragma warning restore CA1307 // Specify StringComparison
Request.GetHashCode() +
(Response == null ? 0 : Response.GetHashCode()) +
(Error == null ? 0 : Error.GetHashCode());
diff --git a/lang/csharp/src/apache/main/Protocol/Protocol.cs b/lang/csharp/src/apache/main/Protocol/Protocol.cs
index cd4e0cc..1f5b541 100644
--- a/lang/csharp/src/apache/main/Protocol/Protocol.cs
+++ b/lang/csharp/src/apache/main/Protocol/Protocol.cs
@@ -269,7 +269,9 @@ namespace Avro
/// <returns></returns>
public override int GetHashCode()
{
+#pragma warning disable CA1307 // Specify StringComparison
return Name.GetHashCode() + Namespace.GetHashCode() +
+#pragma warning restore CA1307 // Specify StringComparison
GetTypesHashCode() + GetMessagesHashCode();
}
@@ -293,7 +295,9 @@ namespace Avro
{
int hash = Messages.Count;
foreach (KeyValuePair<string, Message> pair in Messages)
+#pragma warning disable CA1307 // Specify StringComparison
hash += pair.Key.GetHashCode() + pair.Value.GetHashCode();
+#pragma warning restore CA1307 // Specify StringComparison
return hash;
}
}
diff --git a/lang/csharp/src/apache/main/Schema/EnumSchema.cs b/lang/csharp/src/apache/main/Schema/EnumSchema.cs
index f5b72e0..5a14afd 100644
--- a/lang/csharp/src/apache/main/Schema/EnumSchema.cs
+++ b/lang/csharp/src/apache/main/Schema/EnumSchema.cs
@@ -199,7 +199,9 @@ namespace Avro
public override int GetHashCode()
{
int result = SchemaName.GetHashCode() + getHashCode(Props);
+#pragma warning disable CA1307 // Specify StringComparison
foreach (string s in Symbols) result += 23 * s.GetHashCode();
+#pragma warning restore CA1307 // Specify StringComparison
return result;
}
diff --git a/lang/csharp/src/apache/main/Schema/Field.cs b/lang/csharp/src/apache/main/Schema/Field.cs
index 026b5b2..bdfe928 100644
--- a/lang/csharp/src/apache/main/Schema/Field.cs
+++ b/lang/csharp/src/apache/main/Schema/Field.cs
@@ -251,7 +251,9 @@ namespace Avro
/// <returns></returns>
public override int GetHashCode()
{
+#pragma warning disable CA1307 // Specify StringComparison
return 17 * Name.GetHashCode() + Pos + 19 * getHashCode(Documentation) +
+#pragma warning restore CA1307 // Specify StringComparison
23 * getHashCode(Ordering) + 29 * getHashCode(DefaultValue) + 31 * Schema.GetHashCode() +
37 * getHashCode(Props);
}
diff --git a/lang/csharp/src/apache/main/Schema/Property.cs b/lang/csharp/src/apache/main/Schema/Property.cs
index 5fdb9ef..f424072 100644
--- a/lang/csharp/src/apache/main/Schema/Property.cs
+++ b/lang/csharp/src/apache/main/Schema/Property.cs
@@ -121,7 +121,9 @@ namespace Avro
int hash = this.Count;
int index = 1;
foreach (KeyValuePair<string, string> pair in this)
+#pragma warning disable CA1307 // Specify StringComparison
hash += (pair.Key.GetHashCode() + pair.Value.GetHashCode()) * index++;
+#pragma warning restore CA1307 // Specify StringComparison
return hash;
}
}
diff --git a/lang/csharp/src/apache/main/Schema/SchemaName.cs b/lang/csharp/src/apache/main/Schema/SchemaName.cs
index ae6f0b8..2eb96ee 100644
--- a/lang/csharp/src/apache/main/Schema/SchemaName.cs
+++ b/lang/csharp/src/apache/main/Schema/SchemaName.cs
@@ -63,7 +63,9 @@ namespace Avro
this.Name = this.Space = null;
this.EncSpace = encspace; // need to save enclosing namespace for anonymous types, so named types within the anonymous type can be resolved
}
- else if (!name.Contains("."))
+#pragma warning disable CA1307 // Specify StringComparison
+ else if (name.IndexOf('.') == -1)
+#pragma warning restore CA1307 // Specify StringComparison
{ // unqualified name
this.Space = space; // use default space
this.Name = name;
@@ -135,7 +137,9 @@ namespace Avro
/// <inheritdoc/>
public override int GetHashCode()
{
+#pragma warning disable CA1307 // Specify StringComparison
return string.IsNullOrEmpty(Fullname) ? 0 : 29 * Fullname.GetHashCode();
+#pragma warning restore CA1307 // Specify StringComparison
}
}
diff --git a/lang/csharp/src/apache/main/Specific/ObjectCreator.cs b/lang/csharp/src/apache/main/Specific/ObjectCreator.cs
index 950d111..32dc593 100644
--- a/lang/csharp/src/apache/main/Specific/ObjectCreator.cs
+++ b/lang/csharp/src/apache/main/Specific/ObjectCreator.cs
@@ -111,7 +111,9 @@ namespace Avro.Specific
{
unchecked
{
+#pragma warning disable CA1307 // Specify StringComparison
return ((name != null ? name.GetHashCode() : 0) * 397) ^ type.GetHashCode();
+#pragma warning restore CA1307 // Specify StringComparison
}
}
public static bool operator ==(NameCtorKey left, NameCtorKey right)