You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/07/10 12:00:40 UTC
[24/41] ignite git commit: IGNITE-5716 .NET: Fix 2-byte field offset
handling
IGNITE-5716 .NET: Fix 2-byte field offset handling
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/82e5f8a6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/82e5f8a6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/82e5f8a6
Branch: refs/heads/ignite-5578-1
Commit: 82e5f8a6553323e793c01c54e24dda6d47188ce6
Parents: eb37f53
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Fri Jul 7 19:28:54 2017 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Fri Jul 7 19:28:54 2017 +0300
----------------------------------------------------------------------
.../Apache.Ignite.Core.Tests.csproj | 1 +
.../Binary/BinaryFooterTest.cs | 146 +++++++++++++++++++
.../Binary/BinarySelfTest.cs | 32 ----
.../Impl/Binary/BinaryObject.cs | 2 +-
.../Impl/Binary/BinaryObjectBuilder.cs | 2 +-
.../Impl/Binary/BinaryObjectSchemaField.cs | 3 +
.../Impl/Binary/BinaryObjectSchemaSerializer.cs | 93 +++++++++---
.../Impl/Binary/BinaryReader.cs | 49 +------
8 files changed, 228 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
index 09eac70..d91f0e6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Apache.Ignite.Core.Tests.csproj
@@ -72,6 +72,7 @@
<Compile Include="Binary\BinaryDateTimeTest.cs" />
<Compile Include="Binary\BinaryEqualityComparerTest.cs" />
<Compile Include="Binary\BinaryBuilderSelfTestDynamicRegistration.cs" />
+ <Compile Include="Binary\BinaryFooterTest.cs" />
<Compile Include="Binary\BinaryNameMapperTest.cs" />
<Compile Include="Binary\BinaryReaderWriterTest.cs" />
<Compile Include="Binary\BinarySelfTestSimpleName.cs" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs
new file mode 100644
index 0000000..36f2f65
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Core.Tests.Binary
+{
+ using System;
+ using System.Linq;
+ using Apache.Ignite.Core.Binary;
+ using Apache.Ignite.Core.Impl;
+ using Apache.Ignite.Core.Impl.Binary;
+ using NUnit.Framework;
+
+ /// <summary>
+ /// Tests binary footer integrity (field offsets).
+ /// Writes objects of various sizes to test schema compaction
+ /// (where field offsets can be stored as 1, 2 or 4 bytes).
+ /// </summary>
+ public class BinaryFooterTest
+ {
+ /// <summary>
+ /// Tears down the test.
+ /// </summary>
+ [TearDown]
+ public void TearDown()
+ {
+ Ignition.StopAll(true);
+ }
+
+ /// <summary>
+ /// Tests full footers in offline mode.
+ /// </summary>
+ [Test]
+ public void TestFullFooterOffline()
+ {
+ // Register type to avoid unregistered type name in binary object.
+ // Use new marshaller to read and write to avoid schema caching.
+
+ TestOffsets(() => new Marshaller(new BinaryConfiguration(typeof(OffsetTest))
+ {
+ // Compact footers do not work in offline mode.
+ CompactFooter = false
+ }));
+ }
+
+ /// <summary>
+ /// Tests the full footer online.
+ /// </summary>
+ [Test]
+ public void TestFullFooterOnline()
+ {
+ var ignite = Ignition.Start(new IgniteConfiguration(TestUtils.GetTestConfiguration())
+ {
+ BinaryConfiguration = new BinaryConfiguration
+ {
+ CompactFooter = false
+ }
+ });
+
+ TestOffsets(() => ((Ignite) ignite).Marshaller);
+ }
+
+ /// <summary>
+ /// Tests the full footer online.
+ /// </summary>
+ [Test]
+ public void TestCompactFooterOnline()
+ {
+ var ignite = Ignition.Start(TestUtils.GetTestConfiguration());
+
+ TestOffsets(() => ((Ignite) ignite).Marshaller);
+ }
+
+ /// <summary>
+ /// Tests the offsets.
+ /// </summary>
+ private static void TestOffsets(Func<Marshaller> getMarsh)
+ {
+ // Corner cases are byte/sbyte/short/ushort max values.
+ foreach (var i in new[] {1, sbyte.MaxValue, byte.MaxValue, short.MaxValue, ushort.MaxValue})
+ {
+ foreach (var j in new[] {-1, 0, 1})
+ {
+ var arrSize = i + j;
+
+ var dt = new OffsetTest
+ {
+ Arr = Enumerable.Range(0, arrSize).Select(x => (byte) x).ToArray(),
+ Int = arrSize
+ };
+
+ var bytes = getMarsh().Marshal(dt);
+
+ var res = getMarsh().Unmarshal<OffsetTest>(bytes);
+ var binRes = getMarsh().Unmarshal<IBinaryObject>(bytes, BinaryMode.ForceBinary);
+ var binFieldRes = new OffsetTest
+ {
+ Arr = binRes.GetField<byte[]>("arr"),
+ Int = binRes.GetField<int>("int")
+ };
+
+ foreach (var r in new[] {res, binRes.Deserialize<OffsetTest>(), binFieldRes})
+ {
+ Assert.AreEqual(dt.Arr, r.Arr);
+ Assert.AreEqual(dt.Int, r.Int);
+ }
+ }
+ }
+ }
+
+ /// <summary>
+ /// Offset test.
+ /// </summary>
+ private class OffsetTest : IBinarizable
+ {
+ public byte[] Arr; // Array to enforce field offset.
+ public int Int; // Value at offset.
+
+ public void WriteBinary(IBinaryWriter writer)
+ {
+ writer.WriteByteArray("arr", Arr);
+ writer.WriteInt("int", Int);
+ }
+
+ public void ReadBinary(IBinaryReader reader)
+ {
+ // Read in different order to enforce full schema scan.
+ Int = reader.ReadInt("int");
+ Arr = reader.ReadByteArray("arr");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs
index 2b22d5a..e24dca0 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinarySelfTest.cs
@@ -1534,38 +1534,6 @@ namespace Apache.Ignite.Core.Tests.Binary
Assert.AreEqual(nDateArr, obj2.NDateArr);
}
- /// <summary>
- /// Writes objects of various sizes to test schema compaction
- /// (where field offsets can be stored as 1, 2 or 4 bytes).
- /// </summary>
- [Test]
- public void TestCompactSchema()
- {
- var marsh = new Marshaller(new BinaryConfiguration
- {
- TypeConfigurations = new List<BinaryTypeConfiguration>
- {
- new BinaryTypeConfiguration(typeof (SpecialArray)),
- new BinaryTypeConfiguration(typeof (SpecialArrayMarshalAware))
- }
- });
-
- var dt = new SpecialArrayMarshalAware();
-
- foreach (var i in new[] {1, 5, 10, 13, 14, 15, 100, 200, 1000, 5000, 15000, 30000})
- {
- dt.NGuidArr = Enumerable.Range(1, i).Select(x => (Guid?) Guid.NewGuid()).ToArray();
- dt.NDateArr = Enumerable.Range(1, i).Select(x => (DateTime?) DateTime.Now.AddDays(x)).ToArray();
-
- var bytes = marsh.Marshal(dt);
-
- var res = marsh.Unmarshal<SpecialArrayMarshalAware>(bytes);
-
- CollectionAssert.AreEquivalent(dt.NGuidArr, res.NGuidArr);
- CollectionAssert.AreEquivalent(dt.NDateArr, res.NDateArr);
- }
- }
-
[Test]
public void TestBinaryConfigurationValidation()
{
http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs
index 8c5cee6..370233f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObject.cs
@@ -239,7 +239,7 @@ namespace Apache.Ignite.Core.Impl.Binary
{
var hdr = BinaryObjectHeader.Read(stream, _offset);
- _fields = BinaryObjectSchemaSerializer.ReadSchema(stream, _offset, hdr, desc.Schema,_marsh)
+ _fields = BinaryObjectSchemaSerializer.ReadSchema(stream, _offset, hdr, desc.Schema, _marsh.Ignite)
.ToDictionary() ?? EmptyFields;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs
index 91fe12a..c310b3a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectBuilder.cs
@@ -617,7 +617,7 @@ namespace Apache.Ignite.Core.Impl.Binary
{
// New object, write in full form.
var inSchema = BinaryObjectSchemaSerializer.ReadSchema(inStream, inStartPos, inHeader,
- _desc.Schema, _binary.Marshaller);
+ _desc.Schema, _binary.Marshaller.Ignite);
var outSchema = BinaryObjectSchemaHolder.Current;
var schemaIdx = outSchema.PushSchema();
http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs
index 3c5339a..be6278a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaField.cs
@@ -17,6 +17,7 @@
namespace Apache.Ignite.Core.Impl.Binary
{
+ using System.Diagnostics;
using System.Runtime.InteropServices;
/// <summary>
@@ -41,6 +42,8 @@ namespace Apache.Ignite.Core.Impl.Binary
/// <param name="offset">The offset.</param>
public BinaryObjectSchemaField(int id, int offset)
{
+ Debug.Assert(offset >= 0);
+
Id = id;
Offset = offset;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs
index e2f9ea7..1d699c2 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryObjectSchemaSerializer.cs
@@ -54,18 +54,17 @@ namespace Apache.Ignite.Core.Impl.Binary
/// <param name="position">The position.</param>
/// <param name="hdr">The header.</param>
/// <param name="schema">The schema.</param>
- /// <param name="marsh">The marshaller.</param>
+ /// <param name="ignite">The ignite.</param>
/// <returns>
/// Schema.
/// </returns>
public static BinaryObjectSchemaField[] ReadSchema(IBinaryStream stream, int position, BinaryObjectHeader hdr,
- BinaryObjectSchema schema, Marshaller marsh)
+ BinaryObjectSchema schema, Ignite ignite)
{
Debug.Assert(stream != null);
Debug.Assert(schema != null);
- Debug.Assert(marsh != null);
- return ReadSchema(stream, position, hdr, () => GetFieldIds(hdr, schema, marsh));
+ return ReadSchema(stream, position, hdr, () => GetFieldIds(hdr, schema, ignite));
}
/// <summary>
@@ -78,8 +77,8 @@ namespace Apache.Ignite.Core.Impl.Binary
/// <returns>
/// Schema.
/// </returns>
- public static BinaryObjectSchemaField[] ReadSchema(IBinaryStream stream, int position, BinaryObjectHeader hdr,
- Func<int[]> fieldIdsFunc)
+ public static unsafe BinaryObjectSchemaField[] ReadSchema(IBinaryStream stream, int position,
+ BinaryObjectHeader hdr, Func<int[]> fieldIdsFunc)
{
Debug.Assert(stream != null);
Debug.Assert(fieldIdsFunc != null);
@@ -110,7 +109,7 @@ namespace Apache.Ignite.Core.Impl.Binary
else if (offsetSize == 2)
{
for (var i = 0; i < schemaSize; i++)
- res[i] = new BinaryObjectSchemaField(fieldIds[i], stream.ReadShort());
+ res[i] = new BinaryObjectSchemaField(fieldIds[i], (ushort) stream.ReadShort());
}
else
{
@@ -128,12 +127,22 @@ namespace Apache.Ignite.Core.Impl.Binary
else if (offsetSize == 2)
{
for (var i = 0; i < schemaSize; i++)
- res[i] = new BinaryObjectSchemaField(stream.ReadInt(), stream.ReadShort());
+ res[i] = new BinaryObjectSchemaField(stream.ReadInt(), (ushort) stream.ReadShort());
}
else
{
- for (var i = 0; i < schemaSize; i++)
- res[i] = new BinaryObjectSchemaField(stream.ReadInt(), stream.ReadInt());
+ if (BitConverter.IsLittleEndian)
+ {
+ fixed (BinaryObjectSchemaField* ptr = &res[0])
+ {
+ stream.Read((byte*) ptr, schemaSize * BinaryObjectSchemaField.Size);
+ }
+ }
+ else
+ {
+ for (var i = 0; i < schemaSize; i++)
+ res[i] = new BinaryObjectSchemaField(stream.ReadInt(), stream.ReadInt());
+ }
}
}
@@ -220,7 +229,7 @@ namespace Apache.Ignite.Core.Impl.Binary
{
fixed (BinaryObjectSchemaField* ptr = &fields[offset])
{
- stream.Write((byte*)ptr, count / BinaryObjectSchemaField.Size);
+ stream.Write((byte*)ptr, count * BinaryObjectSchemaField.Size);
}
}
else
@@ -243,22 +252,66 @@ namespace Apache.Ignite.Core.Impl.Binary
/// <summary>
/// Gets the field ids.
/// </summary>
- private static int[] GetFieldIds(BinaryObjectHeader hdr, BinaryObjectSchema schema, Marshaller marsh)
+ private static int[] GetFieldIds(BinaryObjectHeader hdr, Ignite ignite)
{
- var fieldIds = schema.Get(hdr.SchemaId);
+ Debug.Assert(hdr.TypeId != BinaryUtils.TypeUnregistered);
+
+ int[] fieldIds = null;
+
+ if (ignite != null)
+ {
+ fieldIds = ignite.BinaryProcessor.GetSchema(hdr.TypeId, hdr.SchemaId);
+ }
if (fieldIds == null)
{
- Debug.Assert(hdr.TypeId != BinaryUtils.TypeUnregistered);
+ throw new BinaryObjectException("Cannot find schema for object with compact footer [" +
+ "typeId=" + hdr.TypeId + ", schemaId=" + hdr.SchemaId + ']');
+ }
+ return fieldIds;
+ }
- if (marsh.Ignite != null)
- fieldIds = marsh.Ignite.BinaryProcessor.GetSchema(hdr.TypeId, hdr.SchemaId);
+ /// <summary>
+ /// Reads the schema, maintains stream position.
+ /// </summary>
+ public static int[] GetFieldIds(BinaryObjectHeader hdr, Ignite ignite, IBinaryStream stream, int objectPos)
+ {
+ Debug.Assert(stream != null);
- if (fieldIds == null)
- throw new BinaryObjectException("Cannot find schema for object with compact footer [" +
- "typeId=" + hdr.TypeId + ", schemaId=" + hdr.SchemaId + ']');
+ if (hdr.IsCompactFooter)
+ {
+ // Get schema from Java
+ return GetFieldIds(hdr, ignite);
}
- return fieldIds;
+
+ var pos = stream.Position;
+
+ stream.Seek(objectPos + hdr.SchemaOffset, SeekOrigin.Begin);
+
+ var count = hdr.SchemaFieldCount;
+
+ var offsetSize = hdr.SchemaFieldOffsetSize;
+
+ var res = new int[count];
+
+ for (var i = 0; i < count; i++)
+ {
+ res[i] = stream.ReadInt();
+ stream.Seek(offsetSize, SeekOrigin.Current); // Skip offsets.
+ }
+
+ stream.Seek(pos, SeekOrigin.Begin);
+
+ return res;
+ }
+
+
+ /// <summary>
+ /// Gets the field ids.
+ /// </summary>
+ private static int[] GetFieldIds(BinaryObjectHeader hdr, BinaryObjectSchema schema, Ignite ignite)
+ {
+ return schema.Get(hdr.SchemaId) ?? GetFieldIds(hdr, ignite);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/82e5f8a6/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs
index e792dce..76237c4 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReader.cs
@@ -20,7 +20,6 @@ namespace Apache.Ignite.Core.Impl.Binary
using System;
using System.Collections;
using System.Collections.Generic;
- using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using Apache.Ignite.Core.Binary;
@@ -783,7 +782,8 @@ namespace Apache.Ignite.Core.Impl.Binary
if (_frame.Schema == null)
{
- _frame.Schema = ReadSchema(desc.TypeId);
+ _frame.Schema =
+ BinaryObjectSchemaSerializer.GetFieldIds(_frame.Hdr, Marshaller.Ignite, Stream, _frame.Pos);
desc.Schema.Add(_frame.Hdr.SchemaId, _frame.Schema);
}
@@ -795,49 +795,6 @@ namespace Apache.Ignite.Core.Impl.Binary
}
/// <summary>
- /// Reads the schema.
- /// </summary>
- private int[] ReadSchema(int typeId)
- {
- if (_frame.Hdr.IsCompactFooter)
- {
- // Get schema from Java
- var ignite = Marshaller.Ignite;
-
- Debug.Assert(typeId != BinaryUtils.TypeUnregistered);
-
- var schema = ignite == null
- ? null
- : ignite.BinaryProcessor.GetSchema(_frame.Hdr.TypeId, _frame.Hdr.SchemaId);
-
- if (schema == null)
- throw new BinaryObjectException("Cannot find schema for object with compact footer [" +
- "typeId=" + typeId + ", schemaId=" + _frame.Hdr.SchemaId + ']');
-
- return schema;
- }
-
- var pos = Stream.Position;
-
- Stream.Seek(_frame.Pos + _frame.Hdr.SchemaOffset, SeekOrigin.Begin);
-
- var count = _frame.Hdr.SchemaFieldCount;
-
- var offsetSize = _frame.Hdr.SchemaFieldOffsetSize;
-
- var res = new int[count];
-
- for (int i = 0; i < count; i++)
- {
- res[i] = Stream.ReadInt();
- Stream.Seek(offsetSize, SeekOrigin.Current);
- }
-
- Stream.Seek(pos, SeekOrigin.Begin);
-
- return res;
- }
- /// <summary>
/// Reads the handle object.
/// </summary>
private T ReadHandleObject<T>(int pos, Type typeOverride)
@@ -942,7 +899,7 @@ namespace Apache.Ignite.Core.Impl.Binary
int pos;
- if (!_frame.SchemaMap.TryGetValue(fieldId, out pos))
+ if (_frame.SchemaMap == null || !_frame.SchemaMap.TryGetValue(fieldId, out pos))
return false;
Stream.Seek(pos + _frame.Pos, SeekOrigin.Begin);