You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by bl...@apache.org on 2019/07/07 20:30:52 UTC
[avro] branch branch-1.9 updated: AVRO-2459: Add data interop test
for the C# bindings (#574)
This is an automated email from the ASF dual-hosted git repository.
blachniet pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/branch-1.9 by this push:
new 0161682 AVRO-2459: Add data interop test for the C# bindings (#574)
0161682 is described below
commit 016168265871cfad88671b02b9b259b464bfb091
Author: Kengo Seki <se...@apache.org>
AuthorDate: Mon Jul 8 04:29:33 2019 +0900
AVRO-2459: Add data interop test for the C# bindings (#574)
* AVRO-2459: Add data interop test for the C# bindings
* Reflect @blachniet's comments. With these fixes,
* `./build.sh test` runs tests without the "Interop" category
* `./build.sh interop-data-test` runs only the "Interop" category
tests, always resolves the input path correctly, and fails if
the input directory doesn't exist yet
(cherry picked commit from 95f6c12d3ec57ccc79d6a3ff4977947138b679a7)
---
build.sh | 2 +
lang/csharp/build.sh | 13 ++-
lang/csharp/src/apache/test/Avro.test.csproj | 1 +
.../apache/test/Interop/InteropDataConstants.cs | 31 +++++++
.../apache/test/Interop/InteropDataGenerator.cs | 99 ++++++++++++++++++++++
.../src/apache/test/Interop/InteropDataTests.cs | 59 +++++++++++++
6 files changed, 203 insertions(+), 2 deletions(-)
diff --git a/build.sh b/build.sh
index efa4671..c191ac2 100755
--- a/build.sh
+++ b/build.sh
@@ -63,6 +63,7 @@ do
(cd lang/py; ant interop-data-generate)
(cd lang/c; ./build.sh interop-data-generate)
#(cd lang/c++; make interop-data-generate)
+ (cd lang/csharp; ./build.sh interop-data-generate)
(cd lang/ruby; rake generate_interop)
(cd lang/php; ./build.sh interop-data-generate)
@@ -71,6 +72,7 @@ do
(cd lang/py; ant interop-data-test)
(cd lang/c; ./build.sh interop-data-test)
#(cd lang/c++; make interop-data-test)
+ (cd lang/csharp; ./build.sh interop-data-test)
(cd lang/ruby; rake interop)
(cd lang/php; ./build.sh test-interop)
diff --git a/lang/csharp/build.sh b/lang/csharp/build.sh
index d8ca0f5..db344e2 100755
--- a/lang/csharp/build.sh
+++ b/lang/csharp/build.sh
@@ -29,7 +29,8 @@ case "$1" in
dotnet build --configuration Release Avro.sln
# AVRO-2442: Explictly set LANG to work around ICU bug in `dotnet test`
- LANG=en_US.UTF-8 dotnet test --configuration Release --no-build Avro.sln
+ LANG=en_US.UTF-8 dotnet test --configuration Release --no-build \
+ --filter "TestCategory!=Interop" Avro.sln
;;
perf)
@@ -61,6 +62,14 @@ case "$1" in
cp -pr build/doc/* ${ROOT}/build/avro-doc-${VERSION}/api/csharp
;;
+ interop-data-generate)
+ dotnet run --project src/apache/test/Avro.test.csproj --framework netcoreapp2.2 ../../share/test/schemas/interop.avsc ../../build/interop/data
+ ;;
+
+ interop-data-test)
+ LANG=en_US.UTF-8 dotnet test --filter "TestCategory=Interop"
+ ;;
+
clean)
rm -rf src/apache/{main,test,codegen,ipc,msbuild,perf}/{obj,bin}
rm -rf build
@@ -68,7 +77,7 @@ case "$1" in
;;
*)
- echo "Usage: $0 {test|clean|dist|perf}"
+ echo "Usage: $0 {test|clean|dist|perf|interop-data-generate|interop-data-test}"
exit 1
esac
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/test/Avro.test.csproj
index e73878c..6f776fa 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/test/Avro.test.csproj
@@ -22,6 +22,7 @@
<RootNamespace>Avro.test</RootNamespace>
<AssemblyName>Avro.test</AssemblyName>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+ <GenerateProgramFile>false</GenerateProgramFile>
</PropertyGroup>
<ItemGroup>
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs b/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs
new file mode 100644
index 0000000..219dbf0
--- /dev/null
+++ b/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.Collections.Generic;
+using Avro.File;
+
+namespace Avro.Test.Interop
+{
+ public class InteropDataConstants
+ {
+ public static readonly HashSet<string> SupportedCodecNames = new HashSet<string>
+ {
+ DataFileConstants.NullCodec,
+ DataFileConstants.DeflateCodec
+ };
+ }
+}
\ No newline at end of file
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs b/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
new file mode 100644
index 0000000..5562512
--- /dev/null
+++ b/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using Avro.File;
+using Avro.Generic;
+
+namespace Avro.Test.Interop
+{
+ public class InteropDataGenerator
+ {
+ static void GenerateInteropData(string schemaPath, string outputDir)
+ {
+ RecordSchema schema = null;
+ using (var reader = new StreamReader(schemaPath))
+ {
+ schema = Schema.Parse(reader.ReadToEnd()) as RecordSchema;
+ }
+
+ var mapFieldSchema = (schema.Fields.Find(x => x.Name == "mapField").Schema as MapSchema).ValueSchema as RecordSchema;
+ var mapFieldRecord0 = new GenericRecord(mapFieldSchema);
+ var mapFieldRecord1 = new GenericRecord(mapFieldSchema);
+ mapFieldRecord0.Add("label", "a");
+ mapFieldRecord1.Add("label", "cee");
+ var mapFieldValue = new Dictionary<string, GenericRecord>
+ {
+ { "a", mapFieldRecord0 },
+ { "bee", mapFieldRecord1 }
+ };
+
+ var enumFieldValue = new GenericEnum(schema.Fields.Find(x => x.Name == "enumField").Schema as EnumSchema, "C");
+
+ var fixedFieldValue = new GenericFixed(
+ schema.Fields.Find(x => x.Name == "fixedField").Schema as FixedSchema,
+ Encoding.ASCII.GetBytes("1019181716151413"));
+
+ var nodeSchema = schema.Fields.Find(x => x.Name == "recordField").Schema as RecordSchema;
+ var recordFieldValue = new GenericRecord(nodeSchema);
+ var innerRecordFieldValue = new GenericRecord(nodeSchema);
+ innerRecordFieldValue.Add("label", "inner");
+ innerRecordFieldValue.Add("children", new GenericRecord[] { });
+ recordFieldValue.Add("label", "blah");
+ recordFieldValue.Add("children", new GenericRecord[] { innerRecordFieldValue });
+
+ GenericRecord record = new GenericRecord(schema);
+ record.Add("intField", 12);
+ record.Add("longField", 15234324L);
+ record.Add("stringField", "hey");
+ record.Add("boolField", true);
+ record.Add("floatField", 1234.0f);
+ record.Add("doubleField", -1234.0);
+ record.Add("bytesField", Encoding.UTF8.GetBytes("12312adf"));
+ record.Add("nullField", null);
+ record.Add("arrayField", new double[] { 5.0, 0.0, 12.0 });
+ record.Add("mapField", mapFieldValue);
+ record.Add("unionField", 12.0);
+ record.Add("enumField", enumFieldValue);
+ record.Add("fixedField", fixedFieldValue);
+ record.Add("recordField", recordFieldValue);
+
+ var datumWriter = new GenericDatumWriter<GenericRecord>(schema);
+ foreach (var codecName in InteropDataConstants.SupportedCodecNames)
+ {
+ var outputFile = "csharp.avro";
+ if (codecName != DataFileConstants.NullCodec)
+ {
+ outputFile = string.Format("csharp_{0}.avro", codecName);
+ }
+ var outputPath = Path.Combine(outputDir, outputFile);
+ var codec = Codec.CreateCodecFromString(codecName);
+ using (var dataFileWriter = DataFileWriter<GenericRecord>.OpenWriter(datumWriter, outputPath, codec))
+ {
+ dataFileWriter.Append(record);
+ }
+ }
+ }
+
+ static void Main(string[] args)
+ {
+ GenerateInteropData(args[0], args[1]);
+ }
+ }
+}
\ No newline at end of file
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataTests.cs b/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
new file mode 100644
index 0000000..4d8823a
--- /dev/null
+++ b/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.IO;
+using NUnit.Framework;
+using Avro.File;
+using Avro.Generic;
+
+namespace Avro.Test.Interop
+{
+ [TestFixture]
+ [Category("Interop")]
+ public class InteropDataTests
+ {
+ [TestCase("../../../../../../../../build/interop/data")]
+ public void TestInterop(string inputDir)
+ {
+ // Resolve inputDir relative to the TestDirectory
+ inputDir = Path.Combine(TestContext.CurrentContext.TestDirectory, inputDir);
+
+ Assert.True(Directory.Exists(inputDir),
+ "Input directory does not exist. Run `build.sh interop-data-generate` first.");
+
+ foreach (var avroFile in Directory.EnumerateFiles(inputDir, "*.avro"))
+ {
+ var codec = Path.GetFileNameWithoutExtension(avroFile).Split('_');
+ if (1 < codec.Length && !InteropDataConstants.SupportedCodecNames.Contains(codec[1]))
+ {
+ continue;
+ }
+
+ using(var reader = DataFileReader<GenericRecord>.OpenReader(avroFile))
+ {
+ int i = 0;
+ foreach (var record in reader.NextEntries)
+ {
+ i++;
+ Assert.IsNotNull(record);
+ }
+ Assert.AreNotEqual(0, i);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file