You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/08/30 12:04:49 UTC
[avro] branch branch-1.11 updated: AVRO-3001 AVRO-3274 AVRO-3568 AVRO-3613: Add JSON encoder/decoder for C# (#1833)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch branch-1.11
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/branch-1.11 by this push:
new bc5d76e47 AVRO-3001 AVRO-3274 AVRO-3568 AVRO-3613: Add JSON encoder/decoder for C# (#1833)
bc5d76e47 is described below
commit bc5d76e4785716814d258e7d38410782aede97a5
Author: Robert Yokota <ra...@gmail.com>
AuthorDate: Tue Aug 30 05:04:15 2022 -0700
AVRO-3001 AVRO-3274 AVRO-3568 AVRO-3613: Add JSON encoder/decoder for C# (#1833)
* AVRO-3001 AVRO-3274: Add JSON encoder/decoder for C#
* Add more comments for public/protected members
* Make CodeQL happy
* Make CodeQL happy again
* Minor optimization
* Fix cosmetic issues
* Fix JsonEncoder.StartItem accessibility
* Minor doc fix
* Add fixes and test for JSON encoding/decoding logical types
* Fix fullname calculation for logical schemas
* Fix for AVRO-3613
* Fix for AVRO-3568
* Add union with record test
* Fix test
* Incorporate review feedback
* Incorporate review feedback
* More cleanup
* Revert previous cleanup in favor of recommended cleanup
* Incorporate more review feedback
* Incorporate latest review feedback
* Add more unit tests
* Simplify string constant
* Simplify string constant
* Simplify string constant
* Simplify string constant
Co-authored-by: Martin Grigorov <ma...@users.noreply.github.com>
(cherry picked from commit 1841ff115d52727094998b80798d52210b8addb6)
---
.../src/apache/main/Generic/GenericWriter.cs | 1 +
.../apache/main/Generic/PreresolvingDatumWriter.cs | 1 +
lang/csharp/src/apache/main/IO/Encoder.cs | 5 +
lang/csharp/src/apache/main/IO/JsonDecoder.cs | 765 ++++++++++++++++
lang/csharp/src/apache/main/IO/JsonEncoder.cs | 352 ++++++++
.../apache/main/IO/Parsing/JsonGrammarGenerator.cs | 105 +++
lang/csharp/src/apache/main/IO/Parsing/Parser.cs | 229 +++++
.../src/apache/main/IO/Parsing/SkipParser.cs | 107 +++
lang/csharp/src/apache/main/IO/Parsing/Symbol.cs | 984 +++++++++++++++++++++
.../main/IO/Parsing/ValidatingGrammarGenerator.cs | 170 ++++
lang/csharp/src/apache/main/IO/ParsingDecoder.cs | 205 +++++
lang/csharp/src/apache/main/IO/ParsingEncoder.cs | 146 +++
.../csharp/src/apache/main/Schema/LogicalSchema.cs | 12 +
lang/csharp/src/apache/test/IO/JsonCodecTests.cs | 329 +++++++
.../apache/test/Schema/SchemaNormalizationTests.cs | 8 +
15 files changed, 3419 insertions(+)
diff --git a/lang/csharp/src/apache/main/Generic/GenericWriter.cs b/lang/csharp/src/apache/main/Generic/GenericWriter.cs
index 92d5d99f8..b29cb68bf 100644
--- a/lang/csharp/src/apache/main/Generic/GenericWriter.cs
+++ b/lang/csharp/src/apache/main/Generic/GenericWriter.cs
@@ -177,6 +177,7 @@ namespace Avro.Generic
protected virtual void WriteNull(object value, Encoder encoder)
{
if (value != null) throw TypeMismatch(value, "null", "null");
+ encoder.WriteNull();
}
/// <summary>
diff --git a/lang/csharp/src/apache/main/Generic/PreresolvingDatumWriter.cs b/lang/csharp/src/apache/main/Generic/PreresolvingDatumWriter.cs
index f37299d37..dd21f62ed 100644
--- a/lang/csharp/src/apache/main/Generic/PreresolvingDatumWriter.cs
+++ b/lang/csharp/src/apache/main/Generic/PreresolvingDatumWriter.cs
@@ -114,6 +114,7 @@ namespace Avro.Generic
protected void WriteNull(object value, Encoder encoder)
{
if (value != null) throw TypeMismatch(value, "null", "null");
+ encoder.WriteNull();
}
/// <summary>
diff --git a/lang/csharp/src/apache/main/IO/Encoder.cs b/lang/csharp/src/apache/main/IO/Encoder.cs
index 84a2099a1..0c1712af4 100644
--- a/lang/csharp/src/apache/main/IO/Encoder.cs
+++ b/lang/csharp/src/apache/main/IO/Encoder.cs
@@ -187,5 +187,10 @@ namespace Avro.IO
/// <param name="start">Position within data where the contents start.</param>
/// <param name="len">Number of bytes to write.</param>
void WriteFixed(byte[] data, int start, int len);
+
+ /// <summary>
+ /// Flushes the encoder.
+ /// </summary>
+ void Flush();
}
}
diff --git a/lang/csharp/src/apache/main/IO/JsonDecoder.cs b/lang/csharp/src/apache/main/IO/JsonDecoder.cs
new file mode 100644
index 000000000..48d726e30
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/JsonDecoder.cs
@@ -0,0 +1,765 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using Avro.IO.Parsing;
+using Newtonsoft.Json;
+
+namespace Avro.IO
+{
+ /// <summary>
+ /// A <see cref="Decoder"/> for Avro's JSON data encoding.
+ ///
+ /// JsonDecoder is not thread-safe.
+ /// </summary>
+ public class JsonDecoder : ParsingDecoder
+ {
+ private JsonReader reader;
+ private readonly Stack<ReorderBuffer> reorderBuffers = new Stack<ReorderBuffer>();
+ private ReorderBuffer currentReorderBuffer;
+
+ private class ReorderBuffer
+ {
+ public readonly IDictionary<string, IList<JsonElement>> SavedFields =
+ new Dictionary<string, IList<JsonElement>>();
+
+ public JsonReader OrigParser { get; set; }
+ }
+
+ private JsonDecoder(Symbol root, Stream stream) : base(root)
+ {
+ Configure(stream);
+ }
+
+ private JsonDecoder(Symbol root, string str) : base(root)
+ {
+ Configure(str);
+ }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="JsonDecoder"/> class.
+ /// </summary>
+ public JsonDecoder(Schema schema, Stream stream) : this(GetSymbol(schema), stream)
+ {
+ }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="JsonDecoder"/> class.
+ /// </summary>
+ public JsonDecoder(Schema schema, string str) : this(GetSymbol(schema), str)
+ {
+ }
+
+ private static Symbol GetSymbol(Schema schema)
+ {
+ return (new JsonGrammarGenerator()).Generate(schema);
+ }
+
+ /// <summary>
+ /// Reconfigures this JsonDecoder to use the InputStream provided.
+ /// Otherwise, this JsonDecoder will reset its state and then reconfigure its
+ /// input.
+ /// </summary>
+ /// <param name="stream"> The InputStream to read from. Cannot be null. </param>
+ public void Configure(Stream stream)
+ {
+ Parser.Reset();
+ reorderBuffers.Clear();
+ currentReorderBuffer = null;
+ reader = new JsonTextReader(new StreamReader(stream));
+ reader.Read();
+ }
+
+ /// <summary>
+ /// Reconfigures this JsonDecoder to use the String provided for input.
+ /// Otherwise, this JsonDecoder will reset its state and then reconfigure its
+ /// input.
+ /// </summary>
+ /// <param name="str"> The String to read from. Cannot be null. </param>
+ public void Configure(string str)
+ {
+ Parser.Reset();
+ reorderBuffers.Clear();
+ currentReorderBuffer = null;
+ reader = new JsonTextReader(new StringReader(str));
+ reader.Read();
+ }
+
+ private void Advance(Symbol symbol)
+ {
+ Parser.ProcessTrailingImplicitActions();
+ Parser.Advance(symbol);
+ }
+
+ /// <inheritdoc />
+ public override void ReadNull()
+ {
+ Advance(Symbol.Null);
+ if (reader.TokenType == JsonToken.Null)
+ {
+ reader.Read();
+ }
+ else
+ {
+ throw TypeError("null");
+ }
+ }
+
+ /// <inheritdoc />
+ public override bool ReadBoolean()
+ {
+ Advance(Symbol.Boolean);
+ if (reader.TokenType == JsonToken.Boolean)
+ {
+ bool result = Convert.ToBoolean(reader.Value);
+ reader.Read();
+ return result;
+ }
+ else
+ {
+ throw TypeError("boolean");
+ }
+ }
+
+ /// <inheritdoc />
+ public override int ReadInt()
+ {
+ Advance(Symbol.Int);
+ if (reader.TokenType == JsonToken.Integer || reader.TokenType == JsonToken.Float)
+ {
+ int result = Convert.ToInt32(reader.Value);
+ reader.Read();
+ return result;
+ }
+ else
+ {
+ throw TypeError("int");
+ }
+ }
+
+ /// <inheritdoc />
+ public override long ReadLong()
+ {
+ Advance(Symbol.Long);
+ if (reader.TokenType == JsonToken.Integer || reader.TokenType == JsonToken.Float)
+ {
+ long result = Convert.ToInt64(reader.Value);
+ reader.Read();
+ return result;
+ }
+ else
+ {
+ throw TypeError("long");
+ }
+ }
+
+ /// <inheritdoc />
+ public override float ReadFloat()
+ {
+ Advance(Symbol.Float);
+ if (reader.TokenType == JsonToken.Integer || reader.TokenType == JsonToken.Float)
+ {
+ float result = (float)Convert.ToDouble(reader.Value);
+ reader.Read();
+ return result;
+ }
+ else
+ {
+ throw TypeError("float");
+ }
+ }
+
+ /// <inheritdoc />
+ public override double ReadDouble()
+ {
+ Advance(Symbol.Double);
+ if (reader.TokenType == JsonToken.Integer || reader.TokenType == JsonToken.Float)
+ {
+ double result = Convert.ToDouble(reader.Value);
+ reader.Read();
+ return result;
+ }
+ else
+ {
+ throw TypeError("double");
+ }
+ }
+
+ /// <inheritdoc />
+ public override string ReadString()
+ {
+ Advance(Symbol.String);
+ if (Parser.TopSymbol() == Symbol.MapKeyMarker)
+ {
+ Parser.Advance(Symbol.MapKeyMarker);
+ if (reader.TokenType != JsonToken.PropertyName)
+ {
+ throw TypeError("map-key");
+ }
+ }
+ else
+ {
+ if (reader.TokenType != JsonToken.String)
+ {
+ throw TypeError("string");
+ }
+ }
+
+ string result = Convert.ToString(reader.Value);
+ reader.Read();
+ return result;
+ }
+
+ /// <inheritdoc />
+ public override void SkipString()
+ {
+ Advance(Symbol.String);
+ if (Parser.TopSymbol() == Symbol.MapKeyMarker)
+ {
+ Parser.Advance(Symbol.MapKeyMarker);
+ if (reader.TokenType != JsonToken.PropertyName)
+ {
+ throw TypeError("map-key");
+ }
+ }
+ else
+ {
+ if (reader.TokenType != JsonToken.String)
+ {
+ throw TypeError("string");
+ }
+ }
+
+ reader.Read();
+ }
+
+ /// <inheritdoc />
+ public override byte[] ReadBytes()
+ {
+ Advance(Symbol.Bytes);
+ if (reader.TokenType == JsonToken.String)
+ {
+ byte[] result = ReadByteArray();
+ reader.Read();
+ return result;
+ }
+ else
+ {
+ throw TypeError("bytes");
+ }
+ }
+
+ private byte[] ReadByteArray()
+ {
+ Encoding iso = Encoding.GetEncoding("ISO-8859-1");
+ byte[] result = iso.GetBytes(Convert.ToString(reader.Value));
+ return result;
+ }
+
+ /// <inheritdoc />
+ public override void SkipBytes()
+ {
+ Advance(Symbol.Bytes);
+ if (reader.TokenType == JsonToken.String)
+ {
+ reader.Read();
+ }
+ else
+ {
+ throw TypeError("bytes");
+ }
+ }
+
+ private void CheckFixed(int size)
+ {
+ Advance(Symbol.Fixed);
+ Symbol.IntCheckAction top = (Symbol.IntCheckAction)Parser.PopSymbol();
+ if (size != top.Size)
+ {
+ throw new AvroTypeException("Incorrect length for fixed binary: expected " + top.Size +
+ " but received " + size + " bytes.");
+ }
+ }
+
+ /// <inheritdoc />
+ public override void ReadFixed(byte[] bytes)
+ {
+ ReadFixed(bytes, 0, bytes.Length);
+ }
+
+ /// <inheritdoc />
+ public override void ReadFixed(byte[] bytes, int start, int len)
+ {
+ CheckFixed(len);
+ if (reader.TokenType == JsonToken.String)
+ {
+ byte[] result = ReadByteArray();
+ reader.Read();
+ if (result.Length != len)
+ {
+ throw new AvroTypeException("Expected fixed length " + len + ", but got" + result.Length);
+ }
+
+ Array.Copy(result, 0, bytes, start, len);
+ }
+ else
+ {
+ throw TypeError("fixed");
+ }
+ }
+
+ /// <inheritdoc />
+ public override void SkipFixed(int length)
+ {
+ CheckFixed(length);
+ DoSkipFixed(length);
+ }
+
+ private void DoSkipFixed(int length)
+ {
+ if (reader.TokenType == JsonToken.String)
+ {
+ byte[] result = ReadByteArray();
+ reader.Read();
+ if (result.Length != length)
+ {
+ throw new AvroTypeException("Expected fixed length " + length + ", but got" + result.Length);
+ }
+ }
+ else
+ {
+ throw TypeError("fixed");
+ }
+ }
+
+ /// <inheritdoc />
+ protected override void SkipFixed()
+ {
+ Advance(Symbol.Fixed);
+ Symbol.IntCheckAction top = (Symbol.IntCheckAction)Parser.PopSymbol();
+ DoSkipFixed(top.Size);
+ }
+
+ /// <inheritdoc />
+ public override int ReadEnum()
+ {
+ Advance(Symbol.Enum);
+ Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction)Parser.PopSymbol();
+ if (reader.TokenType == JsonToken.String)
+ {
+ string label = Convert.ToString(reader.Value);
+ int n = top.FindLabel(label);
+ if (n >= 0)
+ {
+ reader.Read();
+ return n;
+ }
+
+ throw new AvroTypeException("Unknown symbol in enum " + label);
+ }
+ else
+ {
+ throw TypeError("fixed");
+ }
+ }
+
+ /// <inheritdoc />
+ public override long ReadArrayStart()
+ {
+ Advance(Symbol.ArrayStart);
+ if (reader.TokenType == JsonToken.StartArray)
+ {
+ reader.Read();
+ return DoArrayNext();
+ }
+ else
+ {
+ throw TypeError("array-start");
+ }
+ }
+
+ /// <inheritdoc />
+ public override long ReadArrayNext()
+ {
+ Advance(Symbol.ItemEnd);
+ return DoArrayNext();
+ }
+
+ private long DoArrayNext()
+ {
+ if (reader.TokenType == JsonToken.EndArray)
+ {
+ Parser.Advance(Symbol.ArrayEnd);
+ reader.Read();
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+
+ /// <inheritdoc />
+ public override void SkipArray()
+ {
+ Advance(Symbol.ArrayStart);
+ if (reader.TokenType == JsonToken.StartArray)
+ {
+ reader.Skip();
+ reader.Read();
+ Advance(Symbol.ArrayEnd);
+ }
+ else
+ {
+ throw TypeError("array-start");
+ }
+ }
+
+ /// <inheritdoc />
+ public override long ReadMapStart()
+ {
+ Advance(Symbol.MapStart);
+ if (reader.TokenType == JsonToken.StartObject)
+ {
+ reader.Read();
+ return DoMapNext();
+ }
+ else
+ {
+ throw TypeError("map-start");
+ }
+ }
+
+ /// <inheritdoc />
+ public override long ReadMapNext()
+ {
+ Advance(Symbol.ItemEnd);
+ return DoMapNext();
+ }
+
+ private long DoMapNext()
+ {
+ if (reader.TokenType == JsonToken.EndObject)
+ {
+ reader.Read();
+ Advance(Symbol.MapEnd);
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+
+ /// <inheritdoc />
+ public override void SkipMap()
+ {
+ Advance(Symbol.MapStart);
+ if (reader.TokenType == JsonToken.StartObject)
+ {
+ reader.Skip();
+ reader.Read();
+ Advance(Symbol.MapEnd);
+ }
+ else
+ {
+ throw TypeError("map-start");
+ }
+ }
+
+ /// <inheritdoc />
+ public override int ReadUnionIndex()
+ {
+ Advance(Symbol.Union);
+ Symbol.Alternative a = (Symbol.Alternative)Parser.PopSymbol();
+
+ string label;
+ if (reader.TokenType == JsonToken.Null)
+ {
+ label = "null";
+ }
+ else if (reader.TokenType == JsonToken.StartObject)
+ {
+ reader.Read();
+ if (reader.TokenType == JsonToken.PropertyName)
+ {
+ label = Convert.ToString(reader.Value);
+ reader.Read();
+ Parser.PushSymbol(Symbol.UnionEnd);
+ }
+ else
+ {
+ throw TypeError("start-union");
+ }
+ }
+ else
+ {
+ throw TypeError("start-union");
+ }
+
+ int n = a.FindLabel(label);
+ if (n < 0)
+ {
+ throw new AvroTypeException("Unknown union branch " + label);
+ }
+
+ Parser.PushSymbol(a.GetSymbol(n));
+ return n;
+ }
+
+ /// <inheritdoc />
+ public override void SkipNull()
+ {
+ ReadNull();
+ }
+
+ /// <inheritdoc />
+ public override void SkipBoolean()
+ {
+ ReadBoolean();
+ }
+
+ /// <inheritdoc />
+ public override void SkipInt()
+ {
+ ReadInt();
+ }
+
+ /// <inheritdoc />
+ public override void SkipLong()
+ {
+ ReadLong();
+ }
+
+ /// <inheritdoc />
+ public override void SkipFloat()
+ {
+ ReadFloat();
+ }
+
+ /// <inheritdoc />
+ public override void SkipDouble()
+ {
+ ReadDouble();
+ }
+
+ /// <inheritdoc />
+ public override void SkipEnum()
+ {
+ ReadEnum();
+ }
+
+ /// <inheritdoc />
+ public override void SkipUnionIndex()
+ {
+ ReadUnionIndex();
+ }
+
+ /// <inheritdoc />
+ public override Symbol DoAction(Symbol input, Symbol top)
+ {
+ if (top is Symbol.FieldAdjustAction)
+ {
+ Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction)top;
+ string name = fa.FName;
+ if (currentReorderBuffer != null)
+ {
+ IList<JsonElement> node = currentReorderBuffer.SavedFields[name];
+ if (node != null)
+ {
+ currentReorderBuffer.SavedFields.Remove(name);
+ currentReorderBuffer.OrigParser = reader;
+ reader = MakeParser(node);
+ return null;
+ }
+ }
+
+ if (reader.TokenType == JsonToken.PropertyName)
+ {
+ do
+ {
+ string fn = Convert.ToString(reader.Value);
+ reader.Read();
+ if (name.Equals(fn) || (fa.Aliases != null && fa.Aliases.Contains(fn)))
+ {
+ return null;
+ }
+ else
+ {
+ if (currentReorderBuffer == null)
+ {
+ currentReorderBuffer = new ReorderBuffer();
+ }
+
+ currentReorderBuffer.SavedFields[fn] = GetValueAsTree(reader);
+ }
+ } while (reader.TokenType == JsonToken.PropertyName);
+
+ throw new AvroTypeException("Expected field name not found: " + fa.FName);
+ }
+ }
+ else if (top == Symbol.FieldEnd)
+ {
+ if (currentReorderBuffer != null && currentReorderBuffer.OrigParser != null)
+ {
+ reader = currentReorderBuffer.OrigParser;
+ currentReorderBuffer.OrigParser = null;
+ }
+ }
+ else if (top == Symbol.RecordStart)
+ {
+ if (reader.TokenType == JsonToken.StartObject)
+ {
+ reader.Read();
+ reorderBuffers.Push(currentReorderBuffer);
+ currentReorderBuffer = null;
+ }
+ else
+ {
+ throw TypeError("record-start");
+ }
+ }
+ else if (top == Symbol.RecordEnd || top == Symbol.UnionEnd)
+ {
+ // AVRO-2034 advance to the end of our object
+ while (reader.TokenType != JsonToken.EndObject)
+ {
+ reader.Read();
+ }
+
+ if (top == Symbol.RecordEnd)
+ {
+ if (currentReorderBuffer != null && currentReorderBuffer.SavedFields.Count > 0)
+ {
+ throw TypeError("Unknown fields: " + currentReorderBuffer.SavedFields.Keys
+ .Aggregate((x, y) => x + ", " + y ));
+ }
+
+ currentReorderBuffer = reorderBuffers.Pop();
+ }
+
+ // AVRO-2034 advance beyond the end object for the next record.
+ reader.Read();
+ }
+ else
+ {
+ throw new AvroTypeException("Unknown action symbol " + top);
+ }
+
+ return null;
+ }
+
+
+ private class JsonElement
+ {
+ private readonly JsonToken token;
+ public JsonToken Token => token;
+ private readonly object value;
+ public object Value => value;
+
+ public JsonElement(JsonToken t, object value)
+ {
+ token = t;
+ this.value = value;
+ }
+
+ public JsonElement(JsonToken t) : this(t, null)
+ {
+ }
+ }
+
+ private static IList<JsonElement> GetValueAsTree(JsonReader reader)
+ {
+ int level = 0;
+ IList<JsonElement> result = new List<JsonElement>();
+ do
+ {
+ JsonToken t = reader.TokenType;
+ switch (t)
+ {
+ case JsonToken.StartObject:
+ case JsonToken.StartArray:
+ level++;
+ result.Add(new JsonElement(t));
+ break;
+ case JsonToken.EndObject:
+ case JsonToken.EndArray:
+ level--;
+ result.Add(new JsonElement(t));
+ break;
+ case JsonToken.PropertyName:
+ case JsonToken.String:
+ case JsonToken.Integer:
+ case JsonToken.Float:
+ case JsonToken.Boolean:
+ case JsonToken.Null:
+ result.Add(new JsonElement(t, reader.Value));
+ break;
+ }
+
+ reader.Read();
+ } while (level != 0);
+
+ result.Add(new JsonElement(JsonToken.None));
+ return result;
+ }
+
+ private JsonReader MakeParser(in IList<JsonElement> elements)
+ {
+ return new JsonElementReader(elements);
+ }
+
+ private class JsonElementReader : JsonReader
+ {
+ private readonly IList<JsonElement> elements;
+
+ public JsonElementReader(IList<JsonElement> elements)
+ {
+ this.elements = elements;
+ pos = 0;
+ }
+
+ private int pos;
+
+ public override object Value
+ {
+ get { return elements[pos].Value; }
+ }
+
+ public override JsonToken TokenType
+ {
+ get { return elements[pos].Token; }
+ }
+
+ public override bool Read()
+ {
+ pos++;
+ return true;
+ }
+ }
+
+ private AvroTypeException TypeError(string type)
+ {
+ return new AvroTypeException("Expected " + type + ". Got " + reader.TokenType);
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/IO/JsonEncoder.cs b/lang/csharp/src/apache/main/IO/JsonEncoder.cs
new file mode 100644
index 000000000..c159a013e
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/JsonEncoder.cs
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Avro.IO.Parsing;
+using System.Collections;
+using System.IO;
+using System.Text;
+using Newtonsoft.Json;
+
+namespace Avro.IO
+{
+ /// <summary>
+ /// An <see cref="Encoder"/> for Avro's JSON data encoding.
+ ///
+ /// JsonEncoder buffers output, and data may not appear on the output until
+ /// <see cref="Encoder.Flush()"/> is called.
+ ///
+ /// JsonEncoder is not thread-safe.
+ /// </summary>
+ public class JsonEncoder : ParsingEncoder, Parser.IActionHandler
+ {
+ private readonly Parser parser;
+ private JsonWriter writer;
+ private bool includeNamespace = true;
+
+ // Has anything been written into the collections?
+ private readonly BitArray isEmpty = new BitArray(64);
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="JsonEncoder"/> class.
+ /// </summary>
+ public JsonEncoder(Schema sc, Stream stream) : this(sc, GetJsonWriter(stream, false))
+ {
+ }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="JsonEncoder"/> class.
+ /// </summary>
+ public JsonEncoder(Schema sc, Stream stream, bool pretty) : this(sc, GetJsonWriter(stream, pretty))
+ {
+ }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="JsonEncoder"/> class.
+ /// </summary>
+ public JsonEncoder(Schema sc, JsonWriter writer)
+ {
+ Configure(writer);
+ parser = new Parser((new JsonGrammarGenerator()).Generate(sc), this);
+ }
+
+ /// <inheritdoc />
+ public override void Flush()
+ {
+ parser.ProcessImplicitActions();
+ if (writer != null)
+ {
+ writer.Flush();
+ }
+ }
+
+ // by default, one object per line.
+ // with pretty option use default pretty printer with root line separator.
+ private static JsonWriter GetJsonWriter(Stream stream, bool pretty)
+ {
+ JsonWriter writer = new JsonTextWriter(new StreamWriter(stream));
+ if (pretty)
+ {
+ writer.Formatting = Formatting.Indented;
+ }
+
+ return writer;
+ }
+
+ /// <summary>
+ /// Whether to include a union label when generating JSON.
+ /// </summary>
+ public virtual bool IncludeNamespace
+ {
+ get { return includeNamespace; }
+ set { includeNamespace = value; }
+ }
+
+
+ /// <summary>
+ /// Reconfigures this JsonEncoder to use the output stream provided.
+ /// Otherwise, this JsonEncoder will flush its current output and then
+ /// reconfigure its output to use a default UTF8 JsonWriter that writes to the
+ /// provided Stream.
+ /// </summary>
+ /// <param name="stream"> The Stream to direct output to. Cannot be null. </param>
+ public void Configure(Stream stream)
+ {
+ Configure(GetJsonWriter(stream, false));
+ }
+
+ /// <summary>
+ /// Reconfigures this JsonEncoder to output to the JsonWriter provided.
+ /// Otherwise, this JsonEncoder will flush its current output and then
+ /// reconfigure its output to use the provided JsonWriter.
+ /// </summary>
+ /// <param name="jsonWriter"> The JsonWriter to direct output to. Cannot be null. </param>
+ public void Configure(JsonWriter jsonWriter)
+ {
+ if (null != parser)
+ {
+ Flush();
+ }
+
+ writer = jsonWriter;
+ }
+
+ /// <inheritdoc />
+ public override void WriteNull()
+ {
+ parser.Advance(Symbol.Null);
+ writer.WriteNull();
+ }
+
+ /// <inheritdoc />
+ public override void WriteBoolean(bool b)
+ {
+ parser.Advance(Symbol.Boolean);
+ writer.WriteValue(b);
+ }
+
+ /// <inheritdoc />
+ public override void WriteInt(int n)
+ {
+ parser.Advance(Symbol.Int);
+ writer.WriteValue(n);
+ }
+
+ /// <inheritdoc />
+ public override void WriteLong(long n)
+ {
+ parser.Advance(Symbol.Long);
+ writer.WriteValue(n);
+ }
+
+ /// <inheritdoc />
+ public override void WriteFloat(float f)
+ {
+ parser.Advance(Symbol.Float);
+ writer.WriteValue(f);
+ }
+
+ /// <inheritdoc />
+ public override void WriteDouble(double d)
+ {
+ parser.Advance(Symbol.Double);
+ writer.WriteValue(d);
+ }
+
+ /// <inheritdoc />
+ public override void WriteString(string str)
+ {
+ parser.Advance(Symbol.String);
+ if (parser.TopSymbol() == Symbol.MapKeyMarker)
+ {
+ parser.Advance(Symbol.MapKeyMarker);
+ writer.WritePropertyName(str);
+ }
+ else
+ {
+ writer.WriteValue(str);
+ }
+ }
+
+ /// <inheritdoc />
+ public override void WriteBytes(byte[] bytes)
+ {
+ WriteBytes(bytes, 0, bytes.Length);
+ }
+
+ /// <inheritdoc />
+ public override void WriteBytes(byte[] bytes, int start, int len)
+ {
+ parser.Advance(Symbol.Bytes);
+ WriteByteArray(bytes, start, len);
+ }
+
+ private void WriteByteArray(byte[] bytes, int start, int len)
+ {
+ Encoding iso = Encoding.GetEncoding("ISO-8859-1");
+ writer.WriteValue(iso.GetString(bytes, start, len));
+ }
+
+ /// <inheritdoc />
+ public override void WriteFixed(byte[] bytes)
+ {
+ WriteFixed(bytes, 0, bytes.Length);
+ }
+
+ /// <inheritdoc />
+ public override void WriteFixed(byte[] bytes, int start, int len)
+ {
+ parser.Advance(Symbol.Fixed);
+ Symbol.IntCheckAction top = (Symbol.IntCheckAction)parser.PopSymbol();
+ if (len != top.Size)
+ {
+ throw new AvroTypeException("Incorrect length for fixed binary: expected " + top.Size +
+ " but received " + len + " bytes.");
+ }
+
+ WriteByteArray(bytes, start, len);
+ }
+
+ /// <inheritdoc />
+ public override void WriteEnum(int e)
+ {
+ parser.Advance(Symbol.Enum);
+ Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction)parser.PopSymbol();
+ if (e < 0 || e >= top.Size)
+ {
+ throw new AvroTypeException("Enumeration out of range: max is " + top.Size + " but received " + e);
+ }
+
+ writer.WriteValue(top.GetLabel(e));
+ }
+
+ /// <inheritdoc />
+ public override void WriteArrayStart()
+ {
+ parser.Advance(Symbol.ArrayStart);
+ writer.WriteStartArray();
+ Push();
+ if (Depth() >= isEmpty.Length)
+ {
+ isEmpty.Length += isEmpty.Length;
+ }
+
+ isEmpty.Set(Depth(), true);
+ }
+
+ /// <inheritdoc />
+ public override void WriteArrayEnd()
+ {
+ if (!isEmpty.Get(Pos))
+ {
+ parser.Advance(Symbol.ItemEnd);
+ }
+
+ Pop();
+ parser.Advance(Symbol.ArrayEnd);
+ writer.WriteEndArray();
+ }
+
+ /// <inheritdoc />
+ public override void WriteMapStart()
+ {
+ Push();
+ if (Depth() >= isEmpty.Length)
+ {
+ isEmpty.Length += isEmpty.Length;
+ }
+
+ isEmpty.Set(Depth(), true);
+
+ parser.Advance(Symbol.MapStart);
+ writer.WriteStartObject();
+ }
+
+ /// <inheritdoc />
+ public override void WriteMapEnd()
+ {
+ if (!isEmpty.Get(Pos))
+ {
+ parser.Advance(Symbol.ItemEnd);
+ }
+
+ Pop();
+
+ parser.Advance(Symbol.MapEnd);
+ writer.WriteEndObject();
+ }
+
+ /// <inheritdoc />
+ public override void StartItem()
+ {
+ if (!isEmpty.Get(Pos))
+ {
+ parser.Advance(Symbol.ItemEnd);
+ }
+
+ base.StartItem();
+ if (Depth() >= isEmpty.Length)
+ {
+ isEmpty.Length += isEmpty.Length;
+ }
+
+ isEmpty.Set(Depth(), false);
+ }
+
+ /// <inheritdoc />
+ public override void WriteUnionIndex(int unionIndex)
+ {
+ parser.Advance(Symbol.Union);
+ Symbol.Alternative top = (Symbol.Alternative)parser.PopSymbol();
+ Symbol symbol = top.GetSymbol(unionIndex);
+ if (symbol != Symbol.Null && includeNamespace)
+ {
+ writer.WriteStartObject();
+ writer.WritePropertyName(top.GetLabel(unionIndex));
+ parser.PushSymbol(Symbol.UnionEnd);
+ }
+
+ parser.PushSymbol(symbol);
+ }
+
+ /// <summary>
+ /// Perform an action based on the given input.
+ /// </summary>
+ public virtual Symbol DoAction(Symbol input, Symbol top)
+ {
+ if (top is Symbol.FieldAdjustAction)
+ {
+ Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction)top;
+ writer.WritePropertyName(fa.FName);
+ }
+ else if (top == Symbol.RecordStart)
+ {
+ writer.WriteStartObject();
+ }
+ else if (top == Symbol.RecordEnd || top == Symbol.UnionEnd)
+ {
+ writer.WriteEndObject();
+ }
+ else if (top != Symbol.FieldEnd)
+ {
+ throw new AvroTypeException("Unknown action symbol " + top);
+ }
+
+ return null;
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/IO/Parsing/JsonGrammarGenerator.cs b/lang/csharp/src/apache/main/IO/Parsing/JsonGrammarGenerator.cs
new file mode 100644
index 000000000..508ea264b
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/Parsing/JsonGrammarGenerator.cs
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Collections.Generic;
+
+namespace Avro.IO.Parsing
+{
+ /// <summary>
+ /// The class that generates a grammar suitable to parse Avro data in JSON
+ /// format.
+ /// </summary>
+ public class JsonGrammarGenerator : ValidatingGrammarGenerator
+ {
+ /// <summary>
+ /// Returns the non-terminal that is the start symbol for the grammar for the
+ /// grammar for the given schema <tt>schema</tt>.
+ /// </summary>
+ public override Symbol Generate(Schema schema)
+ {
+ return Symbol.NewRoot(Generate(schema, new Dictionary<LitS, Symbol>()));
+ }
+
+ /// <summary>
+ /// Returns the non-terminal that is the start symbol for grammar of the given
+ /// schema <tt>sc</tt>. If there is already an entry for the given schema in the
+ /// given map <tt>seen</tt> then that entry is returned. Otherwise a new symbol
+ /// is generated and an entry is inserted into the map.
+ /// </summary>
+ /// <param name="sc"> The schema for which the start symbol is required </param>
+ /// <param name="seen"> A map of schema to symbol mapping done so far. </param>
+ /// <returns> The start symbol for the schema </returns>
+ protected override Symbol Generate(Schema sc, IDictionary<LitS, Symbol> seen)
+ {
+ switch (sc.Tag)
+ {
+ case Schema.Type.Null:
+ case Schema.Type.Boolean:
+ case Schema.Type.Int:
+ case Schema.Type.Long:
+ case Schema.Type.Float:
+ case Schema.Type.Double:
+ case Schema.Type.String:
+ case Schema.Type.Bytes:
+ case Schema.Type.Fixed:
+ case Schema.Type.Union:
+ return base.Generate(sc, seen);
+ case Schema.Type.Enumeration:
+ return Symbol.NewSeq(new Symbol.EnumLabelsAction(((EnumSchema)sc).Symbols), Symbol.Enum);
+ case Schema.Type.Array:
+ return Symbol.NewSeq(
+ Symbol.NewRepeat(Symbol.ArrayEnd, Symbol.ItemEnd, Generate(((ArraySchema)sc).ItemSchema, seen)),
+ Symbol.ArrayStart);
+ case Schema.Type.Map:
+ return Symbol.NewSeq(
+ Symbol.NewRepeat(Symbol.MapEnd, Symbol.ItemEnd, Generate(((MapSchema)sc).ValueSchema, seen),
+ Symbol.MapKeyMarker, Symbol.String), Symbol.MapStart);
+ case Schema.Type.Record:
+ {
+ LitS wsc = new LitS(sc);
+ if (!seen.TryGetValue(wsc, out Symbol rresult))
+ {
+ Symbol[] production = new Symbol[((RecordSchema)sc).Fields.Count * 3 + 2];
+ rresult = Symbol.NewSeq(production);
+ seen[wsc] = rresult;
+
+ int i = production.Length;
+ int n = 0;
+ production[--i] = Symbol.RecordStart;
+ foreach (Field f in ((RecordSchema)sc).Fields)
+ {
+ production[--i] = new Symbol.FieldAdjustAction(n, f.Name, f.Aliases);
+ production[--i] = Generate(f.Schema, seen);
+ production[--i] = Symbol.FieldEnd;
+ n++;
+ }
+
+ production[i - 1] = Symbol.RecordEnd;
+ }
+
+ return rresult;
+ }
+ case Schema.Type.Logical:
+ return Generate((sc as LogicalSchema).BaseSchema, seen);
+ default:
+ throw new Exception("Unexpected schema type");
+ }
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/IO/Parsing/Parser.cs b/lang/csharp/src/apache/main/IO/Parsing/Parser.cs
new file mode 100644
index 000000000..ae788ede0
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/Parsing/Parser.cs
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+namespace Avro.IO.Parsing
+{
+ /// <summary>
+ /// Parser is the class that maintains the stack for parsing. This class is used
+ /// by encoders, which are not required to skip.
+ /// </summary>
+ public class Parser
+ {
+ /// <summary>
+ /// The parser knows how to handle the terminal and non-terminal symbols. But it
+ /// needs help from outside to handle implicit and explicit actions. The clients
+ /// implement this interface to provide this help.
+ /// </summary>
+ public interface IActionHandler
+ {
+ /// <summary>
+ /// Handle the action symbol <tt>top</tt> when the <tt>input</tt> is sought to be
+ /// taken off the stack.
+ /// </summary>
+ /// <param name="input"> The input symbol from the caller of Advance </param>
+ /// <param name="top"> The symbol at the top the stack. </param>
+ /// <returns> <tt>null</tt> if Advance() is to continue processing the stack. If
+ /// not <tt>null</tt> the return value will be returned by Advance(). </returns>
+ Symbol DoAction(Symbol input, Symbol top);
+ }
+
+ private readonly IActionHandler symbolHandler;
+ /// <summary>
+ /// Stack of symbols.
+ /// </summary>
+ protected Symbol[] Stack;
+ /// <summary>
+ /// Position of the stack.
+ /// </summary>
+ protected int Pos;
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Parser"/> class.
+ /// </summary>
+ public Parser(Symbol root, IActionHandler symbolHandler)
+ {
+ this.symbolHandler = symbolHandler;
+ Stack = new Symbol[5]; // Start small to make sure expansion code works
+ Stack[0] = root;
+ Pos = 1;
+ }
+
+ /// <summary>
+ /// If there is no sufficient room in the stack, use this expand it.
+ /// </summary>
+ private void ExpandStack()
+ {
+ Array.Resize(ref Stack, Stack.Length + Math.Max(Stack.Length, 1024));
+ }
+
+ /// <summary>
+ /// Recursively replaces the symbol at the top of the stack with its production,
+ /// until the top is a terminal. Then checks if the top symbol matches the
+ /// terminal symbol supplied <tt>input</tt>.
+ /// </summary>
+ /// <param name="input"> The symbol to match against the terminal at the top of the
+ /// stack. </param>
+ /// <returns> The terminal symbol at the top of the stack unless an implicit action
+ /// resulted in another symbol, in which case that symbol is returned. </returns>
+ public Symbol Advance(Symbol input)
+ {
+ for (;;)
+ {
+ Symbol top = Stack[--Pos];
+ if (top == input)
+ {
+ return top; // A common case
+ }
+
+ Symbol.Kind k = top.SymKind;
+ if (k == Symbol.Kind.ImplicitAction)
+ {
+ Symbol result = symbolHandler.DoAction(input, top);
+ if (result != null)
+ {
+ return result;
+ }
+ }
+ else if (k == Symbol.Kind.Terminal)
+ {
+ throw new AvroTypeException("Attempt to process a " + input + " when a " + top + " was expected.");
+ }
+ else if (k == Symbol.Kind.Repeater && input == ((Symbol.Repeater)top).End)
+ {
+ return input;
+ }
+ else
+ {
+ PushProduction(top);
+ }
+ }
+ }
+
+ /// <summary>
+ /// Performs any implicit actions at the top the stack, expanding any production
+ /// (other than the root) that may be encountered. This method will fail if there
+ /// are any repeaters on the stack.
+ /// </summary>
+ public void ProcessImplicitActions()
+ {
+ while (Pos > 1)
+ {
+ Symbol top = Stack[Pos - 1];
+ if (top.SymKind == Symbol.Kind.ImplicitAction)
+ {
+ Pos--;
+ symbolHandler.DoAction(null, top);
+ }
+ else if (top.SymKind != Symbol.Kind.Terminal)
+ {
+ Pos--;
+ PushProduction(top);
+ }
+ else
+ {
+ break;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Performs any "trailing" implicit actions at the top the stack.
+ /// </summary>
+ public void ProcessTrailingImplicitActions()
+ {
+ while (Pos >= 1)
+ {
+ Symbol top = Stack[Pos - 1];
+ if (top.SymKind == Symbol.Kind.ImplicitAction && ((Symbol.ImplicitAction)top).IsTrailing)
+ {
+ Pos--;
+ symbolHandler.DoAction(null, top);
+ }
+ else
+ {
+ break;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Pushes the production for the given symbol <tt>sym</tt>. If <tt>sym</tt> is a
+ /// repeater and <tt>input</tt> is either <see cref="Symbol.ArrayEnd"/> or
+ /// <see cref="Symbol.MapEnd"/> pushes nothing.
+ /// </summary>
+ /// <param name="sym"> </param>
+ public void PushProduction(Symbol sym)
+ {
+ Symbol[] p = sym.Production;
+ while (Pos + p.Length > Stack.Length)
+ {
+ ExpandStack();
+ }
+
+ Array.Copy(p, 0, Stack, Pos, p.Length);
+ Pos += p.Length;
+ }
+
+ /// <summary>
+ /// Pops and returns the top symbol from the stack.
+ /// </summary>
+ public virtual Symbol PopSymbol()
+ {
+ return Stack[--Pos];
+ }
+
+ /// <summary>
+ /// Returns the top symbol from the stack.
+ /// </summary>
+ public virtual Symbol TopSymbol()
+ {
+ return Stack[Pos - 1];
+ }
+
+ /// <summary>
+ /// Pushes <tt>sym</tt> on to the stack.
+ /// </summary>
+ public virtual void PushSymbol(Symbol sym)
+ {
+ if (Pos == Stack.Length)
+ {
+ ExpandStack();
+ }
+
+ Stack[Pos++] = sym;
+ }
+
+ /// <summary>
+ /// Returns the depth of the stack.
+ /// </summary>
+ public virtual int Depth()
+ {
+ return Pos;
+ }
+
+ /// <summary>
+ /// Resets the stack.
+ /// </summary>
+ public virtual void Reset()
+ {
+ Pos = 1;
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/IO/Parsing/SkipParser.cs b/lang/csharp/src/apache/main/IO/Parsing/SkipParser.cs
new file mode 100644
index 000000000..4679215cb
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/Parsing/SkipParser.cs
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System.Diagnostics;
+
+namespace Avro.IO.Parsing
+{
+ /// <summary>
+ /// A parser that capable of skipping as well read and write. This class is used
+ /// by decoders who (unlike encoders) are required to implement methods to skip.
+ /// </summary>
+ public class SkipParser : Parser
+ {
+ /// <summary>
+ /// The clients implement this interface to skip symbols and actions.
+ /// </summary>
+ public interface ISkipHandler
+ {
+ /// <summary>
+ /// Skips the action at the top of the stack.
+ /// </summary>
+ void SkipAction();
+
+ /// <summary>
+ /// Skips the symbol at the top of the stack.
+ /// </summary>
+ void SkipTopSymbol();
+ }
+
+ private readonly ISkipHandler skipHandler;
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="SkipParser"/> class.
+ /// </summary>
+ public SkipParser(Symbol root, IActionHandler symbolHandler, ISkipHandler skipHandler) : base(root, symbolHandler)
+ {
+ this.skipHandler = skipHandler;
+ }
+
+ /// <summary>
+ /// Skips data by calling <code>skipXyz</code> or <code>readXyz</code> methods on
+ /// <code>this</code>, until the parser stack reaches the target level.
+ /// </summary>
+ public void SkipTo(int target)
+ {
+ while (target < Pos)
+ {
+ Symbol top = Stack[Pos - 1];
+ while (top.SymKind != Symbol.Kind.Terminal)
+ {
+ if (top.SymKind == Symbol.Kind.ImplicitAction || top.SymKind == Symbol.Kind.ExplicitAction)
+ {
+ skipHandler.SkipAction();
+ }
+ else
+ {
+ --Pos;
+ PushProduction(top);
+ }
+
+ goto outerContinue;
+ }
+
+ skipHandler.SkipTopSymbol();
+ outerContinue: ;
+ }
+ }
+
+ /// <summary>
+ /// Skips the repeater at the top the stack.
+ /// </summary>
+ public void SkipRepeater()
+ {
+ int target = Pos;
+ Symbol repeater = Stack[--Pos];
+ Debug.Assert(repeater.SymKind == Symbol.Kind.Repeater);
+ PushProduction(repeater);
+ SkipTo(target);
+ }
+
+ /// <summary>
+ /// Pushes the given symbol on to the skip and skips it.
+ /// </summary>
+ /// <param name="symToSkip"> The symbol that should be skipped. </param>
+ public void SkipSymbol(Symbol symToSkip)
+ {
+ int target = Pos;
+ PushSymbol(symToSkip);
+ SkipTo(target);
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/IO/Parsing/Symbol.cs b/lang/csharp/src/apache/main/IO/Parsing/Symbol.cs
new file mode 100644
index 000000000..d5f4ee09c
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/Parsing/Symbol.cs
@@ -0,0 +1,984 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Avro.IO.Parsing
+{
+ /// <summary>
+ /// Symbol is the base of all symbols (terminals and non-terminals) of the
+ /// grammar.
+ /// </summary>
+ public abstract class Symbol
+ {
+ /// <summary>
+ /// The type of symbol.
+ /// </summary>
+ public enum Kind
+ {
+ /// <summary>
+ /// terminal symbols which have no productions </summary>
+ Terminal,
+
+ /// <summary>
+ /// Start symbol for some grammar </summary>
+ Root,
+
+ /// <summary>
+ /// non-terminal symbol which is a sequence of one or more other symbols </summary>
+ Sequence,
+
+ /// <summary>
+ /// non-terminal to represent the contents of an array or map </summary>
+ Repeater,
+
+ /// <summary>
+ /// non-terminal to represent the union </summary>
+ Alternative,
+
+ /// <summary>
+ /// non-terminal action symbol which are automatically consumed </summary>
+ ImplicitAction,
+
+ /// <summary>
+ /// non-terminal action symbol which is explicitly consumed </summary>
+ ExplicitAction
+ }
+
+ /// The kind of this symbol.
+ public Kind SymKind { get; private set; }
+
+ /// <summary>
+ /// The production for this symbol. If this symbol is a terminal this is
+ /// <tt>null</tt>. Otherwise this holds the the sequence of the symbols that
+ /// forms the production for this symbol. The sequence is in the reverse order of
+ /// production. This is useful for easy copying onto parsing stack.
+ ///
+ /// Please note that this is a final. So the production for a symbol should be
+ /// known before that symbol is constructed. This requirement cannot be met for
+ /// those symbols which are recursive (e.g. a record that holds union a branch of
+ /// which is the record itself). To resolve this problem, we initialize the
+ /// symbol with an array of nulls. Later we fill the symbols. Not clean, but
+ /// works. The other option is to not have this field a final. But keeping it
+ /// final and thus keeping symbol immutable gives some comfort. See various
+ /// generators how we generate records.
+ /// </summary>
+ public Symbol[] Production { get; private set; }
+
+ /// <summary>
+ /// Constructs a new symbol of the given kind.
+ /// </summary>
+ protected Symbol(Kind kind) : this(kind, null)
+ {
+ }
+
+ /// <summary>
+ /// Constructs a new symbol of the given kind and production.
+ /// </summary>
+ protected Symbol(Kind kind, Symbol[] production)
+ {
+ Production = production;
+ SymKind = kind;
+ }
+
+ /// <summary>
+ /// A convenience method to construct a root symbol.
+ /// </summary>
+ public static Symbol NewRoot(params Symbol[] symbols) => new Root(symbols);
+
+ /// <summary>
+ /// A convenience method to construct a sequence.
+ /// </summary>
+ /// <param name="production"> The constituent symbols of the sequence. </param>
+ public static Symbol NewSeq(params Symbol[] production) => new Sequence(production);
+
+ /// <summary>
+ /// A convenience method to construct a repeater.
+ /// </summary>
+ /// <param name="endSymbol"> The end symbol. </param>
+ /// <param name="symsToRepeat"> The symbols to repeat in the repeater. </param>
+ public static Symbol NewRepeat(Symbol endSymbol, params Symbol[] symsToRepeat) =>
+ new Repeater(endSymbol, symsToRepeat);
+
+ /// <summary>
+ /// A convenience method to construct a union.
+ /// </summary>
+ public static Symbol NewAlt(Symbol[] symbols, string[] labels) => new Alternative(symbols, labels);
+
+ /// <summary>
+ /// A convenience method to construct an ErrorAction.
+ /// </summary>
+ /// <param name="e"> </param>
+ protected static Symbol Error(string e) => new ErrorAction(e);
+
+ /// <summary>
+ /// A convenience method to construct a ResolvingAction.
+ /// </summary>
+ /// <param name="w"> The writer symbol </param>
+ /// <param name="r"> The reader symbol </param>
+ protected static Symbol Resolve(Symbol w, Symbol r) => new ResolvingAction(w, r);
+
+ /// <summary>
+ /// Fixup symbol.
+ /// </summary>
+ protected class Fixup
+ {
+ private readonly Symbol[] symbols;
+
+ /// <summary>
+ /// The symbols.
+ /// </summary>
+ public Symbol[] Symbols
+ {
+ get { return (Symbol[])symbols.Clone(); }
+ }
+
+ /// <summary>
+ /// The position.
+ /// </summary>
+ public int Pos { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Fixup"/> class.
+ /// </summary>
+ public Fixup(Symbol[] symbols, int pos)
+ {
+ this.symbols = (Symbol[])symbols.Clone();
+ Pos = pos;
+ }
+ }
+
+ /// <summary>
+ /// Flatten the given sub-array of symbols into a sub-array of symbols.
+ /// </summary>
+ protected virtual Symbol Flatten(IDictionary<Sequence, Sequence> map, IDictionary<Sequence, IList<Fixup>> map2) => this;
+
+ /// <summary>
+ /// Returns the flattened size.
+ /// </summary>
+ public virtual int FlattenedSize() => 1;
+
+ /// <summary>
+ /// Flattens the given sub-array of symbols into an sub-array of symbols. Every
+ /// <tt>Sequence</tt> in the input are replaced by its production recursively.
+ /// Non-<tt>Sequence</tt> symbols, they internally have other symbols those
+ /// internal symbols also get flattened. When flattening is done, the only place
+ /// there might be Sequence symbols is in the productions of a Repeater,
+ /// Alternative, or the symToParse and symToSkip in a UnionAdjustAction or
+ /// SkipAction.
+ ///
+ /// Why is this done? We want our parsers to be fast. If we left the grammars
+ /// unflattened, then the parser would be constantly copying the contents of
+ /// nested Sequence productions onto the parsing stack. Instead, because of
+ /// flattening, we have a long top-level production with no Sequences unless the
+ /// Sequence is absolutely needed, e.g., in the case of a Repeater or an
+ /// Alternative.
+ ///
+ /// Well, this is not exactly true when recursion is involved. Where there is a
+ /// recursive record, that record will be "inlined" once, but any internal (ie,
+ /// recursive) references to that record will be a Sequence for the record. That
+ /// Sequence will not further inline itself -- it will refer to itself as a
+ /// Sequence. The same is true for any records nested in this outer recursive
+ /// record. Recursion is rare, and we want things to be fast in the typical case,
+ /// which is why we do the flattening optimization.
+ ///
+ ///
+ /// The algorithm does a few tricks to handle recursive symbol definitions. In
+ /// order to avoid infinite recursion with recursive symbols, we have a map of
+ /// Symbol->Symbol. Before fully constructing a flattened symbol for a
+ /// <tt>Sequence</tt> we insert an empty output symbol into the map and then
+ /// start filling the production for the <tt>Sequence</tt>. If the same
+ /// <tt>Sequence</tt> is encountered due to recursion, we simply return the
+ /// (empty) output <tt>Sequence</tt> from the map. Then we actually fill out
+ /// the production for the <tt>Sequence</tt>. As part of the flattening process
+ /// we copy the production of <tt>Sequence</tt>s into larger arrays. If the
+ /// original <tt>Sequence</tt> has not not be fully constructed yet, we copy a
+ /// bunch of <tt>null</tt>s. Fix-up remembers all those <tt>null</tt> patches.
+ /// The fix-ups gets finally filled when we know the symbols to occupy those
+ /// patches.
+ /// </summary>
+ /// <param name="input"> The array of input symbols to flatten </param>
+ /// <param name="start"> The position where the input sub-array starts. </param>
+ /// <param name="output"> The output that receives the flattened list of symbols. The
+ /// output array should have sufficient space to receive the
+ /// expanded sub-array of symbols. </param>
+ /// <param name="skip"> The position where the output input sub-array starts. </param>
+ /// <param name="map"> A map of symbols which have already been expanded. Useful for
+ /// handling recursive definitions and for caching. </param>
+ /// <param name="map2"> A map to to store the list of fix-ups. </param>
+ protected static void Flatten(Symbol[] input, int start, Symbol[] output, int skip,
+ IDictionary<Sequence, Sequence> map, IDictionary<Sequence, IList<Fixup>> map2)
+ {
+ for (int i = start, j = skip; i < input.Length; i++)
+ {
+ Symbol s = input[i].Flatten(map, map2);
+ if (s is Sequence)
+ {
+ Symbol[] p = s.Production;
+ if (!map2.TryGetValue((Sequence)s, out IList<Fixup> l))
+ {
+ Array.Copy(p, 0, output, j, p.Length);
+ // Copy any fixups that will be applied to p to add missing symbols
+ foreach (IList<Fixup> fixups in map2.Values)
+ {
+ CopyFixups(fixups, output, j, p);
+ }
+ }
+ else
+ {
+ l.Add(new Fixup(output, j));
+ }
+
+ j += p.Length;
+ }
+ else
+ {
+ output[j++] = s;
+ }
+ }
+ }
+
+ private static void CopyFixups(IList<Fixup> fixups, Symbol[] output, int outPos, Symbol[] toCopy)
+ {
+ for (int i = 0, n = fixups.Count; i < n; i += 1)
+ {
+ Fixup fixup = fixups[i];
+ if (fixup.Symbols == toCopy)
+ {
+ fixups.Add(new Fixup(output, fixup.Pos + outPos));
+ }
+ }
+ }
+
+ /// <summary>
+ /// Returns the amount of space required to flatten the given sub-array of
+ /// symbols.
+ /// </summary>
+ /// <param name="symbols"> The array of input symbols. </param>
+ /// <param name="start"> The index where the subarray starts. </param>
+ /// <returns> The number of symbols that will be produced if one expands the given
+ /// input. </returns>
+ protected static int FlattenedSize(Symbol[] symbols, int start)
+ {
+ int result = 0;
+ for (int i = start; i < symbols.Length; i++)
+ {
+ if (symbols[i] is Sequence)
+ {
+ Sequence s = (Sequence)symbols[i];
+ result += s.FlattenedSize();
+ }
+ else
+ {
+ result += 1;
+ }
+ }
+
+ return result;
+ }
+
+ /// <summary>
+ /// Terminal symbol.
+ /// </summary>
+ protected class Terminal : Symbol
+ {
+ /// <summary>
+ /// Printable name.
+ /// </summary>
+ public string PrintName { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.Terminal"/> class.
+ /// </summary>
+ public Terminal(string printName) : base(Kind.Terminal)
+ {
+ PrintName = printName;
+ }
+
+ /// <inheritdoc />
+ public override string ToString() => PrintName;
+ }
+
+ /// <summary>
+ /// Implicit action.
+ /// </summary>
+ public class ImplicitAction : Symbol
+ {
+ /// <summary>
+ /// Set to <tt>true</tt> if and only if this implicit action is a trailing
+ /// action. That is, it is an action that follows real symbol. E.g
+ /// <see cref="Symbol.DefaultEndAction"/>.
+ /// </summary>
+ public bool IsTrailing { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.ImplicitAction"/> class.
+ /// </summary>
+ public ImplicitAction() : this(false)
+ {
+ }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.ImplicitAction"/> class.
+ /// </summary>
+ public ImplicitAction(bool isTrailing) : base(Kind.ImplicitAction)
+ {
+ IsTrailing = isTrailing;
+ }
+ }
+
+ /// <summary>
+ /// Root symbol.
+ /// </summary>
+ protected class Root : Symbol
+ {
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.Root"/> class.
+ /// </summary>
+ public Root(params Symbol[] symbols) : base(Kind.Root, MakeProduction(symbols))
+ {
+ Production[0] = this;
+ }
+
+ private static Symbol[] MakeProduction(Symbol[] symbols)
+ {
+ Symbol[] result = new Symbol[FlattenedSize(symbols, 0) + 1];
+ Flatten(symbols, 0, result, 1, new Dictionary<Sequence, Sequence>(),
+ new Dictionary<Sequence, IList<Fixup>>());
+ return result;
+ }
+ }
+
+ /// <summary>
+ /// Sequence symbol.
+ /// </summary>
+ protected class Sequence : Symbol, IEnumerable<Symbol>
+ {
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.Sequence"/> class.
+ /// </summary>
+ public Sequence(Symbol[] productions) : base(Kind.Sequence, productions)
+ {
+ }
+
+ /// <summary>
+ /// Get the symbol at the given index.
+ /// </summary>
+ public virtual Symbol this[int index] => Production[index];
+
+ /// <summary>
+ /// Get the symbol at the given index.
+ /// </summary>
+ public virtual Symbol Get(int index) => Production[index];
+
+ /// <summary>
+ /// Returns the number of symbols.
+ /// </summary>
+ public virtual int Size() => Production.Length;
+
+ /// <inheritdoc />
+ public IEnumerator<Symbol> GetEnumerator() => Enumerable.Reverse(Production).GetEnumerator();
+
+ IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+
+ /// <inheritdoc />
+ protected override Symbol Flatten(IDictionary<Sequence, Sequence> map,
+ IDictionary<Sequence, IList<Fixup>> map2)
+ {
+ if (!map.TryGetValue(this, out Sequence result))
+ {
+ result = new Sequence(new Symbol[FlattenedSize()]);
+ map[this] = result;
+ IList<Fixup> l = new List<Fixup>();
+ map2[result] = l;
+
+ Flatten(Production, 0, result.Production, 0, map, map2);
+ foreach (Fixup f in l)
+ {
+ Array.Copy(result.Production, 0, f.Symbols, f.Pos, result.Production.Length);
+ }
+
+ map2.Remove(result);
+ }
+
+ return result;
+ }
+
+ /// <inheritdoc />
+ public override int FlattenedSize() => FlattenedSize(Production, 0);
+ }
+
+ /// <summary>
+ /// Repeater symbol.
+ /// </summary>
+ public class Repeater : Symbol
+ {
+ /// <summary>
+ /// The end symbol.
+ /// </summary>
+ public Symbol End { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.Repeater"/> class.
+ /// </summary>
+ public Repeater(Symbol end, params Symbol[] sequenceToRepeat) : base(Kind.Repeater,
+ MakeProduction(sequenceToRepeat))
+ {
+ End = end;
+ Production[0] = this;
+ }
+
+ private static Symbol[] MakeProduction(Symbol[] p)
+ {
+ Symbol[] result = new Symbol[p.Length + 1];
+ Array.Copy(p, 0, result, 1, p.Length);
+ return result;
+ }
+
+ /// <inheritdoc />
+ protected override Symbol Flatten(IDictionary<Sequence, Sequence> map,
+ IDictionary<Sequence, IList<Fixup>> map2)
+ {
+ Repeater result = new Repeater(End, new Symbol[FlattenedSize(Production, 1)]);
+ Flatten(Production, 1, result.Production, 1, map, map2);
+ return result;
+ }
+ }
+
+ /// <summary>
+ /// Returns true if the Parser contains any Error symbol, indicating that it may
+ /// fail for some inputs.
+ /// </summary>
+ private static bool HasErrors(Symbol symbol)
+ {
+ return HasErrors(symbol, new HashSet<Symbol>());
+ }
+
+ private static bool HasErrors(Symbol symbol, ISet<Symbol> visited)
+ {
+ // avoid infinite recursion
+ if (visited.Contains(symbol))
+ {
+ return false;
+ }
+
+ visited.Add(symbol);
+
+ switch (symbol.SymKind)
+ {
+ case Kind.Alternative:
+ return HasErrors(symbol, ((Alternative)symbol).Symbols, visited);
+ case Kind.ExplicitAction:
+ return false;
+ case Kind.ImplicitAction:
+ if (symbol is ErrorAction)
+ {
+ return true;
+ }
+
+ if (symbol is UnionAdjustAction)
+ {
+ return HasErrors(((UnionAdjustAction)symbol).SymToParse, visited);
+ }
+
+ return false;
+ case Kind.Repeater:
+ Repeater r = (Repeater)symbol;
+ return HasErrors(r.End, visited) || HasErrors(symbol, r.Production, visited);
+ case Kind.Root:
+ case Kind.Sequence:
+ return HasErrors(symbol, symbol.Production, visited);
+ case Kind.Terminal:
+ return false;
+ default:
+ throw new Exception("unknown symbol kind: " + symbol.SymKind);
+ }
+ }
+
+ private static bool HasErrors(Symbol root, Symbol[] symbols, ISet<Symbol> visited)
+ {
+ if (null != symbols)
+ {
+ foreach (Symbol s in symbols)
+ {
+ if (s == root)
+ {
+ continue;
+ }
+
+ if (HasErrors(s, visited))
+ {
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
+
+ /// <summary>
+ /// Alternative symbol.
+ /// </summary>
+ public class Alternative : Symbol
+ {
+ /// <summary>
+ /// The symbols.
+ /// </summary>
+ public Symbol[] Symbols { get; private set; }
+
+ /// <summary>
+ /// The labels.
+ /// </summary>
+ public string[] Labels { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.Alternative"/> class.
+ /// </summary>
+ public Alternative(Symbol[] symbols, string[] labels) : base(Kind.Alternative)
+ {
+ Symbols = symbols;
+ Labels = labels;
+ }
+
+ /// <summary>
+ /// Returns the symbol at the given index.
+ /// </summary>
+ public virtual Symbol GetSymbol(int index)
+ {
+ return Symbols[index];
+ }
+
+ /// <summary>
+ /// Returns the label at the given index.
+ /// </summary>
+ public virtual string GetLabel(int index)
+ {
+ return Labels[index];
+ }
+
+ /// <summary>
+ /// Returns the size.
+ /// </summary>
+ public virtual int Size()
+ {
+ return Symbols.Length;
+ }
+
+ /// <summary>
+ /// Returns the index of the given label.
+ /// </summary>
+ public virtual int FindLabel(string label)
+ {
+ if (label != null)
+ {
+ for (int i = 0; i < Labels.Length; i++)
+ {
+ if (label.Equals(Labels[i]))
+ {
+ return i;
+ }
+ }
+ }
+
+ return -1;
+ }
+
+ /// <inheritdoc />
+ protected override Symbol Flatten(IDictionary<Sequence, Sequence> map,
+ IDictionary<Sequence, IList<Fixup>> map2)
+ {
+ Symbol[] ss = new Symbol[Symbols.Length];
+ for (int i = 0; i < ss.Length; i++)
+ {
+ ss[i] = Symbols[i].Flatten(map, map2);
+ }
+
+ return new Alternative(ss, Labels);
+ }
+ }
+
+ /// <summary>
+ /// The error action.
+ /// </summary>
+ public class ErrorAction : ImplicitAction
+ {
+ /// <summary>
+ /// The error message.
+ /// </summary>
+ public string Msg { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.ErrorAction"/> class.
+ /// </summary>
+ public ErrorAction(string msg)
+ {
+ Msg = msg;
+ }
+ }
+
+ /// <summary>
+ /// Int check action.
+ /// </summary>
+ public class IntCheckAction : Symbol
+ {
+ /// <summary>
+ /// The size.
+ /// </summary>
+ public int Size { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.IntCheckAction"/> class.
+ /// </summary>
+ public IntCheckAction(int size) : base(Kind.ExplicitAction)
+ {
+ Size = size;
+ }
+ }
+
+ /// <summary>
+ /// The writer union action.
+ /// </summary>
+ public class WriterUnionAction : ImplicitAction
+ {
+ }
+
+ /// <summary>
+ /// The resolving action.
+ /// </summary>
+ public class ResolvingAction : ImplicitAction
+ {
+ /// <summary>
+ /// The writer.
+ /// </summary>
+ public Symbol Writer { get; private set; }
+
+ /// <summary>
+ /// The reader.
+ /// </summary>
+ public Symbol Reader { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.ResolvingAction"/> class.
+ /// </summary>
+ public ResolvingAction(Symbol writer, Symbol reader)
+ {
+ Writer = writer;
+ Reader = reader;
+ }
+
+ /// <inheritdoc />
+ protected override Symbol Flatten(IDictionary<Sequence, Sequence> map,
+ IDictionary<Sequence, IList<Fixup>> map2)
+ {
+ return new ResolvingAction(Writer.Flatten(map, map2), Reader.Flatten(map, map2));
+ }
+ }
+
+ /// <summary>
+ /// The skip action.
+ /// </summary>
+ public class SkipAction : ImplicitAction
+ {
+ /// <summary>
+ /// The symbol to skip.
+ /// </summary>
+ public Symbol SymToSkip { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.SkipAction"/> class.
+ /// </summary>
+ public SkipAction(Symbol symToSkip) : base(true)
+ {
+ SymToSkip = symToSkip;
+ }
+
+ /// <inheritdoc />
+ protected override Symbol Flatten(IDictionary<Sequence, Sequence> map,
+ IDictionary<Sequence, IList<Fixup>> map2)
+ {
+ return new SkipAction(SymToSkip.Flatten(map, map2));
+ }
+ }
+
+ /// <summary>
+ /// The field adjust action.
+ /// </summary>
+ public class FieldAdjustAction : ImplicitAction
+ {
+ /// <summary>
+ /// The index.
+ /// </summary>
+ public int RIndex { get; private set; }
+
+ /// <summary>
+ /// The field name.
+ /// </summary>
+ public string FName { get; private set; }
+
+ /// <summary>
+ /// The field aliases.
+ /// </summary>
+ public IList<string> Aliases { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.FieldAdjustAction"/> class.
+ /// </summary>
+ public FieldAdjustAction(int rindex, string fname, IList<string> aliases)
+ {
+ RIndex = rindex;
+ FName = fname;
+ Aliases = aliases;
+ }
+ }
+
+ /// <summary>
+ /// THe field order action.
+ /// </summary>
+ public sealed class FieldOrderAction : ImplicitAction
+ {
+ /// <summary>
+ /// Whether no reorder is needed.
+ /// </summary>
+ public bool NoReorder { get; private set; }
+
+ /// <summary>
+ /// The fields.
+ /// </summary>
+ public Field[] Fields { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.FieldOrderAction"/> class.
+ /// </summary>
+ public FieldOrderAction(Field[] fields)
+ {
+ Fields = fields;
+ bool noReorder = true;
+ for (int i = 0; noReorder && i < fields.Length; i++)
+ {
+ noReorder &= (i == fields[i].Pos);
+ }
+
+ NoReorder = noReorder;
+ }
+ }
+
+ /// <summary>
+ /// The default start action.
+ /// </summary>
+ public class DefaultStartAction : ImplicitAction
+ {
+ /// <summary>
+ /// The contents.
+ /// </summary>
+ public byte[] Contents { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.DefaultStartAction"/> class.
+ /// </summary>
+ public DefaultStartAction(byte[] contents)
+ {
+ Contents = contents;
+ }
+ }
+
+ /// <summary>
+ /// The union adjust action.
+ /// </summary>
+ public class UnionAdjustAction : ImplicitAction
+ {
+ /// <summary>
+ /// The index.
+ /// </summary>
+ public int RIndex { get; private set; }
+
+ /// <summary>
+ /// The symbol to parser.
+ /// </summary>
+ public Symbol SymToParse { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.UnionAdjustAction"/> class.
+ /// </summary>
+ public UnionAdjustAction(int rindex, Symbol symToParse)
+ {
+ RIndex = rindex;
+ SymToParse = symToParse;
+ }
+
+ /// <inheritdoc />
+ protected override Symbol Flatten(IDictionary<Sequence, Sequence> map,
+ IDictionary<Sequence, IList<Fixup>> map2)
+ {
+ return new UnionAdjustAction(RIndex, SymToParse.Flatten(map, map2));
+ }
+ }
+
+ /// <summary>
+ /// The enum labels action.
+ /// </summary>
+ public class EnumLabelsAction : IntCheckAction
+ {
+ /// <summary>
+ /// The symbols.
+ /// </summary>
+ public IList<string> Symbols { get; private set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="Symbol.EnumLabelsAction"/> class.
+ /// </summary>
+ public EnumLabelsAction(IList<string> symbols) : base(symbols.Count)
+ {
+ Symbols = symbols;
+ }
+
+ /// <summary>
+ /// Returns the label at the given index.
+ /// </summary>
+ public virtual string GetLabel(int n)
+ {
+ return Symbols[n];
+ }
+
+ /// <summary>
+ /// Returns index of the given label.
+ /// </summary>
+ public virtual int FindLabel(string label)
+ {
+ if (label != null)
+ {
+ for (int i = 0; i < Symbols.Count; i++)
+ {
+ if (label.Equals(Symbols[i]))
+ {
+ return i;
+ }
+ }
+ }
+
+ return -1;
+ }
+ }
+
+ /// <summary>
+ /// The terminal symbols for the grammar.
+ /// </summary>
+ public static Symbol Null { get; } = new Terminal("null");
+
+ /// <summary>
+ /// Boolean
+ /// </summary>
+ public static Symbol Boolean { get; } = new Terminal("boolean");
+
+ /// <summary>
+ /// Int
+ /// </summary>
+ public static Symbol Int { get; } = new Terminal("int");
+ /// <summary>
+ /// Long
+ /// </summary>
+ public static Symbol Long { get; } = new Terminal("long");
+ /// <summary>
+ /// Float
+ /// </summary>
+ public static Symbol Float { get; } = new Terminal("float");
+ /// <summary>
+ /// Double
+ /// </summary>
+ public static Symbol Double { get; } = new Terminal("double");
+ /// <summary>
+ /// String
+ /// </summary>
+ public static Symbol String { get; } = new Terminal("string");
+ /// <summary>
+ /// Bytes
+ /// </summary>
+ public static Symbol Bytes { get; } = new Terminal("bytes");
+ /// <summary>
+ /// Fixed
+ /// </summary>
+ public static Symbol Fixed { get; } = new Terminal("fixed");
+ /// <summary>
+ /// Enum
+ /// </summary>
+ public static Symbol Enum { get; } = new Terminal("enum");
+ /// <summary>
+ /// Union
+ /// </summary>
+ public static Symbol Union { get; } = new Terminal("union");
+
+ /// <summary>
+ /// ArrayStart
+ /// </summary>
+ public static Symbol ArrayStart { get; } = new Terminal("array-start");
+ /// <summary>
+ /// ArrayEnd
+ /// </summary>
+ public static Symbol ArrayEnd { get; } = new Terminal("array-end");
+ /// <summary>
+ /// MapStart
+ /// </summary>
+ public static Symbol MapStart { get; } = new Terminal("map-start");
+ /// <summary>
+ /// MapEnd
+ /// </summary>
+ public static Symbol MapEnd { get; } = new Terminal("map-end");
+ /// <summary>
+ /// ItemEnd
+ /// </summary>
+ public static Symbol ItemEnd { get; } = new Terminal("item-end");
+
+ /// <summary>
+ /// WriterUnion
+ /// </summary>
+ public static Symbol WriterUnion { get; } = new WriterUnionAction();
+
+ /// <summary>
+ /// FieldAction - a pseudo terminal used by parsers
+ /// </summary>
+ public static Symbol FieldAction { get; } = new Terminal("field-action");
+
+ /// <summary>
+ /// RecordStart
+ /// </summary>
+ public static Symbol RecordStart { get; } = new ImplicitAction(false);
+ /// <summary>
+ /// RecordEnd
+ /// </summary>
+ public static Symbol RecordEnd { get; } = new ImplicitAction(true);
+ /// <summary>
+ /// UnionEnd
+ /// </summary>
+ public static Symbol UnionEnd { get; } = new ImplicitAction(true);
+ /// <summary>
+ /// FieldEnd
+ /// </summary>
+ public static Symbol FieldEnd { get; } = new ImplicitAction(true);
+
+ /// <summary>
+ /// DefaultEndAction
+ /// </summary>
+ public static Symbol DefaultEndAction { get; } = new ImplicitAction(true);
+ /// <summary>
+ /// MapKeyMarker
+ /// </summary>
+ public static Symbol MapKeyMarker { get; } = new Terminal("map-key-marker");
+ }
+}
diff --git a/lang/csharp/src/apache/main/IO/Parsing/ValidatingGrammarGenerator.cs b/lang/csharp/src/apache/main/IO/Parsing/ValidatingGrammarGenerator.cs
new file mode 100644
index 000000000..7d1096606
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/Parsing/ValidatingGrammarGenerator.cs
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Collections.Generic;
+using Avro.Generic;
+
+namespace Avro.IO.Parsing
+{
+ /// <summary>
+ /// The class that generates validating grammar.
+ /// </summary>
+ public class ValidatingGrammarGenerator
+ {
+ /// <summary>
+ /// Returns the non-terminal that is the start symbol for the grammar for the
+ /// given schema <tt>sc</tt>.
+ /// </summary>
+ public virtual Symbol Generate(Schema schema)
+ {
+ return Symbol.NewRoot(Generate(schema, new Dictionary<LitS, Symbol>()));
+ }
+
+ /// <summary>
+ /// Returns the non-terminal that is the start symbol for the grammar for the
+ /// given schema <tt>sc</tt>. If there is already an entry for the given schema
+ /// in the given map <tt>seen</tt> then that entry is returned. Otherwise a new
+ /// symbol is generated and an entry is inserted into the map.
+ /// </summary>
+ /// <param name="sc"> The schema for which the start symbol is required </param>
+ /// <param name="seen"> A map of schema to symbol mapping done so far. </param>
+ /// <returns> The start symbol for the schema </returns>
+ protected virtual Symbol Generate(Schema sc, IDictionary<LitS, Symbol> seen)
+ {
+ switch (sc.Tag)
+ {
+ case Schema.Type.Null:
+ return Symbol.Null;
+ case Schema.Type.Boolean:
+ return Symbol.Boolean;
+ case Schema.Type.Int:
+ return Symbol.Int;
+ case Schema.Type.Long:
+ return Symbol.Long;
+ case Schema.Type.Float:
+ return Symbol.Float;
+ case Schema.Type.Double:
+ return Symbol.Double;
+ case Schema.Type.String:
+ return Symbol.String;
+ case Schema.Type.Bytes:
+ return Symbol.Bytes;
+ case Schema.Type.Fixed:
+ return Symbol.NewSeq(new Symbol.IntCheckAction(((FixedSchema)sc).Size), Symbol.Fixed);
+ case Schema.Type.Enumeration:
+ return Symbol.NewSeq(new Symbol.IntCheckAction(((EnumSchema)sc).Symbols.Count), Symbol.Enum);
+ case Schema.Type.Array:
+ return Symbol.NewSeq(
+ Symbol.NewRepeat(Symbol.ArrayEnd, Generate(((ArraySchema)sc).ItemSchema, seen)),
+ Symbol.ArrayStart);
+ case Schema.Type.Map:
+ return Symbol.NewSeq(
+ Symbol.NewRepeat(Symbol.MapEnd, Generate(((MapSchema)sc).ValueSchema, seen), Symbol.String),
+ Symbol.MapStart);
+ case Schema.Type.Record:
+ {
+ LitS wsc = new LitS(sc);
+ if (!seen.TryGetValue(wsc, out Symbol rresult))
+ {
+ Symbol[] production = new Symbol[((RecordSchema)sc).Fields.Count];
+
+ // We construct a symbol without filling the array. Please see
+ // <see cref="Symbol.production"/> for the reason.
+ rresult = Symbol.NewSeq(production);
+ seen[wsc] = rresult;
+
+ int j = production.Length;
+ foreach (Field f in ((RecordSchema)sc).Fields)
+ {
+ production[--j] = Generate(f.Schema, seen);
+ }
+ }
+
+ return rresult;
+ }
+ case Schema.Type.Union:
+ IList<Schema> subs = ((UnionSchema)sc).Schemas;
+ Symbol[] symbols = new Symbol[subs.Count];
+ string[] labels = new string[subs.Count];
+
+ int i = 0;
+ foreach (Schema b in ((UnionSchema)sc).Schemas)
+ {
+ symbols[i] = Generate(b, seen);
+ labels[i] = b.Fullname;
+ i++;
+ }
+
+ return Symbol.NewSeq(Symbol.NewAlt(symbols, labels), Symbol.Union);
+ case Schema.Type.Logical:
+ return Generate((sc as LogicalSchema).BaseSchema, seen);
+ default:
+ throw new Exception("Unexpected schema type");
+ }
+ }
+
+ /// <summary>
+ /// A wrapper around Schema that does "==" equality.
+ /// </summary>
+ protected class LitS
+ {
+ private readonly Schema actual;
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="LitS"/> class.
+ /// </summary>
+ public LitS(Schema actual)
+ {
+ this.actual = actual;
+ }
+
+ /// <summary>
+ /// Two LitS are equal if and only if their underlying schema is the same (not
+ /// merely equal).
+ /// </summary>
+ public override bool Equals(object o)
+ {
+ if (o is null)
+ {
+ return false;
+ }
+
+ if (Object.ReferenceEquals(this, o))
+ {
+ return true;
+ }
+
+ if (GetType() != o.GetType())
+ {
+ return false;
+ }
+
+ return actual.Equals(((LitS)o).actual);
+ }
+
+ /// <summary>
+ /// Returns the hash code for the current <see cref="LitS" />.
+ /// </summary>
+ public override int GetHashCode()
+ {
+ return actual.GetHashCode();
+ }
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/IO/ParsingDecoder.cs b/lang/csharp/src/apache/main/IO/ParsingDecoder.cs
new file mode 100644
index 000000000..ce3276133
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/ParsingDecoder.cs
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Avro.IO.Parsing;
+
+namespace Avro.IO
+{
+ /// <summary>
+ /// Base class for a <see cref="Parsing.Parser"/>-based
+ /// <see cref="Decoder"/>s.
+ /// </summary>
+ public abstract class ParsingDecoder : Decoder, Parser.IActionHandler, SkipParser.ISkipHandler
+ {
+ /// <inheritdoc />
+ public abstract void ReadNull();
+
+ /// <inheritdoc />
+ public abstract bool ReadBoolean();
+
+ /// <inheritdoc />
+ public abstract int ReadInt();
+
+ /// <inheritdoc />
+ public abstract long ReadLong();
+
+ /// <inheritdoc />
+ public abstract float ReadFloat();
+
+ /// <inheritdoc />
+ public abstract double ReadDouble();
+
+ /// <inheritdoc />
+ public abstract byte[] ReadBytes();
+
+ /// <inheritdoc />
+ public abstract string ReadString();
+
+ /// <inheritdoc />
+ public abstract int ReadEnum();
+
+ /// <inheritdoc />
+ public abstract long ReadArrayStart();
+
+ /// <inheritdoc />
+ public abstract long ReadArrayNext();
+
+ /// <inheritdoc />
+ public abstract long ReadMapStart();
+
+ /// <inheritdoc />
+ public abstract long ReadMapNext();
+
+ /// <inheritdoc />
+ public abstract int ReadUnionIndex();
+
+ /// <inheritdoc />
+ public abstract void ReadFixed(byte[] buffer);
+
+ /// <inheritdoc />
+ public abstract void ReadFixed(byte[] buffer, int start, int length);
+
+ /// <inheritdoc />
+ public abstract void SkipNull();
+
+ /// <inheritdoc />
+ public abstract void SkipBoolean();
+
+ /// <inheritdoc />
+ public abstract void SkipInt();
+
+ /// <inheritdoc />
+ public abstract void SkipLong();
+
+ /// <inheritdoc />
+ public abstract void SkipFloat();
+
+ /// <inheritdoc />
+ public abstract void SkipDouble();
+
+ /// <inheritdoc />
+ public abstract void SkipBytes();
+
+ /// <inheritdoc />
+ public abstract void SkipString();
+
+ /// <inheritdoc />
+ public abstract void SkipEnum();
+
+ /// <inheritdoc />
+ public abstract void SkipUnionIndex();
+
+ /// <inheritdoc />
+ public abstract void SkipFixed(int len);
+
+ /// <summary>
+ /// Skips an array on the stream.
+ /// </summary>
+ public abstract void SkipArray();
+
+ /// <summary>
+ /// Skips a map on the stream.
+ /// </summary>
+ public abstract void SkipMap();
+
+ /// <inheritdoc />
+ public abstract Symbol DoAction(Symbol input, Symbol top);
+
+ /// <summary>
+ /// The parser.
+ /// </summary>
+ protected readonly SkipParser Parser;
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="ParsingDecoder"/> class.
+ /// </summary>
+ protected ParsingDecoder(Symbol root)
+ {
+ Parser = new SkipParser(root, this, this);
+ }
+
+ /// <summary>
+ /// Skips a fixed type on the stream.
+ /// </summary>
+ protected abstract void SkipFixed();
+
+ /// <inheritdoc />
+ public virtual void SkipAction()
+ {
+ Parser.PopSymbol();
+ }
+
+ /// <inheritdoc />
+ public virtual void SkipTopSymbol()
+ {
+ Symbol top = Parser.TopSymbol();
+ if (top == Symbol.Null)
+ {
+ ReadNull();
+ }
+ else if (top == Symbol.Boolean)
+ {
+ ReadBoolean();
+ }
+ else if (top == Symbol.Int)
+ {
+ ReadInt();
+ }
+ else if (top == Symbol.Long)
+ {
+ ReadLong();
+ }
+ else if (top == Symbol.Float)
+ {
+ ReadFloat();
+ }
+ else if (top == Symbol.Double)
+ {
+ ReadDouble();
+ }
+ else if (top == Symbol.String)
+ {
+ SkipString();
+ }
+ else if (top == Symbol.Bytes)
+ {
+ SkipBytes();
+ }
+ else if (top == Symbol.Enum)
+ {
+ ReadEnum();
+ }
+ else if (top == Symbol.Fixed)
+ {
+ SkipFixed();
+ }
+ else if (top == Symbol.Union)
+ {
+ ReadUnionIndex();
+ }
+ else if (top == Symbol.ArrayStart)
+ {
+ SkipArray();
+ }
+ else if (top == Symbol.MapStart)
+ {
+ SkipMap();
+ }
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/IO/ParsingEncoder.cs b/lang/csharp/src/apache/main/IO/ParsingEncoder.cs
new file mode 100644
index 000000000..637a6e346
--- /dev/null
+++ b/lang/csharp/src/apache/main/IO/ParsingEncoder.cs
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+namespace Avro.IO
+{
+ /// <summary>
+ /// Base class for a <see cref="Parsing.Parser"/>-based
+ /// <see cref="Encoder"/>s.
+ /// </summary>
+ public abstract class ParsingEncoder : Encoder
+ {
+ /// <summary>
+ /// Tracks the number of items that remain to be written in the collections
+ /// (array or map).
+ /// </summary>
+ private long[] counts = new long[10];
+
+ /// <summary>
+ /// Position into the counts stack.
+ /// </summary>
+ protected int Pos = -1;
+
+ /// <inheritdoc />
+ public abstract void WriteNull();
+
+ /// <inheritdoc />
+ public abstract void WriteBoolean(bool value);
+
+ /// <inheritdoc />
+ public abstract void WriteInt(int value);
+
+ /// <inheritdoc />
+ public abstract void WriteLong(long value);
+
+ /// <inheritdoc />
+ public abstract void WriteFloat(float value);
+
+ /// <inheritdoc />
+ public abstract void WriteDouble(double value);
+
+ /// <inheritdoc />
+ public abstract void WriteBytes(byte[] value);
+
+ /// <inheritdoc />
+ public abstract void WriteBytes(byte[] value, int offset, int length);
+
+ /// <inheritdoc />
+ public abstract void WriteString(string value);
+
+ /// <inheritdoc />
+ public abstract void WriteEnum(int value);
+
+ /// <inheritdoc />
+ public abstract void WriteArrayStart();
+
+ /// <inheritdoc />
+ public abstract void WriteArrayEnd();
+
+ /// <inheritdoc />
+ public abstract void WriteMapStart();
+
+ /// <inheritdoc />
+ public abstract void WriteMapEnd();
+
+ /// <inheritdoc />
+ public abstract void WriteUnionIndex(int value);
+
+ /// <inheritdoc />
+ public abstract void WriteFixed(byte[] data);
+
+ /// <inheritdoc />
+ public abstract void WriteFixed(byte[] data, int start, int len);
+
+ /// <inheritdoc />
+ public abstract void Flush();
+
+ /// <inheritdoc />
+ public virtual void SetItemCount(long value)
+ {
+ if (counts[Pos] != 0)
+ {
+ throw new AvroTypeException("Incorrect number of items written. " + counts[Pos] +
+ " more required.");
+ }
+
+ counts[Pos] = value;
+ }
+
+ /// <inheritdoc />
+ public virtual void StartItem()
+ {
+ counts[Pos]--;
+ }
+
+ /// <summary>
+ /// Push a new collection on to the stack.
+ /// </summary>
+ protected void Push()
+ {
+ if (++Pos == counts.Length)
+ {
+ Array.Resize(ref counts, Pos + 10);
+ }
+
+ counts[Pos] = 0;
+ }
+
+ /// <summary>
+ /// Pop a new collection on to the stack.
+ /// </summary>
+ protected void Pop()
+ {
+ if (counts[Pos] != 0)
+ {
+ throw new AvroTypeException("Incorrect number of items written. " + counts[Pos] + " more required.");
+ }
+
+ Pos--;
+ }
+
+ /// <summary>
+ /// Returns the position into the stack.
+ /// </summary>
+ protected int Depth()
+ {
+ return Pos;
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/main/Schema/LogicalSchema.cs b/lang/csharp/src/apache/main/Schema/LogicalSchema.cs
index 0f23bdf4d..181260f2c 100644
--- a/lang/csharp/src/apache/main/Schema/LogicalSchema.cs
+++ b/lang/csharp/src/apache/main/Schema/LogicalSchema.cs
@@ -75,6 +75,18 @@ namespace Avro
writer.WriteEndObject();
}
+ /// <inheritdoc />
+ public override string Name
+ {
+ get { return BaseSchema.Name; }
+ }
+
+ /// <inheritdoc />
+ public override string Fullname
+ {
+ get { return BaseSchema.Fullname; }
+ }
+
/// <summary>
/// Checks if this schema can read data written by the given schema. Used for decoding data.
/// </summary>
diff --git a/lang/csharp/src/apache/test/IO/JsonCodecTests.cs b/lang/csharp/src/apache/test/IO/JsonCodecTests.cs
new file mode 100644
index 000000000..7c3ec3c2d
--- /dev/null
+++ b/lang/csharp/src/apache/test/IO/JsonCodecTests.cs
@@ -0,0 +1,329 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using NUnit.Framework;
+using System.IO;
+using System.Linq;
+using System.Text;
+using Avro.Generic;
+using Avro.IO;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace Avro.Test
+{
+ using Decoder = Avro.IO.Decoder;
+ using Encoder = Avro.IO.Encoder;
+
+ /// <summary>
+ /// Tests the JsonEncoder and JsonDecoder.
+ /// </summary>
+ [TestFixture]
+ public class JsonCodecTests
+ {
+ [TestCase("{ \"type\": \"record\", \"name\": \"r\", \"fields\": [ " +
+ " { \"name\" : \"f1\", \"type\": \"int\" }, " +
+ " { \"name\" : \"f2\", \"type\": \"float\" } " +
+ "] }",
+ "{ \"f2\": 10.4, \"f1\": 10 } ")]
+ [TestCase("{ \"type\": \"enum\", \"name\": \"e\", \"symbols\": [ \"s1\", \"s2\"] }", " \"s1\" ")]
+ [TestCase("{ \"type\": \"enum\", \"name\": \"e\", \"symbols\": [ \"s1\", \"s2\"] }", " \"s2\" ")]
+ [TestCase("{ \"type\": \"fixed\", \"name\": \"f\", \"size\": 5 }", "\"hello\"")]
+ [TestCase("{ \"type\": \"array\", \"items\": \"int\" }", "[ 10, 20, 30 ]")]
+ [TestCase("{ \"type\": \"map\", \"values\": \"int\" }", "{ \"k1\": 10, \"k2\": 20, \"k3\": 30 }")]
+ [TestCase("[ \"int\", \"long\" ]", "{ \"int\": 10 }")]
+ [TestCase("\"string\"", "\"hello\"")]
+ [TestCase("\"bytes\"", "\"hello\"")]
+ [TestCase("\"int\"", "10")]
+ [TestCase("\"long\"", "10")]
+ [TestCase("\"float\"", "10.0")]
+ [TestCase("\"double\"", "10.0")]
+ [TestCase("\"boolean\"", "true")]
+ [TestCase("\"boolean\"", "false")]
+ [TestCase("\"null\"", "null")]
+ public void TestJsonAllTypesValidValues(String schemaStr, String value)
+ {
+ Schema schema = Schema.Parse(schemaStr);
+ byte[] avroBytes = fromJsonToAvro(value, schema);
+
+ Assert.IsTrue(JToken.DeepEquals(JToken.Parse(value),
+ JToken.Parse(fromAvroToJson(avroBytes, schema, true))));
+ }
+
+ [TestCase("{ \"type\": \"record\", \"name\": \"r\", \"fields\": [ " +
+ " { \"name\" : \"f1\", \"type\": \"int\" }, " +
+ " { \"name\" : \"f2\", \"type\": \"float\" } " +
+ "] }",
+ "{ \"f4\": 10.4, \"f3\": 10 } ")]
+ [TestCase("{ \"type\": \"enum\", \"name\": \"e\", \"symbols\": [ \"s1\", \"s2\"] }", " \"s3\" ")]
+ [TestCase("{ \"type\": \"fixed\", \"name\": \"f\", \"size\": 10 }", "\"hello\"")]
+ [TestCase("{ \"type\": \"array\", \"items\": \"int\" }", "[ \"10\", \"20\", \"30\" ]")]
+ [TestCase("{ \"type\": \"map\", \"values\": \"int\" }", "{ \"k1\": \"10\", \"k2\": \"20\"}")]
+ [TestCase("[ \"int\", \"long\" ]", "10")]
+ [TestCase("\"string\"", "10")]
+ [TestCase("\"bytes\"", "10")]
+ [TestCase("\"int\"", "\"hi\"")]
+ [TestCase("\"long\"", "\"hi\"")]
+ [TestCase("\"float\"", "\"hi\"")]
+ [TestCase("\"double\"", "\"hi\"")]
+ [TestCase("\"boolean\"", "\"hi\"")]
+ [TestCase("\"boolean\"", "\"hi\"")]
+ [TestCase("\"null\"", "\"hi\"")]
+ public void TestJsonAllTypesInvalidValues(String schemaStr, String value)
+ {
+ Schema schema = Schema.Parse(schemaStr);
+ Assert.Throws<AvroTypeException>(() => fromJsonToAvro(value, schema));
+ }
+
+ [TestCase("{ \"type\": \"record\", \"name\": \"r\", \"fields\": [ " +
+ " { \"name\" : \"f1\", \"type\": \"int\" }, " +
+ " { \"name\" : \"f2\", \"type\": \"float\" } " +
+ "] }",
+ "{ \"f2\": 10.4, \"f1")]
+ [TestCase("{ \"type\": \"enum\", \"name\": \"e\", \"symbols\": [ \"s1\", \"s2\"] }", "s1")]
+ [TestCase("\"string\"", "\"hi")]
+ public void TestJsonMalformed(String schemaStr, String value)
+ {
+ Schema schema = Schema.Parse(schemaStr);
+ Assert.Throws<JsonReaderException>(() => fromJsonToAvro(value, schema));
+ }
+
+ [Test]
+ public void TestJsonEncoderWhenIncludeNamespaceOptionIsFalse()
+ {
+ string value = "{\"b\": {\"string\":\"myVal\"}, \"a\": 1}";
+ string schemaStr = "{\"type\": \"record\", \"name\": \"ab\", \"fields\": [" +
+ "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": [\"null\", \"string\"]}" +
+ "]}";
+ Schema schema = Schema.Parse(schemaStr);
+ byte[] avroBytes = fromJsonToAvro(value, schema);
+
+ Assert.IsTrue(JToken.DeepEquals(JObject.Parse("{\"b\":\"myVal\",\"a\":1}"),
+ JObject.Parse(fromAvroToJson(avroBytes, schema, false))));
+ }
+
+ [Test]
+ public void TestJsonEncoderWhenIncludeNamespaceOptionIsTrue()
+ {
+ string value = "{\"b\": {\"string\":\"myVal\"}, \"a\": 1}";
+ string schemaStr = "{\"type\": \"record\", \"name\": \"ab\", \"fields\": [" +
+ "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": [\"null\", \"string\"]}" +
+ "]}";
+ Schema schema = Schema.Parse(schemaStr);
+ byte[] avroBytes = fromJsonToAvro(value, schema);
+
+ Assert.IsTrue(JToken.DeepEquals(JObject.Parse("{\"b\":{\"string\":\"myVal\"},\"a\":1}"),
+ JObject.Parse(fromAvroToJson(avroBytes, schema, true))));
+ }
+
+ [Test]
+ public void TestJsonRecordOrdering()
+ {
+ string value = "{\"b\": 2, \"a\": 1}";
+ Schema schema = Schema.Parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [" +
+ "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": \"int\"}" +
+ "]}");
+ GenericDatumReader<object> reader = new GenericDatumReader<object>(schema, schema);
+ Decoder decoder = new JsonDecoder(schema, value);
+ object o = reader.Read(null, decoder);
+
+ Assert.AreEqual("{\"a\":1,\"b\":2}", fromDatumToJson(o, schema, false));
+ }
+
+ [Test]
+ public void TestJsonRecordOrdering2()
+ {
+ string value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a2\":true, \"a1\": null}}";
+ Schema schema = Schema.Parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" +
+ "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" +
+ "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" +
+ "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" +
+ "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" +
+ "]}");
+ GenericDatumReader<object> reader = new GenericDatumReader<object>(schema, schema);
+ Decoder decoder = new JsonDecoder(schema, value);
+ object o = reader.Read(null, decoder);
+
+ Assert.AreEqual("{\"a\":{\"a1\":null,\"a2\":true},\"b\":{\"b1\":\"h\",\"b2\":3.14,\"b3\":1.4}}",
+ fromDatumToJson(o, schema, false));
+ }
+
+ [Test]
+ public void TestJsonRecordOrderingWithProjection()
+ {
+ String value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a2\":true, \"a1\": null}}";
+ Schema writerSchema = Schema.Parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n"
+ + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
+ + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n"
+ + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n"
+ + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n"
+ + "]}");
+ Schema readerSchema = Schema.Parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n"
+ + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
+ + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n" +
+ "]}");
+ GenericDatumReader<object> reader = new GenericDatumReader<object>(writerSchema, readerSchema);
+ Decoder decoder = new JsonDecoder(writerSchema, value);
+ Object o = reader.Read(null, decoder);
+
+ Assert.AreEqual("{\"a\":{\"a1\":null,\"a2\":true}}",
+ fromDatumToJson(o, readerSchema, false));
+ }
+
+
+ [Test]
+ public void TestJsonRecordOrderingWithProjection2()
+ {
+ String value =
+ "{\"b\": { \"b1\": \"h\", \"b2\": [3.14, 3.56], \"b3\": 1.4}, \"a\": {\"a2\":true, \"a1\": null}}";
+ Schema writerSchema = Schema.Parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n"
+ + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
+ + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n"
+ + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n"
+ + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":{\"type\":\"array\", \"items\":\"float\"}}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n"
+ + "]}");
+
+ Schema readerSchema = Schema.Parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n"
+ + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
+ + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n" +
+ "]}");
+
+ GenericDatumReader<object> reader = new GenericDatumReader<object>(writerSchema, readerSchema);
+ Decoder decoder = new JsonDecoder(writerSchema, value);
+ object o = reader.Read(null, decoder);
+
+ Assert.AreEqual("{\"a\":{\"a1\":null,\"a2\":true}}",
+ fromDatumToJson(o, readerSchema, false));
+ }
+
+ [TestCase("{\"int\":123}")]
+ [TestCase("{\"string\":\"12345678-1234-5678-1234-123456789012\"}")]
+ [TestCase("null")]
+ public void TestJsonUnionWithLogicalTypes(String value)
+ {
+ Schema schema = Schema.Parse(
+ "[\"null\",\n" +
+ " { \"type\": \"int\", \"logicalType\": \"date\" },\n" +
+ " { \"type\": \"string\", \"logicalType\": \"uuid\" }\n" +
+ "]");
+ GenericDatumReader<object> reader = new GenericDatumReader<object>(schema, schema);
+ Decoder decoder = new JsonDecoder(schema, value);
+ object o = reader.Read(null, decoder);
+
+ Assert.AreEqual(value, fromDatumToJson(o, schema, true));
+ }
+
+ [TestCase("{\"int\":123}")]
+ [TestCase("{\"com.myrecord\":{\"f1\":123}}")]
+ [TestCase("null")]
+ public void TestJsonUnionWithRecord(String value)
+ {
+ Schema schema = Schema.Parse(
+
+ "[\"null\",\n" +
+ " { \"type\": \"int\", \"logicalType\": \"date\" },\n" +
+ " {\"type\":\"record\",\"name\":\"myrecord\", \"namespace\":\"com\"," +
+ " \"fields\":[{\"name\":\"f1\",\"type\": \"int\"}]}" +
+ "]");
+ GenericDatumReader<object> reader = new GenericDatumReader<object>(schema, schema);
+ Decoder decoder = new JsonDecoder(schema, value);
+ object o = reader.Read(null, decoder);
+
+ Assert.AreEqual(value, fromDatumToJson(o, schema, true));
+ }
+
+ [TestCase("int", 1)]
+ [TestCase("long", 1L)]
+ [TestCase("float", 1.0F)]
+ [TestCase("double", 1.0)]
+ public void TestJsonDecoderNumeric(string type, object value)
+ {
+ string def = "{\"type\":\"record\",\"name\":\"X\",\"fields\":" + "[{\"type\":\"" + type +
+ "\",\"name\":\"n\"}]}";
+ Schema schema = Schema.Parse(def);
+ DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schema, schema);
+
+ string[] records = { "{\"n\":1}", "{\"n\":1.0}" };
+
+ foreach (GenericRecord g in records.Select(r => reader.Read(null, new JsonDecoder(schema, r))))
+ {
+ Assert.AreEqual(value, g["n"]);
+ }
+ }
+
+ // Ensure that even if the order of fields in JSON is different from the order in schema, it works.
+ [Test]
+ public void TestJsonDecoderReorderFields()
+ {
+ String w = "{\"type\":\"record\",\"name\":\"R\",\"fields\":" + "[{\"type\":\"long\",\"name\":\"l\"},"
+ + "{\"type\":{\"type\":\"array\",\"items\":\"int\"},\"name\":\"a\"}" +
+ "]}";
+ Schema ws = Schema.Parse(w);
+ String data = "{\"a\":[1,2],\"l\":100}";
+ JsonDecoder decoder = new JsonDecoder(ws, data);
+ Assert.AreEqual(100, decoder.ReadLong());
+ decoder.SkipArray();
+ data = "{\"l\": 200, \"a\":[1,2]}";
+ decoder = new JsonDecoder(ws, data);
+ Assert.AreEqual(200, decoder.ReadLong());
+ decoder.SkipArray();
+ }
+
+ private byte[] fromJsonToAvro(string json, Schema schema)
+ {
+ DatumReader<object> reader = new GenericDatumReader<object>(schema, schema);
+ GenericDatumWriter<object> writer = new GenericDatumWriter<object>(schema);
+ MemoryStream output = new MemoryStream();
+
+ Decoder decoder = new JsonDecoder(schema, json);
+ Encoder encoder = new BinaryEncoder(output);
+
+ object datum = reader.Read(null, decoder);
+
+ writer.Write(datum, encoder);
+ encoder.Flush();
+ output.Flush();
+
+ return output.ToArray();
+ }
+
+ private string fromAvroToJson(byte[] avroBytes, Schema schema, bool includeNamespace)
+ {
+ GenericDatumReader<object> reader = new GenericDatumReader<object>(schema, schema);
+
+ Decoder decoder = new BinaryDecoder(new MemoryStream(avroBytes));
+ object datum = reader.Read(null, decoder);
+ return fromDatumToJson(datum, schema, includeNamespace);
+ }
+
+ private string fromDatumToJson(object datum, Schema schema, bool includeNamespace)
+ {
+ DatumWriter<object> writer = new GenericDatumWriter<object>(schema);
+ MemoryStream output = new MemoryStream();
+
+ JsonEncoder encoder = new JsonEncoder(schema, output);
+ encoder.IncludeNamespace = includeNamespace;
+ writer.Write(datum, encoder);
+ encoder.Flush();
+ output.Flush();
+
+ return Encoding.UTF8.GetString(output.ToArray());
+ }
+ }
+}
diff --git a/lang/csharp/src/apache/test/Schema/SchemaNormalizationTests.cs b/lang/csharp/src/apache/test/Schema/SchemaNormalizationTests.cs
index 1e670677a..c62963951 100644
--- a/lang/csharp/src/apache/test/Schema/SchemaNormalizationTests.cs
+++ b/lang/csharp/src/apache/test/Schema/SchemaNormalizationTests.cs
@@ -32,6 +32,14 @@ namespace Avro.Test
private static readonly long One = -9223372036854775808;
private static readonly byte[] Postfix = { 0, 0, 0, 0, 0, 0, 0, 0 };
+ [Test]
+ public void TestLogicalType()
+ {
+ var schema = @"[""int"", {""type"": ""string"", ""logicalType"": ""uuid""}]";
+ string pcf = SchemaNormalization.ToParsingForm(Schema.Parse(schema));
+ Assert.AreEqual(@"[""int"",""string""]", pcf);
+ }
+
[Test, TestCaseSource("ProvideCanonicalTestCases")]
public void CanonicalTest(string input, string expectedOutput)
{