You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@avro.apache.org by "Narayana Swamy Thota (Jira)" <ji...@apache.org> on 2021/11/02 18:16:00 UTC
[jira] [Comment Edited] (AVRO-1851) Serialization of anonymous enum
fails with nullable union
[ https://issues.apache.org/jira/browse/AVRO-1851?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17437520#comment-17437520 ]
Narayana Swamy Thota edited comment on AVRO-1851 at 11/2/21, 6:15 PM:
----------------------------------------------------------------------
[~mgrigorov] Here are the two tests that demonstrate the issues.
What we are looking for is ability to serialize objects with Reflect datum writes that allows null values for some properties in object.
{quote}import lombok.Data;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.reflect.ReflectDatumWriter;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
* Tests that demonstrates the issue with anonymous enums with AVRO serialization.
*/
public class AvroTests {
private static Pojo pojo;
@BeforeAll
static void init()
{ pojo = new Pojo(); Person person = new Person(); person.setAddress("Address"); pojo.setTestEnum(TestEnum.V); pojo.setPerson(person); }
//Test fails with "empty name"
@Test
void avroEnumWithNullTest() throws IOException
{ byte[] output = serialize(pojo); Pojo desePojo = deSerialize(output); }
//Test fails when null value is encountered in Pojo for any field
@Test
void avroEnumWithNotNullTest() throws IOException
{ byte[] output = serializeWithoutNulls(pojo); Pojo desePojo = deSerialize(output); }
private Pojo deSerialize(byte[] input) throws IOException
{ ByteArrayInputStream inputStream = new ByteArrayInputStream(input); Decoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null); ReflectDatumReader<Pojo> reflectDatumReader = new ReflectDatumReader<Pojo>(Pojo.class); return reflectDatumReader.read(null, decoder); }
private byte[] serializeWithoutNulls(Pojo input) throws IOException
{ //Reflect data that doesn't support nulls ReflectData reflectData = ReflectData.get(); System.out.println("Schema: " + reflectData.getSchema(input.getClass())); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null); ReflectDatumWriter<Pojo> datumWriter = new ReflectDatumWriter<Pojo>(Pojo.class, reflectData); datumWriter.write(input, encoder); encoder.flush(); return outputStream.toByteArray(); }
private byte[] serialize(Pojo input) throws IOException
{ //Reflect data that supports nulls ReflectData reflectData = ReflectData.AllowNull.get(); System.out.println("Schema: " + reflectData.getSchema(input.getClass())); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); Encoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null); ReflectDatumWriter<Pojo> datumWriter = new ReflectDatumWriter<Pojo>(Pojo.class, reflectData); datumWriter.write(input, encoder); encoder.flush(); return outputStream.toByteArray(); }
@Data
public static class Pojo
{ private TestEnum testEnum; private Person person; }
@Data
public static class Person
{ private String name; private String address; }
enum TestEnum {
V {
@Override
public boolean is_V()
{ return true; }
};
public boolean is_V()
{ return false; }
}
}
{quote}
was (Author: swamytns):
[~mgrigorov] Here are the two tests that demonstrate the issues.
What we are looking for is ability to serialize objects with Reflect datum writes that allows null values for some properties in object.
import lombok.Data;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.reflect.ReflectDatumWriter;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
* Tests that demonstrates the issue with anonymous enums with AVRO serialization.
*/
public class AvroTests {
private static Pojo pojo;
@BeforeAll
static void init() {
pojo = new Pojo();
Person person = new Person();
person.setAddress("Address");
pojo.setTestEnum(TestEnum.V);
pojo.setPerson(person);
}
//Test fails with "empty name"
@Test
void avroEnumWithNullTest() throws IOException {
byte[] output = serialize(pojo);
Pojo desePojo = deSerialize(output);
}
//Test fails when null value is encountered in Pojo for any field
@Test
void avroEnumWithNotNullTest() throws IOException {
byte[] output = serializeWithoutNulls(pojo);
Pojo desePojo = deSerialize(output);
}
private Pojo deSerialize(byte[] input) throws IOException {
ByteArrayInputStream inputStream = new ByteArrayInputStream(input);
Decoder decoder = DecoderFactory.get().binaryDecoder(inputStream, null);
ReflectDatumReader<Pojo> reflectDatumReader = new ReflectDatumReader<Pojo>(Pojo.class);
return reflectDatumReader.read(null, decoder);
}
private byte[] serializeWithoutNulls(Pojo input) throws IOException {
//Reflect data that doesn't support nulls
ReflectData reflectData = ReflectData.get();
System.out.println("Schema: " + reflectData.getSchema(input.getClass()));
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
Encoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null);
ReflectDatumWriter<Pojo> datumWriter = new ReflectDatumWriter<Pojo>(Pojo.class, reflectData);
datumWriter.write(input, encoder);
encoder.flush();
return outputStream.toByteArray();
}
private byte[] serialize(Pojo input) throws IOException {
//Reflect data that supports nulls
ReflectData reflectData = ReflectData.AllowNull.get();
System.out.println("Schema: " + reflectData.getSchema(input.getClass()));
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
Encoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null);
ReflectDatumWriter<Pojo> datumWriter = new ReflectDatumWriter<Pojo>(Pojo.class, reflectData);
datumWriter.write(input, encoder);
encoder.flush();
return outputStream.toByteArray();
}
@Data
public static class Pojo {
private TestEnum testEnum;
private Person person;
}
@Data
public static class Person {
private String name;
private String address;
}
enum TestEnum {
V {
@Override
public boolean is_V() {
return true;
}
};
public boolean is_V() {
return false;
}
}
}
> Serialization of anonymous enum fails with nullable union
> ---------------------------------------------------------
>
> Key: AVRO-1851
> URL: https://issues.apache.org/jira/browse/AVRO-1851
> Project: Apache Avro
> Issue Type: Bug
> Components: java
> Affects Versions: 1.7.7
> Reporter: Zack Kobza
> Priority: Minor
> Labels: pull-request-available
> Time Spent: 10m
> Remaining Estimate: 0h
>
> Error received when serializing an object containing a nullable anonymous enum:
> {noformat}
> org.apache.avro.SchemaParseException: Empty name
> at org.apache.avro.Schema.validateName(Schema.java:1108)
> at org.apache.avro.Schema.access$200(Schema.java:80)
> at org.apache.avro.Schema$Name.<init>(Schema.java:468)
> at org.apache.avro.Schema.createRecord(Schema.java:151)
> at org.apache.avro.reflect.ReflectData.createSchema(ReflectData.java:466)
> at org.apache.avro.specific.SpecificData.getSchema(SpecificData.java:189)
> at org.apache.avro.reflect.ReflectData.isRecord(ReflectData.java:168)
> at org.apache.avro.generic.GenericData.getSchemaName(GenericData.java:613)
> at org.apache.avro.specific.SpecificData.getSchemaName(SpecificData.java:265)
> at org.apache.avro.generic.GenericData.resolveUnion(GenericData.java:602)
> at org.apache.avro.generic.GenericDatumWriter.resolveUnion(GenericDatumWriter.java:151)
> at org.apache.avro.generic.GenericDatumWriter.write(GenericDatumWriter.java:71)
> at org.apache.avro.reflect.ReflectDatumWriter.write(ReflectDatumWriter.java:143)
> at org.apache.avro.generic.GenericDatumWriter.writeField(GenericDatumWriter.java:114)
> at org.apache.avro.reflect.ReflectDatumWriter.writeField(ReflectDatumWriter.java:175)
> at org.apache.avro.generic.GenericDatumWriter.writeRecord(GenericDatumWriter.java:104)
> at org.apache.avro.generic.GenericDatumWriter.write(GenericDatumWriter.java:66)
> at org.apache.avro.reflect.ReflectDatumWriter.write(ReflectDatumWriter.java:143)
> at org.apache.avro.generic.GenericDatumWriter.write(GenericDatumWriter.java:58)
> at org.apache.avro.reflect.TestReflectDatumReader.serializeWithReflectDatumWriter(TestReflectDatumReader.java:45)
> at org.apache.avro.reflect.TestReflectDatumReader.testWrite_AnonymousEnum(TestReflectDatumReader.java:53)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
> at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
> at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
> at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
> at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
> at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
> at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
> at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
> at org.junit.runner.JUnitCore.run(JUnitCore.java:157)
> at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:117)
> at com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:234)
> at com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:74)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144)
> {noformat}
> Here is a unit test derived from TestReflectDatumReader to reproduce:
> {code}
> @Test
> public void testWrite_AnonymousEnum() throws IOException {
> byte[] serializedBytes =
> serializeWithReflectDatumWriter(new MyObject(AnonymousEnum.A), MyObject.class, AllowNull.get());
> assertNotNull(serializedBytes);
> }
> enum AnonymousEnum {
> A {
> @Override
> public String doSomething() {
> return "A is doing something";
> }
> };
> public abstract String doSomething();
> }
> public static class MyObject {
> private final AnonymousEnum ae;
> public MyObject(AnonymousEnum ae) {
> this.ae = ae;
> }
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)