You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by dk...@apache.org on 2018/12/11 18:10:34 UTC
[avro] branch master updated: AVRO-2184: Unable to decode JSON data
file if a property is renamed in reader schema (#316)
This is an automated email from the ASF dual-hosted git repository.
dkulp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new 595643c AVRO-2184: Unable to decode JSON data file if a property is renamed in reader schema (#316)
595643c is described below
commit 595643cba16a2b4d7be68d46ee8f79c4e380cbf7
Author: nandorKollar <na...@users.noreply.github.com>
AuthorDate: Tue Dec 11 19:10:29 2018 +0100
AVRO-2184: Unable to decode JSON data file if a property is renamed in reader schema (#316)
* AVRO-2184: Unable to decode JSON data file if a property is renamed in reader schema
JsonDecoder doesn't honor aliases
* No need to wrap aliases to unmodifiableSet, since getter Schema#aliases already does it
* Remove unused import to pass Checkstyle check
---
.../main/java/org/apache/avro/io/JsonDecoder.java | 2 +-
.../avro/io/parsing/JsonGrammarGenerator.java | 2 +-
.../java/org/apache/avro/io/parsing/Symbol.java | 8 +--
.../TestReadingWritingDataInEvolvedSchemas.java | 59 +++++++++++++++++++---
4 files changed, 58 insertions(+), 13 deletions(-)
diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java
index 7ffd5cf..ce9beb7 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/io/JsonDecoder.java
@@ -470,7 +470,7 @@ public class JsonDecoder extends ParsingDecoder
do {
String fn = in.getText();
in.nextToken();
- if (name.equals(fn)) {
+ if (name.equals(fn) || fa.aliases.contains(fn)) {
return null;
} else {
if (currentReorderBuffer == null) {
diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java
index 505c094..44fc19b 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java
@@ -84,7 +84,7 @@ public class JsonGrammarGenerator extends ValidatingGrammarGenerator {
int n = 0;
production[--i] = Symbol.RECORD_START;
for (Field f : sc.getFields()) {
- production[--i] = Symbol.fieldAdjustAction(n, f.name());
+ production[--i] = Symbol.fieldAdjustAction(n, f.name(), f.aliases());
production[--i] = generate(f.schema(), seen);
production[--i] = Symbol.FIELD_END;
n++;
diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java
index 4494ec0..df4eade 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java
@@ -572,16 +572,18 @@ public abstract class Symbol {
}
- public static FieldAdjustAction fieldAdjustAction(int rindex, String fname) {
- return new FieldAdjustAction(rindex, fname);
+ public static FieldAdjustAction fieldAdjustAction(int rindex, String fname, Set<String> aliases) {
+ return new FieldAdjustAction(rindex, fname, aliases);
}
public static class FieldAdjustAction extends ImplicitAction {
public final int rindex;
public final String fname;
- @Deprecated public FieldAdjustAction(int rindex, String fname) {
+ public final Set<String> aliases;
+ @Deprecated public FieldAdjustAction(int rindex, String fname, Set<String> aliases) {
this.rindex = rindex;
this.fname = fname;
+ this.aliases = aliases;
}
}
diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestReadingWritingDataInEvolvedSchemas.java b/lang/java/avro/src/test/java/org/apache/avro/TestReadingWritingDataInEvolvedSchemas.java
index d7c0fcb..47fec9a 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/TestReadingWritingDataInEvolvedSchemas.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/TestReadingWritingDataInEvolvedSchemas.java
@@ -19,9 +19,12 @@ package org.apache.avro;
import static org.junit.Assert.*;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericData.EnumSymbol;
@@ -33,7 +36,11 @@ import org.apache.avro.io.*;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+@RunWith(Parameterized.class)
public class TestReadingWritingDataInEvolvedSchemas {
private static final String RECORD_A = "RecordA";
@@ -110,6 +117,23 @@ public class TestReadingWritingDataInEvolvedSchemas {
.name(FIELD_A).type().unionOf().floatType().and().doubleType().endUnion().noDefault() //
.endRecord();
+ @Parameters(name = "encoder = {0}")
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new EncoderType[][]{
+ {EncoderType.BINARY}, {EncoderType.JSON}
+ });
+ }
+
+ public TestReadingWritingDataInEvolvedSchemas(EncoderType encoderType) {
+ this.encoderType = encoderType;
+ }
+
+ private final EncoderType encoderType;
+
+ enum EncoderType {
+ BINARY, JSON
+ }
+
@Test
public void doubleWrittenWithUnionSchemaIsConvertedToDoubleSchema() throws Exception {
Schema writer = UNION_INT_LONG_FLOAT_DOUBLE_RECORD;
@@ -351,33 +375,52 @@ public class TestReadingWritingDataInEvolvedSchemas {
assertEquals(314, decoded.get("newFieldWithDefault"));
}
+ @Test
+ public void aliasesInSchema() throws Exception {
+ Schema writer = new Schema.Parser().parse(
+ "{\"namespace\": \"example.avro\", \"type\": \"record\", \"name\": \"User\", \"fields\": [" +
+ "{\"name\": \"name\", \"type\": \"int\"}\n" +
+ "]}\n");
+ Schema reader = new Schema.Parser().parse(
+ "{\"namespace\": \"example.avro\", \"type\": \"record\", \"name\": \"User\", \"fields\": [" +
+ "{\"name\": \"fname\", \"type\": \"int\", \"aliases\" : [ \"name\" ]}\n" +
+ "]}\n");
+
+ GenericData.Record record = defaultRecordWithSchema(writer, "name", 1);
+ byte[] encoded = encodeGenericBlob(record);
+ GenericData.Record decoded = decodeGenericBlob(reader, reader, encoded);
+
+ assertEquals(1, decoded.get("fname"));
+ }
+
private <T> Record defaultRecordWithSchema(Schema schema, String key, T value) {
Record data = new GenericData.Record(schema);
data.put(key, value);
return data;
}
- private static byte[] encodeGenericBlob(GenericRecord data)
- throws IOException {
+ private byte[] encodeGenericBlob(GenericRecord data) throws IOException {
DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(data.getSchema());
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
- Encoder encoder = EncoderFactory.get().binaryEncoder(outStream, null);
+ Encoder encoder = encoderType == EncoderType.BINARY ?
+ EncoderFactory.get().binaryEncoder(outStream, null) :
+ EncoderFactory.get().jsonEncoder(data.getSchema(), outStream);
writer.write(data, encoder);
encoder.flush();
outStream.close();
return outStream.toByteArray();
}
- private static Record decodeGenericBlob(Schema expectedSchema, Schema schemaOfBlob, byte[] blob) throws IOException {
+ private Record decodeGenericBlob(Schema expectedSchema, Schema schemaOfBlob, byte[] blob) throws IOException {
if (blob == null) {
return null;
}
GenericDatumReader<Record> reader = new GenericDatumReader<>();
reader.setExpected(expectedSchema);
reader.setSchema(schemaOfBlob);
- Decoder decoder = DecoderFactory.get().binaryDecoder(blob, null);
- Record data = null;
- data = reader.read(null, decoder);
- return data;
+ Decoder decoder = encoderType == EncoderType.BINARY ?
+ DecoderFactory.get().binaryDecoder(blob, null) :
+ DecoderFactory.get().jsonDecoder(schemaOfBlob, new ByteArrayInputStream(blob));
+ return reader.read(null, decoder);
}
}