You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2009/08/18 21:43:47 UTC
svn commit: r805549 - in /hadoop/avro/trunk: ./ src/java/org/apache/avro/
src/java/org/apache/avro/io/ src/java/org/apache/avro/io/parsing/
src/test/java/org/apache/avro/
Author: cutting
Date: Tue Aug 18 19:43:46 2009
New Revision: 805549
URL: http://svn.apache.org/viewvc?rev=805549&view=rev
Log:
AVRO-90. Fix Java's JSON codec to correctly encode unions.
Modified:
hadoop/avro/trunk/CHANGES.txt
hadoop/avro/trunk/build.xml
hadoop/avro/trunk/src/java/org/apache/avro/Schema.java
hadoop/avro/trunk/src/java/org/apache/avro/io/JsonDecoder.java
hadoop/avro/trunk/src/java/org/apache/avro/io/JsonEncoder.java
hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java
hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java
Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Tue Aug 18 19:43:46 2009
@@ -46,6 +46,8 @@
change test-java build target to fail on javadoc warnings.
(Thiruvalluvan M. G. and cutting)
+ AVRO-90. Fix Java's JSON codec to correctly encode unions. (cutting)
+
Avro 1.0.0 -- 9 July 2009
INCOMPATIBLE CHANGES
Modified: hadoop/avro/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/build.xml?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/build.xml (original)
+++ hadoop/avro/trunk/build.xml Tue Aug 18 19:43:46 2009
@@ -281,7 +281,7 @@
<fileset dir="@{files.location}" includes="**/${testcase}.java"/>
</batchtest>
</junit>
- <antcall target="checkfailure" />
+ <fail if="tests.failed">Tests Failed!</fail>
</sequential>
</macrodef>
@@ -354,11 +354,6 @@
<test-runner files.location="${test.java.classes}" tests.pattern="**/TestDataFile$InteropTest.class" />
</target>
- <target name="checkfailure" if="tests.failed">
- <touch file="${test.build.dir}/testsfailed"/>
- <fail unless="continueOnFailure">Tests failed!</fail>
- </target>
-
<target name="test-interop-data-py" depends="generate-test-data"
description="Run python data file interoperability tests">
<taskdef name="py-test" classname="org.pyant.tasks.PythonTestTask">
Modified: hadoop/avro/trunk/src/java/org/apache/avro/Schema.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/Schema.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/Schema.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/Schema.java Tue Aug 18 19:43:46 2009
@@ -66,9 +66,12 @@
}
/** The type of a schema. */
- public enum Type
- { RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES,
- INT, LONG, FLOAT, DOUBLE, BOOLEAN, NULL };
+ public enum Type {
+ RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES,
+ INT, LONG, FLOAT, DOUBLE, BOOLEAN, NULL;
+ private String name;
+ private Type() { this.name = this.name().toLowerCase(); }
+ };
private final Type type;
@@ -156,10 +159,9 @@
throw new AvroRuntimeException("Not an enum: "+this);
}
- /** If this is a record, enum or fixed, returns its name, if any. */
- public String getName() {
- throw new AvroRuntimeException("Not a named type: "+this);
- }
+ /** If this is a record, enum or fixed, returns its name, otherwise the name
+ * of the primitive type. */
+ public String getName() { return type.name; }
/** If this is a record, enum or fixed, returns its namespace, if any. */
public String getNamespace() {
@@ -204,7 +206,9 @@
}
}
- abstract void toJson(Names names, JsonGenerator gen) throws IOException;
+ void toJson(Names names, JsonGenerator gen) throws IOException {
+ gen.writeString(getName());
+ }
void fieldsToJson(Names names, JsonGenerator gen) throws IOException {
throw new AvroRuntimeException("Not a record: "+this);
@@ -518,58 +522,34 @@
private static class StringSchema extends Schema {
public StringSchema() { super(Type.STRING); }
- void toJson(Names names, JsonGenerator gen) throws IOException {
- gen.writeString("string");
- }
}
private static class BytesSchema extends Schema {
public BytesSchema() { super(Type.BYTES); }
- void toJson(Names names, JsonGenerator gen) throws IOException {
- gen.writeString("bytes");
- }
}
private static class IntSchema extends Schema {
public IntSchema() { super(Type.INT); }
- void toJson(Names names, JsonGenerator gen) throws IOException {
- gen.writeString("int");
- }
}
private static class LongSchema extends Schema {
public LongSchema() { super(Type.LONG); }
- void toJson(Names names, JsonGenerator gen) throws IOException {
- gen.writeString("long");
- }
}
private static class FloatSchema extends Schema {
public FloatSchema() { super(Type.FLOAT); }
- void toJson(Names names, JsonGenerator gen) throws IOException {
- gen.writeString("float");
- }
}
private static class DoubleSchema extends Schema {
public DoubleSchema() { super(Type.DOUBLE); }
- void toJson(Names names, JsonGenerator gen) throws IOException {
- gen.writeString("double");
- }
}
private static class BooleanSchema extends Schema {
public BooleanSchema() { super(Type.BOOLEAN); }
- void toJson(Names names, JsonGenerator gen) throws IOException {
- gen.writeString("boolean");
- }
}
private static class NullSchema extends Schema {
public NullSchema() { super(Type.NULL); }
- void toJson(Names names, JsonGenerator gen) throws IOException {
- gen.writeString("null");
- }
}
private static final StringSchema STRING_SCHEMA = new StringSchema();
Modified: hadoop/avro/trunk/src/java/org/apache/avro/io/JsonDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/JsonDecoder.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/JsonDecoder.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/JsonDecoder.java Tue Aug 18 19:43:46 2009
@@ -338,20 +338,23 @@
public int readIndex() throws IOException {
parser.advance(Symbol.UNION);
Symbol.Alternative a = (Symbol.Alternative) parser.popSymbol();
- if (in.getCurrentToken() == JsonToken.START_OBJECT &&
- in.nextToken() == JsonToken.FIELD_NAME) {
- String label = in.getText();
+
+ String label;
+ if (in.getCurrentToken() == JsonToken.VALUE_NULL) {
+ label = "null";
+ } else if (in.getCurrentToken() == JsonToken.START_OBJECT &&
+ in.nextToken() == JsonToken.FIELD_NAME) {
+ label = in.getText();
in.nextToken();
- int n = a.findLabel(label);
- if (n < 0) {
- throw new AvroTypeException("Unknown union branch " + label);
- }
parser.pushSymbol(Symbol.UNION_END);
- parser.pushSymbol(a.getSymbol(n));
- return n;
} else {
throw error("start-union");
}
+ int n = a.findLabel(label);
+ if (n < 0)
+ throw new AvroTypeException("Unknown union branch " + label);
+ parser.pushSymbol(a.getSymbol(n));
+ return n;
}
public Symbol doAction(Symbol input, Symbol top) throws IOException {
Modified: hadoop/avro/trunk/src/java/org/apache/avro/io/JsonEncoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/JsonEncoder.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/JsonEncoder.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/JsonEncoder.java Tue Aug 18 19:43:46 2009
@@ -214,10 +214,13 @@
public void writeIndex(int unionIndex) throws IOException {
parser.advance(Symbol.UNION);
Symbol.Alternative top = (Symbol.Alternative) parser.popSymbol();
- out.writeStartObject();
- out.writeFieldName(top.getLabel(unionIndex));
- parser.pushSymbol(Symbol.UNION_END);
- parser.pushSymbol(top.getSymbol(unionIndex));
+ Symbol symbol = top.getSymbol(unionIndex);
+ if (symbol != Symbol.NULL) {
+ out.writeStartObject();
+ out.writeFieldName(top.getLabel(unionIndex));
+ parser.pushSymbol(Symbol.UNION_END);
+ }
+ parser.pushSymbol(symbol);
}
@Override
Modified: hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java Tue Aug 18 19:43:46 2009
@@ -180,7 +180,7 @@
int i = 0;
for (Schema w : alts) {
symbols[i] = generate(w, reader, seen);
- labels[i] = w.getType().name();
+ labels[i] = w.getName();
i++;
}
return Symbol.seq(Symbol.alt(symbols, labels),
Modified: hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java (original)
+++ hadoop/avro/trunk/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java Tue Aug 18 19:43:46 2009
@@ -107,7 +107,7 @@
int i = 0;
for (Schema b : sc.getTypes()) {
symbols[i] = generate(b, seen);
- labels[i] = b.getType().name();
+ labels[i] = b.getName();
i++;
}
return Symbol.seq(Symbol.alt(symbols, labels), Symbol.UNION);
Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java?rev=805549&r1=805548&r2=805549&view=diff
==============================================================================
--- hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java (original)
+++ hadoop/avro/trunk/src/test/java/org/apache/avro/TestSchema.java Tue Aug 18 19:43:46 2009
@@ -157,6 +157,22 @@
public void testUnion() throws Exception {
check("[\"string\", \"long\"]", false);
checkDefault("[\"double\", \"long\"]", "1.1", new Double(1.1));
+
+ // check union json
+ String record = "{\"type\":\"record\",\"name\":\"Foo\",\"fields\":[]}";
+ String fixed = "{\"type\":\"fixed\",\"name\":\"Bar\",\"size\": 1}";
+ String enu = "{\"type\":\"enum\",\"name\":\"Baz\",\"symbols\": [\"X\"]}";
+ Schema union = Schema.parse("[\"null\",\"string\","
+ +record+","+ enu+","+fixed+"]");
+ checkJson(union, null, "null");
+ checkJson(union, new Utf8("foo"), "{\"string\":\"foo\"}");
+ checkJson(union,
+ new GenericData.Record(Schema.parse(record)),
+ "{\"Foo\":{}}");
+ checkJson(union,
+ new GenericData.Fixed(new byte[]{(byte)'a'}),
+ "{\"Bar\":\"a\"}");
+ checkJson(union, "X", "{\"Baz\":\"X\"}");
}
private static void check(String schemaJson, String defaultJson,
@@ -232,6 +248,27 @@
assertEquals("Decoded data does not match.", datum, decoded);
}
+ private static void checkJson(Schema schema, Object datum,
+ String json) throws Exception {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ Encoder encoder = new JsonEncoder(schema, out);
+ DatumWriter<Object> writer = new GenericDatumWriter<Object>();
+ writer.setSchema(schema);
+ writer.write(datum, encoder);
+ encoder.flush();
+ byte[] data = out.toByteArray();
+
+ String encoded = new String(data, "UTF-8");
+ assertEquals("Encoded data does not match.", json, encoded);
+
+ DatumReader<Object> reader = new GenericDatumReader<Object>();
+ reader.setSchema(schema);
+ Object decoded =
+ reader.read(null, new JsonDecoder(schema,new ByteArrayInputStream(data)));
+
+ assertEquals("Decoded data does not match.", datum, decoded);
+ }
+
private static final Schema ACTUAL = // an empty record schema
Schema.parse("{\"type\":\"record\", \"name\":\"Foo\", \"fields\":[]}");