You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2013/03/27 22:33:19 UTC
svn commit: r1461856 - in /avro/trunk: ./ doc/src/content/xdocs/
lang/java/avro/src/main/java/org/apache/avro/generic/
lang/java/avro/src/main/java/org/apache/avro/reflect/
lang/java/avro/src/main/java/org/apache/avro/specific/
lang/java/avro/src/test/...
Author: cutting
Date: Wed Mar 27 21:33:18 2013
New Revision: 1461856
URL: http://svn.apache.org/r1461856
Log:
AVRO-1268. Java: Extend support for stringables from reflect to specific. Contributed by Alexandre Normand and cutting.
Added:
avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/io/
avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/io/Perf.java
- copied, changed from r1459563, avro/trunk/lang/java/avro/src/test/java/org/apache/avro/io/Perf.java
avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/specific/TestSpecificDatumReader.java
- copied, changed from r1459563, avro/trunk/lang/java/avro/src/test/java/org/apache/avro/specific/TestSpecificDatumReader.java
avro/trunk/share/test/schemas/FooBarSpecificRecord.avsc
- copied, changed from r1459563, avro/trunk/lang/java/avro/src/test/resources/FooBarSpecificRecord.avsc
avro/trunk/share/test/schemas/stringables.avdl
Removed:
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/io/Perf.java
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/specific/TestSpecificDatumReader.java
avro/trunk/lang/java/avro/src/test/resources/FooBarSpecificRecord.avsc
Modified:
avro/trunk/CHANGES.txt
avro/trunk/doc/src/content/xdocs/idl.xml
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumWriter.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/package.html
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumWriter.java
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflectDatumReader.java
avro/trunk/lang/java/avro/src/test/java/org/apache/avro/specific/TestSpecificData.java
avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
avro/trunk/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj
avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/specific/TestSpecificData.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Mar 27 21:33:18 2013
@@ -13,6 +13,12 @@ Trunk (not yet released)
AVRO-1272. Ruby: Improve schema namespace handling.
(Martin Kleppmann via cutting)
+ AVRO-1268. Java: Extend support for stringables from reflect to
+ specific. String schemas in generated classes now support the
+ "java-class" and "java-key-class" properties. The built-in Java
+ types BigDecimal, BigInteger, URI, URL, Date and File can now be
+ fields in generated classes. (Alexandre Normand and cutting)
+
BUG FIXES
Avro 1.7.4 (22 February 2012)
Modified: avro/trunk/doc/src/content/xdocs/idl.xml
URL: http://svn.apache.org/viewvc/avro/trunk/doc/src/content/xdocs/idl.xml?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/doc/src/content/xdocs/idl.xml (original)
+++ avro/trunk/doc/src/content/xdocs/idl.xml Wed Mar 27 21:33:18 2013
@@ -352,6 +352,17 @@ record MyRecord {
@java-class("java.util.ArrayList") array<string> myStrings;
}
</source>
+
+ <p>This can be used to support java classes that can be
+ serialized/deserialized via their toString/String constructor, e.g.:</p>
+ <source>
+record MyRecord {
+ @java-class("java.math.BigDecimal") string value;
+ @java-key-class("java.io.File") map<string> fileStates;
+ array<@java-class("java.math.BigDecimal") string> weights;
+}
+ </source>
+
<p>Similarly, a <code>@namespace</code> annotation may be used to modify the namespace
when defining a named schema. For example:
</p>
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java Wed Mar 27 21:33:18 2013
@@ -19,9 +19,12 @@ package org.apache.avro.generic;
import java.io.IOException;
import java.util.HashMap;
+import java.util.IdentityHashMap;
import java.util.Map;
import java.util.Collection;
import java.nio.ByteBuffer;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema;
@@ -335,10 +338,12 @@ public class GenericDatumReader<D> imple
* #readString(Object,Decoder)}.*/
protected Object readString(Object old, Schema expected,
Decoder in) throws IOException {
- if (data.STRING_TYPE_STRING.equals(expected.getProp(data.STRING_PROP)))
+ Class stringClass = getStringClass(expected);
+ if (stringClass == String.class)
return in.readString();
- else
+ if (stringClass == CharSequence.class)
return readString(old, in);
+ return newInstanceFromString(stringClass, in.readString());
}
/** Called to read strings. Subclasses may override to use a different
@@ -353,6 +358,58 @@ public class GenericDatumReader<D> imple
* Utf8#Utf8(String)}.*/
protected Object createString(String value) { return new Utf8(value); }
+ /** Determines the class to used to represent a string Schema. By default
+ * uses {@link #STRING_PROP} to determine whether {@link Utf8} or {@link
+ * String} is used. Subclasses may override for alternate representations.
+ */
+ protected Class findStringClass(Schema schema) {
+ String name = schema.getProp(GenericData.STRING_PROP);
+ if (name == null) return CharSequence.class;
+
+ switch (GenericData.StringType.valueOf(name)) {
+ case String:
+ return String.class;
+ default:
+ return CharSequence.class;
+ }
+ }
+
+ private Map<Schema,Class> stringClassCache =
+ new IdentityHashMap<Schema,Class>();
+
+ private Class getStringClass(Schema s) {
+ Class c = stringClassCache.get(s);
+ if (c == null) {
+ c = findStringClass(s);
+ stringClassCache.put(s, c);
+ }
+ return c;
+ }
+
+ private final Map<Class,Constructor> stringCtorCache =
+ new HashMap<Class,Constructor>();
+
+ @SuppressWarnings("unchecked")
+ private Object newInstanceFromString(Class c, String s) {
+ try {
+ Constructor ctor = stringCtorCache.get(c);
+ if (ctor == null) {
+ ctor = c.getDeclaredConstructor(String.class);
+ ctor.setAccessible(true);
+ stringCtorCache.put(c, ctor);
+ }
+ return ctor.newInstance(s);
+ } catch (NoSuchMethodException e) {
+ throw new AvroRuntimeException(e);
+ } catch (InstantiationException e) {
+ throw new AvroRuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new AvroRuntimeException(e);
+ } catch (InvocationTargetException e) {
+ throw new AvroRuntimeException(e);
+ }
+ }
+
/** Called to read byte arrays. Subclasses may override to use a different
* byte array representation. By default, this calls {@link
* Decoder#readBytes(ByteBuffer)}.*/
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java Wed Mar 27 21:33:18 2013
@@ -33,8 +33,6 @@ import java.util.Collections;
import java.util.concurrent.ConcurrentHashMap;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.Set;
-import java.util.HashSet;
import org.apache.avro.AvroRemoteException;
import org.apache.avro.AvroRuntimeException;
@@ -76,20 +74,6 @@ public class ReflectData extends Specifi
private static final ReflectData INSTANCE = new ReflectData();
- /** Read/write some common builtin classes as strings. Representing these as
- * strings isn't always best, as they aren't always ordered ideally, but at
- * least they're stored. Also note that, for compatibility, only classes
- * that wouldn't be otherwise correctly readable or writable should be added
- * here, e.g., those without a no-arg constructor or those whose fields are
- * all transient. */
- private Set<Class> stringableClasses = new HashSet<Class>(); {
- stringableClasses.add(java.math.BigDecimal.class);
- stringableClasses.add(java.math.BigInteger.class);
- stringableClasses.add(java.net.URI.class);
- stringableClasses.add(java.net.URL.class);
- stringableClasses.add(java.io.File.class);
- }
-
public ReflectData() {}
/** Construct with a particular classloader. */
@@ -215,8 +199,14 @@ public class ReflectData extends Specifi
throw new AvroRuntimeException("No field named "+name+" in: "+original);
}
+ /** @deprecated Replaced by {@link SpecificData#CLASS_PROP} */
+ @Deprecated
static final String CLASS_PROP = "java-class";
+ /** @deprecated Replaced by {@link SpecificData#KEY_CLASS_PROP} */
+ @Deprecated
static final String KEY_CLASS_PROP = "java-key-class";
+ /** @deprecated Replaced by {@link SpecificData#ELEMENT_PROP} */
+ @Deprecated
static final String ELEMENT_PROP = "java-element-class";
static Class getClassProp(Schema schema, String prop) {
@@ -374,9 +364,8 @@ public class ReflectData extends Specifi
return super.createSchema(type, names);
}
- private boolean isStringable(Class<?> c) {
- return c.isAnnotationPresent(Stringable.class) ||
- stringableClasses.contains(c);
+ @Override protected boolean isStringable(Class<?> c) {
+ return c.isAnnotationPresent(Stringable.class) || super.isStringable(c);
}
private static final Schema THROWABLE_MESSAGE =
@@ -512,16 +501,6 @@ public class ReflectData extends Specifi
}
@Override
- protected String getSchemaName(Object datum) {
- if (datum != null) {
- Class c = datum.getClass();
- if (isStringable(c))
- return Schema.Type.STRING.getName();
- }
- return super.getSchemaName(datum);
- }
-
- @Override
protected int compare(Object o1, Object o2, Schema s, boolean equals) {
switch (s.getType()) {
case ARRAY:
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java Wed Mar 27 21:33:18 2013
@@ -21,10 +21,8 @@ import java.io.IOException;
import java.util.Collection;
import java.util.ArrayList;
import java.lang.reflect.Array;
-import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
-import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.io.Decoder;
@@ -107,38 +105,6 @@ public class ReflectDatumReader<T> exten
}
@Override
- protected Object readMapKey(Object old, Schema s, Decoder in)
- throws IOException {
- Class c = ReflectData.getClassProp(s, ReflectData.KEY_CLASS_PROP);
- return readString(in, c);
- }
-
- @Override
- @SuppressWarnings(value="unchecked")
- protected Object readString(Object old, Schema s,
- Decoder in) throws IOException {
- Class c = ReflectData.getClassProp(s, ReflectData.CLASS_PROP);
- return readString(in, c);
- }
-
- private Object readString(Decoder in, Class c) throws IOException {
- String value = (String)readString(null, in);
- if (c != null) // Stringable annotated class
- try { // use String-arg ctor
- return c.getConstructor(String.class).newInstance(value);
- } catch (NoSuchMethodException e) {
- throw new AvroRuntimeException(e);
- } catch (InstantiationException e) {
- throw new AvroRuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new AvroRuntimeException(e);
- } catch (InvocationTargetException e) {
- throw new AvroRuntimeException(e);
- }
- return value;
- }
-
- @Override
protected Object readString(Object old, Decoder in) throws IOException {
return super.readString(null, in).toString();
}
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumWriter.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumWriter.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumWriter.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumWriter.java Wed Mar 27 21:33:18 2013
@@ -78,14 +78,6 @@ public class ReflectDatumWriter<T> exten
}
@Override
- protected void writeString(Schema schema, Object datum, Encoder out)
- throws IOException {
- if (schema.getProp(ReflectData.CLASS_PROP) != null) // Stringable annotated
- datum = datum.toString(); // call toString()
- writeString(datum, out);
- }
-
- @Override
protected void writeBytes(Object datum, Encoder out) throws IOException {
if (datum instanceof byte[])
out.writeBytes((byte[])datum);
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/package.html
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/package.html?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/package.html (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/package.html Wed Mar 27 21:33:18 2013
@@ -53,6 +53,17 @@ classes.
property set to "java.lang.Short", e.g.:
<pre>{"type": "int", "java-class": "java.lang.Short"}</pre>
+<li><b>{@link java.math.BigDecimal}, {@link java.math.BigInteger},
+ {@link java.net.URI}, {@link java.net.URL}, {@link java.io.File}</b>
+ are mapped to an Avro string schema as
+ {@link org.apache.avro.reflect.Stringable Stringable} types and
+ serialized via their {@link java.lang.Object#toString() toString}
+ method and de-serialized via their {@link java.lang.String} constructor.
+ This is done via the "java-class", "java-key-class" or
+ "java-element-class" depending on whether it is a field, or map key
+ or a list/map element, e.g.:
+ <pre>{"type": "string", "java-class": "java.math.BigDecimal"}</pre></li>
+
<li>All other types are mapped as in the {@link org.apache.avro.generic
generic} API.</li>
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java Wed Mar 27 21:33:18 2013
@@ -17,9 +17,11 @@
*/
package org.apache.avro.specific;
+import java.util.HashSet;
import java.util.Map;
import java.util.Collection;
import java.util.List;
+import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.LinkedHashMap;
@@ -47,6 +49,25 @@ public class SpecificData extends Generi
private static final Map<Class,Constructor> CTOR_CACHE =
new ConcurrentHashMap<Class,Constructor>();
+ public static final String CLASS_PROP = "java-class";
+ public static final String KEY_CLASS_PROP = "java-key-class";
+ public static final String ELEMENT_PROP = "java-element-class";
+
+ /** Read/write some common builtin classes as strings. Representing these as
+ * strings isn't always best, as they aren't always ordered ideally, but at
+ * least they're stored. Also note that, for compatibility, only classes
+ * that wouldn't be otherwise correctly readable or writable should be added
+ * here, e.g., those without a no-arg constructor or those whose fields are
+ * all transient. */
+ protected Set<Class> stringableClasses = new HashSet<Class>();
+ {
+ stringableClasses.add(java.math.BigDecimal.class);
+ stringableClasses.add(java.math.BigInteger.class);
+ stringableClasses.add(java.net.URI.class);
+ stringableClasses.add(java.net.URL.class);
+ stringableClasses.add(java.io.File.class);
+ }
+
/** For subclasses. Applications normally use {@link SpecificData#get()}. */
protected SpecificData() { this(SpecificData.class.getClassLoader()); }
@@ -220,6 +241,21 @@ public class SpecificData extends Generi
throw new AvroTypeException("Unknown type: "+type);
}
+ @Override
+ protected String getSchemaName(Object datum) {
+ if (datum != null) {
+ Class c = datum.getClass();
+ if (isStringable(c))
+ return Schema.Type.STRING.getName();
+ }
+ return super.getSchemaName(datum);
+ }
+
+ /** True iff a class should be serialized with toString(). */
+ protected boolean isStringable(Class<?> c) {
+ return stringableClasses.contains(c);
+ }
+
/** Return the protocol for a Java interface. */
public Protocol getProtocol(Class iface) {
try {
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumReader.java Wed Mar 27 21:33:18 2013
@@ -18,6 +18,7 @@
package org.apache.avro.specific;
import org.apache.avro.Schema;
+import org.apache.avro.AvroRuntimeException;
import org.apache.avro.generic.GenericDatumReader;
/** {@link org.apache.avro.io.DatumReader DatumReader} for generated Java classes. */
@@ -79,5 +80,30 @@ public class SpecificDatumReader<T> exte
return Enum.valueOf(c, symbol);
}
+ @Override protected Class findStringClass(Schema schema) {
+ Class stringClass = null;
+ switch (schema.getType()) {
+ case STRING:
+ stringClass = getPropAsClass(schema, SpecificData.CLASS_PROP);
+ break;
+ case MAP:
+ stringClass = getPropAsClass(schema, SpecificData.KEY_CLASS_PROP);
+ break;
+ }
+ if (stringClass != null)
+ return stringClass;
+ return super.findStringClass(schema);
+ }
+
+ private Class getPropAsClass(Schema schema, String prop) {
+ String name = schema.getProp(prop);
+ if (name == null) return null;
+ try {
+ return Class.forName(name);
+ } catch (ClassNotFoundException e) {
+ throw new AvroRuntimeException(e);
+ }
+ }
+
}
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumWriter.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumWriter.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumWriter.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificDatumWriter.java Wed Mar 27 21:33:18 2013
@@ -54,5 +54,13 @@ public class SpecificDatumWriter<T> exte
out.writeEnum(((Enum)datum).ordinal());
}
+ @Override
+ protected void writeString(Schema schema, Object datum, Encoder out)
+ throws IOException {
+ if (!(datum instanceof CharSequence)) // Stringable
+ datum = datum.toString(); // call toString()
+ writeString(datum, out);
+ }
+
}
Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflectDatumReader.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflectDatumReader.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflectDatumReader.java (original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflectDatumReader.java Wed Mar 27 21:33:18 2013
@@ -19,39 +19,20 @@
package org.apache.avro.reflect;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
-import org.apache.avro.FooBarSpecificRecord;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.specific.TestSpecificDatumReader;
import org.junit.Test;
public class TestReflectDatumReader {
- @Test
- public void testRead_SpecificDataRecord() throws IOException {
- FooBarSpecificRecord specificRecord = FooBarSpecificRecord.newBuilder().setId(42)
- .setRelatedids(Arrays.asList(1, 2, 3)).build();
- byte[] specificRecordBytes = TestSpecificDatumReader.serializeRecord(specificRecord);
-
- Decoder decoder = DecoderFactory.get().binaryDecoder(specificRecordBytes, null);
- ReflectDatumReader<FooBarSpecificRecord> reflectDatumReader = new ReflectDatumReader<FooBarSpecificRecord>(
- FooBarSpecificRecord.class);
-
- FooBarSpecificRecord deserialized = new FooBarSpecificRecord();
- reflectDatumReader.read(deserialized, decoder);
-
- assertEquals(specificRecord, deserialized);
- }
-
private static <T> byte[] serializeWithReflectDatumWriter(T toSerialize, Class<T> toSerializeClass)
throws IOException {
ReflectDatumWriter<T> datumWriter = new ReflectDatumWriter<T>(toSerializeClass);
Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/specific/TestSpecificData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/specific/TestSpecificData.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/specific/TestSpecificData.java (original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/specific/TestSpecificData.java Wed Mar 27 21:33:18 2013
@@ -18,20 +18,14 @@
package org.apache.avro.specific;
-import java.io.IOException;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.util.Arrays;
-import org.apache.avro.FooBarSpecificRecord;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Type;
-import org.apache.avro.TypeEnum;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonParser;
-import org.codehaus.jackson.map.ObjectMapper;
import org.junit.Before;
import org.junit.Test;
@@ -79,23 +73,6 @@ public class TestSpecificData {
Reflection.class.getMethod("primitive", integerClass);
}
- @Test
- public void testToString() throws IOException {
- FooBarSpecificRecord foo = FooBarSpecificRecord.newBuilder()
- .setId(123)
- .setRelatedids(Arrays.asList(1,2,3))
- .setTypeEnum(TypeEnum.c)
- .build();
-
- String json = foo.toString();
- JsonFactory factory = new JsonFactory();
- JsonParser parser = factory.createJsonParser(json);
- ObjectMapper mapper = new ObjectMapper();
-
- // will throw exception if string is not parsable json
- mapper.readTree(parser);
- }
-
static class Reflection {
public void primitive(int i) {}
public void primitiveWrapper(Integer i) {}
Modified: avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java (original)
+++ avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java Wed Mar 27 21:33:18 2013
@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.avro.specific.SpecificData;
import org.codehaus.jackson.JsonNode;
import org.apache.avro.Protocol;
@@ -474,7 +475,9 @@ public class SpecificCompiler {
return result;
}
- private String getStringType() {
+ private String getStringType(JsonNode overrideClassProperty) {
+ if (overrideClassProperty != null)
+ return overrideClassProperty.getTextValue();
switch (stringType) {
case String: return "java.lang.String";
case Utf8: return "org.apache.avro.util.Utf8";
@@ -495,14 +498,16 @@ public class SpecificCompiler {
case ARRAY:
return "java.util.List<" + javaType(schema.getElementType()) + ">";
case MAP:
- return "java.util.Map<"+getStringType()+","
- + javaType(schema.getValueType()) + ">";
+ return "java.util.Map<"
+ + getStringType(schema.getJsonProp(SpecificData.KEY_CLASS_PROP))+","
+ + javaType(schema.getValueType()) + ">";
case UNION:
List<Schema> types = schema.getTypes(); // elide unions with null
if ((types.size() == 2) && types.contains(NULL_SCHEMA))
return javaType(types.get(types.get(0).equals(NULL_SCHEMA) ? 1 : 0));
return "java.lang.Object";
- case STRING: return getStringType();
+ case STRING:
+ return getStringType(schema.getJsonProp(SpecificData.CLASS_PROP));
case BYTES: return "java.nio.ByteBuffer";
case INT: return "java.lang.Integer";
case LONG: return "java.lang.Long";
Modified: avro/trunk/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj (original)
+++ avro/trunk/lang/java/compiler/src/main/javacc/org/apache/avro/compiler/idl/idl.jj Wed Mar 27 21:33:18 2013
@@ -1404,8 +1404,11 @@ void FormalParameter(List<Field> fields)
Schema Type():
{
Schema s;
+ Map<String, JsonNode> props = new LinkedHashMap<String, JsonNode>();
}
{
+
+ ( SchemaProperty(props) )*
(
LOOKAHEAD(2) s = ReferenceType()
| s = PrimitiveType()
@@ -1414,6 +1417,8 @@ Schema Type():
| s = MapType()
)
{
+ for (String key : props.keySet())
+ s.addProp(key, props.get(key));
return s;
}
}
Copied: avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/io/Perf.java (from r1459563, avro/trunk/lang/java/avro/src/test/java/org/apache/avro/io/Perf.java)
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/io/Perf.java?p2=avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/io/Perf.java&p1=avro/trunk/lang/java/avro/src/test/java/org/apache/avro/io/Perf.java&r1=1459563&r2=1461856&rev=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/io/Perf.java (original)
+++ avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/io/Perf.java Wed Mar 27 21:33:18 2013
@@ -22,17 +22,23 @@ import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import org.apache.avro.FooBarSpecificRecord;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
+import org.apache.avro.TypeEnum;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.avro.specific.SpecificRecordBase;
import org.apache.avro.util.Utf8;
/**
@@ -70,6 +76,7 @@ public class Perf {
private static final List<TestDescriptor> RECORD = new ArrayList<TestDescriptor>();
private static final List<TestDescriptor> GENERIC = new ArrayList<TestDescriptor>();
private static final List<TestDescriptor> GENERIC_ONETIME = new ArrayList<TestDescriptor>();
+ private static final List<TestDescriptor> SPECIFIC = new ArrayList<TestDescriptor>();
private static final LinkedHashMap<String, TestDescriptor> ALL_TESTS;
private static final LinkedHashMap<String, List<TestDescriptor>> BATCHES;
static {
@@ -95,6 +102,7 @@ public class Perf {
new TestDescriptor(RecordWithPromotion.class, "-Rp").add(RECORD);
BATCHES.put("-generic", GENERIC);
new TestDescriptor(GenericTest.class, "-G").add(GENERIC);
+ new TestDescriptor(GenericStrings.class, "-Gs").add(GENERIC);
new TestDescriptor(GenericNested.class, "-Gn").add(GENERIC);
new TestDescriptor(GenericNestedFake.class, "-Gf").add(GENERIC);
new TestDescriptor(GenericWithDefault.class, "-Gd").add(GENERIC);
@@ -104,6 +112,7 @@ public class Perf {
new TestDescriptor(GenericOneTimeDecoderUse.class, "-Gotd").add(GENERIC_ONETIME);
new TestDescriptor(GenericOneTimeReaderUse.class, "-Gotr").add(GENERIC_ONETIME);
new TestDescriptor(GenericOneTimeUse.class, "-Got").add(GENERIC_ONETIME);
+ new TestDescriptor(FooBarSpecificRecordTest.class, "-Sf").add(SPECIFIC);
}
private static void usage() {
@@ -665,6 +674,14 @@ public class Perf {
}
}
+ private static String randomString(Random r) {
+ char[] data = new char[r.nextInt(70)];
+ for (int j = 0; j < data.length; j++) {
+ data[j] = (char)('a' + r.nextInt('z'-'a'));
+ }
+ return new String(data);
+ }
+
static class StringTest extends BasicTest {
String[] sourceData = null;
public StringTest() throws IOException {
@@ -675,13 +692,8 @@ public class Perf {
void genSourceData() {
Random r = newRandom();
sourceData = new String[count];
- for (int i = 0; i < sourceData.length;) {
- char[] data = new char[r.nextInt(70)];
- for (int j = 0; j < data.length; j++) {
- data[j] = (char)('a' + r.nextInt('z'-'a'));
- }
- sourceData[i++] = new String(data);
- }
+ for (int i = 0; i < sourceData.length;)
+ sourceData[i++] = randomString(r);
}
@Override
@@ -1125,6 +1137,31 @@ public class Perf {
}
}
+ private static final String GENERIC_STRINGS =
+ "{ \"type\": \"record\", \"name\": \"R\", \"fields\": [\n"
+ + "{ \"name\": \"f1\", \"type\": \"string\" },\n"
+ + "{ \"name\": \"f2\", \"type\": \"string\" },\n"
+ + "{ \"name\": \"f3\", \"type\": \"string\" }\n"
+ + "] }";
+
+ static class GenericStrings extends GenericTest {
+ public GenericStrings() throws IOException {
+ super("GenericStrings", GENERIC_STRINGS);
+ }
+ @Override
+ void genSourceData() {
+ Random r = newRandom();
+ sourceData = new GenericRecord[count];
+ for (int i = 0; i < sourceData.length; i++) {
+ GenericRecord rec = new GenericData.Record(schema);
+ rec.put(0, randomString(r));
+ rec.put(1, randomString(r));
+ rec.put(2, randomString(r));
+ sourceData[i] = rec;
+ }
+ }
+ }
+
static class GenericNested extends GenericTest {
public GenericNested() throws IOException {
super("GenericNested_", NESTED_RECORD_SCHEMA);
@@ -1292,5 +1329,86 @@ public class Perf {
return newDecoder();
}
}
-
+
+ static abstract class SpecificTest<T extends SpecificRecordBase> extends BasicTest {
+ protected final SpecificDatumReader<T> reader;
+ protected final SpecificDatumWriter<T> writer;
+ private Object[] sourceData;
+
+ protected SpecificTest(String name, String writerSchema) throws IOException {
+ super(name, writerSchema, 12);
+ reader = newReader();
+ writer = newWriter();
+ }
+ protected SpecificDatumReader<T> getReader() {
+ return reader;
+ }
+ protected SpecificDatumWriter<T> getWriter() {
+ return writer;
+ }
+ protected SpecificDatumReader<T> newReader() {
+ return new SpecificDatumReader<T>(schema);
+ }
+ protected SpecificDatumWriter<T> newWriter() {
+ return new SpecificDatumWriter<T>(schema);
+ }
+ @Override
+ void genSourceData() {
+ Random r = newRandom();
+ sourceData = new Object[count];
+ for (int i = 0; i < sourceData.length; i++) {
+ sourceData[i] = genSingleRecord(r);
+ }
+ }
+
+ protected abstract T genSingleRecord(Random r);
+
+ @Override
+ void readInternal(Decoder d) throws IOException {
+ for (int i = 0; i < count; i++) {
+ getReader().read(null, d);
+ }
+ }
+ @Override
+ void writeInternal(Encoder e) throws IOException {
+ for (int i = 0; i < sourceData.length; i++) {
+ @SuppressWarnings("unchecked")
+ T rec = (T) sourceData[i];
+ getWriter().write(rec, e);
+ }
+ }
+ @Override
+ void reset() {
+ sourceData = null;
+ data = null;
+ }
+ }
+
+ static class FooBarSpecificRecordTest extends SpecificTest<FooBarSpecificRecord> {
+ public FooBarSpecificRecordTest() throws IOException {
+ super("FooBarSpecificRecordTest", FooBarSpecificRecord.SCHEMA$.toString());
+ }
+
+ @Override
+ protected FooBarSpecificRecord genSingleRecord(Random r) {
+ TypeEnum[] typeEnums = TypeEnum.values();
+ List<Integer> relatedIds = new ArrayList<Integer>(10);
+ for (int i = 0; i < 10; i++) {
+ relatedIds.add(r.nextInt());
+ }
+
+ try {
+ return FooBarSpecificRecord.newBuilder().
+ setId(r.nextInt()).
+ setName(randomString(r)).
+ setNicknames(Arrays.asList(randomString(r), randomString(r))).
+ setTypeEnum(typeEnums[r.nextInt(typeEnums.length)]).
+ setRelatedids(relatedIds).
+ build();
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
Modified: avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/specific/TestSpecificData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/specific/TestSpecificData.java?rev=1461856&r1=1461855&r2=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/specific/TestSpecificData.java (original)
+++ avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/specific/TestSpecificData.java Wed Mar 27 21:33:18 2013
@@ -17,10 +17,17 @@
*/
package org.apache.avro.specific;
+import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.ArrayList;
+import org.apache.avro.FooBarSpecificRecord;
+import org.apache.avro.TypeEnum;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonParser;
+import org.codehaus.jackson.map.ObjectMapper;
import org.junit.Test;
import org.junit.Assert;
@@ -94,5 +101,23 @@ public class TestSpecificData {
Assert.assertEquals(Kind.getClassSchema(), Kind.SCHEMA$);
}
+ @Test
+ public void testSpecificRecordToString() throws IOException {
+ FooBarSpecificRecord foo = FooBarSpecificRecord.newBuilder()
+ .setId(123)
+ .setName("foo")
+ .setNicknames(Arrays.asList("bar"))
+ .setRelatedids(Arrays.asList(1, 2, 3))
+ .setTypeEnum(TypeEnum.c)
+ .build();
+
+ String json = foo.toString();
+ JsonFactory factory = new JsonFactory();
+ JsonParser parser = factory.createJsonParser(json);
+ ObjectMapper mapper = new ObjectMapper();
+
+ // will throw exception if string is not parsable json
+ mapper.readTree(parser);
+ }
}
Copied: avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/specific/TestSpecificDatumReader.java (from r1459563, avro/trunk/lang/java/avro/src/test/java/org/apache/avro/specific/TestSpecificDatumReader.java)
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/specific/TestSpecificDatumReader.java?p2=avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/specific/TestSpecificDatumReader.java&p1=avro/trunk/lang/java/avro/src/test/java/org/apache/avro/specific/TestSpecificDatumReader.java&r1=1459563&r2=1461856&rev=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/specific/TestSpecificDatumReader.java (original)
+++ avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/specific/TestSpecificDatumReader.java Wed Mar 27 21:33:18 2013
@@ -22,7 +22,10 @@ import static org.junit.Assert.*;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.util.Arrays;
+import java.util.HashMap;
import org.apache.avro.FooBarSpecificRecord;
import org.apache.avro.FooBarSpecificRecord.Builder;
@@ -32,6 +35,8 @@ import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
import org.junit.Test;
+import test.StringablesRecord;
+
public class TestSpecificDatumReader {
public static byte[] serializeRecord(FooBarSpecificRecord fooBarSpecificRecord) throws IOException {
@@ -44,10 +49,22 @@ public class TestSpecificDatumReader {
return byteArrayOutputStream.toByteArray();
}
+ public static byte[] serializeRecord(StringablesRecord stringablesRecord) throws IOException {
+ SpecificDatumWriter<StringablesRecord> datumWriter =
+ new SpecificDatumWriter<StringablesRecord>(StringablesRecord.SCHEMA$);
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null);
+ datumWriter.write(stringablesRecord, encoder);
+ encoder.flush();
+ return byteArrayOutputStream.toByteArray();
+ }
+
@Test
public void testRead() throws IOException {
Builder newBuilder = FooBarSpecificRecord.newBuilder();
newBuilder.setId(42);
+ newBuilder.setName("foo");
+ newBuilder.setNicknames(Arrays.asList("bar"));
newBuilder.setRelatedids(Arrays.asList(1,2,3));
FooBarSpecificRecord specificRecord = newBuilder.build();
@@ -59,7 +76,30 @@ public class TestSpecificDatumReader {
specificDatumReader.read(deserialized, decoder);
assertEquals(specificRecord, deserialized);
-
+ }
+
+ @Test
+ public void testStringables() throws IOException {
+ StringablesRecord.Builder newBuilder = StringablesRecord.newBuilder();
+ newBuilder.setValue(new BigDecimal("42.11"));
+ HashMap<String, BigDecimal> mapWithBigDecimalElements = new HashMap<String, BigDecimal>();
+ mapWithBigDecimalElements.put("test", new BigDecimal("11.11"));
+ newBuilder.setMapWithBigDecimalElements(mapWithBigDecimalElements);
+ HashMap<BigInteger, String> mapWithBigIntKeys = new HashMap<BigInteger, String>();
+ mapWithBigIntKeys.put(BigInteger.ONE, "test");
+ newBuilder.setMapWithBigIntKeys(mapWithBigIntKeys);
+ StringablesRecord stringablesRecord = newBuilder.build();
+
+ byte[] recordBytes = serializeRecord(stringablesRecord);
+
+ Decoder decoder = DecoderFactory.get().binaryDecoder(recordBytes, null);
+ SpecificDatumReader<StringablesRecord> specificDatumReader =
+ new SpecificDatumReader<StringablesRecord>(StringablesRecord.SCHEMA$);
+ StringablesRecord deserialized = new StringablesRecord();
+ specificDatumReader.read(deserialized, decoder);
+
+ assertEquals(stringablesRecord, deserialized);
+
}
}
Copied: avro/trunk/share/test/schemas/FooBarSpecificRecord.avsc (from r1459563, avro/trunk/lang/java/avro/src/test/resources/FooBarSpecificRecord.avsc)
URL: http://svn.apache.org/viewvc/avro/trunk/share/test/schemas/FooBarSpecificRecord.avsc?p2=avro/trunk/share/test/schemas/FooBarSpecificRecord.avsc&p1=avro/trunk/lang/java/avro/src/test/resources/FooBarSpecificRecord.avsc&r1=1459563&r2=1461856&rev=1461856&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/resources/FooBarSpecificRecord.avsc (original)
+++ avro/trunk/share/test/schemas/FooBarSpecificRecord.avsc Wed Mar 27 21:33:18 2013
@@ -4,6 +4,9 @@
"namespace": "org.apache.avro",
"fields": [
{"name": "id", "type": "int"},
+ {"name": "name", "type": "string"},
+ {"name": "nicknames", "type":
+ {"type": "array", "items": "string"}},
{"name": "relatedids", "type":
{"type": "array", "items": "int"}},
{"name": "typeEnum", "type":
Added: avro/trunk/share/test/schemas/stringables.avdl
URL: http://svn.apache.org/viewvc/avro/trunk/share/test/schemas/stringables.avdl?rev=1461856&view=auto
==============================================================================
--- avro/trunk/share/test/schemas/stringables.avdl (added)
+++ avro/trunk/share/test/schemas/stringables.avdl Wed Mar 27 21:33:18 2013
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A test case to exercise the stringable feature on @java-class, @java-key-class and
+ * @java-element-class
+ */
+@namespace("test")
+protocol AnnotatedStringableTypes {
+
+ record StringablesRecord {
+ /** Each field exercises one of the java-class, key-class or element-class. */
+ @java-class("java.math.BigDecimal") string value;
+ @java-key-class("java.math.BigInteger") map<string> mapWithBigIntKeys;
+ map<@java-class("java.math.BigDecimal") string> mapWithBigDecimalElements;
+ }
+}