You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2012/08/01 23:14:31 UTC
svn commit: r1368249 - in /avro/trunk: ./
lang/java/avro/src/main/java/org/apache/avro/reflect/
lang/java/avro/src/main/java/org/apache/avro/specific/
lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/
lang/java/mapred/src/test/java/org/apache/a...
Author: cutting
Date: Wed Aug 1 21:14:30 2012
New Revision: 1368249
URL: http://svn.apache.org/viewvc?rev=1368249&view=rev
Log:
AVRO-1103. Java: Fix SpecificData and mapreduce to use correct classloader.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Aug 1 21:14:30 2012
@@ -14,6 +14,9 @@ Avro 1.7.2 (unreleased)
AVRO-1131. Generated build makefiles for MSYS/MinGW use Visual
Studio compiler flags (Laurent Moss via thiru)
+ AVRO-1103. Java: Fix SpecificData and mapreduce to use correct
+ classloader. (cutting)
+
Avro 1.7.1 (16 July 2012)
NEW FEATURES
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java Wed Aug 1 21:14:30 2012
@@ -76,6 +76,11 @@ public class ReflectData extends Specifi
protected ReflectData() {}
+ /** Construct with a particular classloader. */
+ public ReflectData(ClassLoader classLoader) {
+ super(classLoader);
+ }
+
/** Return the singleton instance. */
public static ReflectData get() { return INSTANCE; }
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java Wed Aug 1 21:14:30 2012
@@ -52,7 +52,8 @@ public class ReflectDatumReader<T> exten
this(writer, reader, ReflectData.get());
}
- protected ReflectDatumReader(Schema writer, Schema reader, ReflectData data){
+ /** Construct given writer's and reader's schema and the data model. */
+ public ReflectDatumReader(Schema writer, Schema reader, ReflectData data) {
super(writer, reader, data);
}
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java Wed Aug 1 21:14:30 2012
@@ -55,6 +55,9 @@ public class SpecificData extends Generi
this.classLoader = classLoader;
}
+ /** Return the class loader that's used. */
+ public ClassLoader getClassLoader() { return classLoader; }
+
@Override
public DatumReader createDatumReader(Schema schema) {
return new SpecificDatumReader(schema, schema, this);
@@ -267,14 +270,14 @@ public class SpecificData extends Generi
@Override
public Object createFixed(Object old, Schema schema) {
- Class c = SpecificData.get().getClass(schema);
+ Class c = getClass(schema);
if (c == null) return super.createFixed(old, schema); // punt to generic
return c.isInstance(old) ? old : newInstance(c, schema);
}
@Override
public Object newRecord(Object old, Schema schema) {
- Class c = SpecificData.get().getClass(schema);
+ Class c = getClass(schema);
if (c == null) return super.newRecord(old, schema); // punt to generic
return (c.isInstance(old) ? old : newInstance(c, schema));
}
Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java (original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java Wed Aug 1 21:14:30 2012
@@ -26,6 +26,7 @@ import org.apache.avro.io.DatumReader;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.mapred.AvroWrapper;
+import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.hadoop.io.serializer.Deserializer;
@@ -52,7 +53,7 @@ public abstract class AvroDeserializer<T
private final Schema mReaderSchema;
/** The Avro datum reader for deserializing. */
- private final DatumReader<D> mAvroDatumReader;
+ final DatumReader<D> mAvroDatumReader;
/** An Avro binary decoder for deserializing. */
private BinaryDecoder mAvroDecoder;
@@ -63,10 +64,12 @@ public abstract class AvroDeserializer<T
* @param writerSchema The Avro writer schema for the data to deserialize.
* @param readerSchema The Avro reader schema for the data to deserialize (may be null).
*/
- protected AvroDeserializer(Schema writerSchema, Schema readerSchema) {
+ protected AvroDeserializer(Schema writerSchema, Schema readerSchema,
+ ClassLoader classLoader) {
mWriterSchema = writerSchema;
mReaderSchema = null != readerSchema ? readerSchema : writerSchema;
- mAvroDatumReader = new ReflectDatumReader<D>(mWriterSchema, mReaderSchema);
+ mAvroDatumReader = new ReflectDatumReader<D>(mWriterSchema, mReaderSchema,
+ new ReflectData(classLoader));
}
/**
Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java (original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java Wed Aug 1 21:14:30 2012
@@ -36,8 +36,9 @@ public class AvroKeyDeserializer<D> exte
* @param writerSchema The Avro writer schema for the data to deserialize.
* @param readerSchema The Avro reader schema for the data to deserialize.
*/
- public AvroKeyDeserializer(Schema writerSchema, Schema readerSchema) {
- super(writerSchema, readerSchema);
+ public AvroKeyDeserializer(Schema writerSchema, Schema readerSchema,
+ ClassLoader classLoader) {
+ super(writerSchema, readerSchema, classLoader);
}
/**
Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java (original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java Wed Aug 1 21:14:30 2012
@@ -65,9 +65,13 @@ public class AvroSerialization<T> extend
public Deserializer<AvroWrapper<T>> getDeserializer(Class<AvroWrapper<T>> c) {
Configuration conf = getConf();
if (AvroKey.class.isAssignableFrom(c)) {
- return new AvroKeyDeserializer<T>(getKeyWriterSchema(conf), getKeyReaderSchema(conf));
+ return new AvroKeyDeserializer<T>(getKeyWriterSchema(conf),
+ getKeyReaderSchema(conf),
+ conf.getClassLoader());
} else if (AvroValue.class.isAssignableFrom(c)) {
- return new AvroValueDeserializer<T>(getValueWriterSchema(conf), getValueReaderSchema(conf));
+ return new AvroValueDeserializer<T>(getValueWriterSchema(conf),
+ getValueReaderSchema(conf),
+ conf.getClassLoader());
} else {
throw new IllegalStateException("Only AvroKey and AvroValue are supported.");
}
Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java (original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java Wed Aug 1 21:14:30 2012
@@ -36,8 +36,9 @@ public class AvroValueDeserializer<D> ex
* @param writerSchema The Avro writer schema for the data to deserialize.
* @param readerSchema The Avro reader schema for the data to deserialize.
*/
- public AvroValueDeserializer(Schema writerSchema, Schema readerSchema) {
- super(writerSchema, readerSchema);
+ public AvroValueDeserializer(Schema writerSchema, Schema readerSchema,
+ ClassLoader classLoader) {
+ super(writerSchema, readerSchema, classLoader);
}
/**
Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java (original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java Wed Aug 1 21:14:30 2012
@@ -38,8 +38,10 @@ public class TestAvroKeyDeserializer {
// Create a deserializer.
Schema writerSchema = Schema.create(Schema.Type.STRING);
Schema readerSchema = Schema.create(Schema.Type.STRING);
+ ClassLoader classLoader = this.getClass().getClassLoader();
AvroKeyDeserializer<CharSequence> deserializer
- = new AvroKeyDeserializer<CharSequence>(writerSchema, readerSchema);
+ = new AvroKeyDeserializer<CharSequence>(writerSchema, readerSchema,
+ classLoader);
// Check the schemas.
assertEquals(writerSchema, deserializer.getWriterSchema());
Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java (original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java Wed Aug 1 21:14:30 2012
@@ -22,6 +22,8 @@ import static org.junit.Assert.*;
import java.io.IOException;
import org.apache.avro.Schema;
+import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.reflect.ReflectDatumReader;
import org.apache.avro.mapred.AvroKey;
import org.apache.avro.mapred.AvroValue;
import org.apache.avro.mapred.AvroWrapper;
@@ -29,8 +31,10 @@ import org.apache.avro.mapreduce.AvroJob
import org.apache.hadoop.io.serializer.Deserializer;
import org.apache.hadoop.io.serializer.Serializer;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Test;
+import org.junit.Assert;
public class TestAvroSerialization {
@Test
@@ -118,4 +122,17 @@ public class TestAvroSerialization {
// Check that the reader schema is set correctly on the deserializer.
assertEquals(readerSchema, avroDeserializer.getReaderSchema());
}
+
+ @Test public void testClassPath() throws Exception {
+ Configuration conf = new Configuration();
+ ClassLoader loader = new ClassLoader() {};
+ conf.setClassLoader(loader);
+ AvroSerialization serialization = new AvroSerialization();
+ serialization.setConf(conf);
+ AvroDeserializer des =
+ (AvroDeserializer)serialization.getDeserializer(AvroKey.class);
+ ReflectData data =
+ (ReflectData)((ReflectDatumReader)des.mAvroDatumReader).getData();
+ Assert.assertEquals(loader, data.getClassLoader());
+ }
}
Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java (original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java Wed Aug 1 21:14:30 2012
@@ -38,8 +38,10 @@ public class TestAvroValueDeserializer {
// Create a deserializer.
Schema writerSchema = Schema.create(Schema.Type.STRING);
Schema readerSchema = Schema.create(Schema.Type.STRING);
+ ClassLoader classLoader = this.getClass().getClassLoader();
AvroValueDeserializer<CharSequence> deserializer
- = new AvroValueDeserializer<CharSequence>(writerSchema, readerSchema);
+ = new AvroValueDeserializer<CharSequence>(writerSchema, readerSchema,
+ classLoader);
// Check the schemas.
assertEquals(writerSchema, deserializer.getWriterSchema());