You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by gr...@apache.org on 2013/11/01 05:11:26 UTC
git commit: CRUNCH-289 Allow materializing Avro SpecificRecords
Updated Branches:
refs/heads/master a3b56322d -> cb92723af
CRUNCH-289 Allow materializing Avro SpecificRecords
Due to issues outlined in AVRO-1240, the correct classloader must
be supplied to the SpecificData class to allow reading
SpecificRecords. This patch passes a custom classloader to the
SpecificData constructor to allow locally materializing
SpecificRecords when running with "hadoop jar".
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/cb92723a
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/cb92723a
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/cb92723a
Branch: refs/heads/master
Commit: cb92723af09cee63735cbf6b5dac5e86dd480e9c
Parents: a3b5632
Author: Gabriel Reid <gr...@apache.org>
Authored: Wed Oct 30 21:42:14 2013 +0100
Committer: Gabriel Reid <gr...@apache.org>
Committed: Fri Nov 1 05:10:27 2013 +0100
----------------------------------------------------------------------
.../apache/crunch/io/avro/AvroFileReaderFactory.java | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/cb92723a/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java b/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java
index becde73..efde4ec 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java
@@ -26,6 +26,7 @@ import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.io.DatumReader;
import org.apache.avro.mapred.FsInput;
import org.apache.avro.reflect.ReflectDatumReader;
+import org.apache.avro.specific.SpecificData;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -61,7 +62,7 @@ public class AvroFileReaderFactory<T> implements FileReaderFactory<T> {
this.recordReader = new GenericDatumReader<T>(schema);
this.mapFn = IdentityFn.<T>getInstance();
}
-
+
static <T> DatumReader<T> createDatumReader(AvroType<T> avroType) {
if (avroType.hasReflect()) {
if (avroType.hasSpecific()) {
@@ -69,7 +70,16 @@ public class AvroFileReaderFactory<T> implements FileReaderFactory<T> {
}
return new ReflectDatumReader<T>(avroType.getSchema());
} else if (avroType.hasSpecific()) {
- return new SpecificDatumReader<T>(avroType.getSchema());
+ // Use the classloader of the avro type as the classloader
+ // for loading SpecificData classes. This is a best-effort
+ // approach to avoid running into AVRO-1240. We can't just
+ // use the SpecificDatumReader(Class) constructor because the
+ // type class here isn't necessarily a SpecificData class, it
+ // might just contain one as a subtype.
+ return new SpecificDatumReader<T>(
+ avroType.getSchema(),
+ avroType.getSchema(),
+ new SpecificData(avroType.getClass().getClassLoader()));
} else {
return new GenericDatumReader<T>(avroType.getSchema());
}