You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2013/12/19 19:08:25 UTC
svn commit: r1552387 - in /pig/trunk: CHANGES.txt ivy/libraries.properties
src/org/apache/pig/impl/util/avro/AvroBagWrapper.java
src/org/apache/pig/impl/util/avro/AvroTupleWrapper.java
Author: cheolsoo
Date: Thu Dec 19 18:08:25 2013
New Revision: 1552387
URL: http://svn.apache.org/r1552387
Log:
PIG-3584: AvroStorage does not correctly translate arrays of strings (jadler via cheolsoo)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/ivy/libraries.properties
pig/trunk/src/org/apache/pig/impl/util/avro/AvroBagWrapper.java
pig/trunk/src/org/apache/pig/impl/util/avro/AvroTupleWrapper.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1552387&r1=1552386&r2=1552387&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Dec 19 18:08:25 2013
@@ -58,6 +58,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3584: AvroStorage does not correctly translate arrays of strings (jadler via cheolsoo)
+
PIG-3633: AvroStorage tests are failing when running against Avro 1.7.5 (jarcec via cheolsoo)
PIG-3612: Storing schema does not work cross cluster with PigStorage and JsonStorage (rohini)
Modified: pig/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/trunk/ivy/libraries.properties?rev=1552387&r1=1552386&r2=1552387&view=diff
==============================================================================
--- pig/trunk/ivy/libraries.properties (original)
+++ pig/trunk/ivy/libraries.properties Thu Dec 19 18:08:25 2013
@@ -17,7 +17,7 @@
apacheant.version=1.7.1
apacherat.version=0.8
automaton.version=1.11-8
-avro.version=1.7.4
+avro.version=1.7.5
commons-beanutils.version=1.7.0
commons-cli.version=1.0
commons-codec.version=1.4
Modified: pig/trunk/src/org/apache/pig/impl/util/avro/AvroBagWrapper.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/avro/AvroBagWrapper.java?rev=1552387&r1=1552386&r2=1552387&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/avro/AvroBagWrapper.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/avro/AvroBagWrapper.java Thu Dec 19 18:08:25 2013
@@ -18,11 +18,8 @@
package org.apache.pig.impl.util.avro;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Iterator;
-
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
import org.apache.avro.generic.GenericArray;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.IndexedRecord;
@@ -30,8 +27,10 @@ import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterators;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Iterator;
/**
* Class that implements the Pig bag interface, wrapping an Avro array.
@@ -89,7 +88,7 @@ public final class AvroBagWrapper<T> imp
if (arg instanceof IndexedRecord) {
return new AvroTupleWrapper<IndexedRecord>((IndexedRecord) arg);
} else {
- return TupleFactory.getInstance().newTuple(arg);
+ return TupleFactory.getInstance().newTuple(AvroTupleWrapper.unionResolver(arg));
}
}
}
Modified: pig/trunk/src/org/apache/pig/impl/util/avro/AvroTupleWrapper.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/avro/AvroTupleWrapper.java?rev=1552387&r1=1552386&r2=1552387&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/avro/AvroTupleWrapper.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/avro/AvroTupleWrapper.java Thu Dec 19 18:08:25 2013
@@ -18,29 +18,28 @@
package org.apache.pig.impl.util.avro;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
import org.apache.avro.generic.GenericArray;
import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.IndexedRecord;
import org.apache.avro.generic.GenericEnumSymbol;
+import org.apache.avro.generic.IndexedRecord;
import org.apache.commons.logging.LogFactory;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;
-import com.google.common.base.Function;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
/**
* Object that wraps an Avro object in a tuple.
@@ -131,26 +130,31 @@ public final class AvroTupleWrapper <T e
case BYTES:
return new DataByteArray(((ByteBuffer) o).array());
case UNION:
- if (o instanceof org.apache.avro.util.Utf8) {
- return o.toString();
- } else if (o instanceof IndexedRecord) {
- return new AvroTupleWrapper<T>((T) o);
- } else if (o instanceof GenericArray) {
- return new AvroBagWrapper<GenericData.Record>(
- (GenericArray<GenericData.Record>) o);
- } else if (o instanceof Map) {
- return new AvroMapWrapper((Map<CharSequence, Object>) o);
- } else if (o instanceof GenericData.Fixed) {
- return new DataByteArray(((GenericData.Fixed) o).bytes());
- } else if (o instanceof ByteBuffer) {
- return new DataByteArray(((ByteBuffer) o).array());
- } else if (o instanceof GenericEnumSymbol) {
- return o.toString();
- }
+ return unionResolver(o);
default:
return o;
}
+ }
+ public static Object unionResolver(Object o) {
+ if (o instanceof org.apache.avro.util.Utf8) {
+ return o.toString();
+ } else if (o instanceof IndexedRecord) {
+ return new AvroTupleWrapper<IndexedRecord>((IndexedRecord) o);
+ } else if (o instanceof GenericArray) {
+ return new AvroBagWrapper<GenericData.Record>(
+ (GenericArray<GenericData.Record>) o);
+ } else if (o instanceof Map) {
+ return new AvroMapWrapper((Map<CharSequence, Object>) o);
+ } else if (o instanceof GenericData.Fixed) {
+ return new DataByteArray(((GenericData.Fixed) o).bytes());
+ } else if (o instanceof ByteBuffer) {
+ return new DataByteArray(((ByteBuffer) o).array());
+ } else if (o instanceof GenericEnumSymbol) {
+ return o.toString();
+ } else {
+ return o;
+ }
}
@Override