You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/03/24 00:52:28 UTC
svn commit: r1304664 - in /pig/branches/branch-0.10: CHANGES.txt
src/org/apache/pig/data/DefaultAbstractBag.java
src/org/apache/pig/data/DefaultDataBag.java
test/org/apache/pig/test/TestDataBag.java
Author: daijy
Date: Fri Mar 23 23:52:27 2012
New Revision: 1304664
URL: http://svn.apache.org/viewvc?rev=1304664&view=rev
Log:
PIG-2550: Custom tuple results in 'Unexpected datatype 110 while reading tuplefrom binary file' while spilling
Modified:
pig/branches/branch-0.10/CHANGES.txt
pig/branches/branch-0.10/src/org/apache/pig/data/DefaultAbstractBag.java
pig/branches/branch-0.10/src/org/apache/pig/data/DefaultDataBag.java
pig/branches/branch-0.10/test/org/apache/pig/test/TestDataBag.java
Modified: pig/branches/branch-0.10/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/CHANGES.txt?rev=1304664&r1=1304663&r2=1304664&view=diff
==============================================================================
--- pig/branches/branch-0.10/CHANGES.txt (original)
+++ pig/branches/branch-0.10/CHANGES.txt Fri Mar 23 23:52:27 2012
@@ -364,6 +364,8 @@ Release 0.9.3 - Unreleased
BUG FIXES
+PIG-2550: Custom tuple results in "Unexpected datatype 110 while reading tuplefrom binary file" while spilling (daijy)
+
PIG-2442: Multiple Stores in pig streaming causes infinite waiting (daijy)
PIG-2609: e2e harness: make hdfs base path configurable (outside default.conf) (thw via daijy)
Modified: pig/branches/branch-0.10/src/org/apache/pig/data/DefaultAbstractBag.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/src/org/apache/pig/data/DefaultAbstractBag.java?rev=1304664&r1=1304663&r2=1304664&view=diff
==============================================================================
--- pig/branches/branch-0.10/src/org/apache/pig/data/DefaultAbstractBag.java (original)
+++ pig/branches/branch-0.10/src/org/apache/pig/data/DefaultAbstractBag.java Fri Mar 23 23:52:27 2012
@@ -51,7 +51,7 @@ public abstract class DefaultAbstractBag
private static PigLogger pigLogger = PhysicalOperator.getPigLogger();
- private static InterSedes sedes = InterSedesFactory.getInterSedesInstance();
+ protected static InterSedes sedes = InterSedesFactory.getInterSedesInstance();
// Container that holds the tuples. Actual object instantiated by
// subclasses.
protected Collection<Tuple> mContents;
Modified: pig/branches/branch-0.10/src/org/apache/pig/data/DefaultDataBag.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/src/org/apache/pig/data/DefaultDataBag.java?rev=1304664&r1=1304663&r2=1304664&view=diff
==============================================================================
--- pig/branches/branch-0.10/src/org/apache/pig/data/DefaultDataBag.java (original)
+++ pig/branches/branch-0.10/src/org/apache/pig/data/DefaultDataBag.java Fri Mar 23 23:52:27 2012
@@ -99,7 +99,7 @@ public class DefaultDataBag extends Defa
try {
Iterator<Tuple> i = mContents.iterator();
while (i.hasNext()) {
- i.next().write(out);
+ sedes.writeDatum(out, i.next());
spilled++;
// This will spill every 16383 records.
if ((spilled & 0x3fff) == 0) reportProgress();
Modified: pig/branches/branch-0.10/test/org/apache/pig/test/TestDataBag.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestDataBag.java?rev=1304664&r1=1304663&r2=1304664&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestDataBag.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestDataBag.java Fri Mar 23 23:52:27 2012
@@ -1125,6 +1125,31 @@ public class TestDataBag extends junit.f
assertTrue(dfBag.equals(stBag));
}
+ // See PIG-2550
+ static class MyCustomTuple extends DefaultTuple {
+ private static final long serialVersionUID = 8156382697467819543L;
+ public MyCustomTuple() {
+ super();
+ }
+ public MyCustomTuple(Object t) {
+ super();
+ append(t);
+ }
+ }
+
+ @Test
+ public void testSpillCustomTuple() throws Exception {
+ DataBag bag = new DefaultDataBag();
+ Tuple t = new MyCustomTuple();
+ t.append(1);
+ t.append("hello");
+ bag.add(t);
+ bag.spill();
+ Iterator<Tuple> iter = bag.iterator();
+ Tuple t2 = iter.next();
+ assertTrue(t2.equals(t));
+ }
+
void processDataBag(DataBag bg, boolean doSpill) {
Tuple t = TupleFactory.getInstance().newTuple(new Integer(0));
bg.add(t);