You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by dv...@apache.org on 2010/03/21 20:22:23 UTC
svn commit: r925868 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/data/SingleTupleBag.java
test/org/apache/pig/test/TestDataBag.java
Author: dvryaboy
Date: Sun Mar 21 19:22:23 2010
New Revision: 925868
URL: http://svn.apache.org/viewvc?rev=925868&view=rev
Log:
PIG-1285: Allow SingleTupleBag to be serialized (dvryaboy)
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java
hadoop/pig/trunk/test/org/apache/pig/test/TestDataBag.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=925868&r1=925867&r2=925868&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Sun Mar 21 19:22:23 2010
@@ -68,6 +68,8 @@ manner (rding via pradeepkth)
IMPROVEMENTS
+PIG-1285: Allow SingleTupleBag to be serialized (dvryaboy)
+
PIG-1117: Pig reading hive columnar rc tables (gerritjvv via dvryaboy)
PIG-1287: Use hadoop-0.20.2 with pig 0.7.0 release (pradeepkth)
Modified: hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java?rev=925868&r1=925867&r2=925868&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java Sun Mar 21 19:22:23 2010
@@ -43,7 +43,7 @@ public class SingleTupleBag implements D
public SingleTupleBag(Tuple t) {
item = t;
}
-
+
/* (non-Javadoc)
* @see org.apache.pig.data.DataBag#add(org.apache.pig.data.Tuple)
* NOTE: It is the user's responsibility to ensure only a single
@@ -131,10 +131,16 @@ public class SingleTupleBag implements D
*/
@Override
public void readFields(DataInput in) throws IOException {
- // TODO Auto-generated method stub
- int errCode = 2113;
- String msg = "SingleTupleBag should never be serialized or serialized.";
- throw new ExecException(msg, errCode, PigException.BUG);
+ long size = in.readLong();
+
+ for (long i = 0; i < size; i++) {
+ try {
+ Object o = DataReaderWriter.readDatum(in);
+ add((Tuple)o);
+ } catch (ExecException ee) {
+ throw ee;
+ }
+ }
}
/* (non-Javadoc)
@@ -142,10 +148,12 @@ public class SingleTupleBag implements D
*/
@Override
public void write(DataOutput out) throws IOException {
- // TODO Auto-generated method stub
- int errCode = 2113;
- String msg = "SingleTupleBag should never be serialized or serialized.";
- throw new ExecException(msg, errCode, PigException.BUG);
+ out.writeLong(size());
+ Iterator<Tuple> it = iterator();
+ while (it.hasNext()) {
+ Tuple item = it.next();
+ item.write(out);
+ }
}
/* (non-Javadoc)
@@ -175,7 +183,7 @@ public class SingleTupleBag implements D
public boolean hasNext() {
return !nextDone;
}
-
+
/* (non-Javadoc)
* @see java.util.Iterator#next()
*/
@@ -183,9 +191,9 @@ public class SingleTupleBag implements D
public Tuple next() {
nextDone = true;
return item;
-
+
}
-
+
/* (non-Javadoc)
* @see java.util.Iterator#remove()
*/
@@ -194,7 +202,7 @@ public class SingleTupleBag implements D
throw new RuntimeException("SingleTupleBag.iterator().remove() is not allowed");
}
}
-
+
/**
* Write the bag into a string. */
@Override
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestDataBag.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestDataBag.java?rev=925868&r1=925867&r2=925868&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestDataBag.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestDataBag.java Sun Mar 21 19:22:23 2010
@@ -18,7 +18,12 @@
package org.apache.pig.test;
import java.util.*;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
import org.junit.Test;
import org.apache.pig.data.*;
@@ -1105,6 +1110,21 @@ public class TestDataBag extends junit.f
processDataBag(bg5, false);
}
+ // See PIG-1285
+ @Test
+ public void testSerializeSingleTupleBag() throws Exception {
+ Tuple t = Util.createTuple(new String[] {"foo", "bar", "baz"});
+ DataBag stBag = new SingleTupleBag(t);
+ PipedOutputStream pos = new PipedOutputStream();
+ DataOutputStream dos = new DataOutputStream(pos);
+ PipedInputStream pis = new PipedInputStream(pos);
+ DataInputStream dis = new DataInputStream(pis);
+ stBag.write(dos);
+ DataBag dfBag = new DefaultDataBag();
+ dfBag.readFields(dis);
+ assertTrue(dfBag.equals(stBag));
+ }
+
void processDataBag(DataBag bg, boolean doSpill) {
Tuple t = TupleFactory.getInstance().newTuple(new Integer(0));
bg.add(t);