You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by jc...@apache.org on 2012/09/28 02:04:05 UTC
svn commit: r1391267 - in /pig/trunk/test/org/apache/pig/test:
TestBinInterSedes.java TestTuple.java
Author: jcoveney
Date: Fri Sep 28 00:04:05 2012
New Revision: 1391267
URL: http://svn.apache.org/viewvc?rev=1391267&view=rev
Log:
Adding tests for PIG-2936 (jcoveney)
Modified:
pig/trunk/test/org/apache/pig/test/TestBinInterSedes.java
pig/trunk/test/org/apache/pig/test/TestTuple.java
Modified: pig/trunk/test/org/apache/pig/test/TestBinInterSedes.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBinInterSedes.java?rev=1391267&r1=1391266&r2=1391267&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBinInterSedes.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBinInterSedes.java Fri Sep 28 00:04:05 2012
@@ -18,19 +18,19 @@
package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
+import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
-import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.BinInterSedes;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.InterSedes;
@@ -41,11 +41,12 @@ import org.apache.pig.impl.util.TupleFor
import org.junit.Test;
public class TestBinInterSedes {
+ private static final TupleFactory mTupleFactory = TupleFactory.getInstance();
+ private static final BinInterSedes bis = new BinInterSedes();
+ private static final Random random = new Random(100L);
@Test
public void testTupleWriteRead1() throws IOException {
-
- try {
//create a tuple with columns of different type
Tuple tuplein = TupleFactory.getInstance().newTuple(7);
tuplein.set(0, 12);
@@ -68,11 +69,6 @@ public class TestBinInterSedes {
assertEquals(
"(12,[pig#scalability],,12,1.2,(innerTuple),{(innerTuple)})",
TupleFormat.format(tuplein));
- } catch (ExecException e) {
- e.printStackTrace();
- fail();
- }
-
}
@@ -82,8 +78,6 @@ public class TestBinInterSedes {
*/
@Test
public void testTupleWriteReadIntDiffSizes() throws IOException {
-
- try {
//create a tuple with integer columns of different sizes
Tuple tuple = TupleFactory.getInstance().newTuple();
tuple.append(new Integer(0)); //boolean rep
@@ -93,12 +87,6 @@ public class TestBinInterSedes {
tuple.append(new Integer(1024*1024*1024)); //fits into int (=~ 2 ^30)
testTupleSedes(tuple);
-
- } catch (ExecException e) {
- e.printStackTrace();
- fail();
- }
-
}
/**
@@ -107,8 +95,6 @@ public class TestBinInterSedes {
*/
@Test
public void testTupleWriteReadByteArrStringDiffSizes() throws IOException {
-
- try {
// tuple with ByteArray and strings of different sizes
Tuple tuple = TupleFactory.getInstance().newTuple();
byte [] tinyBA = new byte[10];
@@ -133,24 +119,14 @@ public class TestBinInterSedes {
tuple.append(new String(largeBytearray));
testTupleSedes(tuple);
-
- } catch (ExecException e) {
- e.printStackTrace();
- fail();
}
- }
-
-
-
/**
* test sedes with bags of diff sizes
* @throws IOException
*/
@Test
public void testTupleWriteReadBagDiffSizes() throws IOException {
-
- try {
// tuple with ByteArray and strings of different sizes
Tuple tuple = TupleFactory.getInstance().newTuple();
DataBag tinyBag = createBag(10);
@@ -162,13 +138,6 @@ public class TestBinInterSedes {
tuple.append(largeBag);
testTupleSedes(tuple);
-
-
- } catch (ExecException e) {
- e.printStackTrace();
- fail();
- }
-
}
/*
@@ -177,7 +146,6 @@ public class TestBinInterSedes {
*/
@Test
public void testTupleWriteReadLongDiffSizes() throws IOException {
- try {
Random r = new Random(100L);
Tuple tuple = TupleFactory.getInstance().newTuple();
@@ -197,21 +165,14 @@ public class TestBinInterSedes {
}
testTupleSedes(tuple);
-
- } catch (ExecException e) {
- e.printStackTrace();
- fail();
- }
}
-
/**
* create bag having given number of tuples
* @param size
* @return
*/
private DataBag createBag(int size) {
-
Tuple innerTuple = TupleFactory.getInstance().newTuple();
innerTuple.append(Integer.valueOf(1));
DataBag bag = BagFactory.getInstance().newDefaultBag();
@@ -221,28 +182,18 @@ public class TestBinInterSedes {
return bag;
}
-
/**
* test sedes tuple of diff sizes
* @throws IOException
*/
@Test
public void testTupleWriteReadDiffSizes() throws IOException {
-
- try {
// tuple with ByteArray and strings of different sizes
Tuple smallTuple = createTupleWithManyCols(1000);
testTupleSedes(smallTuple);
Tuple largeTuple = createTupleWithManyCols(100*1000);
testTupleSedes(largeTuple);
-
-
- } catch (ExecException e) {
- e.printStackTrace();
- fail();
- }
-
}
private Tuple createTupleWithManyCols(int size) {
@@ -254,16 +205,12 @@ public class TestBinInterSedes {
return t;
}
-
-
/**
* test sedes with maps of diff sizes
* @throws IOException
*/
@Test
public void testTupleWriteReadMapDiffSizes() throws IOException {
-
- try {
// tuple with ByteArray and strings of different sizes
Tuple tuple = TupleFactory.getInstance().newTuple();
Map<String, Object> tinyMap = createMap(10);
@@ -274,16 +221,8 @@ public class TestBinInterSedes {
tuple.append(largeMap);
testTupleSedes(tuple);
-
-
- } catch (ExecException e) {
- e.printStackTrace();
- fail();
- }
-
}
-
private Map<String, Object> createMap(int size) {
Map<String,Object> map = new HashMap<String, Object>(size);
String key = String.valueOf('A');
@@ -321,6 +260,50 @@ public class TestBinInterSedes {
}
+ /**
+ * See PIG-2936. The purpose of this test is to ensure that Tuples are being serialized in
+ * the specific way that we expect.
+ */
+ @Test
+ public void testTupleSerializationSpecific() throws Exception {
+ byte[] flags = {
+ BinInterSedes.TUPLE_0,
+ BinInterSedes.TUPLE_1,
+ BinInterSedes.TUPLE_2,
+ BinInterSedes.TUPLE_3,
+ BinInterSedes.TUPLE_4,
+ BinInterSedes.TUPLE_5,
+ BinInterSedes.TUPLE_6,
+ BinInterSedes.TUPLE_7,
+ BinInterSedes.TUPLE_8,
+ BinInterSedes.TUPLE_9,
+ };
+
+ for (int i = 0; i < flags.length; i++) {
+ Tuple t = mTupleFactory.newTuple(i);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutput out = new DataOutputStream(baos);
+ out.writeByte(flags[i]);
+
+ for (int j = 0; j < i; j++) {
+ Integer val = Integer.valueOf(random.nextInt());
+ bis.writeDatum(out, val);
+ t.set(j, val);
+ }
+ testSerTuple(t, baos.toByteArray());
+ }
+ }
+
+ private void testSerTuple(Tuple t, byte[] expected) throws Exception {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutput out = new DataOutputStream(baos);
+
+ bis.writeDatum(out, t);
+ Tuple t2 = (Tuple) bis.readDatum(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));
+
+ assertEquals(t, t2);
+ }
}
Modified: pig/trunk/test/org/apache/pig/test/TestTuple.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestTuple.java?rev=1391267&r1=1391266&r2=1391267&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestTuple.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestTuple.java Fri Sep 28 00:04:05 2012
@@ -17,28 +17,40 @@
*/
package org.apache.pig.test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
-import junit.framework.TestCase;
-
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.BinInterSedes;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DefaultBagFactory;
-import org.apache.pig.data.DefaultDataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.util.TupleFormat;
+import org.junit.Test;
-public class TestTuple extends TestCase {
+public class TestTuple {
+ private static final TupleFactory mTupleFactory = TupleFactory.getInstance();
+ private static final Random random = new Random(100L);
+ private static final BinInterSedes bis = new BinInterSedes();
+ @Test
public void testTupleFormat() {
try {
- Tuple tuple = TupleFactory.getInstance().newTuple(7);
+ Tuple tuple = mTupleFactory.newTuple(7);
tuple.set(0, 12);
Map<String, String> map = new HashMap<String, String>();
map.put("pig", "scalability");
@@ -47,7 +59,7 @@ public class TestTuple extends TestCase
tuple.set(3, 12L);
tuple.set(4, 1.2F);
- Tuple innerTuple = TupleFactory.getInstance().newTuple(1);
+ Tuple innerTuple = mTupleFactory.newTuple(1);
innerTuple.set(0, "innerTuple");
tuple.set(5, innerTuple);
@@ -65,18 +77,21 @@ public class TestTuple extends TestCase
}
+ @Test
public void testEmptyTupleSize() {
- Tuple t = TupleFactory.getInstance().newTuple();
+ Tuple t = mTupleFactory.newTuple();
long size = t.getMemorySize();
assertEquals("tuple size",size, 96);
}
+ @Test
public void testEmptyBagSize() {
DataBag bag = DefaultBagFactory.getInstance().newDefaultBag();
long size = bag.getMemorySize();
assertEquals("bag size",size, 124);
}
+ @Test
// See PIG-1443
public void testTupleSizeWithString() {
Tuple t = Util.createTuple(new String[] {"1234567", "bar"});
@@ -84,48 +99,54 @@ public class TestTuple extends TestCase
assertEquals("tuple size",size, 200);
}
+ @Test
public void testTupleSizeWithByteArrays() {
- Tuple t = TupleFactory.getInstance().newTuple();
+ Tuple t = mTupleFactory.newTuple();
t.append(new DataByteArray("1234567"));
t.append(new DataByteArray("bar"));
long size = t.getMemorySize();
assertEquals("tuple size",size, 168);
}
+ @Test
public void testTupleSizeWithDoubles() {
- Tuple t = TupleFactory.getInstance().newTuple();
+ Tuple t = mTupleFactory.newTuple();
t.append(new Double(0.1));
t.append(new Double(2000.10001));
long size = t.getMemorySize();
assertEquals("tuple size",size, 128);
}
+ @Test
public void testTupleSizeWithFloats() {
- Tuple t = TupleFactory.getInstance().newTuple();
+ Tuple t = mTupleFactory.newTuple();
t.append(new Float(0.1F));
t.append(new Float(2000.10001F));
long size = t.getMemorySize();
assertEquals("tuple size",size, 128);
}
+ @Test
public void testTupleSizeWithLongs() {
- Tuple t = TupleFactory.getInstance().newTuple();
+ Tuple t = mTupleFactory.newTuple();
t.append(new Long(100));
t.append(new Long(2000));
long size = t.getMemorySize();
assertEquals("tuple size",size, 128);
}
+ @Test
public void testTupleSizeWithBooleans() {
- Tuple t = TupleFactory.getInstance().newTuple();
+ Tuple t = mTupleFactory.newTuple();
t.append(new Boolean(true));
t.append(new Boolean(false));
long size = t.getMemorySize();
assertEquals("tuple size",size, 128);
}
+ @Test
public void testTupleIterator() {
- Tuple t = TupleFactory.getInstance().newTuple();
+ Tuple t = mTupleFactory.newTuple();
Random r = new Random();
for (int i = 0; i < 1000; i++) {
t.append(r.nextLong());
@@ -143,8 +164,9 @@ public class TestTuple extends TestCase
}
}
+ @Test
public void testToDelimitedString() {
- Tuple t = TupleFactory.getInstance().newTuple();
+ Tuple t = mTupleFactory.newTuple();
t.append(new Integer(1));
t.append(new Long(2));
t.append(new Float(1.1f));
@@ -164,4 +186,22 @@ public class TestTuple extends TestCase
}
}
+ @Test
+ public void testTupleSerDe() throws Exception {
+ for (int i = 0; i < 1000; i++) {
+ Tuple t = mTupleFactory.newTuple(i);
+ for (int j = 0; j < i; j++) {
+ t.set(j,Integer.valueOf(random.nextInt()));
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutput out = new DataOutputStream(baos);
+ bis.writeDatum(out, t);
+
+ DataInput in = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
+ Tuple t2 = (Tuple) bis.readDatum(in);
+
+ assertEquals(t, t2);
+ }
+ }
+
}