You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by jc...@apache.org on 2012/10/05 00:26:31 UTC
svn commit: r1394301 - in /pig/trunk: CHANGES.txt
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java
Author: jcoveney
Date: Thu Oct 4 22:26:31 2012
New Revision: 1394301
URL: http://svn.apache.org/viewvc?rev=1394301&view=rev
Log:
PIG-2737: [piggybank] TestIndexedStorage is failing, should be refactored (jcoveney)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1394301&r1=1394300&r2=1394301&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Oct 4 22:26:31 2012
@@ -28,7 +28,7 @@ IMPROVEMENTS
PIG-2923: Lazily register bags with SpillableMemoryManager (dvryaboy)
PIG-2929: Improve documentation around AVG, CONCAT, MIN, MAX (cheolsoo via billgraham)
-
+
PIG-2852: Update documentation regarding parallel local mode execution (cheolsoo via jcoveney)
PIG-2879: Pig current releases lack a UDF startsWith.This UDF tests if a given string starts with the specified prefix. (initialcontext via azaroth)
@@ -285,6 +285,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-2737: [piggybank] TestIndexedStorage is failing, should be refactored (jcoveney)
+
PIG-2935: Catch NoSuchMethodError when StoreFuncInterface's new cleanupOnSuccess method isn't implemented. (gates via dvryaboy)
PIG-2920: e2e tests override PERL5LIB environment variable (azaroth)
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java?rev=1394301&r1=1394300&r2=1394301&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestIndexedStorage.java Thu Oct 4 22:26:31 2012
@@ -18,6 +18,10 @@
package org.apache.pig.piggybank.test.storage;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.io.Files;
+
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -45,90 +49,85 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.piggybank.storage.IndexedStorage;
import org.apache.pig.test.Util;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
/**
* Tests IndexedStorage.
*/
public class TestIndexedStorage {
+ private File outputDir;
+
public TestIndexedStorage () throws IOException {
}
- @BeforeClass
+ @Before
/**
* Creates indexed data sets.
- */
- public static void setUpBeforeClass() throws Exception {
+ */
+ public void setUp() throws Exception {
PigServer pigServer = new PigServer(ExecType.LOCAL);
-
+
+ outputDir = Files.createTempDir();
+ outputDir.deleteOnExit();
+
String[] input1 = new String[] {
"2\t2", "3\t3", "4\t3", "5\t5", "6\t6", "10\t10"
};
-
- createInputFile(pigServer, input1, 1);
-
+
+ createInputFile(pigServer, input1, 1, outputDir);
+
String[] input2 = new String[] {
"3\t2", "4\t4", "5\t5", "11\t11", "13\t13"
};
-
- createInputFile(pigServer, input2, 2);
+
+ createInputFile(pigServer, input2, 2, outputDir);
String[] input3 = new String[] {
"7\t7", "8\t8", "9\t9"
};
-
- createInputFile(pigServer, input3, 3);
+
+ createInputFile(pigServer, input3, 3, outputDir);
}
-
- private static void createInputFile(PigServer pigServer, String[] inputs, int id) throws IOException {
+
+ private static void createInputFile(PigServer pigServer, String[] inputs, int id, File outputDir) throws IOException {
File input = File.createTempFile("tmp", "");
input.delete();
Util.createLocalInputFile(input.getAbsolutePath(), inputs);
-
+
pigServer.registerQuery("A = load '" + input.getAbsolutePath() + "' as (a0:int, a1:int);");
-
- File output = File.createTempFile("tmp", "");
- output.delete();
+
+ File output = new File(outputDir, "/" + id);
pigServer.store("A", output.getAbsolutePath(), "org.apache.pig.piggybank.storage.IndexedStorage('\t','0,1')");
-
- File outputFile = new File(output.getAbsoluteFile()+"/part-m-00000");
- new File("out/"+id).mkdirs();
- outputFile.renameTo(new File("out/"+id+"/part-m-00000"));
-
- File outputIndexFile = new File(output.getAbsoluteFile()+"/.part-m-00000.index");
- outputIndexFile.renameTo(new File("out/"+id+"/.part-m-00000.index"));
}
- @AfterClass
+ @After
/**
* Deletes all data directories.
- */
- public static void tearDownAfterClass() throws Exception {
- Configuration conf = new Configuration();
- LocalFileSystem fs = FileSystem.getLocal(conf);
- fs.delete(new Path("out"), true);
+ */
+ public void tearDown() throws Exception {
+ outputDir.delete();
}
@Test
/**
* Tests getNext called repeatedly returns items in sorted order.
- */
+ */
public void testGetNext() throws IOException, InterruptedException {
IndexedStorage storage = new IndexedStorage("\t","0,1");
Configuration conf = new Configuration();
conf.set("fs.default.name", "file:///");
LocalFileSystem fs = FileSystem.getLocal(conf);
-
+
TaskAttemptID taskId = HadoopShims.createTaskAttemptID("jt", 1, true, 1, 1);
conf.set("mapred.task.id", taskId.toString());
-
- conf.set("mapred.input.dir","out");
+
+ conf.set("mapred.input.dir", outputDir.getAbsolutePath());
storage.initialize(conf);
-
- Integer key;
+
+ Integer key;
int [][] correctValues = {{2,2},{3,2},{3,3},{4,3},{4,4},{5,5},{5,5},{6,6},{7,7},{8,8},{9,9},{10,10},{11,11},{13,13}};
for (int [] outer : correctValues) {
System.out.println("Testing: (" + outer[0] + "," + outer[1] + ")");
@@ -136,28 +135,28 @@ public class TestIndexedStorage {
System.out.println("Read: " + read);
key = Integer.decode(((DataByteArray)read.get(0)).toString());
- Assert.assertTrue("GetNext did not return the correct value. Received: " + read, key.equals(outer[0]));
+ assertTrue("GetNext did not return the correct value. Received: " + read, key.equals(outer[0]));
key = Integer.decode(((DataByteArray)read.get(1)).toString());
- Assert.assertTrue("GetNext did not return the correct value. Received: " + read, key.equals(outer[1]));
+ assertTrue("GetNext did not return the correct value. Received: " + read, key.equals(outer[1]));
}
Tuple read = storage.getNext();
- Assert.assertTrue("GetNext did not return the correct value", (read == null));
+ assertTrue("GetNext did not return the correct value", (read == null));
}
@Test
/**
* Tests seekNear and getNext work correctly.
- */
+ */
public void testSeek() throws IOException, InterruptedException {
IndexedStorage storage = new IndexedStorage("\t","0,1");
Configuration conf = new Configuration();
conf.set("fs.default.name", "file:///");
LocalFileSystem fs = FileSystem.getLocal(conf);
-
+
TaskAttemptID taskId = HadoopShims.createTaskAttemptID("jt", 2, true, 2, 2);
conf.set("mapred.task.id", taskId.toString());
-
- conf.set("mapred.input.dir","out");
+
+ conf.set("mapred.input.dir", outputDir.getAbsolutePath());
storage.initialize(conf);
TupleFactory tupleFactory = TupleFactory.getInstance();
@@ -166,82 +165,82 @@ public class TestIndexedStorage {
Tuple read;
//Seek to 1,1 (not in index). getNext should return 2
- seek.set(0, new Integer(1));
- seek.set(1, new Integer(1));
+ seek.set(0, new Integer(1));
+ seek.set(1, new Integer(1));
storage.seekNear(seek);
read = storage.getNext();
key = Integer.decode(((DataByteArray)read.get(0)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(2));
+ assertTrue("GetNext did not return the correct value", key.equals(2));
key = Integer.decode(((DataByteArray)read.get(1)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(2));
+ assertTrue("GetNext did not return the correct value", key.equals(2));
//Seek to 3,2. getNext should return 3,2
- seek.set(0, new Integer(3));
- seek.set(1, new Integer(2));
+ seek.set(0, new Integer(3));
+ seek.set(1, new Integer(2));
storage.seekNear(seek);
read = storage.getNext();
key = Integer.decode(((DataByteArray)read.get(0)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(3));
+ assertTrue("GetNext did not return the correct value", key.equals(3));
key = Integer.decode(((DataByteArray)read.get(1)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(2));
+ assertTrue("GetNext did not return the correct value", key.equals(2));
//getNext should return 3,3
read = storage.getNext();
key = Integer.decode(((DataByteArray)read.get(0)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(3));
+ assertTrue("GetNext did not return the correct value", key.equals(3));
key = Integer.decode(((DataByteArray)read.get(1)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(3));
+ assertTrue("GetNext did not return the correct value", key.equals(3));
//Seek to 6,6. getNext should return 6,6
- seek.set(0, new Integer(6));
- seek.set(1, new Integer(6));
+ seek.set(0, new Integer(6));
+ seek.set(1, new Integer(6));
storage.seekNear(seek);
read = storage.getNext();
key = Integer.decode(((DataByteArray)read.get(0)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(6));
+ assertTrue("GetNext did not return the correct value", key.equals(6));
key = Integer.decode(((DataByteArray)read.get(1)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(6));
+ assertTrue("GetNext did not return the correct value", key.equals(6));
//getNext should return 7,7
read = storage.getNext();
key = Integer.decode(((DataByteArray)read.get(0)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(7));
+ assertTrue("GetNext did not return the correct value", key.equals(7));
key = Integer.decode(((DataByteArray)read.get(1)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(7));
+ assertTrue("GetNext did not return the correct value", key.equals(7));
//Seek to 9,9. getNext should return 9,9
- seek.set(0, new Integer(9));
- seek.set(1, new Integer(9));
+ seek.set(0, new Integer(9));
+ seek.set(1, new Integer(9));
storage.seekNear(seek);
read = storage.getNext();
key = Integer.decode(((DataByteArray)read.get(0)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(9));
+ assertTrue("GetNext did not return the correct value", key.equals(9));
key = Integer.decode(((DataByteArray)read.get(1)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(9));
+ assertTrue("GetNext did not return the correct value", key.equals(9));
//getNext should return 10,10
read = storage.getNext();
key = Integer.decode(((DataByteArray)read.get(0)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(10));
+ assertTrue("GetNext did not return the correct value", key.equals(10));
key = Integer.decode(((DataByteArray)read.get(1)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(10));
+ assertTrue("GetNext did not return the correct value", key.equals(10));
//Seek to 13,12 (Not in index). getNext should return 13,13
- seek.set(0, new Integer(13));
- seek.set(1, new Integer(12));
+ seek.set(0, new Integer(13));
+ seek.set(1, new Integer(12));
storage.seekNear(seek);
read = storage.getNext();
key = Integer.decode(((DataByteArray)read.get(0)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(13));
+ assertTrue("GetNext did not return the correct value", key.equals(13));
key = Integer.decode(((DataByteArray)read.get(1)).toString());
- Assert.assertTrue("GetNext did not return the correct value", key.equals(13));
+ assertTrue("GetNext did not return the correct value", key.equals(13));
- //Seek to 20 (Not in index). getNext should return null
- seek.set(0, new Integer(20));
- seek.set(1, new Integer(20));
+ //Seek to 20 (Not in index). getNext should return null
+ seek.set(0, new Integer(20));
+ seek.set(1, new Integer(20));
storage.seekNear(seek);
read = storage.getNext();
- Assert.assertTrue("GetNext did not return the correct value", (read == null));
+ assertTrue("GetNext did not return the correct value", (read == null));
}
}