You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/04/09 06:39:00 UTC
svn commit: r646189 - in /incubator/pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
test/org/apache/pig/test/TestMapReduceResultRecycling.java
Author: gates
Date: Tue Apr 8 21:38:56 2008
New Revision: 646189
URL: http://svn.apache.org/viewvc?rev=646189&view=rev
Log:
PIG-153: Incorrect result caused by dump in between statements.
Added:
incubator/pig/trunk/test/org/apache/pig/test/TestMapReduceResultRecycling.java
Modified:
incubator/pig/trunk/CHANGES.txt
incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
Modified: incubator/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=646189&r1=646188&r2=646189&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Tue Apr 8 21:38:56 2008
@@ -207,3 +207,6 @@
PIG-182: streaming bug fix (arunc via olgan)
PIG-184: streaming bug fixes
+
+ PIG-153: Incorrect result caused by dump in between statements (pi_song
+ via gates).
Modified: incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=646189&r1=646188&r2=646189&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Tue Apr 8 21:38:56 2008
@@ -280,7 +280,7 @@
pom.close();
- this.materializedResults.put(pom.sourceLogicalKey,
+ this.materializedResults.put(plan.getRoot(),
new MapRedResult(pom.outputFileSpec,
pom.reduceParallelism));
}
Added: incubator/pig/trunk/test/org/apache/pig/test/TestMapReduceResultRecycling.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/test/org/apache/pig/test/TestMapReduceResultRecycling.java?rev=646189&view=auto
==============================================================================
--- incubator/pig/trunk/test/org/apache/pig/test/TestMapReduceResultRecycling.java (added)
+++ incubator/pig/trunk/test/org/apache/pig/test/TestMapReduceResultRecycling.java Tue Apr 8 21:38:56 2008
@@ -0,0 +1,67 @@
+package org.apache.pig.test;
+
+import static org.apache.pig.PigServer.ExecType ;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.Iterator;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.data.Tuple;
+import org.junit.Test;
+
+import junit.framework.TestCase;
+
+public class TestMapReduceResultRecycling extends TestCase {
+ MiniCluster cluster = MiniCluster.buildCluster();
+
+ @Test
+ public void testPlanRecycling() throws Throwable {
+ PigServer pig = new PigServer(ExecType.MAPREDUCE);
+ File tmpFile = this.createTempFile();
+ {
+ String query = "a = load 'file:" + tmpFile + "'; " ;
+ System.out.println(query);
+ pig.registerQuery(query);
+ pig.explain("a", System.out) ;
+ Iterator<Tuple> it = pig.openIterator("a");
+ assertTrue(it.next().getAtomField(0).strval().equals("a1")) ;
+ assertTrue(it.next().getAtomField(0).strval().equals("b1")) ;
+ assertTrue(it.next().getAtomField(0).strval().equals("c1")) ;
+ assertFalse(it.hasNext()) ;
+ }
+
+ {
+ String query = "b = filter a by $0 eq 'a1';" ;
+ System.out.println(query);
+ pig.registerQuery(query);
+ pig.explain("b", System.out) ;
+ Iterator<Tuple> it = pig.openIterator("b");
+ assertTrue(it.next().getAtomField(0).strval().equals("a1")) ;
+ assertFalse(it.hasNext()) ;
+ }
+
+ {
+ String query = "c = filter a by $0 eq 'b1';" ;
+ System.out.println(query);
+ pig.registerQuery(query);
+ pig.explain("c", System.out) ;
+ Iterator<Tuple> it = pig.openIterator("c");
+ assertTrue(it.next().getAtomField(0).strval().equals("b1")) ;
+ assertFalse(it.hasNext()) ;
+ }
+
+ }
+
+ private File createTempFile() throws Throwable {
+ File tmpFile = File.createTempFile("pi_test1", "txt");
+ tmpFile.deleteOnExit() ;
+ PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
+ ps.println("a1\t1\t1000") ;
+ ps.println("b1\t2\t1000") ;
+ ps.println("c1\t3\t1000") ;
+ ps.close();
+ return tmpFile ;
+ }
+}