You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/04/02 03:36:27 UTC
svn commit: r761145 [2/2] - in /hadoop/pig/branches/multiquery: ./
src/org/apache/pig/ src/org/apache/pig/backend/hadoop/datastorage/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/ph...
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestLogToPhyCompiler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestLogToPhyCompiler.java?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestLogToPhyCompiler.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestLogToPhyCompiler.java Thu Apr 2 01:36:25 2009
@@ -69,7 +69,7 @@
Random r = new Random();
PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
-
+
private boolean generate = false;
@@ -114,6 +114,7 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld");
@@ -125,6 +126,7 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
+ goldenPlan = goldenPlan.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -143,7 +145,8 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
-
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
+
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/Sort.gld");
fos.write(baos.toByteArray());
@@ -154,7 +157,7 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
-
+ goldenPlan = goldenPlan.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -174,6 +177,7 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/Distinct.gld");
@@ -185,7 +189,7 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
-
+ goldenPlan = goldenPlan.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -206,6 +210,7 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld");
@@ -217,7 +222,7 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
-
+ goldenPlan = goldenPlan.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -242,7 +247,8 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
-
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
+
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/Arithmetic.gld");
fos.write(baos.toByteArray());
@@ -253,7 +259,7 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
-
+ goldenPlan = goldenPlan.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -275,6 +281,7 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/Comparison.gld");
@@ -286,7 +293,7 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
-
+ goldenPlan = goldenPlan.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -308,7 +315,8 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
-
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
+
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/BinCond.gld");
fos.write(baos.toByteArray());
@@ -319,7 +327,7 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
-
+ goldenPlan = goldenPlan.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -342,7 +350,8 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
-
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
+
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/Generate.gld");
fos.write(baos.toByteArray());
@@ -353,7 +362,7 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
-
+ goldenPlan = goldenPlan.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -373,7 +382,8 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
-
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
+
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/Union.gld");
fos.write(baos.toByteArray());
@@ -384,7 +394,7 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
-
+ goldenPlan = goldenPlan.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -405,7 +415,8 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
-
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
+
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/Split1.gld");
fos.write(baos.toByteArray());
@@ -421,7 +432,8 @@
//System.out.println("Length of first plan = " + len + " of second = " + test);
String goldenPlan1 = new String(b1, 0, len);
String goldenPlan2 = new String(b2, 0, len);
-
+ goldenPlan1 = goldenPlan1.replaceAll("Load(.*)","Load()");
+ goldenPlan2 = goldenPlan2.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -458,7 +470,8 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
-
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
+
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld");
fos.write(baos.toByteArray());
@@ -474,7 +487,8 @@
//System.out.println("Length of first plan = " + len + " of second = " + test + " Length of compiled plan = " + compiledPlan.length());
String goldenPlan1 = new String(b1, 0, len);
String goldenPlan2 = new String(b2, 0, len);
-
+ goldenPlan1 = goldenPlan1.replaceAll("Load(.*)","Load()");
+ goldenPlan2 = goldenPlan2.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -507,6 +521,7 @@
pp.explain(baos);
baos.write((int)'\n');
String compiledPlan = baos.toString();
+ compiledPlan = compiledPlan.replaceAll("Load(.*)","Load()");
if(generate){
FileOutputStream fos = new FileOutputStream("test/org/apache/pig/test/data/GoldenFiles/Limit.gld");
@@ -518,7 +533,7 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
-
+ goldenPlan = goldenPlan.replaceAll("Load(.*)","Load()");
System.out.println();
System.out.println(compiledPlan);
@@ -622,14 +637,19 @@
public LogicalPlan buildPlan(String query, ClassLoader cldr) {
LogicalPlanBuilder.classloader = cldr;
PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
- LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //
-
try {
+
+ pigContext.connect();
+
+ LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //
+
+
LogicalPlan lp = builder.parse("Test-Plan-Builder",
query,
aliases,
logicalOpTable,
- aliasOp);
+ aliasOp,
+ fileNameMap);
List<LogicalOperator> roots = lp.getRoots();
@@ -679,4 +699,5 @@
Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>();
+ Map<String, String> fileNameMap = new HashMap<String, String>();
}
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestLogicalPlanBuilder.java Thu Apr 2 01:36:25 2009
@@ -1954,14 +1954,17 @@
public LogicalPlan buildPlan(String query, ClassLoader cldr) {
LogicalPlanBuilder.classloader = cldr;
- LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //
try {
+ pigContext.connect();
+ LogicalPlanBuilder builder = new LogicalPlanBuilder(pigContext); //
+
LogicalPlan lp = builder.parse("Test-Plan-Builder",
query,
aliases,
logicalOpTable,
- aliasOp);
+ aliasOp,
+ fileNameMap);
List<LogicalOperator> roots = lp.getRoots();
if(roots.size() > 0) {
@@ -1995,5 +1998,6 @@
Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>();
+ Map<String, String> fileNameMap = new HashMap<String, String>();
PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
}
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQuery.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQuery.java?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQuery.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQuery.java Thu Apr 2 01:36:25 2009
@@ -32,7 +32,11 @@
import org.apache.pig.backend.executionengine.util.ExecTools;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSplit;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
@@ -44,6 +48,7 @@
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.apache.hadoop.fs.Path;
public class TestMultiQuery extends TestCase {
@@ -53,6 +58,7 @@
@Before
public void setUp() throws Exception {
+ cluster.setProperty("opt.multiquery", ""+true);
myPig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
deleteOutputFiles();
}
@@ -968,6 +974,167 @@
}
}
+ @Test
+ public void testEmptyFilterRemoval() {
+ System.out.println("===== multi-query empty filters =====");
+
+ try {
+ myPig.setBatchOn();
+ myPig.registerQuery("a = load 'file:test/org/apache/pig/test/data/passwd' " +
+ "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);");
+ myPig.registerQuery("b = filter a by uid>0;");
+ myPig.registerQuery("c = filter a by uid>5;");
+ myPig.registerQuery("d = filter c by uid<10;");
+ myPig.registerQuery("store b into '/tmp/output1';");
+ myPig.registerQuery("store b into '/tmp/output2';");
+ myPig.registerQuery("store b into '/tmp/output3';");
+
+ LogicalPlan lp = checkLogicalPlan(1, 3, 10);
+ PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 10);
+ MROperPlan mp = checkMRPlan(pp, 1, 1, 1);
+
+ MapReduceOper mo = mp.getRoots().get(0);
+
+ checkPhysicalPlan(mo.mapPlan, 1, 1, 4);
+ PhysicalOperator leaf = mo.mapPlan.getLeaves().get(0);
+
+ Assert.assertTrue(leaf instanceof POSplit);
+
+ POSplit split = (POSplit)leaf;
+
+ int i = 0;
+ for (PhysicalPlan p: split.getPlans()) {
+ checkPhysicalPlan(p, 1, 1, 1);
+ ++i;
+ }
+
+ Assert.assertEquals(i,3);
+
+ myPig.executeBatch();
+ myPig.discardBatch();
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+
+ }
+
+ @Test
+ public void testUnnecessaryStoreRemoval() {
+ System.out.println("===== multi-query unnecessary stores =====");
+
+ try {
+ myPig.setBatchOn();
+ myPig.registerQuery("a = load 'file:test/org/apache/pig/test/data/passwd' " +
+ "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);");
+ myPig.registerQuery("b = group a by uname;");
+ myPig.registerQuery("store b into '/tmp/output1';");
+ myPig.registerQuery("store b into '/tmp/output2';");
+ myPig.registerQuery("c = load '/tmp/output1';");
+ myPig.registerQuery("d = group c by $0;");
+ myPig.registerQuery("e = store d into '/tmp/output3';");
+ myPig.registerQuery("f = load '/tmp/output2';");
+ myPig.registerQuery("g = group f by $0;");
+ myPig.registerQuery("store g into '/tmp/output4';");
+
+ LogicalPlan lp = checkLogicalPlan(1, 4, 14);
+ PhysicalPlan pp = checkPhysicalPlan(lp, 1, 4, 20);
+ MROperPlan mp = checkMRPlan(pp, 1, 2, 3);
+
+ MapReduceOper mo1 = mp.getRoots().get(0);
+ MapReduceOper mo2 = mp.getLeaves().get(0);
+ MapReduceOper mo3 = mp.getLeaves().get(1);
+
+ checkPhysicalPlan(mo1.mapPlan, 1, 1, 3);
+ checkPhysicalPlan(mo1.reducePlan, 1, 1, 2);
+ PhysicalOperator leaf = mo1.reducePlan.getLeaves().get(0);
+
+ Assert.assertTrue(leaf instanceof POSplit);
+
+ POSplit split = (POSplit)leaf;
+
+ int i = 0;
+ for (PhysicalPlan p: split.getPlans()) {
+ checkPhysicalPlan(p, 1, 1, 1);
+ ++i;
+ }
+
+ Assert.assertEquals(i,2);
+
+ checkPhysicalPlan(mo2.mapPlan, 1, 1, 2);
+ checkPhysicalPlan(mo2.reducePlan, 1, 1, 2);
+ leaf = mo2.reducePlan.getLeaves().get(0);
+
+ Assert.assertTrue(leaf instanceof POStore);
+
+ checkPhysicalPlan(mo3.mapPlan, 1, 1, 2);
+ checkPhysicalPlan(mo3.reducePlan, 1, 1, 2);
+ leaf = mo3.reducePlan.getLeaves().get(0);
+
+ Assert.assertTrue(leaf instanceof POStore);
+
+ myPig.executeBatch();
+ myPig.discardBatch();
+
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output1"));
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output2"));
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output3"));
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output4"));
+
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testUnnecessaryStoreRemovalCollapseSplit() {
+ System.out.println("===== multi-query unnecessary stores collapse split =====");
+
+ try {
+ myPig.setBatchOn();
+ myPig.registerQuery("a = load 'file:test/org/apache/pig/test/data/passwd' " +
+ "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);");
+ myPig.registerQuery("b = group a by uname;");
+ myPig.registerQuery("store b into '/tmp/output1';");
+ myPig.registerQuery("c = load '/tmp/output1';");
+ myPig.registerQuery("d = group c by $0;");
+ myPig.registerQuery("e = store d into '/tmp/output2';");
+
+ LogicalPlan lp = checkLogicalPlan(1, 2, 9);
+ PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 13);
+ MROperPlan mp = checkMRPlan(pp, 1, 1, 2);
+
+ MapReduceOper mo1 = mp.getRoots().get(0);
+ MapReduceOper mo2 = mp.getLeaves().get(0);
+
+ checkPhysicalPlan(mo1.mapPlan, 1, 1, 3);
+ checkPhysicalPlan(mo1.reducePlan, 1, 1, 2);
+ PhysicalOperator leaf = mo1.reducePlan.getLeaves().get(0);
+
+ Assert.assertTrue(leaf instanceof POStore);
+
+ checkPhysicalPlan(mo2.mapPlan, 1, 1, 2);
+ checkPhysicalPlan(mo2.reducePlan, 1, 1, 2);
+ leaf = mo2.reducePlan.getLeaves().get(0);
+
+ Assert.assertTrue(leaf instanceof POStore);
+
+ myPig.executeBatch();
+ myPig.discardBatch();
+
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output1"));
+ Assert.assertTrue(myPig.getPigContext().getDfs().isContainer("/tmp/output2"));
+
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ }
+ }
+
// --------------------------------------------------------------------------
// Helper methods
@@ -1028,6 +1195,25 @@
return lp;
}
+ private void checkPhysicalPlan(PhysicalPlan pp, int expectedRoots,
+ int expectedLeaves, int expectedSize) throws IOException {
+
+ System.out.println("===== check physical plan =====");
+
+ showPlanOperators(pp);
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ pp.explain(out);
+
+ System.out.println("===== Display Physical Plan =====");
+ System.out.println(out.toString());
+
+ Assert.assertEquals(expectedRoots, pp.getRoots().size());
+ Assert.assertEquals(expectedLeaves, pp.getLeaves().size());
+ Assert.assertEquals(expectedSize, pp.size());
+
+ }
+
private PhysicalPlan checkPhysicalPlan(LogicalPlan lp, int expectedRoots,
int expectedLeaves, int expectedSize) throws IOException {
@@ -1097,6 +1283,7 @@
FileLocalizer.delete("/tmp/output1", myPig.getPigContext());
FileLocalizer.delete("/tmp/output2", myPig.getPigContext());
FileLocalizer.delete("/tmp/output3", myPig.getPigContext());
+ FileLocalizer.delete("/tmp/output4", myPig.getPigContext());
} catch (IOException e) {
e.printStackTrace();
Assert.fail();
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQueryLocal.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQueryLocal.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestMultiQueryLocal.java Thu Apr 2 01:36:25 2009
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Properties;
import junit.framework.Assert;
import junit.framework.TestCase;
@@ -39,8 +40,10 @@
import org.apache.pig.impl.logicalLayer.LogicalPlan;
import org.apache.pig.impl.plan.Operator;
import org.apache.pig.impl.plan.OperatorPlan;
+import org.apache.pig.impl.PigContext;
import org.apache.pig.tools.grunt.GruntParser;
import org.apache.pig.impl.util.LogUtils;
+import org.apache.pig.ExecType;
import org.apache.pig.tools.pigscript.parser.ParseException;
import org.junit.After;
import org.junit.Before;
@@ -52,7 +55,9 @@
@Before
public void setUp() throws Exception {
- myPig = new PigServer("local");
+ PigContext context = new PigContext(ExecType.LOCAL, new Properties());
+ context.getProperties().setProperty("opt.multiquery", ""+true);
+ myPig = new PigServer(context);
}
@After
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestPigScriptParser.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestPigScriptParser.java?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestPigScriptParser.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestPigScriptParser.java Thu Apr 2 01:36:25 2009
@@ -46,7 +46,9 @@
Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
Map<OperatorKey, LogicalOperator> opTable = new HashMap<OperatorKey, LogicalOperator>() ;
Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>() ;
+ Map<String, String> fileNameMap = new HashMap<String, String>();
PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties()) ;
+ pigContext.connect();
String tempFile = this.prepareTempFile() ;
@@ -56,49 +58,51 @@
// Initial statement
String query = String.format("A = LOAD '%s' ;", Util.encodeEscape(tempFile)) ;
ByteArrayInputStream in = new ByteArrayInputStream(query.getBytes());
- QueryParser parser = new QueryParser(in, pigContext, "scope", aliases, opTable, aliasOp) ;
+ QueryParser parser = new QueryParser(in, pigContext, "scope", aliases, opTable, aliasOp, fileNameMap) ;
LogicalPlan lp = parser.Parse() ;
}
{
// Normal condition
String query = "B1 = filter A by $0 eq 'This is a test string' ;" ;
- checkParsedConstContent(aliases, opTable, pigContext, aliasOp,
+ checkParsedConstContent(aliases, opTable, pigContext, aliasOp, fileNameMap,
query, "This is a test string") ;
}
{
// single-quote condition
String query = "B2 = filter A by $0 eq 'This is a test \\'string' ;" ;
- checkParsedConstContent(aliases, opTable, pigContext, aliasOp,
+ checkParsedConstContent(aliases, opTable, pigContext, aliasOp, fileNameMap,
query, "This is a test 'string") ;
}
{
// newline condition
String query = "B3 = filter A by $0 eq 'This is a test \\nstring' ;" ;
- checkParsedConstContent(aliases, opTable, pigContext, aliasOp,
+ checkParsedConstContent(aliases, opTable, pigContext, aliasOp, fileNameMap,
query, "This is a test \nstring") ;
}
{
// Unicode
String query = "B4 = filter A by $0 eq 'This is a test \\uD30C\\uC774string' ;" ;
- checkParsedConstContent(aliases, opTable, pigContext, aliasOp,
+ checkParsedConstContent(aliases, opTable, pigContext, aliasOp, fileNameMap,
query, "This is a test \uD30C\uC774string") ;
}
}
private void checkParsedConstContent(Map<LogicalOperator, LogicalPlan> aliases,
- Map<OperatorKey, LogicalOperator> opTable,
- PigContext pigContext,
- Map<String, LogicalOperator> aliasOp,
- String query,
- String expectedContent)
+ Map<OperatorKey, LogicalOperator> opTable,
+ PigContext pigContext,
+ Map<String, LogicalOperator> aliasOp,
+ Map<String, String> fileNameMap,
+ String query,
+ String expectedContent)
throws Exception {
// Run the parser
ByteArrayInputStream in = new ByteArrayInputStream(query.getBytes());
- QueryParser parser = new QueryParser(in, pigContext, "scope", aliases, opTable, aliasOp) ;
+ QueryParser parser = new QueryParser(in, pigContext, "scope", aliases, opTable, aliasOp,
+ fileNameMap) ;
LogicalPlan lp = parser.Parse() ;
// Digging down the tree
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestStore.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestStore.java?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestStore.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestStore.java Thu Apr 2 01:36:25 2009
@@ -19,6 +19,10 @@
import static org.junit.Assert.assertEquals;
+import java.util.*;
+
+import org.apache.pig.ExecType;
+
import java.io.File;
import java.io.BufferedReader;
import java.io.FileReader;
@@ -27,6 +31,9 @@
import java.util.List;
import java.util.Random;
+import junit.framework.Assert;
+import org.apache.pig.impl.plan.OperatorKey;
+
import org.apache.pig.FuncSpec;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.builtin.PigStorage;
@@ -37,6 +44,7 @@
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.PigServer;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.backend.local.executionengine.LocalPigLauncher;
@@ -47,9 +55,18 @@
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
import org.apache.pig.test.utils.GenPhyOp;
import org.apache.pig.test.utils.GenRandomData;
import org.apache.pig.test.utils.TestHelper;
+import org.apache.pig.impl.logicalLayer.LOStore;
+import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.impl.logicalLayer.LogicalPlan;
+import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
+import org.apache.pig.backend.datastorage.ContainerDescriptor;
+import org.apache.pig.backend.datastorage.DataStorage;
+import org.apache.pig.backend.datastorage.DataStorageException;
+import org.apache.pig.backend.datastorage.ElementDescriptor;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -61,6 +78,7 @@
static MiniCluster cluster = MiniCluster.buildCluster();
PigContext pc;
POProject proj;
+ PigServer pig;
@Before
public void setUp() throws Exception {
@@ -68,8 +86,11 @@
fSpec = new FileSpec("file:/tmp/storeTest.txt",
new FuncSpec(PigStorage.class.getName(), new String[]{":"}));
st.setSFile(fSpec);
- pc = new PigContext();
- pc.connect();
+
+ FileLocalizer.deleteTempFiles();
+ pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ pc = pig.getPigContext();
+
st.setStoreImpl(new LocalPOStoreImpl(pc));
proj = GenPhyOp.exprProject();
@@ -184,4 +205,80 @@
FileLocalizer.delete(fSpec.getFileName(), pc);
}
+ @Test
+ public void testStoreRemoteRel() throws Exception {
+ checkStorePath("test","/tmp/test");
+ }
+
+ @Test
+ public void testStoreRemoteAbs() throws Exception {
+ checkStorePath("/tmp/test","/tmp/test");
+ }
+
+ @Test
+ public void testStoreRemoteRelScheme() throws Exception {
+ checkStorePath("hdfs:test","/tmp/test");
+ }
+
+ @Test
+ public void testStoreRemoteAbsScheme() throws Exception {
+ checkStorePath("hdfs:/tmp/test","/tmp/test");
+ }
+
+ @Test
+ public void testStoreRemoteAbsAuth() throws Exception {
+ checkStorePath("hdfs://localhost:9000/test","/test");
+ }
+
+ @Test
+ public void testStoreRemoteNormalize() throws Exception {
+ checkStorePath("/tmp/foo/../././","/tmp");
+ }
+
+ private void checkStorePath(String orig, String expected) throws Exception {
+ checkStorePath(orig, expected, false);
+ }
+
+ private void checkStorePath(String orig, String expected, boolean isTmp) throws Exception {
+ pc.getProperties().setProperty("opt.multiquery",""+true);
+
+ DataStorage dfs = pc.getDfs();
+ dfs.setActiveContainer(dfs.asContainer("/tmp"));
+ Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
+ Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
+ Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>();
+ Map<String, String> fileNameMap = new HashMap<String, String>();
+
+ LogicalPlanBuilder builder = new LogicalPlanBuilder(pc);
+
+ String query = "a = load 'foo';";
+ LogicalPlan lp = builder.parse("Test-Store",
+ query,
+ aliases,
+ logicalOpTable,
+ aliasOp,
+ fileNameMap);
+ query = "store a into '"+orig+"';";
+ lp = builder.parse("Test-Store",
+ query,
+ aliases,
+ logicalOpTable,
+ aliasOp,
+ fileNameMap);
+
+ Assert.assertTrue(lp.size()>1);
+ LogicalOperator op = lp.getLeaves().get(0);
+
+ Assert.assertTrue(op instanceof LOStore);
+ LOStore store = (LOStore)op;
+
+ String p = store.getOutputFile().getFileName();
+ p = p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/","/");
+
+ if (isTmp) {
+ Assert.assertTrue(p.matches("/tmp.*"));
+ } else {
+ Assert.assertEquals(p, expected);
+ }
+ }
}
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestUnion.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestUnion.java?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestUnion.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/TestUnion.java Thu Apr 2 01:36:25 2009
@@ -35,6 +35,7 @@
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileSpec;
+import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
@@ -193,6 +194,7 @@
Util.createInputFile(cluster, "a.txt", new String[] {"1\t2\t3", "4\t5\t6"});
Util.createInputFile(cluster, "b.txt", new String[] {"7\t8\t9", "1\t200\t300"});
Util.createInputFile(cluster, "c.txt", new String[] {"1\t20\t30"});
+ FileLocalizer.deleteTempFiles();
PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
pig.registerQuery("a = load 'a.txt' ;");
pig.registerQuery("b = load 'b.txt';");
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC1.gld Thu Apr 2 01:36:25 2009
@@ -6,10 +6,10 @@
| |---Package[tuple]{Unknown} - --6430355428631435461
| Local Rearrange[tuple]{Unknown}(false) - -8729990799028586395
| |
-| |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -77
+| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -77
|
|---MapReduce(-1) - -75:
- | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -76
+ | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -76
| |
| |---Package[tuple]{Unknown} - -4721502244557927278
| Local Rearrange[tuple]{Unknown}(false) - --7681398237172009051
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC10.gld Thu Apr 2 01:36:25 2009
@@ -4,12 +4,12 @@
| |
| |---Union[tuple] - -4061122832660258194
| |
-| |---Load(/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -12
+| |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -12
| |
-| |---Load(/tmp/temp-1456742965/tmp774375955:org.apache.pig.builtin.BinStorage) - -14
+| |---Load(file:/tmp/temp-1456742965/tmp774375955:org.apache.pig.builtin.BinStorage) - -14
|
|---MapReduce(-1) - -2:
-| | Store(/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -13
+| | Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -13
| | |
| | |---Filter[tuple] - --171729478481529687
| | |
@@ -27,15 +27,15 @@
| | |---Load(DummyFil:DummyLdr) - --838807233869503381
|
|---MapReduce(-1) - -6:
- | Store(/tmp/temp-1456742965/tmp774375955:org.apache.pig.builtin.BinStorage) - -15
+ | Store(file:/tmp/temp-1456742965/tmp774375955:org.apache.pig.builtin.BinStorage) - -15
| |
| |---Package[tuple]{Unknown} - -990040854696137546
- | Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -7
+ | Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -7
|
- | Load(/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -9
+ | Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -9
|
|---MapReduce(30) - -4:
- | | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -8
+ | | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -8
| | |
| | |---Local Rearrange[tuple]{Unknown}(false) - --5623550231721294978
| | |
@@ -45,7 +45,7 @@
| | |---Load(DummyFil:DummyLdr) - -6748240903696823165
|
|---MapReduce(20) - -5:
- | Store(/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -10
+ | Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -10
| |
| |---Local Rearrange[tuple]{Unknown}(false) - -5679595123645092366
| |
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC11.gld Thu Apr 2 01:36:25 2009
@@ -4,12 +4,12 @@
| |
| |---Union[tuple] - -913150185705910016
| |
-| |---Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -27
+| |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -27
| |
-| |---Load(/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -29
+| |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -29
|
|---MapReduce(-1) - -18:
-| | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -28
+| | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -28
| | |
| | |---Filter[tuple] - -3055962796670893316
| | |
@@ -27,12 +27,12 @@
| | |---Load(DummyFil:DummyLdr) - --1115934782004129477
|
|---MapReduce(-1) - -22:
- | Store(/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -30
+ | Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -30
| |
| |---Package[tuple]{Unknown} - -6108697957385222190
| Union[tuple] - -23
| |
- | |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -24
+ | |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -24
| |
| |---Local Rearrange[tuple]{Unknown}(false) - -7490898804471997380
| |
@@ -41,7 +41,7 @@
| |---Load(DummyFil:DummyLdr) - --8248982303554009
|
|---MapReduce(-1) - -20:
- | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -25
+ | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -25
| |
| |---Package[tuple]{Unknown} - --8723304958939002625
| Local Rearrange[tuple]{Unknown}(false) - -689137294940608050
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC12.gld Thu Apr 2 01:36:25 2009
@@ -8,16 +8,16 @@
| | |
| | |---Filter[tuple] - -7391599663069134339
| | |
-| | |---Load(/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -33
+| | |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -33
| |
| |---Local Rearrange[tuple]{Unknown}(false) - -8760996681222683693
| |
| |---Filter[tuple] - -956528893337238225
| |
-| |---Load(/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -35
+| |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -35
|
|---MapReduce(-1) - -31:
Reduce Plan Empty
- | Store(/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -32
+ | Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -32
| |
| |---Load(DummyFil:DummyLdr) - --5840807574562740596
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC13.gld Thu Apr 2 01:36:25 2009
@@ -6,14 +6,14 @@
| |
| |---Filter[tuple] - -1351115285619019432
| | |
-| | |---Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -48
+| | |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -48
| |
| |---Filter[tuple] - -6869916393445222618
| |
-| |---Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -50
+| |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -50
|
|---MapReduce(-1) - -45:
- | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -47
+ | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -47
| |
| |---Package[tuple]{Unknown} - --1596982214453270401
| Union[tuple] - -46
@@ -22,16 +22,16 @@
| | |
| | |---Filter[tuple] - --3509976263222494134
| | |
- | | |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -41
+ | | |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -41
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --4071762447953696591
| |
| |---Filter[tuple] - -7123161826157220327
| |
- | |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -43
+ | |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -43
|
|---MapReduce(-1) - -39:
Reduce Plan Empty
- | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -40
+ | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -40
| |
| |---Load(DummyFil:DummyLdr) - --5941754500044119766
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC14.gld Thu Apr 2 01:36:25 2009
@@ -6,14 +6,14 @@
| |
| |---Filter[tuple] - --7091196022628815072
| | |
-| | |---Load(/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -70
+| | |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -70
| |
| |---Filter[tuple] - -5984722006991913573
| |
-| |---Load(/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -72
+| |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -72
|
|---MapReduce(40) - -67:
- | Store(/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -69
+ | Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -69
| |
| |---Package[tuple]{Unknown} - -8067897495211048679
| Union[tuple] - -68
@@ -22,44 +22,44 @@
| | |
| | |---Filter[tuple] - -551977918718096509
| | |
- | | |---Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -58
+ | | |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -58
| |
| |---Local Rearrange[tuple]{Unknown}(false) - -6411189422016119355
| | |
| | |---Filter[tuple] - -2056844330428347996
| | |
- | | |---Load(/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -63
+ | | |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -63
| |
| |---Local Rearrange[tuple]{Unknown}(false) - -7225341208466719305
| |
| |---Filter[tuple] - --7987250529196129674
| |
- | |---Load(/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -65
+ | |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -65
|
|---MapReduce(10) - -56:
| Reduce Plan Empty
- | | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -57
+ | | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -57
| | |
| | |---Filter[tuple] - -2351257193210757136
| | |
- | | |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -55
+ | | |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -55
| |
| |---MapReduce(-1) - -53:
| Reduce Plan Empty
- | | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -54
+ | | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -54
| | |
| | |---Load(DummyFil:DummyLdr) - -728209667098611077
|
|---MapReduce(20) - -61:
Reduce Plan Empty
- | Store(/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -62
+ | Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -62
| |
| |---Filter[tuple] - --1369132270304461038
| |
- | |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -60
+ | |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -60
|
|---MapReduce(-1) - -53:
Reduce Plan Empty
- | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -54
+ | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -54
| |
| |---Load(DummyFil:DummyLdr) - -728209667098611077
\ No newline at end of file
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC15.gld Thu Apr 2 01:36:25 2009
@@ -10,10 +10,10 @@
| |---Package[tuple]{Unknown} - --885269774183211482
| Local Rearrange[tuple]{Unknown}(false) - --776319888013965510
| |
-| |---Load(/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -152
+| |---Load(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -152
|
|---MapReduce(1,AVG) - -150:
- | Store(/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -151
+ | Store(file:/tmp/temp-1456742965/tmp-586682361:org.apache.pig.builtin.BinStorage) - -151
| |
| |---New For Each(false)[tuple] - -7965768498188214494
| | |
@@ -24,10 +24,10 @@
| |---Package[tuple]{Unknown} - --7335024873119453444
| Local Rearrange[tuple]{Unknown}(false) - -4589138876054328603
| |
- | |---Load(/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -149
+ | |---Load(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -149
|
|---MapReduce(20,TestMRCompiler$WeirdComparator,COUNT,SUM) - -142:
- | Store(/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -148
+ | Store(file:/tmp/temp-1456742965/tmp-26634357:org.apache.pig.builtin.BinStorage) - -148
| |
| |---New For Each(false,false)[tuple] - --4248200967728536480
| | |
@@ -48,10 +48,10 @@
| | |
| | Project[tuple][*] - -143
| |
- | |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -141
+ | |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -141
|
|---MapReduce(1,TestMRCompiler$WeirdComparator) - -130:
- | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -140
+ | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -140
| |
| |---New For Each(false,false)[tuple] - -139
| | |
@@ -72,11 +72,11 @@
| | |
| | Project[tuple][*] - -131
| |
- | |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.builtin.RandomSampleLoader) - -129
+ | |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.impl.builtin.RandomSampleLoader) - -129
|
|---MapReduce(-1) - -127:
Reduce Plan Empty
- | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -128
+ | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -128
| |
| |---Filter[tuple] - -9193928674704944093
| |
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC16.gld Thu Apr 2 01:36:25 2009
@@ -10,18 +10,18 @@
| | |
| | Project[tuple][*] - -163
| |
-| |---Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -166
+| |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -166
|
|---MapReduce(-1) - -162:
- | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -165
+ | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -165
| |
| |---Package[tuple]{Unknown} - -2082992246427879202
| Local Rearrange[tuple]{Unknown}(false) - --3148893660811981376
| |
- | |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -161
+ | |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -161
|
|---MapReduce(-1) - -154:
- | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -160
+ | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -160
| |
| |---New For Each(true)[bag] - -159
| | |
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC2.gld Thu Apr 2 01:36:25 2009
@@ -4,12 +4,12 @@
| |
| |---Union[tuple] - -2910267342587130351
| |
-| |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -82
+| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -82
| |
-| |---Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -84
+| |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -84
|
|---MapReduce(-1) - -79:
-| | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -83
+| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -83
| | |
| | |---Package[tuple]{Unknown} - -370083002387034672
| | Local Rearrange[tuple]{Unknown}(false) - -6305091296204163466
@@ -17,7 +17,7 @@
| | |---Load(DummyFil:DummyLdr) - -1236980712440527800
|
|---MapReduce(-1) - -80:
- | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -85
+ | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -85
| |
| |---Package[tuple]{Unknown} - -7123718043666602037
| Local Rearrange[tuple]{Unknown}(false) - --6380139708906526553
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC3.gld Thu Apr 2 01:36:25 2009
@@ -8,12 +8,12 @@
| | |
| | |---Load(DummyFil:DummyLdr) - -5515425171581967372
| |
-| |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -90
+| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -90
| |
-| |---Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -92
+| |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -92
|
|---MapReduce(-1) - -87:
-| | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -91
+| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -91
| | |
| | |---Package[tuple]{Unknown} - --6061281703859425960
| | Local Rearrange[tuple]{Unknown}(false) - --1158897849427419546
@@ -21,7 +21,7 @@
| | |---Load(DummyFil:DummyLdr) - -3709512757404691843
|
|---MapReduce(-1) - -88:
- | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -93
+ | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -93
| |
| |---Package[tuple]{Unknown} - --2057425961601007773
| Local Rearrange[tuple]{Unknown}(false) - --8361563503038121624
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC6.gld Thu Apr 2 01:36:25 2009
@@ -2,12 +2,12 @@
| Store(DummyFil:DummyLdr) - --3624738819541751735
| |
| |---Package[tuple]{Unknown} - --4776862487779398064
-| Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -100
+| Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -100
|
-| Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -102
+| Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -102
|
|---MapReduce(-1) - -97:
-| | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -101
+| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -101
| | |
| | |---Local Rearrange[tuple]{Unknown}(false) - --3671186819751472084
| | |
@@ -17,7 +17,7 @@
| | |---Load(DummyFil:DummyLdr) - --8240903279973257769
|
|---MapReduce(-1) - -98:
- | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -103
+ | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -103
| |
| |---Local Rearrange[tuple]{Unknown}(false) - --3008031181120208412
| |
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC7.gld Thu Apr 2 01:36:25 2009
@@ -8,12 +8,12 @@
| | |
| | |---Load(DummyFil:DummyLdr) - -7219092146411102494
| |
-| |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -109
+| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -109
| |
-| |---Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -111
+| |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -111
|
|---MapReduce(-1) - -104:
-| | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -110
+| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -110
| | |
| | |---Local Rearrange[tuple]{Unknown}(false) - --1036765666284482668
| | |
@@ -23,7 +23,7 @@
| | |---Load(DummyFil:DummyLdr) - -1761200043915140459
|
|---MapReduce(-1) - -106:
- | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -112
+ | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -112
| |
| |---Local Rearrange[tuple]{Unknown}(false) - -7127277012934370361
| |
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/data/GoldenFiles/MRC8.gld Thu Apr 2 01:36:25 2009
@@ -4,9 +4,9 @@
| |---Package[tuple]{Unknown} - -2997708366016271267
| Union[tuple] - -118
| |
-| |---Load(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -119
+| |---Load(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -119
| |
-| |---Load(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -121
+| |---Load(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -121
| |
| |---Filter[tuple] - -3615014757987062850
| | |
@@ -17,7 +17,7 @@
| |---Load(DummyFil:DummyLdr) - --1195368474144577564
|
|---MapReduce(-1) - -113:
-| | Store(/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -120
+| | Store(file:/tmp/temp-1456742965/tmp-1456742965:org.apache.pig.builtin.BinStorage) - -120
| | |
| | |---Local Rearrange[tuple]{Unknown}(false) - --3347320220748403847
| | |
@@ -27,7 +27,7 @@
| | |---Load(DummyFil:DummyLdr) - --2465563180022385815
|
|---MapReduce(-1) - -115:
- | Store(/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -122
+ | Store(file:/tmp/temp-1456742965/tmp2077335416:org.apache.pig.builtin.BinStorage) - -122
| |
| |---Local Rearrange[tuple]{Unknown}(false) - -6693653468874366867
| |
Modified: hadoop/pig/branches/multiquery/test/org/apache/pig/test/utils/LogicalPlanTester.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/multiquery/test/org/apache/pig/test/utils/LogicalPlanTester.java?rev=761145&r1=761144&r2=761145&view=diff
==============================================================================
--- hadoop/pig/branches/multiquery/test/org/apache/pig/test/utils/LogicalPlanTester.java (original)
+++ hadoop/pig/branches/multiquery/test/org/apache/pig/test/utils/LogicalPlanTester.java Thu Apr 2 01:36:25 2009
@@ -54,6 +54,7 @@
private Map<LogicalOperator, LogicalPlan> aliases = null ;
private Map<OperatorKey, LogicalOperator> logicalOpTable = null ;
private Map<String, LogicalOperator> aliasOp = null ;
+ private Map<String, String> fileNameMap = null ;
public LogicalPlanTester() {
reset() ;
@@ -66,6 +67,7 @@
aliases = new HashMap<LogicalOperator, LogicalPlan>();
logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
aliasOp = new HashMap<String, LogicalOperator>();
+ fileNameMap = new HashMap<String, String>();
NodeIdGenerator.reset(SCOPE);
}
@@ -206,7 +208,8 @@
query,
aliases,
logicalOpTable,
- aliasOp);
+ aliasOp,
+ fileNameMap);
List<LogicalOperator> roots = lp.getRoots();