You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by xu...@apache.org on 2011/05/03 18:53:42 UTC
svn commit: r1099121 [7/8] - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/impl/ src/org/apache/pig/impl/logicalLayer/ src/o...
Modified: pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java?rev=1099121&r1=1099120&r2=1099121&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java Tue May 3 16:53:40 2011
@@ -26,6 +26,8 @@ import java.util.List;
import java.util.Properties;
import java.util.Set;
+import junit.framework.AssertionFailedError;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputFormat;
@@ -35,16 +37,14 @@ import org.apache.pig.ExecType;
import org.apache.pig.Expression;
import org.apache.pig.LoadFunc;
import org.apache.pig.LoadMetadata;
+import org.apache.pig.PigServer;
import org.apache.pig.ResourceSchema;
import org.apache.pig.ResourceStatistics;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
import org.apache.pig.data.Tuple;
-import org.apache.pig.newplan.logical.LogicalPlanMigrationVistor;
import org.apache.pig.newplan.logical.expression.LogicalExpression;
-import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
import org.apache.pig.newplan.logical.relational.LOFilter;
-import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.rules.PartitionFilterOptimizer;
import org.apache.pig.newplan.logical.rules.LoadTypeCastInserter;
@@ -53,18 +53,12 @@ import org.apache.pig.newplan.OperatorPl
import org.apache.pig.newplan.OperatorSubPlan;
import org.apache.pig.newplan.PColFilterExtractor;
import org.apache.pig.newplan.optimizer.PlanOptimizer;
-import org.apache.pig.newplan.optimizer.PlanTransformListener;
import org.apache.pig.newplan.optimizer.Rule;
import org.apache.pig.newplan.optimizer.Transformer;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.PlanSetter;
import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.util.LogUtils;
-import org.apache.pig.test.TestPartitionFilterOptimization.TestLoader;
-import org.apache.pig.test.utils.LogicalPlanTester;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -77,12 +71,10 @@ import org.junit.Test;
*/
public class TestPartitionFilterPushDown {
static PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
- static LogicalPlanTester lpTester;
-
+ String query = "a = load 'foo' as (srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int);";
+
@BeforeClass
public static void setup() throws Exception {
- lpTester = new LogicalPlanTester(pc);
- lpTester.buildPlan("a = load 'foo' as (srcid, mrkt, dstid, name, age);");
}
@AfterClass
@@ -92,66 +84,61 @@ public class TestPartitionFilterPushDown
/**
* test case where there is a single expression on partition columns in
* the filter expression along with an expression on non partition column
- * @throws IOException
+ * @throws Exception
*/
@Test
- public void testSimpleMixed() throws IOException {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by srcid == 10 and name == 'foo';");
- test(lp, Arrays.asList("srcid"), "(srcid == 10)", "(name == 'foo')");
+ public void testSimpleMixed() throws Exception {
+ String q = query + "b = filter a by srcid == 10 and name == 'foo';" + "store b into 'out';";
+ test(q, Arrays.asList("srcid"), "(srcid == 10)", "(name == 'foo')");
}
-
+
/**
* test case where filter does not contain any condition on partition cols
* @throws Exception
*/
@Test
public void testNoPartFilter() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by age == 20 and name == 'foo';");
- test(lp, Arrays.asList("srcid"), null,
- "((age == 20) and (name == 'foo'))");
+ String q = query + "b = filter a by age == 20 and name == 'foo';" + "store b into 'out';";
+ test(q, Arrays.asList("srcid"), null,
+ "((age == 20) and (name == 'foo'))");
}
-
+
/**
* test case where filter only contains condition on partition cols
* @throws Exception
*/
@Test
public void testOnlyPartFilter1() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by srcid > 20 and mrkt == 'us';");
- test(lp, Arrays.asList("srcid", "mrkt"),
- "((srcid > 20) and (mrkt == 'us'))", null);
-
+ String q = query + "b = filter a by srcid > 20 and mrkt == 'us';" + "store b into 'out';";
+ test(q, Arrays.asList("srcid", "mrkt"),
+ "((srcid > 20) and (mrkt == 'us'))", null);
+
}
-
+
/**
* test case where filter only contains condition on partition cols
* @throws Exception
*/
@Test
public void testOnlyPartFilter2() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by mrkt == 'us';");
- test(lp, Arrays.asList("srcid", "mrkt"),
- "(mrkt == 'us')", null);
-
+ String q = query + "b = filter a by mrkt == 'us';" + "store b into 'out';";
+ test(q, Arrays.asList("srcid", "mrkt"),
+ "(mrkt == 'us')", null);
+
}
-
+
/**
* test case where filter only contains condition on partition cols
* @throws Exception
*/
@Test
public void testOnlyPartFilter3() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by srcid == 20 or mrkt == 'us';");
- test(lp, Arrays.asList("srcid", "mrkt"),
- "((srcid == 20) or (mrkt == 'us'))", null);
-
+ String q = query + "b = filter a by srcid == 20 or mrkt == 'us';" + "store b into 'out';";
+ test(q, Arrays.asList("srcid", "mrkt"),
+ "((srcid == 20) or (mrkt == 'us'))", null);
+
}
-
+
/**
* test case where filter has both conditions on partition cols and non
* partition cols and the filter condition will be split to extract the
@@ -159,16 +146,15 @@ public class TestPartitionFilterPushDown
*/
@Test
public void testMixed1() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by " +
- "(age < 20 and mrkt == 'us') and (srcid == 10 and " +
- "name == 'foo');");
- test(lp, Arrays.asList("srcid", "mrkt"),
+ String q = query + "b = filter a by " +
+ "(age < 20 and mrkt == 'us') and (srcid == 10 and " +
+ "name == 'foo');" + "store b into 'out';";
+ test(q, Arrays.asList("srcid", "mrkt"),
"((mrkt == 'us') and (srcid == 10))",
- "((age < 20) and (name == 'foo'))");
+ "((age < 20) and (name == 'foo'))");
}
-
-
+
+
/**
* test case where filter has both conditions on partition cols and non
* partition cols and the filter condition will be split to extract the
@@ -176,15 +162,14 @@ public class TestPartitionFilterPushDown
*/
@Test
public void testMixed2() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by " +
- "(age >= 20 and mrkt == 'us') and (srcid == 10 and " +
- "dstid == 15);");
- test(lp, Arrays.asList("srcid", "dstid", "mrkt"),
+ String q = query + "b = filter a by " +
+ "(age >= 20 and mrkt == 'us') and (srcid == 10 and " +
+ "dstid == 15);" + "store b into 'out';";
+ test(q, Arrays.asList("srcid", "dstid", "mrkt"),
"((mrkt == 'us') and ((srcid == 10) and (dstid == 15)))",
- "(age >= 20)");
+ "(age >= 20)");
}
-
+
/**
* test case where filter has both conditions on partition cols and non
* partition cols and the filter condition will be split to extract the
@@ -192,13 +177,12 @@ public class TestPartitionFilterPushDown
*/
@Test
public void testMixed3() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by " +
- "age >= 20 and mrkt == 'us' and srcid == 10;");
- test(lp, Arrays.asList("srcid", "dstid", "mrkt"),
+ String q = query + "b = filter a by " +
+ "age >= 20 and mrkt == 'us' and srcid == 10;" + "store b into 'out';";
+ test(q, Arrays.asList("srcid", "dstid", "mrkt"),
"((mrkt == 'us') and (srcid == 10))", "(age >= 20)");
}
-
+
/**
* test case where filter has both conditions on partition cols and non
* partition cols and the filter condition will be split to extract the
@@ -207,15 +191,14 @@ public class TestPartitionFilterPushDown
*/
@Test
public void testMixed4() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by " +
- "age >= 20 and mrkt == 'us' and name == 'foo' and " +
- "srcid == dstid;");
- test(lp, Arrays.asList("srcid", "dstid", "mrkt"),
+ String q = query + "b = filter a by " +
+ "age >= 20 and mrkt == 'us' and name == 'foo' and " +
+ "srcid == dstid;" + "store b into 'out';";
+ test(q, Arrays.asList("srcid", "dstid", "mrkt"),
"((mrkt == 'us') and (srcid == dstid))",
- "((age >= 20) and (name == 'foo'))");
+ "((age >= 20) and (name == 'foo'))");
}
-
+
/**
* test case where filter has both conditions on partition cols and non
* partition cols and the filter condition will be split to extract the
@@ -225,15 +208,14 @@ public class TestPartitionFilterPushDown
*/
@Test
public void testMixed5() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by " +
- "(srcid == 10 or mrkt == 'us') and name == 'foo' and " +
- "dstid == 30;");
- test(lp, Arrays.asList("srcid", "dstid", "mrkt"),
+ String q = query + "b = filter a by " +
+ "(srcid == 10 or mrkt == 'us') and name == 'foo' and " +
+ "dstid == 30;" + "store b into 'out';";
+ test(q, Arrays.asList("srcid", "dstid", "mrkt"),
"(((srcid == 10) or (mrkt == 'us')) and (dstid == 30))",
- "(name == 'foo')");
+ "(name == 'foo')");
}
-
+
/**
* test case where filter has both conditions on partition cols and non
* partition cols and the filter condition will be split to extract the
@@ -243,23 +225,20 @@ public class TestPartitionFilterPushDown
*/
@Test
public void testMixed6() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by " +
- "dstid == 30 and (srcid == 10 or mrkt == 'us') and name == 'foo';");
- test(lp, Arrays.asList("srcid", "dstid", "mrkt"),
+ String q = query + "b = filter a by " +
+ "dstid == 30 and (srcid == 10 or mrkt == 'us') and name == 'foo';" + "store b into 'out';";
+ test(q, Arrays.asList("srcid", "dstid", "mrkt"),
"((dstid == 30) and ((srcid == 10) or (mrkt == 'us')))",
- "(name == 'foo')");
+ "(name == 'foo')");
}
@Test
public void test7() throws Exception {
- LogicalPlanTester tester = new LogicalPlanTester(pc);
- tester.buildPlan("a = load 'foo' using " + TestLoader.class.getName()
- + "('srcid, mrkt, dstid, name, age', 'srcid, name');");
- org.apache.pig.impl.logicalLayer.LogicalPlan lp = tester
- .buildPlan("b = filter a by "
- + "(srcid < 20 and age < 30) or (name == 'foo' and age > 40);");
- LogicalPlan plan = migratePlan(lp);
+ String query = "a = load 'foo' using " + TestLoader.class.getName() +
+ "('srcid, mrkt, dstid, name, age', 'srcid, name');" +
+ "b = filter a by (srcid < 20 and age < 30) or (name == 'foo' and age > 40);" +
+ "store b into 'output';";
+ LogicalPlan plan = buildPlan(new PigServer(pc), query);
Rule rule = new PartitionFilterOptimizer("test");
List<OperatorPlan> matches = rule.match(plan);
@@ -279,13 +258,11 @@ public class TestPartitionFilterPushDown
@Test
public void test8() throws Exception {
- LogicalPlanTester tester = new LogicalPlanTester(pc);
- tester.buildPlan("a = load 'foo' using " + TestLoader.class.getName()
- + "('srcid, mrkt, dstid, name, age', 'srcid,name');");
- org.apache.pig.impl.logicalLayer.LogicalPlan lp = tester
- .buildPlan("b = filter a by "
- + "(srcid < 20) or (name == 'foo');");
- LogicalPlan plan = migratePlan(lp);
+ String query = "a = load 'foo' using " + TestLoader.class.getName() +
+ "('srcid, mrkt, dstid, name, age', 'srcid,name');" +
+ "b = filter a by (srcid < 20) or (name == 'foo');" +
+ "store b into 'output';";
+ LogicalPlan plan = Util.buildLp(new PigServer(pc), query);
Rule rule = new PartitionFilterOptimizer("test");
List<OperatorPlan> matches = rule.match(plan);
@@ -298,7 +275,7 @@ public class TestPartitionFilterPushDown
}
OperatorSubPlan newPlan = (OperatorSubPlan)transformer.reportChanges();
- Assert.assertTrue(newPlan.getBasePlan().size() == 1);
+ Assert.assertTrue(newPlan.getBasePlan().size() == 3);
}
}
@@ -312,100 +289,102 @@ public class TestPartitionFilterPushDown
*/
@Test
public void testMixedArith() throws Exception {
- org.apache.pig.impl.logicalLayer.LogicalPlan lp =
- lpTester.buildPlan("b = filter a by " +
- "mrkt == 'us' and srcid * 10 == 150 + 20 and age != 15;");
- test(lp, Arrays.asList("srcid", "dstid", "mrkt"),
+ String q = query + "b = filter a by " +
+ "mrkt == 'us' and srcid * 10 == 150 + 20 and age != 15;" + "store b into 'out';";
+ test(q, Arrays.asList("srcid", "dstid", "mrkt"),
"((mrkt == 'us') and ((srcid * 10) == (150 + 20)))",
- "(age != 15)");
+ "(age != 15)");
}
-
+
@Test
public void testNegPColConditionWithNonPCol() throws Exception {
// use of partition column condition and non partition column in
// same condition should fail
- org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester.buildPlan("b = filter a by " +
- "srcid > age;");
- negativeTest(lp, Arrays.asList("srcid"));
- lp = lpTester.buildPlan("b = filter a by " +
- "srcid + age == 20;");
- negativeTest(lp, Arrays.asList("srcid"));
+ String q = query + "b = filter a by " +
+ "srcid > age;" + "store b into 'out';";
+ negativeTest(q, Arrays.asList("srcid"), 1111);
+ q = query + "b = filter a by " +
+ "srcid + age == 20;" + "store b into 'out';";
+ negativeTest(q, Arrays.asList("srcid"), 1111);
// OR of partition column condition and non partiton col condition
// should fail
- lp = lpTester.buildPlan("b = filter a by " +
- "srcid > 10 or name == 'foo';");
- negativeTest(lp, Arrays.asList("srcid"));
+ q = query + "b = filter a by " +
+ "srcid > 10 or name == 'foo';" +
+ "store b into 'out';";
+ negativeTest(q, Arrays.asList("srcid"), 1111);
}
-
+
@Test
public void testNegPColInWrongPlaces() throws Exception {
+ int expectedErrCode = 1112;
- org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester.buildPlan("b = filter a by " +
- "(srcid > 10 and name == 'foo') or dstid == 10;");
- negativeTest(lp, Arrays.asList("srcid", "dstid"));
-
- lp = lpTester.buildPlan("b = filter a by " +
- "CONCAT(mrkt, '_10') == 'US_10' and age == 20;");
- negativeTest(lp, Arrays.asList("srcid", "dstid", "mrkt"));
-
- lp = lpTester.buildPlan("b = filter a by " +
- "mrkt matches '.*us.*' and age < 15;");
- negativeTest(lp, Arrays.asList("srcid", "dstid", "mrkt"));
-
- lp = lpTester.buildPlan("b = filter a by " +
- "(int)mrkt == 10 and name matches '.*foo.*';");
- negativeTest(lp, Arrays.asList("srcid", "dstid", "mrkt"));
-
- lp = lpTester.buildPlan("b = filter a by " +
- "(mrkt == 'us' ? age : age + 10) == 40 and name matches '.*foo.*';");
- negativeTest(lp, Arrays.asList("srcid", "dstid", "mrkt"));
-
- lp = lpTester.buildPlan("b = filter a by " +
- "(mrkt is null) and name matches '.*foo.*';");
- negativeTest(lp, Arrays.asList("srcid", "dstid", "mrkt"));
-
- lp = lpTester.buildPlan("b = filter a by " +
- "(mrkt is not null) and name matches '.*foo.*';");
- negativeTest(lp, Arrays.asList("srcid", "dstid", "mrkt"));
- }
-
- @Test
- public void testNegPColInWrongPlaces2() throws Exception {
-
- LogicalPlanTester tester = new LogicalPlanTester(pc);
- tester.buildPlan("a = load 'foo' using " + TestLoader.class.getName()
- + "('srcid, mrkt, dstid, name, age', 'srcid,dstid,mrkt');");
-
- org.apache.pig.impl.logicalLayer.LogicalPlan lp = tester
- .buildPlan("b = filter a by "
- + "(srcid > 10 and name == 'foo') or dstid == 10;");
- negativeTest(lp);
-
- lp = tester.buildPlan("b = filter a by " +
- "CONCAT(mrkt, '_10') == 'US_10' and age == 20;");
- negativeTest(lp);
-
- lp = tester.buildPlan("b = filter a by " +
- "mrkt matches '.*us.*' and age < 15;");
- negativeTest(lp);
-
- lp = tester.buildPlan("b = filter a by " +
- "(int)mrkt == 10 and name matches '.*foo.*';");
- negativeTest(lp);
-
- lp = tester.buildPlan("b = filter a by " +
- "(mrkt == 'us' ? age : age + 10) == 40 and name matches '.*foo.*';");
- negativeTest(lp);
-
- lp = tester.buildPlan("b = filter a by " +
- "(mrkt is null) and name matches '.*foo.*';");
- negativeTest(lp);
-
- lp = tester.buildPlan("b = filter a by " +
- "(mrkt is not null) and name matches '.*foo.*';");
- negativeTest(lp);
- }
+ String q = query + "b = filter a by " +
+ "(srcid > 10 and name == 'foo') or dstid == 10;" + "store b into 'out';";
+ negativeTest(q, Arrays.asList("srcid", "dstid"), expectedErrCode);
+
+ expectedErrCode = 1110;
+ q = query + "b = filter a by " +
+ "CONCAT(mrkt, '_10') == 'US_10' and age == 20;" + "store b into 'out';";
+ negativeTest(q, Arrays.asList("srcid", "dstid", "mrkt"), expectedErrCode);
+
+ q = query + "b = filter a by " +
+ "mrkt matches '.*us.*' and age < 15;" + "store b into 'out';";
+ negativeTest(q, Arrays.asList("srcid", "dstid", "mrkt"), expectedErrCode);
+
+ q = query + "b = filter a by " +
+ "(int)mrkt == 10 and name matches '.*foo.*';" + "store b into 'out';";
+ negativeTest(q, Arrays.asList("srcid", "dstid", "mrkt"),expectedErrCode);
+
+ q = query + "b = filter a by " +
+ "(mrkt == 'us' ? age : age + 10) == 40 and name matches '.*foo.*';" + "store b into 'out';";
+ negativeTest(q, Arrays.asList("srcid", "dstid", "mrkt"), expectedErrCode);
+
+ q = query + "b = filter a by " +
+ "(mrkt is null) and name matches '.*foo.*';" + "store b into 'out';";
+ negativeTest(q, Arrays.asList("srcid", "dstid", "mrkt"), expectedErrCode);
+
+ q = query + "b = filter a by " +
+ "(mrkt is not null) and name matches '.*foo.*';" + "store b into 'out';";
+ negativeTest(q, Arrays.asList("srcid", "dstid", "mrkt"), expectedErrCode);
+ }
+
+// @Test
+// public void testNegPColInWrongPlaces2() throws Exception {
+//
+// LogicalPlanTester tester = new LogicalPlanTester(pc);
+// tester.buildPlan("a = load 'foo' using " + TestLoader.class.getName()
+// + "('srcid, mrkt, dstid, name, age', 'srcid,dstid,mrkt');");
+//
+// org.apache.pig.impl.logicalLayer.LogicalPlan lp = tester
+// .buildPlan("b = filter a by "
+// + "(srcid > 10 and name == 'foo') or dstid == 10;");
+// negativeTest(lp);
+//
+// lp = tester.buildPlan("b = filter a by " +
+// "CONCAT(mrkt, '_10') == 'US_10' and age == 20;");
+// negativeTest(lp);
+//
+// lp = tester.buildPlan("b = filter a by " +
+// "mrkt matches '.*us.*' and age < 15;");
+// negativeTest(lp);
+//
+// lp = tester.buildPlan("b = filter a by " +
+// "(int)mrkt == 10 and name matches '.*foo.*';");
+// negativeTest(lp);
+//
+// lp = tester.buildPlan("b = filter a by " +
+// "(mrkt == 'us' ? age : age + 10) == 40 and name matches '.*foo.*';");
+// negativeTest(lp);
+//
+// lp = tester.buildPlan("b = filter a by " +
+// "(mrkt is null) and name matches '.*foo.*';");
+// negativeTest(lp);
+//
+// lp = tester.buildPlan("b = filter a by " +
+// "(mrkt is not null) and name matches '.*foo.*';");
+// negativeTest(lp);
+// }
/**
@@ -417,19 +396,21 @@ public class TestPartitionFilterPushDown
@Test
public void testColNameMapping1() throws Exception {
TestLoader.partFilter = null;
- lpTester.buildPlan("a = load 'foo' using "
+ String q = "a = load 'foo' using "
+ TestLoader.class.getName() +
"('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
- "'srcid,mrkt') as (f1, f2, f3, f4, f5);");
- org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester.buildPlan("b = filter a by " +
- "(f5 >= 20 and f2 == 'us') and (f1 == 10 and f3 == 15);");
-
- LogicalPlan newLogicalPlan = migrateAndOptimizePlan( lp );
-
+ "'srcid,mrkt') as (f1, f2, f3, f4, f5);" +
+ "b = filter a by " +
+ "(f5 >= 20 and f2 == 'us') and (f1 == 10 and f3 == 15);" +
+ "store b into 'out';";
+
+ LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );
+
Assert.assertEquals("checking partition filter:",
- "((mrkt == 'us') and (srcid == 10))",
- TestLoader.partFilter.toString());
- LOFilter filter = (LOFilter)newLogicalPlan.getSinks().get(0);
+ "((mrkt == 'us') and (srcid == 10))",
+ TestLoader.partFilter.toString());
+ Operator op = newLogicalPlan.getSinks().get(0);
+ LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
@@ -439,15 +420,16 @@ public class TestPartitionFilterPushDown
Assert.assertEquals("checking trimmed filter expression:",
"((f5 >= 20) and (f3 == 15))", actual);
}
-
- private LogicalPlan migrateAndOptimizePlan(org.apache.pig.impl.logicalLayer.LogicalPlan plan) throws IOException {
- LogicalPlan newLogicalPlan = migratePlan( plan );
+
+ private LogicalPlan migrateAndOptimizePlan(String query) throws Exception {
+ PigServer pigServer = new PigServer( pc );
+ LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
PlanOptimizer optimizer = new MyPlanOptimizer( newLogicalPlan, 3 );
optimizer.optimize();
return newLogicalPlan;
}
-
-
+
+
/**
* Test that pig sends correct partition column names in setPartitionFilter
* when the user has a schema in the load statement which renames partition
@@ -459,19 +441,21 @@ public class TestPartitionFilterPushDown
@Test
public void testColNameMapping2() throws Exception {
TestLoader.partFilter = null;
- lpTester.buildPlan("a = load 'foo' using "
+ String q = "a = load 'foo' using "
+ TestLoader.class.getName() +
"('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
- "'srcid') as (f1, f2, f3, f4, f5);");
- org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester.buildPlan("b = filter a by " +
- "f5 >= 20 and f2 == 'us' and f3 == 15;");
+ "'srcid') as (f1, f2, f3, f4, f5);" +
+ "b = filter a by " +
+ "f5 >= 20 and f2 == 'us' and f3 == 15;" +
+ "store b into 'out';";
- LogicalPlan newLogicalPlan = migrateAndOptimizePlan( lp );
+ LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );
Assert.assertEquals("checking partition filter:",
- null,
- TestLoader.partFilter);
- LOFilter filter = (LOFilter) newLogicalPlan.getSinks().get(0);
+ null,
+ TestLoader.partFilter);
+ Operator op = newLogicalPlan.getSinks().get(0);
+ LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
@@ -482,7 +466,7 @@ public class TestPartitionFilterPushDown
Assert.assertEquals("checking trimmed filter expression:",
"(((f5 >= 20) and (f2 == 'us')) and (f3 == 15))", actual);
}
-
+
/**
* Test that pig sends correct partition column names in setPartitionFilter
* when the user has a schema in the load statement which renames partition
@@ -493,29 +477,28 @@ public class TestPartitionFilterPushDown
@Test
public void testColNameMapping3() throws Exception {
TestLoader.partFilter = null;
- lpTester.buildPlan("a = load 'foo' using "
+ String query = "a = load 'foo' using "
+ TestLoader.class.getName() +
"('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
- "'srcid,mrkt,dstid,age') as (f1, f2, f3, f4, f5);");
- org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester.buildPlan("b = filter a by " +
- "(f5 >= 20 or f2 == 'us') and (f1 == 10 and f3 == 15);");
+ "'srcid,mrkt,dstid,age') as (f1, f2, f3, f4, f5);" +
+ "b = filter a by " +
+ "(f5 >= 20 or f2 == 'us') and (f1 == 10 and f3 == 15);" +
+ "store b into 'out';";
- LogicalPlan newLogicalPlan = migrateAndOptimizePlan( lp );
+ LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Assert.assertEquals("checking partition filter:",
- "(((age >= 20) or (mrkt == 'us')) and ((srcid == 10) and " +
- "(dstid == 15)))",
- TestLoader.partFilter.toString());
+ "(((age >= 20) or (mrkt == 'us')) and ((srcid == 10) and " +
+ "(dstid == 15)))",
+ TestLoader.partFilter.toString());
Iterator<Operator> it = newLogicalPlan.getOperators();
- Assert.assertTrue("Checking that filter has been removed since it contained" +
- " only conditions on partition cols:",
- (it.next() instanceof LOLoad));
- Assert.assertFalse("Checking that filter has been removed since it contained" +
- " only conditions on partition cols:",
- it.hasNext());
-
+ while( it.hasNext() ) {
+ Assert.assertFalse("Checking that filter has been removed since it contained" +
+ " only conditions on partition cols:",
+ (it.next() instanceof LOFilter));
+ }
}
-
+
/**
* Test that pig sends correct partition column names in setPartitionFilter
* when the user has a schema in the load statement which renames partition
@@ -527,19 +510,20 @@ public class TestPartitionFilterPushDown
@Test
public void testColNameMapping4() throws Exception {
TestLoader.partFilter = null;
- lpTester.buildPlan("a = load 'foo' using "
+ String q = "a = load 'foo' using "
+ TestLoader.class.getName() +
"('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
- "'srcid,mrkt') as (f1, f2, f3);");
- org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester.buildPlan("b = filter a by " +
- "(age >= 20 and f2 == 'us') and (f1 == 10 and f3 == 15);");
+ "'srcid,mrkt') as (f1:int, f2:chararray, f3:int, name:chararray, age:int);" +
+ "b = filter a by " +
+ "(age >= 20 and f2 == 'us') and (f1 == 10 and f3 == 15);" + "store b into 'out';";
+
+ LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );
- LogicalPlan newLogicalPlan = migrateAndOptimizePlan( lp );
-
Assert.assertEquals("checking partition filter:",
- "((mrkt == 'us') and (srcid == 10))",
- TestLoader.partFilter.toString());
- LOFilter filter = (LOFilter) newLogicalPlan.getSinks().get(0);
+ "((mrkt == 'us') and (srcid == 10))",
+ TestLoader.partFilter.toString());
+ Operator op = newLogicalPlan.getSinks().get(0);
+ LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
@@ -549,7 +533,7 @@ public class TestPartitionFilterPushDown
Assert.assertEquals("checking trimmed filter expression:",
"((age >= 20) and (f3 == 15))", actual);
}
-
+
/**
* Test PIG-1267
* @throws Exception
@@ -557,109 +541,87 @@ public class TestPartitionFilterPushDown
@Test
public void testColNameMapping5() throws Exception {
TestLoader.partFilter = null;
- lpTester.buildPlan("a = load 'foo' using "
+ String q = "a = load 'foo' using "
+ TestLoader.class.getName() +
"('mrkt:chararray, a1:chararray, a2:chararray, srcid:int, bcookie:chararray', " +
- "'srcid');");
- lpTester.buildPlan("b = load 'bar' using "
- + TestLoader.class.getName() +
- "('dstid:int, b1:int, b2:int, srcid:int, bcookie:chararray, mrkt:chararray'," +
- "'srcid');");
- lpTester.buildPlan("a1 = filter a by srcid == 10;");
- lpTester.buildPlan("b1 = filter b by srcid == 20;");
- lpTester.buildPlan("c = join a1 by bcookie, b1 by bcookie;");
- org.apache.pig.impl.logicalLayer.LogicalPlan lp = lpTester
- .buildPlan("d = foreach c generate $4 as bcookie:chararray, " +
- "$5 as dstid:int, $0 as mrkt:chararray;");
-
- new PlanSetter(lp).visit();
-
- LogicalPlan newLogicalPlan = migrateAndOptimizePlan( lp );
-
+ "'srcid');" +
+ "b = load 'bar' using "
+ + TestLoader.class.getName() +
+ "('dstid:int, b1:int, b2:int, srcid:int, bcookie:chararray, mrkt:chararray'," +
+ "'srcid');" +
+ "a1 = filter a by srcid == 10;" +
+ "b1 = filter b by srcid == 20;"+
+ "c = join a1 by bcookie, b1 by bcookie;" +
+ "d = foreach c generate $4 as bcookie:chararray, " +
+ "$5 as dstid:int, $0 as mrkt:chararray;" +
+ "store d into 'out';";
+
+ LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );
+
String partFilter = TestLoader.partFilter.toString();
Assert.assertTrue( "(srcid == 20)".equals( partFilter ) || "(srcid == 10)".equals( partFilter ) );
-
+
int counter = 0;
Iterator<Operator> iter = newLogicalPlan.getOperators();
while (iter.hasNext()) {
- Assert.assertTrue(!(iter.next() instanceof LOFilter));
+ Assert.assertTrue(!(iter.next() instanceof LOFilter));
counter++;
}
- Assert.assertEquals(counter, 6);
+ Assert.assertEquals(counter, 7);
}
-
+
//// helper methods ///////
-
- private PColFilterExtractor test(org.apache.pig.impl.logicalLayer.LogicalPlan lp, List<String> partitionCols,
+
+ private PColFilterExtractor test(String query, List<String> partitionCols,
String expPartFilterString, String expFilterString)
- throws IOException {
- LogicalPlan newLogicalPlan = migratePlan( lp );
- LOFilter filter = (LOFilter)newLogicalPlan.getSinks().get(0);
+ throws Exception {
+ PigServer pigServer = new PigServer( pc );
+ LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
+ Operator op = newLogicalPlan.getSinks().get(0);
+ LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
PColFilterExtractor pColExtractor = new PColFilterExtractor(
filter.getFilterPlan(), partitionCols);
pColExtractor.visit();
-
+
if(expPartFilterString == null) {
- Assert.assertEquals("Checking partition column filter:", null,
+ Assert.assertEquals("Checking partition column filter:", null,
pColExtractor.getPColCondition());
} else {
- Assert.assertEquals("Checking partition column filter:",
+ Assert.assertEquals("Checking partition column filter:",
expPartFilterString.toLowerCase(),
pColExtractor.getPColCondition().toString().toLowerCase());
}
-
+
if(expFilterString == null) {
- Assert.assertTrue("Check that filter can be removed:",
+ Assert.assertTrue("Check that filter can be removed:",
pColExtractor.isFilterRemovable());
} else {
String actual = pColExtractor.getExpression(
- (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
- toString().toLowerCase();
+ (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
+ toString().toLowerCase();
Assert.assertEquals("checking trimmed filter expression:", expFilterString,
actual);
}
return pColExtractor;
}
-
- private void negativeTest(org.apache.pig.impl.logicalLayer.LogicalPlan lp,
- List<String> partitionCols) throws FrontendException {
- LogicalPlan newLogicalPlan = migratePlan(lp);
- LOFilter filter = (LOFilter) newLogicalPlan.getSinks().get(0);
-
- LogicalExpressionPlan plan = filter.getFilterPlan();
-
- PColFilterExtractor pColExtractor = new PColFilterExtractor(plan,
- partitionCols);
- pColExtractor.visit();
-
- Assert.assertTrue(!pColExtractor.canPushDown());
- }
-
- private void negativeTest(org.apache.pig.impl.logicalLayer.LogicalPlan lp)
- throws FrontendException {
- LogicalPlan plan = migratePlan(lp);
-
- LOFilter filter = (LOFilter) plan.getSinks().get(0);
-
- Rule rule = new PartitionFilterOptimizer("test");
- List<OperatorPlan> matches = rule.match(plan);
- if (matches != null) {
- Transformer transformer = rule.getNewTransformer();
- for (OperatorPlan m : matches) {
- if (transformer.check(m)) {
- transformer.transform(m);
- }
- }
- OperatorSubPlan newPlan = (OperatorSubPlan)transformer.reportChanges();
-
- LOFilter filter2 = (LOFilter) newPlan.getBasePlan().getSinks().get(0);
-
- Assert.assertTrue(filter.isEqual(filter2));
- Assert.assertTrue(newPlan.getBasePlan().isEqual(plan));
+ private void negativeTest(String query, List<String> partitionCols,
+ int expectedErrorCode) throws Exception {
+ PigServer pigServer = new PigServer( pc );
+ LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
+ Operator op = newLogicalPlan.getSinks().get(0);
+ LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
+ PColFilterExtractor pColExtractor = new PColFilterExtractor(
+ filter.getFilterPlan(), partitionCols);
+ try {
+ pColExtractor.visit();
+ } catch(Exception e) {
+ Assert.assertEquals("Checking if exception has right error code",
+ expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
+ return;
}
}
-
+
/**
* this loader is only used to test that parition column filters are given
* in the manner expected in terms of column names - hence it does not
@@ -670,13 +632,13 @@ public class TestPartitionFilterPushDown
Schema schema;
String[] partCols;
static Expression partFilter = null;
-
+
public TestLoader(String schemaString, String commaSepPartitionCols)
throws ParseException {
schema = Util.getSchemaFromString(schemaString);
partCols = commaSepPartitionCols.split(",");
}
-
+
@Override
public InputFormat getInputFormat() throws IOException {
return null;
@@ -689,7 +651,7 @@ public class TestPartitionFilterPushDown
@Override
public void prepareToRead(RecordReader reader, PigSplit split)
- throws IOException {
+ throws IOException {
}
@Override
@@ -698,13 +660,13 @@ public class TestPartitionFilterPushDown
@Override
public String[] getPartitionKeys(String location, Job job)
- throws IOException {
+ throws IOException {
return partCols;
}
@Override
public ResourceSchema getSchema(String location, Job job)
- throws IOException {
+ throws IOException {
return new ResourceSchema(schema);
}
@@ -716,40 +678,42 @@ public class TestPartitionFilterPushDown
@Override
public void setPartitionFilter(Expression partitionFilter)
- throws IOException {
+ throws IOException {
partFilter = partitionFilter;
}
-
+
}
public class MyPlanOptimizer extends LogicalPlanOptimizer {
protected MyPlanOptimizer(OperatorPlan p, int iterations) {
super( p, iterations, new HashSet<String>() );
}
-
+
protected List<Set<Rule>> buildRuleSets() {
List<Set<Rule>> ls = new ArrayList<Set<Rule>>();
-
+
Set<Rule> s = new HashSet<Rule>();
// add split filter rule
Rule r = new PartitionFilterOptimizer("PartitionFilterPushDown");
s = new HashSet<Rule>();
s.add(r);
ls.add(s);
-
+
r = new LoadTypeCastInserter( "LoadTypeCastInserter" );
s = new HashSet<Rule>();
s.add(r);
ls.add(s);
return ls;
}
- }
+ }
- private LogicalPlan migratePlan(org.apache.pig.impl.logicalLayer.LogicalPlan lp) throws VisitorException{
- LogicalPlanMigrationVistor visitor = new LogicalPlanMigrationVistor(lp);
- visitor.visit();
- LogicalPlan newPlan = visitor.getNewLogicalPlan();
- return newPlan;
+ // Helper Functions
+ public LogicalPlan buildPlan(PigServer pigServer, String query) throws Exception {
+ try {
+ return Util.buildLp(pigServer, query);
+ } catch(Throwable t) {
+ throw new AssertionFailedError(t.getMessage());
+ }
}
-
+
}
Modified: pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java?rev=1099121&r1=1099120&r2=1099121&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigScriptParser.java Tue May 3 16:53:40 2011
@@ -36,8 +36,13 @@ import org.apache.pig.ExecType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.plan.OperatorKey;
-import org.apache.pig.impl.logicalLayer.* ;
import org.apache.pig.impl.logicalLayer.parser.* ;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.expression.ConstantExpression;
+import org.apache.pig.newplan.logical.expression.LogicalExpression;
+import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
+import org.apache.pig.newplan.logical.relational.LOFilter;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
public class TestPigScriptParser extends TestCase {
@@ -45,37 +50,30 @@ public class TestPigScriptParser extends
public void testParserWithEscapeCharacters() throws Exception {
// All the needed variables
- Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
- Map<OperatorKey, LogicalOperator> opTable = new HashMap<OperatorKey, LogicalOperator>() ;
- Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>() ;
- Map<String, String> fileNameMap = new HashMap<String, String>();
PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties()) ;
+ PigServer pigServer = new PigServer( pigContext );
pigContext.connect();
String tempFile = this.prepareTempFile() ;
+ String query = String.format("A = LOAD '%s' ;", Util.encodeEscape(tempFile)) ;
// Start the real parsing job
{
-
// Initial statement
- String query = String.format("A = LOAD '%s' ;", Util.encodeEscape(tempFile)) ;
- ByteArrayInputStream in = new ByteArrayInputStream(query.getBytes());
- QueryParser parser = new QueryParser(in, pigContext, "scope", aliases, opTable, aliasOp, fileNameMap) ;
- LogicalPlan lp = parser.Parse() ;
+ Util.buildLp(pigServer, query);
}
{
// Normal condition
- String query = "B1 = filter A by $0 eq 'This is a test string' ;" ;
- checkParsedConstContent(aliases, opTable, pigContext, aliasOp, fileNameMap,
- query, "This is a test string") ;
+ String q = query + "B = filter A by $0 eq 'This is a test string' ;" ;
+ checkParsedConstContent(pigServer, pigContext, q, "This is a test string") ;
}
{
// single-quote condition
- String query = "B2 = filter A by $0 eq 'This is a test \\'string' ;" ;
- checkParsedConstContent(aliases, opTable, pigContext, aliasOp, fileNameMap,
- query, "This is a test 'string") ;
+ String q = query + "B = filter A by $0 eq 'This is a test \\'string' ;" ;
+ checkParsedConstContent(pigServer, pigContext,
+ q, "This is a test 'string") ;
}
{
@@ -84,23 +82,23 @@ public class TestPigScriptParser extends
// since this is to be represented in a Java String, we escape each backslash with one more
// backslash - hence 4. In a pig script in a file, this would be
// \\.string
- String query = "B2 = filter A by $0 eq 'This is a test \\\\.string' ;" ;
- checkParsedConstContent(aliases, opTable, pigContext, aliasOp, fileNameMap,
- query, "This is a test \\.string") ;
+ String q = query + "B = filter A by $0 eq 'This is a test \\\\.string' ;" ;
+ checkParsedConstContent(pigServer, pigContext,
+ q, "This is a test \\.string") ;
}
{
// newline condition
- String query = "B3 = filter A by $0 eq 'This is a test \\nstring' ;" ;
- checkParsedConstContent(aliases, opTable, pigContext, aliasOp, fileNameMap,
- query, "This is a test \nstring") ;
+ String q = query + "B = filter A by $0 eq 'This is a test \\nstring' ;" ;
+ checkParsedConstContent(pigServer, pigContext,
+ q, "This is a test \nstring") ;
}
{
// Unicode
- String query = "B4 = filter A by $0 eq 'This is a test \\uD30C\\uC774string' ;" ;
- checkParsedConstContent(aliases, opTable, pigContext, aliasOp, fileNameMap,
- query, "This is a test \uD30C\uC774string") ;
+ String q = query + "B = filter A by $0 eq 'This is a test \\uD30C\\uC774string' ;" ;
+ checkParsedConstContent(pigServer, pigContext,
+ q, "This is a test \uD30C\uC774string") ;
}
}
@@ -135,38 +133,31 @@ public class TestPigScriptParser extends
}
}
- private void checkParsedConstContent(Map<LogicalOperator, LogicalPlan> aliases,
- Map<OperatorKey, LogicalOperator> opTable,
+ private void checkParsedConstContent(PigServer pigServer,
PigContext pigContext,
- Map<String, LogicalOperator> aliasOp,
- Map<String, String> fileNameMap,
String query,
String expectedContent)
throws Exception {
- // Run the parser
- ByteArrayInputStream in = new ByteArrayInputStream(query.getBytes());
- QueryParser parser = new QueryParser(in, pigContext, "scope", aliases, opTable, aliasOp,
- fileNameMap) ;
- LogicalPlan lp = parser.Parse() ;
-
+ pigContext.connect();
+ LogicalPlan lp = Util.buildLp(pigServer, query + "store B into 'output';");
// Digging down the tree
- LogicalOperator root = lp.getRoots().get(0) ;
- LogicalOperator filter = lp.getSuccessors(root).get(0);
- LogicalPlan comparisonPlan = ((LOFilter)filter).getComparisonPlan();
- List<LogicalOperator> comparisonPlanRoots = comparisonPlan.getRoots();
- LogicalOperator compRootOne = comparisonPlanRoots.get(0);
- LogicalOperator compRootTwo = comparisonPlanRoots.get(1);
+ Operator load = lp.getSources().get(0);
+ Operator filter = lp.getSuccessors( load ).get(0);
+ LogicalExpressionPlan comparisonPlan = ((LOFilter)filter).getFilterPlan();
+ List<Operator> comparisonPlanRoots = comparisonPlan.getSinks();
+ Operator compRootOne = comparisonPlanRoots.get(0);
+ Operator compRootTwo = comparisonPlanRoots.get(1);
// Here is the actual check logic
- if (compRootOne instanceof LOConst) {
+ if (compRootOne instanceof ConstantExpression) {
assertTrue("Must be equal",
- ((String)((LOConst)compRootOne).getValue()).equals(expectedContent)) ;
+ ((String)((ConstantExpression)compRootOne).getValue()).equals(expectedContent)) ;
}
// If not left, it must be right.
else {
assertTrue("Must be equal",
- ((String)((LOConst)compRootTwo).getValue()).equals(expectedContent)) ;
+ ((String)((ConstantExpression)compRootTwo).getValue()).equals(expectedContent)) ;
}
}
Modified: pig/trunk/test/org/apache/pig/test/TestSampleOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSampleOptimizer.java?rev=1099121&r1=1099120&r2=1099121&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSampleOptimizer.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSampleOptimizer.java Tue May 3 16:53:40 2011
@@ -37,8 +37,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.test.utils.LogicalPlanTester;
+
import org.junit.AfterClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -48,10 +47,12 @@ import org.junit.runners.JUnit4;
public class TestSampleOptimizer {
static PigContext pc;
+ static PigServer pigServer;
static{
pc = new PigContext(ExecType.MAPREDUCE,MiniCluster.buildCluster().getProperties());
try {
pc.connect();
+ pigServer = new PigServer( pc );
} catch (ExecException e) {
e.printStackTrace();
}
@@ -64,12 +65,9 @@ public class TestSampleOptimizer {
@Test
public void testOptimizerFired() throws Exception{
-
- LogicalPlanTester planTester = new LogicalPlanTester() ;
- planTester.buildPlan(" A = load 'input' using PigStorage('\t');");
- planTester.buildPlan(" B = order A by $0;");
- LogicalPlan lp = planTester.buildPlan("store B into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ String query = " A = load 'input' using PigStorage('\t');" +
+ " B = order A by $0;" + "store B into 'output';";
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
int count = 1;
@@ -111,13 +109,9 @@ public class TestSampleOptimizer {
@Test
public void testOptimizerNotFired() throws Exception{
-
- LogicalPlanTester planTester = new LogicalPlanTester() ;
- planTester.buildPlan(" A = load 'input' using PigStorage('\t');");
- planTester.buildPlan("B = group A by $0;");
- planTester.buildPlan(" C = order B by $0;");
- LogicalPlan lp = planTester.buildPlan("store C into 'output';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ String query = " A = load 'input' using PigStorage('\t');" + "B = group A by $0;" +
+ " C = order B by $0;" + "store C into 'output';";
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
int count = 1;
@@ -194,12 +188,11 @@ public class TestSampleOptimizer {
@Test
public void testPoissonSampleOptimizer() throws Exception {
- LogicalPlanTester planTester = new LogicalPlanTester() ;
- planTester.buildPlan(" A = load 'input' using PigStorage('\t');");
- planTester.buildPlan("B = load 'input' using PigStorage('\t');");
- planTester.buildPlan(" C = join A by $0, B by $0 using \"skewed\";");
- LogicalPlan lp = planTester.buildPlan("store C into 'output';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ String query = " A = load 'input' using PigStorage('\t');" +
+ "B = load 'input' using PigStorage('\t');" +
+ " C = join A by $0, B by $0 using 'skewed';" +
+ "store C into 'output';";
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
int count = 1;
@@ -226,12 +219,10 @@ public class TestSampleOptimizer {
@Test
public void testOrderByUDFSet() throws Exception {
- LogicalPlanTester planTester = new LogicalPlanTester() ;
- planTester.buildPlan("a = load 'input1' using BinStorage();");
- planTester.buildPlan("b = order a by $0;");
- LogicalPlan lp = planTester.buildPlan("store b into '/tmp';");
+ String query = "a = load 'input1' using BinStorage();" +
+ "b = order a by $0;" + "store b into 'output';";
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
int count = 1;
Modified: pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java?rev=1099121&r1=1099120&r2=1099121&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSecondarySort.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSecondarySort.java Tue May 3 16:53:40 2011
@@ -33,13 +33,9 @@ import org.apache.pig.backend.executione
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DefaultBagFactory;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
-import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.test.utils.LogicalPlanTester;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
@@ -73,37 +69,34 @@ public class TestSecondarySort extends T
pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
}
- @Test
- public void testDistinctOptimization1() throws Exception {
- // Limit in the foreach plan
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = LOAD 'input2' AS (b0, b1, b2);");
- planTester.buildPlan("C = cogroup A by a0, B by b0;");
- planTester.buildPlan("D = foreach C { E = limit A 10; F = E.a1; G = DISTINCT F; generate group, COUNT(G);};");
-
- LogicalPlan lp = planTester.buildPlan("store D into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
- MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
-
- SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
- so.visit();
-
- assertTrue(so.getNumMRUseSecondaryKey() == 1);
- assertTrue(so.getNumSortRemoved() == 0);
- assertTrue(so.getDistinctChanged() == 1);
- }
+// @Test // Currently failing due to PIG-2009
+// public void testDistinctOptimization1() throws Exception {
+// // Limit in the foreach plan
+// String query = ("A=LOAD 'input1' AS (a0, a1, a2);"+
+// "B = LOAD 'input2' AS (b0, b1, b2);" +
+// "C = cogroup A by a0, B by b0;" +
+// "D = foreach C { E = limit A 10; F = E.a1; G = DISTINCT F; generate group, COUNT(G);};" +
+// "store D into 'output';");
+// PhysicalPlan pp = Util.buildPp(pigServer, query);
+// MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
+//
+// SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
+// so.visit();
+//
+// assertEquals( 1, so.getNumMRUseSecondaryKey() );
+// assertTrue(so.getNumSortRemoved() == 0);
+// assertTrue(so.getDistinctChanged() == 1);
+// }
@Test
public void testDistinctOptimization2() throws Exception {
// Distinct on one entire input
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0;");
- planTester.buildPlan("C = foreach B { D = distinct A; generate group, D;};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);"+
+ "B = group A by $0;"+
+ "C = foreach B { D = distinct A; generate group, D;};"+
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -117,13 +110,12 @@ public class TestSecondarySort extends T
@Test
public void testDistinctOptimization3() throws Exception {
// Distinct on the prefix of main sort key
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0;");
- planTester.buildPlan("C = foreach B { D = A.a0; E = distinct D; generate group, E;};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);"+
+ "B = group A by $0;"+
+ "C = foreach B { D = A.a0; E = distinct D; generate group, E;};"+
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -137,13 +129,12 @@ public class TestSecondarySort extends T
@Test
public void testDistinctOptimization4() throws Exception {
// Distinct on secondary key again, should remove
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0;");
- planTester.buildPlan("C = foreach B { D = A.a1; E = distinct D; F = distinct E; generate group, F;};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);"+
+ "B = group A by $0;"+
+ "C = foreach B { D = A.a1; E = distinct D; F = distinct E; generate group, F;};"+
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -157,13 +148,12 @@ public class TestSecondarySort extends T
@Test
public void testDistinctOptimization5() throws Exception {
// Filter in foreach plan
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0;");
- planTester.buildPlan("C = foreach B { D = A.a1; E = distinct D; F = filter E by $0=='1'; generate group, F;};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
+ "B = group A by $0;" +
+ "C = foreach B { D = A.a1; E = distinct D; F = filter E by $0=='1'; generate group, F;};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -177,13 +167,12 @@ public class TestSecondarySort extends T
@Test
public void testDistinctOptimization6() throws Exception {
// group by * with no schema, and distinct key is not part of main key
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1';");
- planTester.buildPlan("B = group A by *;");
- planTester.buildPlan("C = foreach B { D = limit A 10; E = D.$1; F = DISTINCT E; generate group, COUNT(F);};");
+ String query = ("A=LOAD 'input1';" +
+ "B = group A by *;" +
+ "C = foreach B { D = limit A 10; E = D.$1; F = DISTINCT E; generate group, COUNT(F);};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -197,13 +186,12 @@ public class TestSecondarySort extends T
@Test
public void testDistinctOptimization7() throws Exception {
// group by * with no schema, distinct key is more specific than the main key
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1';");
- planTester.buildPlan("B = group A by *;");
- planTester.buildPlan("C = foreach B { D = limit A 10; E = D.$0; F = DISTINCT E; generate group, COUNT(F);};");
+ String query = ("A=LOAD 'input1';" +
+ "B = group A by *;" +
+ "C = foreach B { D = limit A 10; E = D.$0; F = DISTINCT E; generate group, COUNT(F);};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -217,13 +205,12 @@ public class TestSecondarySort extends T
@Test
public void testDistinctOptimization8() throws Exception {
// local arrange plan is an expression
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0+$1;");
- planTester.buildPlan("C = foreach B { D = limit A 10; E = D.$0; F = DISTINCT E; generate group, COUNT(F);};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
+ "B = group A by $0+$1;" +
+ "C = foreach B { D = limit A 10; E = D.$0; F = DISTINCT E; generate group, COUNT(F);};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -237,13 +224,12 @@ public class TestSecondarySort extends T
@Test
public void testDistinctOptimization9() throws Exception {
// local arrange plan is nested project
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' as (a:tuple(a0:int, a1:chararray));");
- planTester.buildPlan("B = group A by a.a1;");
- planTester.buildPlan("C = foreach B { D = A.a; E = DISTINCT D; generate group, COUNT(E);};");
+ String query = ("A=LOAD 'input1' as (a:tuple(a0:int, a1:chararray));" +
+ "B = group A by a.a1;" +
+ "C = foreach B { D = A.a; E = DISTINCT D; generate group, COUNT(E);};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -257,13 +243,12 @@ public class TestSecondarySort extends T
@Test
public void testSortOptimization1() throws Exception {
// Sort on something other than the main key
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0;");
- planTester.buildPlan("C = foreach B { D = limit A 10; E = order D by $1; generate group, E;};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
+ "B = group A by $0;" +
+ "C = foreach B { D = limit A 10; E = order D by $1; generate group, E;};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -277,13 +262,12 @@ public class TestSecondarySort extends T
@Test
public void testSortOptimization2() throws Exception {
// Sort on the prefix of the main key
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0;");
- planTester.buildPlan("C = foreach B { D = limit A 10; E = order D by $0; generate group, E;};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
+ "B = group A by $0;" +
+ "C = foreach B { D = limit A 10; E = order D by $0; generate group, E;};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -297,14 +281,12 @@ public class TestSecondarySort extends T
@Test
public void testSortOptimization3() throws Exception {
// Sort on the main key prefix / non main key prefix mixed
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0;");
- planTester
- .buildPlan("C = foreach B { D = limit A 10; E = order D by $1; F = order E by $0; generate group, F;};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
+ "B = group A by $0;" +
+ "C = foreach B { D = limit A 10; E = order D by $1; F = order E by $0; generate group, F;};"+
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -318,13 +300,12 @@ public class TestSecondarySort extends T
@Test
public void testSortOptimization4() throws Exception {
// Sort on the main key again
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0;");
- planTester.buildPlan("C = foreach B { D = limit A 10; E = order D by $0, $1, $2; generate group, E;};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
+ "B = group A by $0;" +
+ "C = foreach B { D = limit A 10; E = order D by $0, $1, $2; generate group, E;};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -338,14 +319,11 @@ public class TestSecondarySort extends T
@Test
public void testSortOptimization5() throws Exception {
// Sort on the two keys, we can only take off 1
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0;");
- planTester
- .buildPlan("C = foreach B { D = limit A 10; E = order D by $1; F = order E by $2; generate group, F;};");
-
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
+ "B = group A by $0;" +
+ "C = foreach B { D = limit A 10; E = order D by $1; F = order E by $2; generate group, F;};" +
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -359,13 +337,12 @@ public class TestSecondarySort extends T
@Test
public void testSortOptimization6() throws Exception {
// Sort desc
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by $0;");
- planTester.buildPlan("C = foreach B { D = order A by $0 desc; generate group, D;};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
+ "B = group A by $0;" +
+ "C = foreach B { D = order A by $0 desc; generate group, D;};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -379,13 +356,12 @@ public class TestSecondarySort extends T
@Test
public void testSortOptimization7() throws Exception {
// Sort asc on 1st key, desc on 2nd key
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0, a1, a2);");
- planTester.buildPlan("B = group A by ($0, $1);");
- planTester.buildPlan("C = foreach B { D = order A by $0, $1 desc; generate group, D;};");
+ String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
+ "B = group A by ($0, $1);" +
+ "C = foreach B { D = order A by $0, $1 desc; generate group, D;};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
@@ -400,13 +376,12 @@ public class TestSecondarySort extends T
@Test
public void testSortOptimization8() throws Exception {
// Sort desc, used in UDF twice
- LogicalPlanTester planTester = new LogicalPlanTester();
- planTester.buildPlan("A = LOAD 'input1' AS (a0);");
- planTester.buildPlan("B = group A all;");
- planTester.buildPlan("C = foreach B { D = order A by $0 desc; generate DIFF(D, D);};");
+ String query = ("A=LOAD 'input1' AS (a0);" +
+ "B = group A all;" +
+ "C = foreach B { D = order A by $0 desc; generate DIFF(D, D);};" +
- LogicalPlan lp = planTester.buildPlan("store C into '/tmp';");
- PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+ "store C into 'output';");
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
Modified: pig/trunk/test/org/apache/pig/test/TestStore.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestStore.java?rev=1099121&r1=1099120&r2=1099121&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestStore.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestStore.java Tue May 3 16:53:40 2011
@@ -45,7 +45,6 @@ import org.apache.pig.backend.executione
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
import org.apache.pig.builtin.BinStorage;
@@ -57,20 +56,17 @@ import org.apache.pig.data.DefaultTuple;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
-import org.apache.pig.impl.logicalLayer.LOStore;
-import org.apache.pig.impl.logicalLayer.LogicalOperator;
-import org.apache.pig.impl.logicalLayer.LogicalPlan;
-import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.validators.InputOutputFileVisitor;
-import org.apache.pig.impl.plan.CompilationMessageCollector;
-import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.PlanValidationException;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.relational.LOStore;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.rules.InputOutputFileValidator;
+import org.apache.pig.parser.QueryParserDriver;
import org.apache.pig.test.utils.GenRandomData;
-import org.apache.pig.test.utils.LogicalPlanTester;
import org.apache.pig.test.utils.TestHelper;
-import org.apache.pig.tools.pigstats.PigStats;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -130,28 +126,15 @@ public class TestStore extends junit.fra
cluster.shutDown();
}
- private PigStats store() throws Exception {
- PhysicalPlan pp = new PhysicalPlan();
- pp.add(proj);
- pp.add(st);
- //pp.connect(proj, st);
- pp.connect(proj, st);
- pc.setExecType(ExecType.LOCAL);
- return new MapReduceLauncher().launchPig(pp, "TestStore", pc);
- }
-
@Test
public void testValidation() throws Exception{
-
String outputFileName = "test-output.txt";
try {
- LogicalPlanTester lpt = new LogicalPlanTester();
- lpt.buildPlan("a = load '" + inputFileName + "' as (c:chararray, " +
- "i:int,d:double);");
- LogicalPlan lp = lpt.buildPlan("store a into '" + outputFileName + "' using " +
- "PigStorage();");
- InputOutputFileVisitor visitor = new InputOutputFileVisitor(lp, null, pig.getPigContext());
- visitor.visit();
+ String query = "a = load '" + inputFileName + "' as (c:chararray, " +
+ "i:int,d:double);" +
+ "store a into '" + outputFileName + "' using " + "PigStorage();";
+ org.apache.pig.newplan.logical.relational.LogicalPlan lp = Util.buildLp( pig, query );
+ new InputOutputFileValidator(lp, pig.getPigContext()).validate();
} catch (PlanValidationException e){
// Since output file is not present, validation should pass
// and not throw this exception.
@@ -163,21 +146,17 @@ public class TestStore extends junit.fra
@Test
public void testValidationFailure() throws Exception{
-
String input[] = new String[] { "some data" };
String outputFileName = "test-output.txt";
boolean sawException = false;
try {
Util.createInputFile(pig.getPigContext(),outputFileName, input);
- LogicalPlanTester lpt = new LogicalPlanTester(pig.getPigContext());
- lpt.buildPlan("a = load '" + inputFileName + "' as (c:chararray, " +
- "i:int,d:double);");
- LogicalPlan lp = lpt.buildPlan("store a into '" + outputFileName +
- "' using PigStorage();");
- InputOutputFileVisitor visitor = new InputOutputFileVisitor(lp,
- new CompilationMessageCollector(), pig.getPigContext());
- visitor.visit();
- } catch (PlanValidationException pve){
+ String query = "a = load '" + inputFileName + "' as (c:chararray, " +
+ "i:int,d:double);" +
+ "store a into '" + outputFileName + "' using PigStorage();";
+ org.apache.pig.newplan.logical.relational.LogicalPlan lp = Util.buildLp( pig, query );
+ new InputOutputFileValidator(lp, pig.getPigContext()).validate();
+ } catch (FrontendException pve){
// Since output file is present, validation should fail
// and throw this exception
assertEquals(6000,pve.getErrorCode());
@@ -351,14 +330,6 @@ public class TestStore extends junit.fra
}
}
- private static void randomizeBytes(byte[] data, int offset, int length) {
- Random random = new Random();
- for(int i=offset + length - 1; i >= offset; --i) {
- data[i] = (byte) random.nextInt(256);
- }
- }
-
-
@Test
public void testStoreRemoteRel() throws Exception {
checkStorePath("test","/tmp/test");
@@ -800,35 +771,20 @@ public class TestStore extends junit.fra
DataStorage dfs = pc.getDfs();
dfs.setActiveContainer(dfs.asContainer("/tmp"));
- Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
- Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
- Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>();
Map<String, String> fileNameMap = new HashMap<String, String>();
- LogicalPlanBuilder builder = new LogicalPlanBuilder(pc);
+ QueryParserDriver builder = new QueryParserDriver(pc, "Test-Store", fileNameMap);
- String query = "a = load 'foo';";
- LogicalPlan lp = builder.parse("Test-Store",
- query,
- aliases,
- logicalOpTable,
- aliasOp,
- fileNameMap);
- query = "store a into '"+orig+"';";
- lp = builder.parse("Test-Store",
- query,
- aliases,
- logicalOpTable,
- aliasOp,
- fileNameMap);
+ String query = "a = load 'foo';" + "store a into '"+orig+"';";
+ LogicalPlan lp = builder.parse(query);
Assert.assertTrue(lp.size()>1);
- LogicalOperator op = lp.getLeaves().get(0);
+ Operator op = lp.getSinks().get(0);
Assert.assertTrue(op instanceof LOStore);
LOStore store = (LOStore)op;
- String p = store.getOutputFile().getFileName();
+ String p = store.getFileSpec().getFileName();
p = p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/","/");
if (isTmp) {
Modified: pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1099121&r1=1099120&r2=1099121&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Tue May 3 16:53:40 2011
@@ -115,7 +115,6 @@ import org.apache.pig.newplan.logical.vi
import org.apache.pig.newplan.logical.visitor.TypeCheckingRelVisitor;
import org.apache.pig.newplan.logical.visitor.UnionOnSchemaSetter;
import org.apache.pig.parser.ParserTestingUtils;
-import org.apache.pig.test.utils.LogicalPlanTester;
import org.junit.Before;
import org.junit.Test;
import static org.apache.pig.test.Util.*;
@@ -123,7 +122,6 @@ import static org.apache.pig.test.Util.*
public class TestTypeCheckingValidatorNewLP {
PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
- LogicalPlanTester planTester;
private static final String CAST_LOAD_NOT_FOUND =
"Cannot resolve load function to use for casting from bytearray";
@@ -133,10 +131,6 @@ public class TestTypeCheckingValidatorNe
*/
@Before
public void setUp() throws Exception {
- // create a new instance of the plan tester
- // for each test so that different tests do not
- // interact with each other's plans
- planTester = new LogicalPlanTester(pc) ;
}
private static final String simpleEchoStreamingCommand;