You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/11/05 22:14:19 UTC
svn commit: r1198058 - in /pig/branches/branch-0.9: CHANGES.txt
src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java
test/org/apache/pig/test/TestPlanGeneration.java
Author: daijy
Date: Sat Nov 5 21:14:18 2011
New Revision: 1198058
URL: http://svn.apache.org/viewvc?rev=1198058&view=rev
Log:
PIG-2339: HCatLoader loads all the partitions in a partitioned table even though a filter clause on the partitions is specified in the Pig script
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestPlanGeneration.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1198058&r1=1198057&r2=1198058&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Sat Nov 5 21:14:18 2011
@@ -26,6 +26,9 @@ PIG-2320: Error: "projection with nothin
PIG-2346: TypeCastInsert should not insert Foreach if there is no as statement (daijy)
+PIG-2339: HCatLoader loads all the partitions in a partitioned table even though
+ a filter clause on the partitions is specified in the Pig script (daijy)
+
Release 0.9.1 - Unreleased
IMPROVEMENTS
Modified: pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java?rev=1198058&r1=1198057&r2=1198058&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java Sat Nov 5 21:14:18 2011
@@ -91,15 +91,6 @@ public class LogicalPlanOptimizer extend
checkAndAddRule(s, r);
if (!s.isEmpty())
ls.add(s);
-
- // Partition filter set
- // This set of rules push partition filter to LoadFunc
- s = new HashSet<Rule>();
- // Optimize partition filter
- r = new PartitionFilterOptimizer("PartitionFilterOptimizer");
- checkAndAddRule(s, r);
- if (!s.isEmpty())
- ls.add(s);
// Limit Set
// This set of rules push up limit
@@ -141,6 +132,15 @@ public class LogicalPlanOptimizer extend
if (!s.isEmpty())
ls.add(s);
+ // Partition filter set
+ // This set of rules push partition filter to LoadFunc
+ s = new HashSet<Rule>();
+ // Optimize partition filter
+ r = new PartitionFilterOptimizer("PartitionFilterOptimizer");
+ checkAndAddRule(s, r);
+ if (!s.isEmpty())
+ ls.add(s);
+
// PushDownForEachFlatten set
s = new HashSet<Rule>();
// Add the PushDownForEachFlatten
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestPlanGeneration.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestPlanGeneration.java?rev=1198058&r1=1198057&r2=1198058&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestPlanGeneration.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestPlanGeneration.java Sat Nov 5 21:14:18 2011
@@ -235,4 +235,64 @@ public class TestPlanGeneration extends
Assert.assertTrue(lp.getSuccessors(loStore)==null);
}
+
+ public static class PartitionedLoader extends PigStorage implements LoadMetadata {
+
+ Schema schema;
+ String[] partCols;
+ static Expression partFilter = null;
+
+ public PartitionedLoader(String schemaString, String commaSepPartitionCols)
+ throws ParserException {
+ schema = Utils.getSchemaFromString(schemaString);
+ partCols = commaSepPartitionCols.split(",");
+ }
+
+ @Override
+ public ResourceSchema getSchema(String location, Job job)
+ throws IOException {
+ return new ResourceSchema(schema);
+ }
+
+ @Override
+ public ResourceStatistics getStatistics(String location,
+ Job job) throws IOException {
+ return null;
+ }
+
+ @Override
+ public void setPartitionFilter(Expression partitionFilter)
+ throws IOException {
+ partFilter = partitionFilter;
+ }
+
+ @Override
+ public String[] getPartitionKeys(String location, Job job)
+ throws IOException {
+ return partCols;
+ }
+
+ public Expression getPartFilter() {
+ return partFilter;
+ }
+
+ }
+
+ @Test
+ // See PIG-2339
+ public void testPartitionFilterOptimizer() throws Exception {
+ String query = "a = load 'foo' using " + PartitionedLoader.class.getName() +
+ "('name:chararray, dt:chararray', 'dt');\n" +
+ "b = filter a by dt=='2011';\n" +
+ "store b into 'output';";
+
+ LogicalPlan lp = Util.parse(query, pc);
+ Util.optimizeNewLP(lp);
+
+ LOLoad loLoad = (LOLoad)lp.getSources().get(0);
+ LOForEach loForEach = (LOForEach)lp.getSuccessors(loLoad).get(0);
+ LOStore loStore = (LOStore)lp.getSuccessors(loForEach).get(0);
+ Assert.assertTrue(((PartitionedLoader)loLoad.getLoadFunc()).getPartFilter()!=null);
+ Assert.assertTrue(loStore.getAlias().equals("b"));
+ }
}