You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/08/11 19:34:05 UTC
svn commit: r684831 - in /incubator/pig/branches/types:
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/impl/logicalLayer/optimizer/ test/org/apache/pig/test/
test/org/apache/pig/test/data/DotFiles/
Author: gates
Date: Mon Aug 11 10:34:02 2008
New Revision: 684831
URL: http://svn.apache.org/viewvc?rev=684831&view=rev
Log:
PIG-338. Daniel's patch to fix limit after distinct.
Modified:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot
incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot
Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java Mon Aug 11 10:34:02 2008
@@ -679,7 +679,7 @@
log.warn("Something in the reduce plan while map plan is not done. Something wrong!");
}
} else if (mro.isMapDone() && !mro.isReduceDone()) {
- // limit should add into reduce reduce function
+ // limit should add into reduce plan
mro.reducePlan.addAsLeaf(op);
} else {
log.warn("Both map and reduce phases have been done. This is unexpected while compiling!");
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/OpLimitOptimizer.java Mon Aug 11 10:34:02 2008
@@ -97,7 +97,7 @@
// Limit cannot be pushed up
if (predecessor instanceof LOCogroup || predecessor instanceof LOFilter ||
predecessor instanceof LOLoad || predecessor instanceof LOSplit ||
- predecessor instanceof LOSplitOutput)
+ predecessor instanceof LOSplitOutput || predecessor instanceof LODistinct)
{
return;
}
@@ -129,19 +129,19 @@
{
throw new OptimizerException("LOFilter should have one input");
}
+ // we can move LOLimit even further, recursively optimize LOLimit
+ processNode(limit);
}
- // we can move LOLimit even further, recursively optimize LOLimit
- processNode(limit);
}
// Limit can be duplicated, and the new instance pushed in front of an operator for the following operators
// (that is, if you have X->limit, you can transform that to limit->X->limit):
- else if (predecessor instanceof LOCross || predecessor instanceof LODistinct ||
+ else if (predecessor instanceof LOCross ||
predecessor instanceof LOForEach || predecessor instanceof LOUnion)
{
LOLimit newLimit = null;
// Process the predecessors with only one input. LOForEach should now have at least
// one flaten
- if (predecessor instanceof LODistinct || predecessor instanceof LOForEach)
+ if (predecessor instanceof LOForEach)
{
LogicalOperator prepredecessor = mPlan.getPredecessors(predecessor).get(0);
try {
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java Mon Aug 11 10:34:02 2008
@@ -69,7 +69,9 @@
int len = fis.read(b);
String goldenPlan = new String(b, 0, len);
- String actualPlan = printLimitGraph(plan);
+ String actualPlan = printLimitGraph(plan);
+ System.out.println("We get:");
+ System.out.println(actualPlan);
assertEquals(goldenPlan, actualPlan);
}
@@ -118,8 +120,9 @@
// Duplicte limit with one input
public void testOPLimit4Optimizer() throws Exception {
planTester.buildPlan("A = load 'myfile1';");
- planTester.buildPlan("B = distinct A;");
- LogicalPlan plan = planTester.buildPlan("C = limit B 100;");
+ planTester.buildPlan("B = group A by $0;");
+ planTester.buildPlan("C = foreach B generate flatten(A);");
+ LogicalPlan plan = planTester.buildPlan("D = limit C 100;");
optimizePlan(plan);
compareWithGoldenFile(plan, FILE_BASE_LOCATION + "optlimitplan4.dot");
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot Mon Aug 11 10:34:02 2008
@@ -1,7 +1,7 @@
digraph graph1 {
- LOLoad31 -> LOLimit34;
- LOLimit34 [limit="100"];
- LOLimit34 -> LODistinct32;
- LODistinct32 -> LOLimit33;
- LOLimit33 [limit="100"];
+ LOLoad31 -> LOCogroup33;
+ LOCogroup33 -> LOForEach36;
+ LOForEach36 [hasflat="true"];
+ LOForEach36 -> LOLimit37;
+ LOLimit37 [limit="100"];
}
\ No newline at end of file
Modified: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot Mon Aug 11 10:34:02 2008
@@ -1,6 +1,6 @@
digraph graph1 {
- LOLoad35 -> LOLimit39;
- LOLimit39 [limit="100"];
- LOLimit39 -> LOForEach38;
- LOForEach38 [hasflat="false"];
+ LOLoad38 -> LOLimit42;
+ LOLimit42 [limit="100"];
+ LOLimit42 -> LOForEach41;
+ LOForEach41 [hasflat="false"];
}
\ No newline at end of file
Modified: incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot?rev=684831&r1=684830&r2=684831&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot Mon Aug 11 10:34:02 2008
@@ -1,4 +1,4 @@
digraph graph1 {
- LOLoad40 -> LOLimit41;
- LOLimit41 [limit="20"];
+ LOLoad43 -> LOLimit44;
+ LOLimit44 [limit="20"];
}
\ No newline at end of file