You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/12/15 00:20:41 UTC
svn commit: r1049324 - in /pig/branches/branch-0.8: CHANGES.txt
src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java
test/org/apache/pig/test/TestEvalPipeline2.java
Author: daijy
Date: Tue Dec 14 23:20:41 2010
New Revision: 1049324
URL: http://svn.apache.org/viewvc?rev=1049324&view=rev
Log:
PIG-1766: New logical plan: ImplicitSplitInserter should before DuplicateForEachColumnRewrite
Modified:
pig/branches/branch-0.8/CHANGES.txt
pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java
pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java
Modified: pig/branches/branch-0.8/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/CHANGES.txt?rev=1049324&r1=1049323&r2=1049324&view=diff
==============================================================================
--- pig/branches/branch-0.8/CHANGES.txt (original)
+++ pig/branches/branch-0.8/CHANGES.txt Tue Dec 14 23:20:41 2010
@@ -211,6 +211,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1766: New logical plan: ImplicitSplitInserter should before DuplicateForEachColumnRewrite (daijy)
+
PIG-1762: Logical simplification fails on map key referenced values (yanz)
PIG-1761: New logical plan: Exception when bag dereference in the middle of expression (daijy)
Modified: pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java?rev=1049324&r1=1049323&r2=1049324&view=diff
==============================================================================
--- pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java (original)
+++ pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/optimizer/LogicalPlanOptimizer.java Tue Dec 14 23:20:41 2010
@@ -57,23 +57,23 @@ public class LogicalPlanOptimizer extend
List<Set<Rule>> ls = new ArrayList<Set<Rule>>();
- // DuplicateForEachColumnRewrite set
- // This insert Identity UDF in the case foreach duplicate field.
- // This is because we need unique uid through out the plan
+ // ImplicitSplitInserter set
+ // This set of rules Insert Foreach dedicated for casting after load
Set<Rule> s = new HashSet<Rule>();
- Rule r = new DuplicateForEachColumnRewrite("DuplicateForEachColumnRewrite");
+ Rule r = new ImplicitSplitInserter("ImplicitSplitInserter");
checkAndAddRule(s, r);
if (!s.isEmpty())
ls.add(s);
-
- // ImplicitSplitInserter set
- // This set of rules Insert Foreach dedicated for casting after load
+
+ // DuplicateForEachColumnRewrite set
+ // This insert Identity UDF in the case foreach duplicate field.
+ // This is because we need unique uid through out the plan
s = new HashSet<Rule>();
- r = new ImplicitSplitInserter("ImplicitSplitInserter");
+ r = new DuplicateForEachColumnRewrite("DuplicateForEachColumnRewrite");
checkAndAddRule(s, r);
if (!s.isEmpty())
ls.add(s);
-
+
// Logical expression simplifier
s = new HashSet<Rule>();
// add logical expression simplification rule
Modified: pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1049324&r1=1049323&r2=1049324&view=diff
==============================================================================
--- pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java Tue Dec 14 23:20:41 2010
@@ -912,7 +912,7 @@ public class TestEvalPipeline2 extends T
fail();
}
- // See PIG-1732
+ // See PIG-1745
@Test
public void testForEachDupColumn() throws Exception{
String[] input1 = {
@@ -965,4 +965,35 @@ public class TestEvalPipeline2 extends T
assertTrue(t.size()==1);
assertTrue(t.get(0).equals("APACHE"));
}
+
+ // See PIG-1766
+ @Test
+ public void testForEachSameOriginColumn() throws Exception{
+ String[] input1 = {
+ "1\t2",
+ "1\t3",
+ "2\t4",
+ "2\t5",
+ };
+
+ String[] input2 = {
+ "1\tone",
+ "2\ttwo",
+ };
+
+ Util.createInputFile(cluster, "table_testForEachSameOriginColumn1", input1);
+ Util.createInputFile(cluster, "table_testForEachSameOriginColumn2", input2);
+ pigServer.registerQuery("A = load 'table_testForEachSameOriginColumn1' AS (a0:int, a1:int);");
+ pigServer.registerQuery("B = load 'table_testForEachSameOriginColumn2' AS (b0:int, b1:chararray);");
+ pigServer.registerQuery("C = join A by a0, B by b0;");
+ pigServer.registerQuery("D = foreach B generate b0 as d0, b1 as d1;");
+ pigServer.registerQuery("E = join C by a1, D by d0;");
+ pigServer.registerQuery("F = foreach E generate b1, d1;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("F");
+ Tuple t = iter.next();
+ assertTrue(t.size()==2);
+ assertTrue(t.get(0).equals("one"));
+ assertTrue(t.get(1).equals("two"));
+ }
}