You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2016/09/13 21:32:43 UTC
svn commit: r1760601 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java
test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-4.gld
test/org/apache/pig/tez/TestTezCompiler.java
Author: knoguchi
Date: Tue Sep 13 21:32:43 2016
New Revision: 1760601
URL: http://svn.apache.org/viewvc?rev=1760601&view=rev
Log:
PIG-5031: Tez failing to compile when replicate join is done with a limit vertex on left (knoguchi)
Added:
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-4.gld
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java
pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1760601&r1=1760600&r2=1760601&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Sep 13 21:32:43 2016
@@ -46,6 +46,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5031: Tez failing to compile when replicate join is done with a limit vertex on left (knoguchi)
+
PIG-5019: Pig generates tons of warnings for udf with enabled warnings aggregation (murshyd via rohini)
PIG-4974: A simple map reference fail to cast (knoguchi)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java?rev=1760601&r1=1760600&r2=1760601&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/TezCompiler.java Tue Sep 13 21:32:43 2016
@@ -849,6 +849,7 @@ public class TezCompiler extends PhyPlan
} else {
curTezOp.plan.addAsLeaf(op);
}
+ phyToTezOpMap.put(op, curTezOp);
} catch (Exception e) {
int errCode = 2034;
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-4.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-4.gld?rev=1760601&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-4.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-4.gld Tue Sep 13 21:32:43 2016
@@ -0,0 +1,61 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-42 -> Tez vertex scope-39,
+Tez vertex scope-32 -> Tez vertex scope-39,
+Tez vertex scope-39
+
+Tez vertex scope-42
+# Plan on vertex
+Local Rearrange[tuple]{int}(false) - scope-28 -> scope-39
+| |
+| Project[int][0] - scope-24
+|
+|---b: New For Each(false,false)[bag] - scope-22
+ | |
+ | Cast[int] - scope-17
+ | |
+ | |---Project[bytearray][0] - scope-16
+ | |
+ | Cast[int] - scope-20
+ | |
+ | |---Project[bytearray][1] - scope-19
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-15
+Tez vertex scope-32
+# Plan on vertex
+a: Split - scope-43
+| |
+| a: Store(file:///tmp/pigoutput/a:org.apache.pig.builtin.PigStorage) - scope-11
+| |
+| POValueOutputTez - scope-38 -> [scope-39]
+| |
+| |---c: Limit - scope-14
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-39
+# Plan on vertex
+d: Store(file:///tmp/pigoutput/d:org.apache.pig.builtin.PigStorage) - scope-31
+|
+|---d: FRJoin[tuple] - scope-25 <- scope-42
+ | |
+ | Project[int][0] - scope-23
+ | |
+ | Project[int][0] - scope-24
+ |
+ |---c: Limit - scope-41
+ |
+ |---POValueInputTez - scope-40 <- scope-32
Modified: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1760601&r1=1760600&r2=1760601&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Tue Sep 13 21:32:43 2016
@@ -353,6 +353,19 @@ public class TestTezCompiler {
}
@Test
+ public void testLimitReplJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input' as (x:int, y:int);" +
+ "c = limit a 1;" +
+ "d = join c by x, b by x using 'replicated';" +
+ "store a into 'file:///tmp/pigoutput/a';" +
+ "store d into 'file:///tmp/pigoutput/d';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-4.gld");
+ }
+
+ @Test
public void testDistinct() throws Exception {
String query =
"a = load 'file:///tmp/input' as (x:int, y:int);" +