You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/02/27 07:13:27 UTC
svn commit: r1572442 - in /pig/branches/tez:
src/org/apache/pig/backend/hadoop/executionengine/tez/CombinerOptimizer.java
test/org/apache/pig/test/data/GoldenFiles/TEZC18.gld
test/org/apache/pig/tez/TestTezCompiler.java
Author: cheolsoo
Date: Thu Feb 27 06:13:27 2014
New Revision: 1572442
URL: http://svn.apache.org/r1572442
Log:
PIG-3781: Fix TestTezCompiler#testReplicatedJoinInReducer (cheolsoo)
Added:
pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC18.gld
Modified:
pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/CombinerOptimizer.java
pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/CombinerOptimizer.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/CombinerOptimizer.java?rev=1572442&r1=1572441&r2=1572442&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/CombinerOptimizer.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/tez/CombinerOptimizer.java Thu Feb 27 06:13:27 2014
@@ -60,7 +60,7 @@ public class CombinerOptimizer extends T
if (predecessors == null) {
return;
}
- if (predecessors.size()>1) {
+ if (to.isCogroup()) {
return;
}
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC18.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC18.gld?rev=1572442&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC18.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/TEZC18.gld Thu Feb 27 06:13:27 2014
@@ -0,0 +1,65 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-37
+#--------------------------------------------------
+Tez vertex scope-34 -> Tez vertex scope-36,
+Tez vertex scope-35 -> Tez vertex scope-36,
+Tez vertex scope-36
+
+Tez vertex scope-34
+# Plan on vertex
+c: Local Rearrange[tuple]{int}(false) - scope-18 -> scope-36
+| |
+| Project[int][0] - scope-19
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-35
+# Plan on vertex
+c: Local Rearrange[tuple]{int}(false) - scope-20 -> scope-36
+| |
+| Project[int][0] - scope-21
+|
+|---b: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][0] - scope-9
+ | |
+ | Cast[int] - scope-13
+ | |
+ | |---Project[bytearray][1] - scope-12
+ |
+ |---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-36
+# Plan on vertex
+d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33
+|
+|---d: New For Each(false,false,false)[bag] - scope-32
+ | |
+ | Project[int][0] - scope-22
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT)[long] - scope-26
+ | |
+ | |---Project[bag][1] - scope-25
+ | |
+ | |---Project[bag][1] - scope-24
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT)[long] - scope-30
+ | |
+ | |---Project[bag][1] - scope-29
+ | |
+ | |---Project[bag][2] - scope-28
+ |
+ |---c: Package(Packager)[tuple]{int} - scope-17
Modified: pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java?rev=1572442&r1=1572441&r2=1572442&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java Thu Feb 27 06:13:27 2014
@@ -299,6 +299,20 @@ public class TestTezCompiler {
run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld");
}
+ // PIG-3759, PIG-3781
+ // Combiner should not be added in case of co-group
+ @Test
+ public void testCogroupWithAlgebraiceUDF() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input2' as (x:int, z:int);" +
+ "c = cogroup a by x, b by x;" +
+ "d = foreach c generate group, COUNT(a.y), COUNT(b.z);" +
+ "store d into 'file:///tmp/output/d';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC18.gld");
+ }
+
private void run(String query, String expectedFile) throws Exception {
PhysicalPlan pp = Util.buildPp(pigServer, query);
TezLauncher launcher = new TezLauncher();