You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/04/22 16:59:44 UTC
svn commit: r1589152 [2/2] - in /pig/branches/tez:
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/
src/org/apache/pig/backend/hadoop/executionengine/tez/
src/org/apache/pig/backend/hadoop/executionengine/tez/optimiz...
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld?rev=1589152&r1=1589151&r2=1589152&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld Tue Apr 22 14:59:44 2014
@@ -2,90 +2,90 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-115
+# TEZ DAG plan: scope-117
#--------------------------------------------------
-Tez vertex scope-86 -> Tez vertex scope-88,
-Tez vertex scope-87 -> Tez vertex scope-88,
-Tez vertex scope-88 -> Tez vertex scope-108,Tez vertex scope-98,
-Tez vertex scope-98 -> Tez vertex scope-108,
-Tez vertex scope-108 -> Tez vertex scope-110,
-Tez vertex scope-110
+Tez vertex scope-88 -> Tez vertex scope-90,
+Tez vertex scope-89 -> Tez vertex scope-90,
+Tez vertex scope-90 -> Tez vertex scope-110,Tez vertex scope-100,
+Tez vertex scope-100 -> Tez vertex scope-110,
+Tez vertex scope-110 -> Tez vertex scope-112,
+Tez vertex scope-112
-Tez vertex scope-86
+Tez vertex scope-88
# Plan on vertex
-POValueOutputTez - scope-90 -> [scope-88]
+POValueOutputTez - scope-92 -> [scope-90]
|
-|---a: New For Each(false,false)[bag] - scope-73
+|---a: New For Each(false,false)[bag] - scope-75
| |
- | Cast[int] - scope-68
+ | Cast[int] - scope-70
| |
- | |---Project[bytearray][0] - scope-67
+ | |---Project[bytearray][0] - scope-69
| |
- | Cast[chararray] - scope-71
+ | Cast[chararray] - scope-73
| |
- | |---Project[bytearray][1] - scope-70
+ | |---Project[bytearray][1] - scope-72
|
- |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-66
-Tez vertex scope-87
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-68
+Tez vertex scope-89
# Plan on vertex
-POValueOutputTez - scope-91 -> [scope-88]
+POValueOutputTez - scope-93 -> [scope-90]
|
-|---c: New For Each(false,false)[bag] - scope-81
+|---c: New For Each(false,false)[bag] - scope-83
| |
- | Cast[int] - scope-76
+ | Cast[int] - scope-78
| |
- | |---Project[bytearray][1] - scope-75
+ | |---Project[bytearray][1] - scope-77
| |
- | Cast[chararray] - scope-79
+ | Cast[chararray] - scope-81
| |
- | |---Project[bytearray][0] - scope-78
+ | |---Project[bytearray][0] - scope-80
|
- |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-74
-Tez vertex scope-88
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-76
+Tez vertex scope-90
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-94 -> scope-98
+Local Rearrange[tuple]{tuple}(false) - scope-96 -> scope-100
| |
-| Constant(DummyVal) - scope-93
+| Constant(DummyVal) - scope-95
|
-|---ReservoirSample - scope-97
+|---ReservoirSample - scope-99
|
- |---New For Each(false)[tuple] - scope-96
+ |---New For Each(false)[tuple] - scope-98
| |
- | Project[int][0] - scope-95
+ | Project[int][0] - scope-97
|
- |---d: Local Rearrange[tuple]{int}(false) - scope-92 -> scope-108
+ |---d: Local Rearrange[tuple]{int}(false) - scope-94 -> scope-110
| |
- | Project[int][0] - scope-83
+ | Project[int][0] - scope-85
|
- |---POShuffledValueInputTez - scope-89 <- [scope-86, scope-87]
-Tez vertex scope-98
+ |---POShuffledValueInputTez - scope-91 <- [scope-88, scope-89]
+Tez vertex scope-100
# Plan on vertex
-POValueOutputTez - scope-107 -> [scope-108]
+POValueOutputTez - scope-109 -> [scope-110]
|
-|---New For Each(false)[tuple] - scope-106
+|---New For Each(false)[tuple] - scope-108
| |
- | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-105
+ | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-107
| |
- | |---Project[tuple][*] - scope-104
+ | |---Project[tuple][*] - scope-106
|
- |---New For Each(false,false)[tuple] - scope-103
+ |---New For Each(false,false)[tuple] - scope-105
| |
- | Constant(1) - scope-102
+ | Constant(1) - scope-104
| |
- | Project[bag][1] - scope-100
+ | Project[bag][1] - scope-102
|
- |---Package(Packager)[tuple]{bytearray} - scope-99
-Tez vertex scope-108
+ |---Package(Packager)[tuple]{bytearray} - scope-101
+Tez vertex scope-110
# Plan on vertex
-POIdentityInOutTez - scope-109 <- scope-88 -> scope-110
+POIdentityInOutTez - scope-111 <- scope-90 -> scope-112
| |
-| Project[int][0] - scope-83
-Tez vertex scope-110
+| Project[int][0] - scope-85
+Tez vertex scope-112
# Plan on vertex
-d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-85
+d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-87
|
-|---New For Each(true)[tuple] - scope-113
+|---New For Each(true)[tuple] - scope-115
| |
- | Project[bag][1] - scope-112
+ | Project[bag][1] - scope-114
|
- |---Package(LitePackager)[tuple]{int} - scope-111
+ |---Package(LitePackager)[tuple]{int} - scope-113
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld?rev=1589152&r1=1589151&r2=1589152&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld Tue Apr 22 14:59:44 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-65
+# TEZ DAG plan: scope-67
#--------------------------------------------------
Tez vertex scope-20 -> Tez vertex scope-49,Tez vertex scope-50,
Tez vertex scope-21 -> Tez vertex scope-49,Tez vertex scope-50,
@@ -24,9 +24,9 @@ Local Rearrange[tuple]{tuple}(false) - s
| |
| Project[int][0] - scope-51
|
- |---d: Local Rearrange[tuple]{int}(false) - scope-56 -> scope-42
+ |---d: Local Rearrange[tuple]{int}(false) - scope-57 -> scope-42
| |
- | Project[int][0] - scope-57
+ | Project[int][0] - scope-58
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -41,19 +41,19 @@ Local Rearrange[tuple]{tuple}(false) - s
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-21
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-61 -> scope-32
+Local Rearrange[tuple]{tuple}(false) - scope-62 -> scope-32
| |
-| Constant(DummyVal) - scope-62
+| Constant(DummyVal) - scope-63
|
-|---ReservoirSample - scope-60
+|---ReservoirSample - scope-61
|
- |---New For Each(false)[tuple] - scope-59
+ |---New For Each(false)[tuple] - scope-60
| |
- | Project[int][0] - scope-58
+ | Project[int][0] - scope-59
|
- |---d: Local Rearrange[tuple]{int}(false) - scope-63 -> scope-42
+ |---d: Local Rearrange[tuple]{int}(false) - scope-65 -> scope-42
| |
- | Project[int][0] - scope-64
+ | Project[int][0] - scope-66
|
|---c: New For Each(false,false)[bag] - scope-15
| |
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld?rev=1589152&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld Tue Apr 22 14:59:44 2014
@@ -0,0 +1,89 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-121
+#--------------------------------------------------
+Tez vertex scope-109 -> Tez vertex scope-110,
+Tez vertex scope-103 -> Tez vertex scope-110,
+Tez vertex scope-110
+
+Tez vertex scope-109
+# Plan on vertex
+POValueOutputTez - scope-113 -> [scope-110]
+|
+|---c: New For Each(false,false)[bag] - scope-89
+ | |
+ | Cast[int] - scope-84
+ | |
+ | |---Project[bytearray][1] - scope-83
+ | |
+ | Cast[chararray] - scope-87
+ | |
+ | |---Project[bytearray][0] - scope-86
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-82
+Tez vertex scope-103
+# Plan on vertex
+1-12: Split - scope-120
+| |
+| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-77
+| |
+| |---a2: Filter[bag] - scope-72
+| | |
+| | Not[boolean] - scope-76
+| | |
+| | |---Greater Than[boolean] - scope-75
+| | |
+| | |---Project[int][0] - scope-73
+| | |
+| | |---Constant(100) - scope-74
+| |
+| POValueOutputTez - scope-112 -> [scope-110]
+| |
+| |---a1: Filter[bag] - scope-78
+| | |
+| | Greater Than[boolean] - scope-81
+| | |
+| | |---Project[int][0] - scope-79
+| | |
+| | |---Constant(100) - scope-80
+|
+|---a: New For Each(false,false)[bag] - scope-70
+ | |
+ | Cast[int] - scope-65
+ | |
+ | |---Project[bytearray][0] - scope-64
+ | |
+ | Cast[chararray] - scope-68
+ | |
+ | |---Project[bytearray][1] - scope-67
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-63
+Tez vertex scope-110
+# Plan on vertex
+1-13: Split - scope-119
+| |
+| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-96
+| |
+| |---d: Filter[bag] - scope-92
+| | |
+| | Greater Than[boolean] - scope-95
+| | |
+| | |---Project[int][0] - scope-93
+| | |
+| | |---Constant(500) - scope-94
+| |
+| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-102
+| |
+| |---e: Filter[bag] - scope-97
+| | |
+| | Not[boolean] - scope-101
+| | |
+| | |---Greater Than[boolean] - scope-100
+| | |
+| | |---Project[int][0] - scope-98
+| | |
+| | |---Constant(500) - scope-99
+|
+|---POShuffledValueInputTez - scope-111 <- [scope-109, scope-103]
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld?rev=1589152&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld Tue Apr 22 14:59:44 2014
@@ -0,0 +1,111 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-62
+#--------------------------------------------------
+Tez vertex scope-40 -> Tez vertex scope-58,Tez vertex scope-59,
+Tez vertex scope-46 -> Tez vertex scope-58,Tez vertex scope-59,
+Tez vertex scope-59
+Tez vertex scope-58
+
+Tez vertex scope-40
+# Plan on vertex
+1-2: Split - scope-57
+| |
+| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-14
+| |
+| |---a2: Filter[bag] - scope-9
+| | |
+| | Not[boolean] - scope-13
+| | |
+| | |---Greater Than[boolean] - scope-12
+| | |
+| | |---Project[int][0] - scope-10
+| | |
+| | |---Constant(100) - scope-11
+| |
+| 1-3: Split - scope-60
+| | |
+| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33
+| | |
+| | |---d: Filter[bag] - scope-29
+| | | |
+| | | Greater Than[boolean] - scope-32
+| | | |
+| | | |---Project[int][0] - scope-30
+| | | |
+| | | |---Constant(500) - scope-31
+| | |
+| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-39
+| | |
+| | |---e: Filter[bag] - scope-34
+| | | |
+| | | Not[boolean] - scope-38
+| | | |
+| | | |---Greater Than[boolean] - scope-37
+| | | |
+| | | |---Project[int][0] - scope-35
+| | | |
+| | | |---Constant(500) - scope-36
+| |
+| |---a1: Filter[bag] - scope-15
+| | |
+| | Greater Than[boolean] - scope-18
+| | |
+| | |---Project[int][0] - scope-16
+| | |
+| | |---Constant(100) - scope-17
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-46
+# Plan on vertex
+1-3: Split - scope-61
+| |
+| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33
+| |
+| |---d: Filter[bag] - scope-29
+| | |
+| | Greater Than[boolean] - scope-32
+| | |
+| | |---Project[int][0] - scope-30
+| | |
+| | |---Constant(500) - scope-31
+| |
+| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-39
+| |
+| |---e: Filter[bag] - scope-34
+| | |
+| | Not[boolean] - scope-38
+| | |
+| | |---Greater Than[boolean] - scope-37
+| | |
+| | |---Project[int][0] - scope-35
+| | |
+| | |---Constant(500) - scope-36
+|
+|---c: New For Each(false,false)[bag] - scope-26
+ | |
+ | Cast[int] - scope-21
+ | |
+ | |---Project[bytearray][1] - scope-20
+ | |
+ | Cast[chararray] - scope-24
+ | |
+ | |---Project[bytearray][0] - scope-23
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-19
+Tez vertex group scope-59 <- [scope-40, scope-46] -> null
+# No plan on vertex group
+Tez vertex group scope-58 <- [scope-40, scope-46] -> null
+# No plan on vertex group
Modified: pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java?rev=1589152&r1=1589151&r2=1589152&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java Tue Apr 22 14:59:44 2014
@@ -72,14 +72,18 @@ public class TestTezCompiler {
@Before
public void setUp() throws ExecException {
- NodeIdGenerator.reset();
- PigServer.resetScope();
+ resetScope();
pc.getProperties().remove(PigConfiguration.OPT_MULTIQUERY);
pc.getProperties().remove(PigConfiguration.TEZ_OPT_UNION);
pc.getProperties().remove(PigConfiguration.PIG_EXEC_NO_SECONDARY_KEY);
pigServer = new PigServer(pc);
}
+ private void resetScope() {
+ NodeIdGenerator.reset();
+ PigServer.resetScope();
+ }
+
@Test
public void testFilter() throws Exception {
String query =
@@ -466,6 +470,60 @@ public class TestTezCompiler {
}
@Test
+ public void testUnionSplit() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "split a into a1 if x > 100, a2 otherwise;" +
+ "c = union onschema a1, b;" +
+ "split c into d if x > 500, e otherwise;" +
+ "store a2 into 'file:///tmp/output/a2';" +
+ "store d into 'file:///tmp/output/d';" +
+ "store e into 'file:///tmp/output/e';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionUnion() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, y:chararray);" +
+ "e = union onschema c, d;" +
+ "f = group e by x;" +
+ "store f into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld");
+ resetScope();
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld");
+ }
+
+ //TODO: union followed by union followed by store does not work.
+ //@Test
+ public void testUnionUnionStore() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, y:chararray);" +
+ "e = union onschema c, d;" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld");
+ resetScope();
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11-OPTOFF.gld");
+ }
+
+ @Test
public void testRank() throws Exception {
String query =
"a = load 'file:///tmp/input1' as (x:int, y:int);" +