You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2015/04/08 01:21:02 UTC
svn commit: r1671973 [3/3] - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/
src/org/apache/pig/backend/hadoop/executionengine/tez/
src/org/apache/pig/backend/hadoop/executionengine/tez/plan/ src/o...
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15.gld?rev=1671973&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15.gld Tue Apr 7 23:21:01 2015
@@ -0,0 +1,121 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-30 -> Tez vertex scope-48,Tez vertex scope-58,
+Tez vertex scope-48 -> Tez vertex scope-40,Tez vertex scope-58,
+Tez vertex scope-58 -> Tez vertex scope-62,
+Tez vertex scope-40 -> Tez vertex scope-62,
+Tez vertex scope-62
+
+Tez vertex scope-30
+# Plan on vertex
+a: Split - scope-68
+| |
+| Local Rearrange[tuple]{tuple}(false) - scope-71 -> scope-48
+| | |
+| | Constant(DummyVal) - scope-72
+| |
+| |---New For Each(true,true)[tuple] - scope-78
+| | |
+| | Project[int][0] - scope-75
+| | |
+| | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-77
+| | |
+| | |---Project[tuple][*] - scope-76
+| |
+| |---PoissonSample - scope-74
+| |
+| |---Local Rearrange[tuple]{int}(false) - scope-69 -> scope-58
+| | |
+| | Project[int][0] - scope-70
+| |
+| Local Rearrange[tuple]{tuple}(false) - scope-81 -> scope-48
+| | |
+| | Constant(DummyVal) - scope-82
+| |
+| |---New For Each(true,true)[tuple] - scope-88
+| | |
+| | Project[int][0] - scope-85
+| | |
+| | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-87
+| | |
+| | |---Project[tuple][*] - scope-86
+| |
+| |---PoissonSample - scope-84
+| |
+| |---Local Rearrange[tuple]{int}(false) - scope-79 -> scope-58
+| | |
+| | Project[int][0] - scope-80
+| |
+| |---b: Filter[bag] - scope-13
+| | |
+| | Equal To[boolean] - scope-16
+| | |
+| | |---Project[int][0] - scope-14
+| | |
+| | |---Constant(2) - scope-15
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-48
+# Plan on vertex
+POValueOutputTez - scope-57 -> [scope-40, scope-58]
+|
+|---New For Each(false)[tuple] - scope-56
+ | |
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-55
+ | |
+ | |---Project[tuple][*] - scope-54
+ |
+ |---New For Each(false,false)[tuple] - scope-53
+ | |
+ | Constant(-1) - scope-52
+ | |
+ | Project[bag][1] - scope-50
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-49
+Tez vertex scope-58
+# Plan on vertex
+POIdentityInOutTez - scope-59 <- scope-30 -> scope-62
+| |
+| Project[int][0] - scope-26
+Tez vertex scope-40
+# Plan on vertex
+Partition Rearrange[tuple]{int}(false) - scope-60 -> scope-62
+| |
+| Project[int][0] - scope-27
+|
+|---d: New For Each(false,false)[bag] - scope-25
+ | |
+ | Cast[int] - scope-20
+ | |
+ | |---Project[bytearray][0] - scope-19
+ | |
+ | Cast[chararray] - scope-23
+ | |
+ | |---Project[bytearray][1] - scope-22
+ |
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-18
+Tez vertex scope-62
+# Plan on vertex
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-29
+|
+|---New For Each(true,true)[tuple] - scope-66
+ | |
+ | Project[bag][1] - scope-64
+ | |
+ | Project[bag][2] - scope-65
+ |
+ |---Package(Packager)[tuple]{int} - scope-63
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16-OPTOFF.gld?rev=1671973&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16-OPTOFF.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16-OPTOFF.gld Tue Apr 7 23:21:01 2015
@@ -0,0 +1,117 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-30 -> Tez vertex scope-48,Tez vertex scope-58,
+Tez vertex scope-48 -> Tez vertex scope-37,Tez vertex scope-58,
+Tez vertex scope-58 -> Tez vertex scope-62,
+Tez vertex scope-31 -> Tez vertex scope-35,Tez vertex scope-37,
+Tez vertex scope-35 -> Tez vertex scope-37,
+Tez vertex scope-37 -> Tez vertex scope-62,
+Tez vertex scope-62
+
+Tez vertex scope-30
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-43 -> scope-48
+| |
+| Constant(DummyVal) - scope-42
+|
+|---New For Each(true,true)[tuple] - scope-47
+ | |
+ | Project[int][0] - scope-26
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-46
+ | |
+ | |---Project[tuple][*] - scope-45
+ |
+ |---PoissonSample - scope-44
+ |
+ |---Local Rearrange[tuple]{int}(false) - scope-41 -> scope-58
+ | |
+ | Project[int][0] - scope-26
+ |
+ |---d: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-48
+# Plan on vertex
+POValueOutputTez - scope-57 -> [scope-37, scope-58]
+|
+|---New For Each(false)[tuple] - scope-56
+ | |
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-55
+ | |
+ | |---Project[tuple][*] - scope-54
+ |
+ |---New For Each(false,false)[tuple] - scope-53
+ | |
+ | Constant(-1) - scope-52
+ | |
+ | Project[bag][1] - scope-50
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-49
+Tez vertex scope-58
+# Plan on vertex
+POIdentityInOutTez - scope-59 <- scope-30 -> scope-62
+| |
+| Project[int][0] - scope-26
+Tez vertex scope-31
+# Plan on vertex
+a: Split - scope-68
+| |
+| POValueOutputTez - scope-39 -> [scope-37]
+| |
+| POValueOutputTez - scope-32 -> [scope-35]
+|
+|---a: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][0] - scope-9
+ | |
+ | Cast[chararray] - scope-13
+ | |
+ | |---Project[bytearray][1] - scope-12
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-35
+# Plan on vertex
+POValueOutputTez - scope-40 -> [scope-37]
+|
+|---b: Filter[bag] - scope-21
+ | |
+ | Equal To[boolean] - scope-24
+ | |
+ | |---Project[int][0] - scope-22
+ | |
+ | |---Constant(2) - scope-23
+ |
+ |---POValueInputTez - scope-36 <- scope-31
+Tez vertex scope-37
+# Plan on vertex
+Partition Rearrange[tuple]{int}(false) - scope-60 -> scope-62
+| |
+| Project[int][0] - scope-27
+|
+|---POShuffledValueInputTez - scope-38 <- [scope-31, scope-35]
+Tez vertex scope-62
+# Plan on vertex
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-29
+|
+|---New For Each(true,true)[tuple] - scope-66
+ | |
+ | Project[bag][1] - scope-64
+ | |
+ | Project[bag][2] - scope-65
+ |
+ |---Package(Packager)[tuple]{int} - scope-63
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16.gld?rev=1671973&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16.gld Tue Apr 7 23:21:01 2015
@@ -0,0 +1,107 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: pig-0_scope-0
+#--------------------------------------------------
+Tez vertex scope-30 -> Tez vertex scope-48,Tez vertex scope-58,
+Tez vertex scope-48 -> Tez vertex scope-31,Tez vertex scope-58,
+Tez vertex scope-58 -> Tez vertex scope-62,
+Tez vertex scope-31 -> Tez vertex scope-62,
+Tez vertex scope-62
+
+Tez vertex scope-30
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-43 -> scope-48
+| |
+| Constant(DummyVal) - scope-42
+|
+|---New For Each(true,true)[tuple] - scope-47
+ | |
+ | Project[int][0] - scope-26
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-46
+ | |
+ | |---Project[tuple][*] - scope-45
+ |
+ |---PoissonSample - scope-44
+ |
+ |---Local Rearrange[tuple]{int}(false) - scope-41 -> scope-58
+ | |
+ | Project[int][0] - scope-26
+ |
+ |---d: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-48
+# Plan on vertex
+POValueOutputTez - scope-57 -> [scope-31, scope-58]
+|
+|---New For Each(false)[tuple] - scope-56
+ | |
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-55
+ | |
+ | |---Project[tuple][*] - scope-54
+ |
+ |---New For Each(false,false)[tuple] - scope-53
+ | |
+ | Constant(-1) - scope-52
+ | |
+ | Project[bag][1] - scope-50
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-49
+Tez vertex scope-58
+# Plan on vertex
+POIdentityInOutTez - scope-59 <- scope-30 -> scope-62
+| |
+| Project[int][0] - scope-26
+Tez vertex scope-31
+# Plan on vertex
+a: Split - scope-68
+| |
+| Partition Rearrange[tuple]{int}(false) - scope-69 -> scope-62
+| | |
+| | Project[int][0] - scope-70
+| |
+| Partition Rearrange[tuple]{int}(false) - scope-71 -> scope-62
+| | |
+| | Project[int][0] - scope-72
+| |
+| |---b: Filter[bag] - scope-21
+| | |
+| | Equal To[boolean] - scope-24
+| | |
+| | |---Project[int][0] - scope-22
+| | |
+| | |---Constant(2) - scope-23
+|
+|---a: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][0] - scope-9
+ | |
+ | Cast[chararray] - scope-13
+ | |
+ | |---Project[bytearray][1] - scope-12
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-62
+# Plan on vertex
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-29
+|
+|---New For Each(true,true)[tuple] - scope-66
+ | |
+ | Project[bag][1] - scope-64
+ | |
+ | Project[bag][2] - scope-65
+ |
+ |---Package(Packager)[tuple]{int} - scope-63
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld?rev=1671973&r1=1671972&r2=1671973&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld Tue Apr 7 23:21:01 2015
@@ -4,30 +4,20 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-46 -> Tez vertex scope-47,
-Tez vertex scope-40 -> Tez vertex scope-47,
-Tez vertex scope-47
+Tez vertex scope-45 -> Tez vertex scope-52,Tez vertex scope-57,
+Tez vertex scope-52 -> Tez vertex scope-57,
+Tez vertex scope-56 -> Tez vertex scope-57,
+Tez vertex scope-57
-Tez vertex scope-46
+Tez vertex scope-45
# Plan on vertex
-POValueOutputTez - scope-50 -> [scope-47]
-|
-|---c: New For Each(false,false)[bag] - scope-26
- | |
- | Cast[int] - scope-21
- | |
- | |---Project[bytearray][1] - scope-20
- | |
- | Cast[chararray] - scope-24
- | |
- | |---Project[bytearray][0] - scope-23
- |
- |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-19
-Tez vertex scope-40
-# Plan on vertex
-1-2: Split - scope-57
+1-2: Split - scope-69
| |
-| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-14
+| a2: Split - scope-68
+| | |
+| | a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-17
+| | |
+| | POValueOutputTez - scope-60 -> [scope-57]
| |
| |---a2: Filter[bag] - scope-9
| | |
@@ -39,15 +29,7 @@ Tez vertex scope-40
| | |
| | |---Constant(100) - scope-11
| |
-| POValueOutputTez - scope-49 -> [scope-47]
-| |
-| |---a1: Filter[bag] - scope-15
-| | |
-| | Greater Than[boolean] - scope-18
-| | |
-| | |---Project[int][0] - scope-16
-| | |
-| | |---Constant(100) - scope-17
+| POValueOutputTez - scope-46 -> [scope-52]
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -60,30 +42,58 @@ Tez vertex scope-40
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-47
+Tez vertex scope-52
+# Plan on vertex
+POValueOutputTez - scope-59 -> [scope-57]
+|
+|---a1: Filter[bag] - scope-18
+ | |
+ | Greater Than[boolean] - scope-21
+ | |
+ | |---Project[int][0] - scope-19
+ | |
+ | |---Constant(100) - scope-20
+ |
+ |---POValueInputTez - scope-53 <- scope-45
+Tez vertex scope-56
+# Plan on vertex
+POValueOutputTez - scope-61 -> [scope-57]
+|
+|---c: New For Each(false,false)[bag] - scope-31
+ | |
+ | Cast[int] - scope-26
+ | |
+ | |---Project[bytearray][1] - scope-25
+ | |
+ | Cast[chararray] - scope-29
+ | |
+ | |---Project[bytearray][0] - scope-28
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-24
+Tez vertex scope-57
# Plan on vertex
-1-3: Split - scope-56
+1-3: Split - scope-67
| |
-| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33
+| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-38
| |
-| |---d: Filter[bag] - scope-29
+| |---d: Filter[bag] - scope-34
| | |
-| | Greater Than[boolean] - scope-32
+| | Greater Than[boolean] - scope-37
| | |
-| | |---Project[int][0] - scope-30
+| | |---Project[int][0] - scope-35
| | |
-| | |---Constant(500) - scope-31
+| | |---Constant(500) - scope-36
| |
-| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-39
+| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-44
| |
-| |---e: Filter[bag] - scope-34
+| |---e: Filter[bag] - scope-39
| | |
-| | Not[boolean] - scope-38
+| | Not[boolean] - scope-43
| | |
-| | |---Greater Than[boolean] - scope-37
+| | |---Greater Than[boolean] - scope-42
| | |
-| | |---Project[int][0] - scope-35
+| | |---Project[int][0] - scope-40
| | |
-| | |---Constant(500) - scope-36
+| | |---Constant(500) - scope-41
|
-|---POShuffledValueInputTez - scope-48 <- [scope-40, scope-46]
+|---POShuffledValueInputTez - scope-58 <- [scope-45, scope-52, scope-56]
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld?rev=1671973&r1=1671972&r2=1671973&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld Tue Apr 7 23:21:01 2015
@@ -4,16 +4,42 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-40 -> Tez vertex group scope-58,Tez vertex group scope-59,
-Tez vertex scope-46 -> Tez vertex group scope-58,Tez vertex group scope-59,
-Tez vertex group scope-58
-Tez vertex group scope-59
+Tez vertex scope-45 -> Tez vertex group scope-70,Tez vertex group scope-70,Tez vertex group scope-71,Tez vertex group scope-71,
+Tez vertex scope-56 -> Tez vertex group scope-70,Tez vertex group scope-71,
+Tez vertex group scope-70
+Tez vertex group scope-71
-Tez vertex scope-40
+Tez vertex scope-45
# Plan on vertex
-1-2: Split - scope-57
+1-2: Split - scope-69
| |
-| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-14
+| a2: Split - scope-68
+| | |
+| | a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-17
+| | |
+| | 1-3: Split - scope-72
+| | | |
+| | | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-77 -> scope-38
+| | | |
+| | | |---d: Filter[bag] - scope-73
+| | | | |
+| | | | Greater Than[boolean] - scope-76
+| | | | |
+| | | | |---Project[int][0] - scope-74
+| | | | |
+| | | | |---Constant(500) - scope-75
+| | | |
+| | | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-83 -> scope-44
+| | | |
+| | | |---e: Filter[bag] - scope-78
+| | | | |
+| | | | Not[boolean] - scope-82
+| | | | |
+| | | | |---Greater Than[boolean] - scope-81
+| | | | |
+| | | | |---Project[int][0] - scope-79
+| | | | |
+| | | | |---Constant(500) - scope-80
| |
| |---a2: Filter[bag] - scope-9
| | |
@@ -25,37 +51,37 @@ Tez vertex scope-40
| | |
| | |---Constant(100) - scope-11
| |
-| 1-3: Split - scope-60
+| 1-3: Split - scope-84
| | |
-| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-65
+| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-89 -> scope-38
| | |
-| | |---d: Filter[bag] - scope-61
+| | |---d: Filter[bag] - scope-85
| | | |
-| | | Greater Than[boolean] - scope-64
+| | | Greater Than[boolean] - scope-88
| | | |
-| | | |---Project[int][0] - scope-62
+| | | |---Project[int][0] - scope-86
| | | |
-| | | |---Constant(500) - scope-63
+| | | |---Constant(500) - scope-87
| | |
-| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-71
+| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-95 -> scope-44
| | |
-| | |---e: Filter[bag] - scope-66
+| | |---e: Filter[bag] - scope-90
| | | |
-| | | Not[boolean] - scope-70
+| | | Not[boolean] - scope-94
| | | |
-| | | |---Greater Than[boolean] - scope-69
+| | | |---Greater Than[boolean] - scope-93
| | | |
-| | | |---Project[int][0] - scope-67
+| | | |---Project[int][0] - scope-91
| | | |
-| | | |---Constant(500) - scope-68
+| | | |---Constant(500) - scope-92
| |
-| |---a1: Filter[bag] - scope-15
+| |---a1: Filter[bag] - scope-18
| | |
-| | Greater Than[boolean] - scope-18
+| | Greater Than[boolean] - scope-21
| | |
-| | |---Project[int][0] - scope-16
+| | |---Project[int][0] - scope-19
| | |
-| | |---Constant(100) - scope-17
+| | |---Constant(100) - scope-20
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -68,44 +94,44 @@ Tez vertex scope-40
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-46
+Tez vertex scope-56
# Plan on vertex
-1-3: Split - scope-72
+1-3: Split - scope-96
| |
-| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-77
+| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-101 -> scope-38
| |
-| |---d: Filter[bag] - scope-73
+| |---d: Filter[bag] - scope-97
| | |
-| | Greater Than[boolean] - scope-76
+| | Greater Than[boolean] - scope-100
| | |
-| | |---Project[int][0] - scope-74
+| | |---Project[int][0] - scope-98
| | |
-| | |---Constant(500) - scope-75
+| | |---Constant(500) - scope-99
| |
-| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-83
+| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-107 -> scope-44
| |
-| |---e: Filter[bag] - scope-78
+| |---e: Filter[bag] - scope-102
| | |
-| | Not[boolean] - scope-82
+| | Not[boolean] - scope-106
| | |
-| | |---Greater Than[boolean] - scope-81
+| | |---Greater Than[boolean] - scope-105
| | |
-| | |---Project[int][0] - scope-79
+| | |---Project[int][0] - scope-103
| | |
-| | |---Constant(500) - scope-80
+| | |---Constant(500) - scope-104
|
-|---c: New For Each(false,false)[bag] - scope-26
+|---c: New For Each(false,false)[bag] - scope-31
| |
- | Cast[int] - scope-21
+ | Cast[int] - scope-26
| |
- | |---Project[bytearray][1] - scope-20
+ | |---Project[bytearray][1] - scope-25
| |
- | Cast[chararray] - scope-24
+ | Cast[chararray] - scope-29
| |
- | |---Project[bytearray][0] - scope-23
+ | |---Project[bytearray][0] - scope-28
|
- |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-19
-Tez vertex group scope-58 <- [scope-40, scope-46] -> null
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-24
+Tez vertex group scope-70 <- [scope-45, scope-45, scope-56] -> null
# No plan on vertex group
-Tez vertex group scope-59 <- [scope-40, scope-46] -> null
+Tez vertex group scope-71 <- [scope-45, scope-45, scope-56] -> null
# No plan on vertex group
Modified: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1671973&r1=1671972&r2=1671973&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Tue Apr 7 23:21:01 2015
@@ -120,6 +120,68 @@ public class TestTezCompiler {
}
@Test
+ public void testSelfJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = filter a by x < 5;" +
+ "c = filter a by x == 10;" +
+ "d = filter a by x > 10;" +
+ "e = join b by x, c by x, d by x;" +
+ "store e into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-1.gld");
+ }
+
+ @Test
+ public void testSelfJoinSkewed() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = filter a by x < 5;" +
+ "c = filter a by x == 10;" +
+ "d = join b by x, c by x using 'skewed';" +
+ "store d into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-2.gld");
+ }
+
+ @Test
+ public void testSelfJoinReplicated() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = filter a by x < 5;" +
+ "c = filter a by x == 10;" +
+ "d = filter a by x > 10;" +
+ "e = join b by x, c by x, d by x using 'replicated';" +
+ "store e into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-3.gld");
+ }
+
+ @Test
+ public void testCross() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input2' as (x:int, z:int);" +
+ "c = cross a, b;" +
+ "d = foreach c generate a::x as x, y, z;" +
+ "store d into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-1.gld");
+ }
+
+ @Test
+ public void testSelfCross() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = filter a by x < 5;" +
+ "c = filter a by x == 10;" +
+ "d = cross b, c;" +
+ "store d into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-2.gld");
+ }
+
+ @Test
public void testSkewedJoin() throws Exception {
String query =
"a = load 'file:///tmp/input1' as (x:int, y:int);" +
@@ -370,6 +432,40 @@ public class TestTezCompiler {
}
@Test
+ public void testMultiQueryScalar() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = group a by x;" +
+ "c = foreach b generate group, COUNT(a) as cnt;" +
+ "SPLIT a into d if (2 * c.cnt) < y, e OTHERWISE;" +
+ "store d into 'file:///tmp/output1';" +
+ "store e into 'file:///tmp/output2';";
+
+ setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6-OPTOFF.gld");
+ }
+
+ @Test
+ public void testMultiQueryMultipleReplicateJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input' as (x:int, y:int);" +
+ "c = join a by $0, b by $0 using 'replicated';" +
+ "d = join a by $1, b by $1 using 'replicated';" +
+ "e = union c,d;" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7-OPTOFF.gld");
+ }
+
+ @Test
public void testUnionStore() throws Exception {
String query =
"a = load 'file:///tmp/input' as (x:int, y:chararray);" +
@@ -500,7 +596,7 @@ public class TestTezCompiler {
"a = load 'file:///tmp/input' as (x:int, y:chararray);" +
"b = load 'file:///tmp/input' as (y:chararray, x:int);" +
"split a into a1 if x > 100, a2 otherwise;" +
- "c = union onschema a1, b;" +
+ "c = union onschema a1, a2, b;" +
"split c into d if x > 500, e otherwise;" +
"store a2 into 'file:///tmp/output/a2';" +
"store d into 'file:///tmp/output/d';" +
@@ -531,8 +627,7 @@ public class TestTezCompiler {
run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld");
}
- //TODO: union followed by union followed by store does not work.
- //@Test
+ @Test
public void testUnionUnionStore() throws Exception {
String query =
"a = load 'file:///tmp/input' as (x:int, y:chararray);" +
@@ -550,6 +645,94 @@ public class TestTezCompiler {
}
@Test
+ public void testMultipleUnionSplitJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = filter a by x == 2;" +
+ "b1 = foreach b generate *;" +
+ "b2 = foreach b generate *;" +
+ "b3 = union onschema b1, b2;" +
+ "c = filter a by x == 3;" +
+ "c1 = foreach c generate y, x;" +
+ "c2 = foreach c generate y, x;" +
+ "c3 = union c1, c2;" +
+ "a1 = union onschema b3, c3;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join a1 by x, d by x;" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionSplitReplicateJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = filter a by x == 2;" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join c by x, d by x using 'replicated';" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13-OPTOFF.gld");
+
+ query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = filter a by x == 2;" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join d by x, c by x using 'replicated';" +
+ "store e into 'file:///tmp/output';";
+
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionSplitSkewedJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = filter a by x == 2;" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join c by x, d by x using 'skewed';" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15-OPTOFF.gld");
+
+ query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = filter a by x == 2;" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join d by x, c by x using 'skewed';" +
+ "store e into 'file:///tmp/output';";
+
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16.gld");
+ resetScope();
+ setProperty(PigConfiguration.PIG_TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-16-OPTOFF.gld");
+ }
+
+ @Test
public void testRank() throws Exception {
String query =
"a = load 'file:///tmp/input1' as (x:int, y:int);" +