You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/05/30 21:07:29 UTC
svn commit: r1598702 [23/23] - in /pig/trunk: ./ ivy/
shims/src/hadoop23/org/apache/pig/backend/hadoop23/
shims/test/hadoop20/org/apache/pig/test/
shims/test/hadoop23/org/apache/pig/test/ src/META-INF/services/
src/org/apache/pig/ src/org/apache/pig/ba...
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld Fri May 30 19:07:23 2014
@@ -0,0 +1,115 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-149
+#--------------------------------------------------
+Tez vertex scope-115 -> Tez vertex scope-117,
+Tez vertex scope-116 -> Tez vertex scope-117,
+Tez vertex scope-117 -> Tez vertex scope-139,Tez vertex scope-129,
+Tez vertex scope-129 -> Tez vertex scope-139,Tez vertex scope-121,
+Tez vertex scope-139 -> Tez vertex scope-143,
+Tez vertex scope-121 -> Tez vertex scope-143,
+Tez vertex scope-143
+
+Tez vertex scope-115
+# Plan on vertex
+POValueOutputTez - scope-119 -> [scope-117]
+|
+|---a: New For Each(false,false)[bag] - scope-93
+ | |
+ | Cast[int] - scope-88
+ | |
+ | |---Project[bytearray][0] - scope-87
+ | |
+ | Cast[chararray] - scope-91
+ | |
+ | |---Project[bytearray][1] - scope-90
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-86
+Tez vertex scope-116
+# Plan on vertex
+POValueOutputTez - scope-120 -> [scope-117]
+|
+|---c: New For Each(false,false)[bag] - scope-101
+ | |
+ | Cast[int] - scope-96
+ | |
+ | |---Project[bytearray][1] - scope-95
+ | |
+ | Cast[chararray] - scope-99
+ | |
+ | |---Project[bytearray][0] - scope-98
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-94
+Tez vertex scope-117
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-124 -> scope-129
+| |
+| Constant(DummyVal) - scope-123
+|
+|---New For Each(true,true)[tuple] - scope-128
+ | |
+ | Project[int][0] - scope-111
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-127
+ | |
+ | |---Project[tuple][*] - scope-126
+ |
+ |---PoissonSample - scope-125
+ |
+ |---Local Rearrange[tuple]{int}(false) - scope-122 -> scope-139
+ | |
+ | Project[int][0] - scope-111
+ |
+ |---POShuffledValueInputTez - scope-118 <- [scope-116, scope-115]
+Tez vertex scope-129
+# Plan on vertex
+POValueOutputTez - scope-138 -> [scope-139, scope-121]
+|
+|---New For Each(false)[tuple] - scope-137
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.PartitionSkewedKeys)[tuple] - scope-136
+ | |
+ | |---Project[tuple][*] - scope-135
+ |
+ |---New For Each(false,false)[tuple] - scope-134
+ | |
+ | Constant(1) - scope-133
+ | |
+ | Project[bag][1] - scope-131
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-130
+Tez vertex scope-139
+# Plan on vertex
+POIdentityInOutTez - scope-140 <- scope-117 -> scope-143
+| |
+| Project[int][0] - scope-111
+Tez vertex scope-121
+# Plan on vertex
+Partition Rearrange[tuple]{int}(false) - scope-141 -> scope-143
+| |
+| Project[int][0] - scope-112
+|
+|---d: New For Each(false,false)[bag] - scope-110
+ | |
+ | Cast[int] - scope-105
+ | |
+ | |---Project[bytearray][0] - scope-104
+ | |
+ | Cast[chararray] - scope-108
+ | |
+ | |---Project[bytearray][1] - scope-107
+ |
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-103
+Tez vertex scope-143
+# Plan on vertex
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-114
+|
+|---New For Each(true,true)[tuple] - scope-147
+ | |
+ | Project[bag][1] - scope-145
+ | |
+ | Project[bag][2] - scope-146
+ |
+ |---Package(Packager)[tuple]{int} - scope-144
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld Fri May 30 19:07:23 2014
@@ -0,0 +1,131 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-85
+#--------------------------------------------------
+Tez vertex scope-29 -> Tez vertex scope-63,Tez vertex scope-64,
+Tez vertex scope-30 -> Tez vertex scope-63,Tez vertex scope-64,
+Tez vertex scope-64 -> Tez vertex scope-43,
+Tez vertex scope-43 -> Tez vertex scope-53,Tez vertex scope-35,
+Tez vertex scope-63 -> Tez vertex scope-53,
+Tez vertex scope-53 -> Tez vertex scope-57,
+Tez vertex scope-35 -> Tez vertex scope-57,
+Tez vertex scope-57
+
+Tez vertex scope-29
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-67 -> scope-43
+| |
+| Constant(DummyVal) - scope-68
+|
+|---New For Each(true,true)[tuple] - scope-74
+ | |
+ | Project[int][0] - scope-71
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-72
+ | |
+ | |---Project[tuple][*] - scope-73
+ |
+ |---PoissonSample - scope-70
+ |
+ |---Local Rearrange[tuple]{int}(false) - scope-65 -> scope-53
+ | |
+ | Project[int][0] - scope-66
+ |
+ |---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-30
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-77 -> scope-43
+| |
+| Constant(DummyVal) - scope-78
+|
+|---New For Each(true,true)[tuple] - scope-84
+ | |
+ | Project[int][0] - scope-81
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-82
+ | |
+ | |---Project[tuple][*] - scope-83
+ |
+ |---PoissonSample - scope-80
+ |
+ |---Local Rearrange[tuple]{int}(false) - scope-75 -> scope-53
+ | |
+ | Project[int][0] - scope-76
+ |
+ |---c: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][1] - scope-9
+ | |
+ | Cast[chararray] - scope-13
+ | |
+ | |---Project[bytearray][0] - scope-12
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex group scope-64 <- [scope-29, scope-30] -> scope-43
+# No plan on vertex group
+Tez vertex scope-43
+# Plan on vertex
+POValueOutputTez - scope-52 -> [scope-53, scope-35]
+|
+|---New For Each(false)[tuple] - scope-51
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.PartitionSkewedKeys)[tuple] - scope-50
+ | |
+ | |---Project[tuple][*] - scope-49
+ |
+ |---New For Each(false,false)[tuple] - scope-48
+ | |
+ | Constant(1) - scope-47
+ | |
+ | Project[bag][1] - scope-45
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-44
+Tez vertex group scope-63 <- [scope-29, scope-30] -> scope-53
+# No plan on vertex group
+Tez vertex scope-53
+# Plan on vertex
+POIdentityInOutTez - scope-54 <- scope-63 -> scope-57
+| |
+| Project[int][0] - scope-25
+Tez vertex scope-35
+# Plan on vertex
+Partition Rearrange[tuple]{int}(false) - scope-55 -> scope-57
+| |
+| Project[int][0] - scope-26
+|
+|---d: New For Each(false,false)[bag] - scope-24
+ | |
+ | Cast[int] - scope-19
+ | |
+ | |---Project[bytearray][0] - scope-18
+ | |
+ | Cast[chararray] - scope-22
+ | |
+ | |---Project[bytearray][1] - scope-21
+ |
+ |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-17
+Tez vertex scope-57
+# Plan on vertex
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-28
+|
+|---New For Each(true,true)[tuple] - scope-61
+ | |
+ | Project[bag][1] - scope-59
+ | |
+ | Project[bag][2] - scope-60
+ |
+ |---Package(Packager)[tuple]{int} - scope-58
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld Fri May 30 19:07:23 2014
@@ -0,0 +1,91 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-117
+#--------------------------------------------------
+Tez vertex scope-88 -> Tez vertex scope-90,
+Tez vertex scope-89 -> Tez vertex scope-90,
+Tez vertex scope-90 -> Tez vertex scope-110,Tez vertex scope-100,
+Tez vertex scope-100 -> Tez vertex scope-110,
+Tez vertex scope-110 -> Tez vertex scope-112,
+Tez vertex scope-112
+
+Tez vertex scope-88
+# Plan on vertex
+POValueOutputTez - scope-92 -> [scope-90]
+|
+|---a: New For Each(false,false)[bag] - scope-75
+ | |
+ | Cast[int] - scope-70
+ | |
+ | |---Project[bytearray][0] - scope-69
+ | |
+ | Cast[chararray] - scope-73
+ | |
+ | |---Project[bytearray][1] - scope-72
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-68
+Tez vertex scope-89
+# Plan on vertex
+POValueOutputTez - scope-93 -> [scope-90]
+|
+|---c: New For Each(false,false)[bag] - scope-83
+ | |
+ | Cast[int] - scope-78
+ | |
+ | |---Project[bytearray][1] - scope-77
+ | |
+ | Cast[chararray] - scope-81
+ | |
+ | |---Project[bytearray][0] - scope-80
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-76
+Tez vertex scope-90
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-96 -> scope-100
+| |
+| Constant(DummyVal) - scope-95
+|
+|---ReservoirSample - scope-99
+ |
+ |---New For Each(false)[tuple] - scope-98
+ | |
+ | Project[int][0] - scope-97
+ |
+ |---d: Local Rearrange[tuple]{int}(false) - scope-94 -> scope-110
+ | |
+ | Project[int][0] - scope-85
+ |
+ |---POShuffledValueInputTez - scope-91 <- [scope-88, scope-89]
+Tez vertex scope-100
+# Plan on vertex
+POValueOutputTez - scope-109 -> [scope-110]
+|
+|---New For Each(false)[tuple] - scope-108
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-107
+ | |
+ | |---Project[tuple][*] - scope-106
+ |
+ |---New For Each(false,false)[tuple] - scope-105
+ | |
+ | Constant(1) - scope-104
+ | |
+ | Project[bag][1] - scope-102
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-101
+Tez vertex scope-110
+# Plan on vertex
+POIdentityInOutTez - scope-111 <- scope-90 -> scope-112
+| |
+| Project[int][0] - scope-85
+Tez vertex scope-112
+# Plan on vertex
+d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-87
+|
+|---New For Each(true)[tuple] - scope-115
+ | |
+ | Project[bag][1] - scope-114
+ |
+ |---Package(LitePackager)[tuple]{int} - scope-113
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld Fri May 30 19:07:23 2014
@@ -0,0 +1,103 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-67
+#--------------------------------------------------
+Tez vertex scope-20 -> Tez vertex scope-49,Tez vertex scope-50,
+Tez vertex scope-21 -> Tez vertex scope-49,Tez vertex scope-50,
+Tez vertex scope-50 -> Tez vertex scope-32,
+Tez vertex scope-32 -> Tez vertex scope-42,
+Tez vertex scope-49 -> Tez vertex scope-42,
+Tez vertex scope-42 -> Tez vertex scope-44,
+Tez vertex scope-44
+
+Tez vertex scope-20
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-54 -> scope-32
+| |
+| Constant(DummyVal) - scope-55
+|
+|---ReservoirSample - scope-53
+ |
+ |---New For Each(false)[tuple] - scope-52
+ | |
+ | Project[int][0] - scope-51
+ |
+ |---d: Local Rearrange[tuple]{int}(false) - scope-57 -> scope-42
+ | |
+ | Project[int][0] - scope-58
+ |
+ |---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-21
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-62 -> scope-32
+| |
+| Constant(DummyVal) - scope-63
+|
+|---ReservoirSample - scope-61
+ |
+ |---New For Each(false)[tuple] - scope-60
+ | |
+ | Project[int][0] - scope-59
+ |
+ |---d: Local Rearrange[tuple]{int}(false) - scope-65 -> scope-42
+ | |
+ | Project[int][0] - scope-66
+ |
+ |---c: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][1] - scope-9
+ | |
+ | Cast[chararray] - scope-13
+ | |
+ | |---Project[bytearray][0] - scope-12
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex group scope-50 <- [scope-20, scope-21] -> scope-32
+# No plan on vertex group
+Tez vertex scope-32
+# Plan on vertex
+POValueOutputTez - scope-41 -> [scope-42]
+|
+|---New For Each(false)[tuple] - scope-40
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-39
+ | |
+ | |---Project[tuple][*] - scope-38
+ |
+ |---New For Each(false,false)[tuple] - scope-37
+ | |
+ | Constant(1) - scope-36
+ | |
+ | Project[bag][1] - scope-34
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-33
+Tez vertex group scope-49 <- [scope-20, scope-21] -> scope-42
+# No plan on vertex group
+Tez vertex scope-42
+# Plan on vertex
+POIdentityInOutTez - scope-43 <- scope-49 -> scope-44
+| |
+| Project[int][0] - scope-17
+Tez vertex scope-44
+# Plan on vertex
+d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-19
+|
+|---New For Each(true)[tuple] - scope-47
+ | |
+ | Project[bag][1] - scope-46
+ |
+ |---Package(LitePackager)[tuple]{int} - scope-45
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8-OPTOFF.gld?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8-OPTOFF.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8-OPTOFF.gld Fri May 30 19:07:23 2014
@@ -0,0 +1,97 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-108
+#--------------------------------------------------
+Tez vertex scope-81 -> Tez vertex scope-84,
+Tez vertex scope-84 -> Tez vertex scope-97,
+Tez vertex scope-89 -> Tez vertex scope-92,
+Tez vertex scope-92 -> Tez vertex scope-97,
+Tez vertex scope-97 -> Tez vertex scope-103,
+Tez vertex scope-103
+
+Tez vertex scope-81
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-83 -> scope-84
+| |
+| Project[tuple][*] - scope-82
+|
+|---Limit - scope-59
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-58
+Tez vertex scope-84
+# Plan on vertex
+POValueOutputTez - scope-99 -> [scope-97]
+|
+|---a: New For Each(false,false)[bag] - scope-67
+ | |
+ | Cast[int] - scope-62
+ | |
+ | |---Project[bytearray][0] - scope-61
+ | |
+ | Cast[chararray] - scope-65
+ | |
+ | |---Project[bytearray][1] - scope-64
+ |
+ |---Limit - scope-60
+ |
+ |---Limit - scope-88
+ |
+ |---New For Each(true)[bag] - scope-87
+ | |
+ | Project[tuple][1] - scope-86
+ |
+ |---Package(Packager)[tuple]{tuple} - scope-85
+Tez vertex scope-89
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-91 -> scope-92
+| |
+| Project[tuple][*] - scope-90
+|
+|---Limit - scope-69
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-68
+Tez vertex scope-92
+# Plan on vertex
+POValueOutputTez - scope-100 -> [scope-97]
+|
+|---c: New For Each(false,false)[bag] - scope-77
+ | |
+ | Cast[int] - scope-72
+ | |
+ | |---Project[bytearray][1] - scope-71
+ | |
+ | Cast[chararray] - scope-75
+ | |
+ | |---Project[bytearray][0] - scope-74
+ |
+ |---Limit - scope-70
+ |
+ |---Limit - scope-96
+ |
+ |---New For Each(true)[bag] - scope-95
+ | |
+ | Project[tuple][1] - scope-94
+ |
+ |---Package(Packager)[tuple]{tuple} - scope-93
+Tez vertex scope-97
+# Plan on vertex
+d: Local Rearrange[tuple]{tuple}(false) - scope-102 -> scope-103
+| |
+| Project[tuple][*] - scope-101
+|
+|---d: Limit - scope-79
+ |
+ |---POShuffledValueInputTez - scope-98 <- [scope-92, scope-84]
+Tez vertex scope-103
+# Plan on vertex
+d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-80
+|
+|---d: Limit - scope-107
+ |
+ |---d: New For Each(true)[bag] - scope-106
+ | |
+ | Project[tuple][1] - scope-105
+ |
+ |---d: Package(Packager)[tuple]{tuple} - scope-104
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8.gld?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8.gld Fri May 30 19:07:23 2014
@@ -0,0 +1,98 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-57
+#--------------------------------------------------
+Tez vertex scope-23 -> Tez vertex scope-26,
+Tez vertex scope-26 -> Tez vertex scope-50,
+Tez vertex scope-31 -> Tez vertex scope-34,
+Tez vertex scope-34 -> Tez vertex scope-50,
+Tez vertex scope-50 -> Tez vertex scope-45,
+Tez vertex scope-45
+
+Tez vertex scope-23
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-25 -> scope-26
+| |
+| Project[tuple][*] - scope-24
+|
+|---Limit - scope-1
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-26
+# Plan on vertex
+d: Local Rearrange[tuple]{tuple}(false) - scope-51 -> scope-45
+| |
+| Project[tuple][*] - scope-52
+|
+|---d: Limit - scope-53
+ |
+ |---a: New For Each(false,false)[bag] - scope-9
+ | |
+ | Cast[int] - scope-4
+ | |
+ | |---Project[bytearray][0] - scope-3
+ | |
+ | Cast[chararray] - scope-7
+ | |
+ | |---Project[bytearray][1] - scope-6
+ |
+ |---Limit - scope-2
+ |
+ |---Limit - scope-30
+ |
+ |---New For Each(true)[bag] - scope-29
+ | |
+ | Project[tuple][1] - scope-28
+ |
+ |---Package(Packager)[tuple]{tuple} - scope-27
+Tez vertex scope-31
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-33 -> scope-34
+| |
+| Project[tuple][*] - scope-32
+|
+|---Limit - scope-11
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-10
+Tez vertex scope-34
+# Plan on vertex
+d: Local Rearrange[tuple]{tuple}(false) - scope-54 -> scope-45
+| |
+| Project[tuple][*] - scope-55
+|
+|---d: Limit - scope-56
+ |
+ |---c: New For Each(false,false)[bag] - scope-19
+ | |
+ | Cast[int] - scope-14
+ | |
+ | |---Project[bytearray][1] - scope-13
+ | |
+ | Cast[chararray] - scope-17
+ | |
+ | |---Project[bytearray][0] - scope-16
+ |
+ |---Limit - scope-12
+ |
+ |---Limit - scope-38
+ |
+ |---New For Each(true)[bag] - scope-37
+ | |
+ | Project[tuple][1] - scope-36
+ |
+ |---Package(Packager)[tuple]{tuple} - scope-35
+Tez vertex group scope-50 <- [scope-26, scope-34] -> scope-45
+# No plan on vertex group
+Tez vertex scope-45
+# Plan on vertex
+d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-22
+|
+|---d: Limit - scope-49
+ |
+ |---d: New For Each(true)[bag] - scope-48
+ | |
+ | Project[tuple][1] - scope-47
+ |
+ |---d: Package(Packager)[tuple]{tuple} - scope-46
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld Fri May 30 19:07:23 2014
@@ -0,0 +1,89 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-121
+#--------------------------------------------------
+Tez vertex scope-109 -> Tez vertex scope-110,
+Tez vertex scope-103 -> Tez vertex scope-110,
+Tez vertex scope-110
+
+Tez vertex scope-109
+# Plan on vertex
+POValueOutputTez - scope-113 -> [scope-110]
+|
+|---c: New For Each(false,false)[bag] - scope-89
+ | |
+ | Cast[int] - scope-84
+ | |
+ | |---Project[bytearray][1] - scope-83
+ | |
+ | Cast[chararray] - scope-87
+ | |
+ | |---Project[bytearray][0] - scope-86
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-82
+Tez vertex scope-103
+# Plan on vertex
+1-12: Split - scope-120
+| |
+| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-77
+| |
+| |---a2: Filter[bag] - scope-72
+| | |
+| | Not[boolean] - scope-76
+| | |
+| | |---Greater Than[boolean] - scope-75
+| | |
+| | |---Project[int][0] - scope-73
+| | |
+| | |---Constant(100) - scope-74
+| |
+| POValueOutputTez - scope-112 -> [scope-110]
+| |
+| |---a1: Filter[bag] - scope-78
+| | |
+| | Greater Than[boolean] - scope-81
+| | |
+| | |---Project[int][0] - scope-79
+| | |
+| | |---Constant(100) - scope-80
+|
+|---a: New For Each(false,false)[bag] - scope-70
+ | |
+ | Cast[int] - scope-65
+ | |
+ | |---Project[bytearray][0] - scope-64
+ | |
+ | Cast[chararray] - scope-68
+ | |
+ | |---Project[bytearray][1] - scope-67
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-63
+Tez vertex scope-110
+# Plan on vertex
+1-13: Split - scope-119
+| |
+| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-96
+| |
+| |---d: Filter[bag] - scope-92
+| | |
+| | Greater Than[boolean] - scope-95
+| | |
+| | |---Project[int][0] - scope-93
+| | |
+| | |---Constant(500) - scope-94
+| |
+| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-102
+| |
+| |---e: Filter[bag] - scope-97
+| | |
+| | Not[boolean] - scope-101
+| | |
+| | |---Greater Than[boolean] - scope-100
+| | |
+| | |---Project[int][0] - scope-98
+| | |
+| | |---Constant(500) - scope-99
+|
+|---POShuffledValueInputTez - scope-111 <- [scope-109, scope-103]
Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld (added)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld Fri May 30 19:07:23 2014
@@ -0,0 +1,111 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-62
+#--------------------------------------------------
+Tez vertex scope-40 -> Tez vertex scope-58,Tez vertex scope-59,
+Tez vertex scope-46 -> Tez vertex scope-58,Tez vertex scope-59,
+Tez vertex scope-59
+Tez vertex scope-58
+
+Tez vertex scope-40
+# Plan on vertex
+1-2: Split - scope-57
+| |
+| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-14
+| |
+| |---a2: Filter[bag] - scope-9
+| | |
+| | Not[boolean] - scope-13
+| | |
+| | |---Greater Than[boolean] - scope-12
+| | |
+| | |---Project[int][0] - scope-10
+| | |
+| | |---Constant(100) - scope-11
+| |
+| 1-3: Split - scope-60
+| | |
+| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33
+| | |
+| | |---d: Filter[bag] - scope-29
+| | | |
+| | | Greater Than[boolean] - scope-32
+| | | |
+| | | |---Project[int][0] - scope-30
+| | | |
+| | | |---Constant(500) - scope-31
+| | |
+| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-39
+| | |
+| | |---e: Filter[bag] - scope-34
+| | | |
+| | | Not[boolean] - scope-38
+| | | |
+| | | |---Greater Than[boolean] - scope-37
+| | | |
+| | | |---Project[int][0] - scope-35
+| | | |
+| | | |---Constant(500) - scope-36
+| |
+| |---a1: Filter[bag] - scope-15
+| | |
+| | Greater Than[boolean] - scope-18
+| | |
+| | |---Project[int][0] - scope-16
+| | |
+| | |---Constant(100) - scope-17
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-46
+# Plan on vertex
+1-3: Split - scope-61
+| |
+| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33
+| |
+| |---d: Filter[bag] - scope-29
+| | |
+| | Greater Than[boolean] - scope-32
+| | |
+| | |---Project[int][0] - scope-30
+| | |
+| | |---Constant(500) - scope-31
+| |
+| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-39
+| |
+| |---e: Filter[bag] - scope-34
+| | |
+| | Not[boolean] - scope-38
+| | |
+| | |---Greater Than[boolean] - scope-37
+| | |
+| | |---Project[int][0] - scope-35
+| | |
+| | |---Constant(500) - scope-36
+|
+|---c: New For Each(false,false)[bag] - scope-26
+ | |
+ | Cast[int] - scope-21
+ | |
+ | |---Project[bytearray][1] - scope-20
+ | |
+ | Cast[chararray] - scope-24
+ | |
+ | |---Project[bytearray][0] - scope-23
+ |
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-19
+Tez vertex group scope-59 <- [scope-40, scope-46] -> null
+# No plan on vertex group
+Tez vertex group scope-58 <- [scope-40, scope-46] -> null
+# No plan on vertex group
Added: pig/trunk/test/org/apache/pig/tez/TestSecondarySortTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestSecondarySortTez.java?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestSecondarySortTez.java (added)
+++ pig/trunk/test/org/apache/pig/tez/TestSecondarySortTez.java Fri May 30 19:07:23 2014
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.tez;
+
+import org.apache.pig.PigConfiguration;
+import org.apache.pig.backend.hadoop.executionengine.optimizer.SecondaryKeyOptimizer;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.CombinerOptimizer;
+import org.apache.pig.backend.hadoop.executionengine.tez.SecondaryKeyOptimizerTez;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezCompiler;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.test.MiniGenericCluster;
+import org.apache.pig.test.TestSecondarySort;
+import org.apache.pig.test.Util;
+
+public class TestSecondarySortTez extends TestSecondarySort {
+
+ public TestSecondarySortTez() {
+ super();
+ }
+
+ @Override
+ public MiniGenericCluster getCluster() {
+ return MiniGenericCluster.buildCluster(MiniGenericCluster.EXECTYPE_TEZ);
+ }
+
+ @Override
+ public SecondaryKeyOptimizer visitSecondaryKeyOptimizer(String query)
+ throws Exception, VisitorException {
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
+ TezCompiler comp = new TezCompiler(pp, pc, null);
+ TezOperPlan tezPlan = comp.compile();
+ boolean nocombiner = Boolean.parseBoolean(pc.getProperties().getProperty(
+ PigConfiguration.PROP_NO_COMBINER, "false"));
+
+ // Run CombinerOptimizer on Tez plan
+ if (!nocombiner) {
+ boolean doMapAgg = Boolean.parseBoolean(pc.getProperties()
+ .getProperty(PigConfiguration.PROP_EXEC_MAP_PARTAGG,
+ "false"));
+ CombinerOptimizer co = new CombinerOptimizer(tezPlan, doMapAgg);
+ co.visit();
+ }
+
+ SecondaryKeyOptimizerTez skOptimizer = new SecondaryKeyOptimizerTez(
+ tezPlan);
+ skOptimizer.visit();
+ return skOptimizer;
+ }
+
+}
+
Added: pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java (added)
+++ pig/trunk/test/org/apache/pig/tez/TestTezCompiler.java Fri May 30 19:07:23 2014
@@ -0,0 +1,590 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.tez;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.util.Properties;
+
+import org.apache.pig.PigConfiguration;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezPlanContainer;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezPlanContainerPrinter;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.plan.NodeIdGenerator;
+import org.apache.pig.test.Util;
+import org.apache.pig.test.utils.TestHelper;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test cases to test the TezCompiler. VERY IMPORTANT NOTE: The tests here
+ * compare results with a "golden" set of outputs. In each test case here, the
+ * operators generated have a random operator key which uses Java's Random
+ * class. So if there is a code change which changes the number of operators
+ * created in a plan, then the "golden" file for that test case
+ * need to be changed.
+ */
+
+public class TestTezCompiler {
+ private static PigContext pc;
+ private static PigServer pigServer;
+ private static final int MAX_SIZE = 100000;
+
+ // If for some reason, the golden files need to be regenerated, set this to
+ // true - THIS WILL OVERWRITE THE GOLDEN FILES - So use this with caution
+ // and only for the test cases you need and are sure of.
+ private boolean generate = false;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ pc = new PigContext(new TezLocalExecType(), new Properties());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ }
+
+ @Before
+ public void setUp() throws ExecException {
+ resetScope();
+ pc.getProperties().remove(PigConfiguration.OPT_MULTIQUERY);
+ pc.getProperties().remove(PigConfiguration.TEZ_OPT_UNION);
+ pc.getProperties().remove(PigConfiguration.PIG_EXEC_NO_SECONDARY_KEY);
+ pigServer = new PigServer(pc);
+ }
+
+ private void resetScope() {
+ NodeIdGenerator.reset();
+ PigServer.resetScope();
+ }
+
+ @Test
+ public void testFilter() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = filter a by x > 0;" +
+ "c = foreach b generate y;" +
+ "store c into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC1.gld");
+ }
+
+ @Test
+ public void testGroupBy() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = group a by x;" +
+ "c = foreach b generate group, COUNT(a.x);" +
+ "store c into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC2.gld");
+ }
+
+ @Test
+ public void testJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input2' as (x:int, z:int);" +
+ "c = join a by x, b by x;" +
+ "d = foreach c generate a::x as x, y, z;" +
+ "store d into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC3.gld");
+ }
+
+ @Test
+ public void testSkewedJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input2' as (x:int, z:int);" +
+ "c = join a by x, b by x using 'skewed';" +
+ "d = foreach c generate a::x as x, y, z;" +
+ "store d into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld");
+ }
+
+ @Test
+ public void testLimit() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = limit a 10;" +
+ "c = foreach b generate y;" +
+ "store c into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC4.gld");
+ }
+
+ @Test
+ public void testDistinct() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = distinct a;" +
+ "c = foreach b generate y;" +
+ "store c into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC5.gld");
+ }
+
+ @Test
+ public void testDistinctAlgebraicUdfCombiner() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = group a by x;" +
+ "c = foreach b { d = distinct a; generate COUNT(d); };" +
+ "store c into 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC13.gld");
+ }
+
+
+
+ @Test
+ public void testReplicatedJoinInMapper() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input2' as (x:int, z:int);" +
+ "c = load 'file:///tmp/input3' as (x:int, z:int);" +
+ "d = join a by x, b by x, c by x using 'replicated';" +
+ "store d into 'file:///tmp/output/d';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC10.gld");
+ }
+
+ @Test
+ public void testReplicatedJoinInReducer() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = group a by x;" +
+ "b1 = foreach b generate group, COUNT(a.y);" +
+ "c = load 'file:///tmp/input2' as (x:int, z:int);" +
+ "d = join b1 by group, c by x using 'replicated';" +
+ "store d into 'file:///tmp/output/e';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC11.gld");
+ }
+
+ @Test
+ public void testStream() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' using PigStorage(',') as (x:int, y:int);" +
+ "b = stream a through `stream.pl -n 5`;" +
+ "STORE b INTO 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC12.gld");
+ }
+
+ @Test
+ public void testSecondaryKeySort() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' using PigStorage(',') as (x:int, y:int, z:int);" +
+ "b = group a by $0;" +
+ "c = foreach b { d = limit a 10; e = order d by $1; f = order e by $0; generate group, f;};"+
+ "store c INTO 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC14.gld");
+
+ // With optimization turned off
+ setProperty(PigConfiguration.PIG_EXEC_NO_SECONDARY_KEY, "true");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC15.gld");
+ }
+
+ @Test
+ public void testOrderBy() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' using PigStorage(',') as (x:int, y:int);" +
+ "b = order a by x;" +
+ "STORE b INTO 'file:///tmp/output';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld");
+ }
+
+ // PIG-3759, PIG-3781
+ // Combiner should not be added in case of co-group
+ @Test
+ public void testCogroupWithAlgebraiceUDF() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input2' as (x:int, z:int);" +
+ "c = cogroup a by x, b by x;" +
+ "d = foreach c generate group, COUNT(a.y), COUNT(b.z);" +
+ "store d into 'file:///tmp/output/d';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC18.gld");
+ }
+
+ @Test
+ public void testMulitQueryWithSplitSingleVertex() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "split a into b if x <= 5, c if x <= 10, d if x >10;" +
+ "store b into 'file:///tmp/output/b';" +
+ "store c into 'file:///tmp/output/c';" +
+ "store d into 'file:///tmp/output/d';";
+
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld");
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld");
+ }
+
+ @Test
+ public void testMulitQueryWithSplitMultiVertex() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "split a into b if x <= 5, c if x <= 10, d if x >10;" +
+ "split b into e if x < 3, f if x >= 3;" +
+ // No Combiner on the edge to b1/b2 vertex as both b1 and b2 are stored
+ "b1 = group b by x;" +
+ "b2 = foreach b1 generate group, SUM(b.x);" +
+ // Case of two outputs within a split going to same edge as input
+ "c1 = join c by x, b by x;" +
+ "c2 = group c by x;" +
+ // Combiner on the edge to c3 vertex
+ "c3 = foreach c2 generate group, SUM(c.x);" +
+ "d1 = filter d by x == 5;" +
+ "e1 = order e by x;" +
+ // TODO: Physical plan has extra split for f1 - 1-2: Split - scope-80
+ // POSplit has only 1 sub plan. Optimized and removed in MR plan.
+ // Needs to be removed in Tez plan as well.
+ "f1 = limit f 1;" +
+ "f2 = union d1, f1;" +
+ "store b1 into 'file:///tmp/output/b1';" +
+ "store b2 into 'file:///tmp/output/b2';" +
+ "store c1 into 'file:///tmp/output/c1';" +
+ "store c3 into 'file:///tmp/output/c1';" +
+ "store d1 into 'file:///tmp/output/d1';" +
+ "store e1 into 'file:///tmp/output/e1';" +
+ "store f1 into 'file:///tmp/output/f1';" +
+ "store f2 into 'file:///tmp/output/f2';";
+
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld");
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld");
+ }
+
+ @Test
+ public void testMultiQueryWithGroupBy() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = group a by x;" +
+ "b = foreach b generate group, COUNT(a.x);" +
+ "c = group a by (x,y);" +
+ "c = foreach c generate group, COUNT(a.y);" +
+ "store b into 'file:///tmp/output/b';" +
+ "store c into 'file:///tmp/output/c';";
+
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld");
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld");
+ }
+
+ @Test
+ public void testMultiQueryWithJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input2' as (x:int, z:int);" +
+ "c = join a by x, b by x;" +
+ "d = foreach c generate $0, $1, $3;" +
+ "e = foreach c generate $0, $1, $2, $3;" +
+ "store c into 'file:///tmp/output/c';" +
+ "store d into 'file:///tmp/output/d';" +
+ "store e into 'file:///tmp/output/e';";
+
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld");
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4-OPTOFF.gld");
+ }
+
+ @Test
+ public void testMultiQueryWithNestedSplit() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = group a by x;" + //b: {group: int,a: {(x: int,y: int)}}
+ "store b into 'file:///tmp/output/b';" +
+ "c = foreach b generate a.x, a.y;" + //c: {{(x: int)},{(y: int)}}
+ "store c into 'file:///tmp/output/c';" +
+ "d = foreach b GENERATE FLATTEN(a);" + //d: {a::x: int,a::y: int}
+ "store d into 'file:///tmp/output/d';" +
+ "e = foreach d GENERATE a::x, a::y;" +
+ "store e into 'file:///tmp/output/e';";
+
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld");
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionStore() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "store c into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionGroupBy() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input' as (y:int, x:int);" +
+ "c = union onschema a, b;" +
+ "d = group c by x;" +
+ "e = foreach d generate group, SUM(c.y);" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join c by x, d by x;" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-3.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-3-OPTOFF.gld");
+ }
+
+
+ @Test
+ public void testUnionReplicateJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join c by x, d by x using 'replicated';" +
+ "store e into 'file:///tmp/output';";
+
+ //TODO: PIG-3856 Not optimized
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-4.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-4-OPTOFF.gld");
+
+ query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join d by x, c by x using 'replicated';" +
+ "store e into 'file:///tmp/output';";
+
+ // Optimized
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-5.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-5-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionSkewedJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join c by x, d by x using 'skewed';" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionOrderby() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = order c by x;" +
+ "store d into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld");
+ }
+
+ //TODO: PIG-3854 Limit is too convoluted and can be simplified.
+ @Test
+ public void testUnionLimit() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = limit c 1;" +
+ "store d into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionSplit() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "split a into a1 if x > 100, a2 otherwise;" +
+ "c = union onschema a1, b;" +
+ "split c into d if x > 500, e otherwise;" +
+ "store a2 into 'file:///tmp/output/a2';" +
+ "store d into 'file:///tmp/output/d';" +
+ "store e into 'file:///tmp/output/e';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld");
+ }
+
+ @Test
+ public void testUnionUnion() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, y:chararray);" +
+ "e = union onschema c, d;" +
+ "f = group e by x;" +
+ "store f into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld");
+ resetScope();
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld");
+ }
+
+ //TODO: union followed by union followed by store does not work.
+ //@Test
+ public void testUnionUnionStore() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, y:chararray);" +
+ "e = union onschema c, d;" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld");
+ resetScope();
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11-OPTOFF.gld");
+ }
+
+ @Test
+ public void testRank() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = rank a;" +
+ "store b into 'file:///tmp/output/d';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-1.gld");
+ }
+
+ @Test
+ public void testRankBy() throws Exception {
+ //TODO: Physical plan (affects both MR and Tez) has extra job before order by. Does not look right.
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = rank a by x;" +
+ "store b into 'file:///tmp/output/d';";
+
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld");
+ }
+
+ private void setProperty(String property, String value) {
+ pigServer.getPigContext().getProperties().setProperty(property, value);
+ }
+
+ private void run(String query, String expectedFile) throws Exception {
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
+ TezLauncher launcher = new TezLauncher();
+ TezPlanContainer tezPlanContainer = launcher.compile(pp, pc, null);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(baos);
+ TezPlanContainerPrinter printer = new TezPlanContainerPrinter(ps, tezPlanContainer);
+ printer.visit();
+ String compiledPlan = baos.toString();
+ System.out.println();
+ System.out.println("<<<" + compiledPlan + ">>>");
+
+ if (generate) {
+ FileOutputStream fos = new FileOutputStream(expectedFile);
+ fos.write(baos.toByteArray());
+ fos.close();
+ return;
+ }
+ FileInputStream fis = new FileInputStream(expectedFile);
+ byte[] b = new byte[MAX_SIZE];
+ int len = fis.read(b);
+ fis.close();
+ String goldenPlan = new String(b, 0, len);
+ if (goldenPlan.charAt(len-1) == '\n') {
+ goldenPlan = goldenPlan.substring(0, len-1);
+ }
+
+ System.out.println("-------------");
+ System.out.println("Golden");
+ System.out.println("<<<" + goldenPlan + ">>>");
+ System.out.println("-------------");
+
+ String goldenPlanClean = Util.standardizeNewline(goldenPlan).trim();
+ String compiledPlanClean = Util.standardizeNewline(compiledPlan).trim();
+ assertEquals(TestHelper.sortUDFs(Util.removeSignature(goldenPlanClean)),
+ TestHelper.sortUDFs(Util.removeSignature(compiledPlanClean)));
+ }
+}
+
Added: pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java (added)
+++ pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java Fri May 30 19:07:23 2014
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.tez;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezCompiler;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezJobControlCompiler;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.util.Pair;
+import org.apache.pig.test.Util;
+import org.apache.pig.test.junit.OrderedJUnit4Runner;
+import org.apache.pig.test.junit.OrderedJUnit4Runner.TestOrder;
+import org.apache.tez.dag.api.DAG;
+import org.apache.tez.dag.api.Vertex;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Test cases to test the TezJobControlCompiler.
+ */
+@RunWith(OrderedJUnit4Runner.class)
+@TestOrder({
+ "testRun1",
+ "testRun2",
+ "testRun3"
+})
+public class TestTezJobControlCompiler {
+ private static PigContext pc;
+ private static PigServer pigServer;
+ private static URI input1;
+ private static URI input2;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ pc = new PigContext(new TezLocalExecType(), new Properties());
+ input1 = Util.createTempFileDelOnExit("input1", "txt").toURI();
+ input2 = Util.createTempFileDelOnExit("input2", "txt").toURI();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ }
+
+ @Before
+ public void setUp() throws ExecException {
+ pigServer = new PigServer(pc);
+ }
+
+ @Test
+ public void testRun1() throws Exception {
+ String query =
+ "a = load '" + input1 +"' as (x:int, y:int);" +
+ "b = filter a by x > 0;" +
+ "c = foreach b generate y;" +
+ "store c into 'file:///tmp/output';";
+
+ Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+
+ // Make sure DAG has a single vertex.
+ List<TezOperator> roots = compiledPlan.first.getRoots();
+ assertEquals(1, roots.size());
+
+ Vertex root = compiledPlan.second.getVertex(roots.get(0).getOperatorKey().toString());
+ assertNotNull(root);
+ assertEquals(0, root.getInputVertices().size());
+ assertEquals(0, root.getOutputVertices().size());
+ }
+
+ @Test
+ public void testRun2() throws Exception {
+ String query =
+ "a = load '" + input1 +"' as (x:int, y:int);" +
+ "b = group a by x;" +
+ "c = foreach b generate group, a;" +
+ "store c into 'file:///tmp/output';";
+
+ Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+
+ // Make sure DAG has two vertices, and the root vertex is the input of
+ // the leaf vertex.
+ List<TezOperator> roots = compiledPlan.first.getRoots();
+ assertEquals(1, roots.size());
+ List<TezOperator> leaves = compiledPlan.first.getLeaves();
+ assertEquals(1, leaves.size());
+
+ Vertex root = compiledPlan.second.getVertex(roots.get(0).getOperatorKey().toString());
+ assertNotNull(root);
+ assertEquals(0, root.getInputVertices().size());
+ assertEquals(1, root.getOutputVertices().size());
+
+ Vertex leaf = compiledPlan.second.getVertex(leaves.get(0).getOperatorKey().toString());
+ assertNotNull(leaf);
+ assertEquals(1, leaf.getInputVertices().size());
+ assertEquals(0, leaf.getOutputVertices().size());
+
+ assertEquals(root.getOutputVertices().get(0), leaf);
+ assertEquals(root, leaf.getInputVertices().get(0));
+ }
+
+ @Test
+ public void testRun3() throws Exception {
+ String query =
+ "a = load '" + input1 +"' as (x:int, y:int);" +
+ "b = load '" + input2 +"' as (x:int, z:int);" +
+ "c = join a by x, b by x;" +
+ "d = foreach c generate a::x as x, y, z;" +
+ "store d into 'file:///tmp/output';";
+
+ Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+
+ // Make sure DAG has three vertices, and the two root vertices are the
+ // input of the leaf vertex.
+ List<TezOperator> roots = compiledPlan.first.getRoots();
+ assertEquals(2, roots.size());
+ List<TezOperator> leaves = compiledPlan.first.getLeaves();
+ assertEquals(1, leaves.size());
+
+ Vertex root0 = compiledPlan.second.getVertex(roots.get(0).getOperatorKey().toString());
+ Vertex root1 = compiledPlan.second.getVertex(roots.get(1).getOperatorKey().toString());
+ assertNotNull(root0);
+ assertNotNull(root1);
+ assertEquals(0, root0.getInputVertices().size());
+ assertEquals(1, root1.getOutputVertices().size());
+
+ Vertex leaf = compiledPlan.second.getVertex(leaves.get(0).getOperatorKey().toString());
+ assertNotNull(leaf);
+ assertEquals(2, leaf.getInputVertices().size());
+ assertEquals(0, leaf.getOutputVertices().size());
+
+ assertEquals(root0.getOutputVertices().get(0), leaf);
+ assertEquals(root1.getOutputVertices().get(0), leaf);
+ assertTrue(leaf.getInputVertices().contains(root0));
+ assertTrue(leaf.getInputVertices().contains(root1));
+ }
+
+ private Pair<TezOperPlan, DAG> compile(String query) throws Exception {
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
+ TezCompiler comp = new TezCompiler(pp, pc, null);
+ TezOperPlan tezPlan = comp.compile();
+ TezJobControlCompiler jobComp = new TezJobControlCompiler(pc, new Configuration());
+ DAG dag = jobComp.buildDAG(tezPlan, new HashMap<String, LocalResource>());
+ return new Pair<TezOperPlan, DAG>(tezPlan, dag);
+ }
+}
+
Added: pig/trunk/test/org/apache/pig/tez/TestTezLauncher.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezLauncher.java?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezLauncher.java (added)
+++ pig/trunk/test/org/apache/pig/tez/TestTezLauncher.java Fri May 30 19:07:23 2014
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.tez;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezExecType;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.test.MiniGenericCluster;
+import org.apache.pig.test.Util;
+import org.apache.pig.tools.pigstats.PigStats;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test cases to test the TezLauncher.
+ */
+public class TestTezLauncher {
+ private static PigContext pc;
+ private static PigServer pigServer;
+ private static MiniGenericCluster cluster;
+
+ private static final String INPUT_FILE = "TestTezLauncherInput";
+ private static final String[] INPUT_RECORDS = {
+ "100\tapple",
+ "200\torange",
+ "300\tstrawberry",
+ "300\tpear",
+ "100\tapple",
+ "300\tpear",
+ "400\tapple",
+ };
+
+ private static final String OUTPUT_FILE = "TestTezLauncherOutput";
+ private static final String[] OUTPUT_RECORDS = {
+ "all\t{(apple),(pear),(pear),(strawberry),(orange)}"
+ };
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ cluster = MiniGenericCluster.buildCluster(MiniGenericCluster.EXECTYPE_TEZ);
+ pc = new PigContext(new TezExecType(), cluster.getProperties());
+ Util.createInputFile(cluster, INPUT_FILE, INPUT_RECORDS);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ cluster.shutDown();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ pigServer = new PigServer(pc);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ Util.deleteFile(cluster, OUTPUT_FILE);
+ }
+
+ @Test
+ public void testRun1() throws Exception {
+ String query =
+ "a = load '" + INPUT_FILE + "' as (x:int, y:chararray);" +
+ "b = filter a by x > 100;" +
+ "c = foreach b generate y;" +
+ "d = group c all;" +
+ "store d into '" + OUTPUT_FILE + "';";
+
+ PhysicalPlan pp = Util.buildPp(pigServer, query);
+ TezLauncher launcher = new TezLauncher();
+ PigStats pigStats = launcher.launchPig(pp, "testRun1", pc);
+ assertTrue(pigStats.isSuccessful());
+
+ String[] output = Util.readOutput(cluster.getFileSystem(), OUTPUT_FILE);
+ for (int i = 0; i < output.length; i++) {
+ assertEquals(OUTPUT_RECORDS[i], output[i]);
+ }
+
+ assertEquals(1, pigStats.getInputStats().size());
+ assertEquals(INPUT_FILE, pigStats.getInputStats().get(0).getName());
+
+ assertEquals(1, pigStats.getOutputStats().size());
+ assertEquals(OUTPUT_FILE, pigStats.getOutputStats().get(0).getName());
+ }
+}
+
Added: pig/trunk/test/tez-tests
URL: http://svn.apache.org/viewvc/pig/trunk/test/tez-tests?rev=1598702&view=auto
==============================================================================
--- pig/trunk/test/tez-tests (added)
+++ pig/trunk/test/tez-tests Fri May 30 19:07:23 2014
@@ -0,0 +1,49 @@
+**/TestAccumulator.java
+**/TestAlgebraicEval.java
+**/TestBZip.java
+**/TestBestFitCast.java
+**/TestBinaryExpressionOps.java
+**/TestBuiltin.java
+**/TestCharArrayToNumeric.java
+**/TestCombiner.java
+**/TestCompressedFiles.java
+**/TestCustomPartitioner.java
+**/TestEvalPipeline.java
+**/TestFilterUDF.java
+**/TestFinish.java
+**/TestForEachNestedPlan.java
+**/TestFRJoin.java
+**/TestFRJoinNullValue.java
+**/TestGrunt.java
+**/TestImplicitSplit.java
+**/TestInputOutputMiniClusterFileValidator.java
+**/TestJoin.java
+**/TestJoinSmoke.java
+**/TestKeyTypeDiscoveryVisitor.java
+**/TestLoad.java
+**/TestLocalRearrange.java
+**/TestMapReduce.java
+**/TestMapReduce2.java
+**/TestNestedForeach.java
+**/TestNewPlanImplicitSplit.java
+**/TestPigContext.java
+**/TestPigServer.java
+**/TestPigServerWithMacros.java
+**/TestPigStorage.java
+**/TestRegisteredJarVisibility.java
+**/TestSample.java
+**/TestSchema.java
+**/TestScriptLanguageJavaScript.java
+**/TestScriptUDF.java
+**/TestSkewedJoin.java
+**/TestSplitStore.java
+**/TestStoreInstances.java
+**/TestStoreOld.java
+**/TestStreaming.java
+**/TestToolsPigServer.java
+**/TestUDF.java
+**/TestUDFContext.java
+**/TestSecondarySortTez.java
+**/TestTezCompiler.java
+**/TestTezJobControlCompiler.java
+**/TestTezLauncher.java
\ No newline at end of file