You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/04/11 16:57:16 UTC
svn commit: r1586670 [2/2] - in /pig/branches/tez: ivy/
src/org/apache/pig/backend/hadoop/executionengine/tez/
src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/
src/org/apache/pig/tools/pigstats/
src/org/apache/pig/tools/pigstats/mapred...
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1586670&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Fri Apr 11 14:57:15 2014
@@ -0,0 +1,253 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-209
+#--------------------------------------------------
+Tez vertex scope-106 -> Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-156,Tez vertex scope-146,Tez vertex scope-166,Tez vertex scope-178,
+Tez vertex scope-113
+Tez vertex scope-119 -> Tez vertex scope-126,Tez vertex scope-129,
+Tez vertex scope-126
+Tez vertex scope-129
+Tez vertex scope-146 -> Tez vertex scope-156,
+Tez vertex scope-156 -> Tez vertex scope-158,
+Tez vertex scope-158
+Tez vertex scope-166 -> Tez vertex scope-178,
+Tez vertex scope-178
+
+Tez vertex scope-106
+# Plan on vertex
+1-1: Split - scope-208
+| |
+| b: Split - scope-205
+| | |
+| | b1: Local Rearrange[tuple]{int}(false) - scope-18 -> scope-113
+| | | |
+| | | Project[int][0] - scope-19
+| | |
+| | c1: Local Rearrange[tuple]{int}(false) - scope-49 -> scope-126
+| | | |
+| | | Project[int][0] - scope-50
+| | |
+| | 1-2: Split - scope-204
+| | | |
+| | | Local Rearrange[tuple]{tuple}(false) - scope-142 -> scope-146
+| | | | |
+| | | | Constant(DummyVal) - scope-141
+| | | |
+| | | |---ReservoirSample - scope-145
+| | | |
+| | | |---New For Each(false)[tuple] - scope-144
+| | | | |
+| | | | Project[int][0] - scope-143
+| | | |
+| | | |---e1: Local Rearrange[tuple]{int}(false) - scope-140 -> scope-156
+| | | | |
+| | | | Project[int][0] - scope-88
+| | | |
+| | | |---e: Filter[bag] - scope-84
+| | | | |
+| | | | Less Than[boolean] - scope-87
+| | | | |
+| | | | |---Project[int][0] - scope-85
+| | | | |
+| | | | |---Constant(3) - scope-86
+| | | |
+| | | f1: Local Rearrange[tuple]{tuple}(false) - scope-165 -> scope-166
+| | | | |
+| | | | Project[tuple][*] - scope-164
+| | | |
+| | | |---f1: Limit - scope-95
+| | | |
+| | | |---f: Filter[bag] - scope-91
+| | | | |
+| | | | Greater Than or Equal[boolean] - scope-94
+| | | | |
+| | | | |---Project[int][0] - scope-92
+| | | | |
+| | | | |---Constant(3) - scope-93
+| |
+| |---b: Filter[bag] - scope-9
+| | |
+| | Less Than or Equal[boolean] - scope-12
+| | |
+| | |---Project[int][0] - scope-10
+| | |
+| | |---Constant(5) - scope-11
+| |
+| d1: Split - scope-207
+| | |
+| | d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-80
+| | |
+| | POValueOutputTez - scope-180 -> [scope-178]
+| |
+| |---d1: Filter[bag] - scope-73
+| | |
+| | Equal To[boolean] - scope-76
+| | |
+| | |---Project[int][0] - scope-74
+| | |
+| | |---Constant(5) - scope-75
+| |
+| |---d: Filter[bag] - scope-69
+| | |
+| | Greater Than[boolean] - scope-72
+| | |
+| | |---Project[int][0] - scope-70
+| | |
+| | |---Constant(10) - scope-71
+| |
+| POValueOutputTez - scope-107 -> [scope-119]
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-113
+# Plan on vertex
+b1: Split - scope-202
+| |
+| b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-23
+| |
+| b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-33
+| |
+| |---b2: New For Each(false,false)[bag] - scope-32
+| | |
+| | Project[int][0] - scope-26
+| | |
+| | POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-30
+| | |
+| | |---Project[bag][0] - scope-29
+| | |
+| | |---Project[bag][1] - scope-28
+|
+|---b1: Package(Packager)[tuple]{int} - scope-17
+Tez vertex scope-119
+# Plan on vertex
+c: Split - scope-206
+| |
+| c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126
+| | |
+| | Project[int][0] - scope-48
+| |
+| c2: Local Rearrange[tuple]{int}(false) - scope-194 -> scope-129
+| | |
+| | Project[int][0] - scope-196
+| |
+| |---c3: New For Each(false,false)[bag] - scope-182
+| | |
+| | Project[int][0] - scope-183
+| | |
+| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-184
+| | |
+| | |---Project[bag][0] - scope-185
+| | |
+| | |---Project[bag][1] - scope-186
+| |
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-197
+|
+|---c: Filter[bag] - scope-34
+ | |
+ | Less Than or Equal[boolean] - scope-37
+ | |
+ | |---Project[int][0] - scope-35
+ | |
+ | |---Constant(10) - scope-36
+ |
+ |---POValueInputTez - scope-120 <- scope-106
+Tez vertex scope-126
+# Plan on vertex
+c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-54
+|
+|---c1: New For Each(true,true)[tuple] - scope-53
+ | |
+ | Project[bag][1] - scope-51
+ | |
+ | Project[bag][2] - scope-52
+ |
+ |---c1: Package(Packager)[tuple]{int} - scope-46
+Tez vertex scope-129
+# Combine plan on edge <scope-119>
+c2: Local Rearrange[tuple]{int}(false) - scope-198 -> scope-129
+| |
+| Project[int][0] - scope-200
+|
+|---c3: New For Each(false,false)[bag] - scope-187
+ | |
+ | Project[int][0] - scope-188
+ | |
+ | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-189
+ | |
+ | |---Project[bag][1] - scope-190
+ |
+ |---c2: Package(CombinerPackager)[tuple]{int} - scope-193
+# Plan on vertex
+c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-68
+|
+|---c3: New For Each(false,false)[bag] - scope-67
+ | |
+ | Project[int][0] - scope-61
+ | |
+ | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-65
+ | |
+ | |---Project[bag][1] - scope-191
+ |
+ |---c2: Package(CombinerPackager)[tuple]{int} - scope-58
+Tez vertex scope-146
+# Plan on vertex
+POValueOutputTez - scope-155 -> [scope-156]
+|
+|---New For Each(false)[tuple] - scope-154
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-153
+ | |
+ | |---Project[tuple][*] - scope-152
+ |
+ |---New For Each(false,false)[tuple] - scope-151
+ | |
+ | Constant(1) - scope-150
+ | |
+ | Project[bag][1] - scope-148
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-147
+Tez vertex scope-156
+# Plan on vertex
+POIdentityInOutTez - scope-157 <- scope-106 -> scope-158
+| |
+| Project[int][0] - scope-88
+Tez vertex scope-158
+# Plan on vertex
+e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-90
+|
+|---New For Each(true)[tuple] - scope-161
+ | |
+ | Project[bag][1] - scope-160
+ |
+ |---Package(LitePackager)[tuple]{int} - scope-159
+Tez vertex scope-166
+# Plan on vertex
+f1: Split - scope-203
+| |
+| f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99
+| |
+| POValueOutputTez - scope-181 -> [scope-178]
+|
+|---f1: Limit - scope-170
+ |
+ |---f1: New For Each(true)[bag] - scope-169
+ | |
+ | Project[tuple][1] - scope-168
+ |
+ |---f1: Package(Packager)[tuple]{tuple} - scope-167
+Tez vertex scope-178
+# Plan on vertex
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-105
+|
+|---POShuffledValueInputTez - scope-179 <- [scope-106, scope-166]
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld?rev=1586670&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld Fri Apr 11 14:57:15 2014
@@ -0,0 +1,119 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-174
+#--------------------------------------------------
+Tez vertex scope-126 -> Tez vertex scope-128,Tez vertex scope-131,
+Tez vertex scope-131 -> Tez vertex scope-133,
+Tez vertex scope-133
+Tez vertex scope-128 -> Tez vertex scope-130,
+Tez vertex scope-130
+
+Tez vertex scope-126
+# Plan on vertex
+POValueOutputTez - scope-127 -> [scope-128, scope-131]
+|
+|---a: New For Each(false,false)[bag] - scope-95
+ | |
+ | Cast[int] - scope-90
+ | |
+ | |---Project[bytearray][0] - scope-89
+ | |
+ | Cast[int] - scope-93
+ | |
+ | |---Project[bytearray][1] - scope-92
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-88
+Tez vertex scope-131
+# Plan on vertex
+c: Local Rearrange[tuple]{tuple}(false) - scope-165 -> scope-133
+| |
+| Project[tuple][0] - scope-168
+|
+|---c: New For Each(false,false)[bag] - scope-153
+ | |
+ | Project[tuple][0] - scope-154
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-155
+ | |
+ | |---Project[bag][1] - scope-156
+ | |
+ | |---Project[bag][1] - scope-157
+ |
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-169
+ |
+ |---POValueInputTez - scope-132 <- scope-126
+Tez vertex scope-133
+# Combine plan on edge <scope-131>
+c: Local Rearrange[tuple]{tuple}(false) - scope-170 -> scope-133
+| |
+| Project[tuple][0] - scope-173
+|
+|---c: New For Each(false,false)[bag] - scope-158
+ | |
+ | Project[tuple][0] - scope-159
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-160
+ | |
+ | |---Project[bag][1] - scope-161
+ |
+ |---c: Package(CombinerPackager)[tuple]{tuple} - scope-164
+# Plan on vertex
+c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-125
+|
+|---c: New For Each(false,false)[bag] - scope-124
+ | |
+ | Project[tuple][0] - scope-118
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-122
+ | |
+ | |---Project[bag][1] - scope-162
+ |
+ |---c: Package(CombinerPackager)[tuple]{tuple} - scope-114
+Tez vertex scope-128
+# Plan on vertex
+b: Local Rearrange[tuple]{int}(false) - scope-146 -> scope-130
+| |
+| Project[int][0] - scope-148
+|
+|---b: New For Each(false,false)[bag] - scope-134
+ | |
+ | Project[int][0] - scope-135
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-136
+ | |
+ | |---Project[bag][0] - scope-137
+ | |
+ | |---Project[bag][1] - scope-138
+ |
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-149
+ |
+ |---POValueInputTez - scope-129 <- scope-126
+Tez vertex scope-130
+# Combine plan on edge <scope-128>
+b: Local Rearrange[tuple]{int}(false) - scope-150 -> scope-130
+| |
+| Project[int][0] - scope-152
+|
+|---b: New For Each(false,false)[bag] - scope-139
+ | |
+ | Project[int][0] - scope-140
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-141
+ | |
+ | |---Project[bag][1] - scope-142
+ |
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-145
+# Plan on vertex
+b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-110
+|
+|---b: New For Each(false,false)[bag] - scope-109
+ | |
+ | Project[int][0] - scope-103
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-107
+ | |
+ | |---Project[bag][1] - scope-143
+ |
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-100
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld?rev=1586670&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld Fri Apr 11 14:57:15 2014
@@ -0,0 +1,111 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-87
+#--------------------------------------------------
+Tez vertex scope-38 -> Tez vertex scope-42,Tez vertex scope-45,
+Tez vertex scope-45
+Tez vertex scope-42
+
+Tez vertex scope-38
+# Plan on vertex
+a: Split - scope-86
+| |
+| b: Local Rearrange[tuple]{int}(false) - scope-58 -> scope-42
+| | |
+| | Project[int][0] - scope-60
+| |
+| |---b: New For Each(false,false)[bag] - scope-46
+| | |
+| | Project[int][0] - scope-47
+| | |
+| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-48
+| | |
+| | |---Project[bag][0] - scope-49
+| | |
+| | |---Project[bag][1] - scope-50
+| |
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-61
+| |
+| c: Local Rearrange[tuple]{tuple}(false) - scope-77 -> scope-45
+| | |
+| | Project[tuple][0] - scope-80
+| |
+| |---c: New For Each(false,false)[bag] - scope-65
+| | |
+| | Project[tuple][0] - scope-66
+| | |
+| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-67
+| | |
+| | |---Project[bag][1] - scope-68
+| | |
+| | |---Project[bag][1] - scope-69
+| |
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-81
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-45
+# Combine plan on edge <scope-38>
+c: Local Rearrange[tuple]{tuple}(false) - scope-82 -> scope-45
+| |
+| Project[tuple][0] - scope-85
+|
+|---c: New For Each(false,false)[bag] - scope-70
+ | |
+ | Project[tuple][0] - scope-71
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-72
+ | |
+ | |---Project[bag][1] - scope-73
+ |
+ |---c: Package(CombinerPackager)[tuple]{tuple} - scope-76
+# Plan on vertex
+c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-37
+|
+|---c: New For Each(false,false)[bag] - scope-36
+ | |
+ | Project[tuple][0] - scope-30
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-34
+ | |
+ | |---Project[bag][1] - scope-74
+ |
+ |---c: Package(CombinerPackager)[tuple]{tuple} - scope-26
+Tez vertex scope-42
+# Combine plan on edge <scope-38>
+b: Local Rearrange[tuple]{int}(false) - scope-62 -> scope-42
+| |
+| Project[int][0] - scope-64
+|
+|---b: New For Each(false,false)[bag] - scope-51
+ | |
+ | Project[int][0] - scope-52
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-53
+ | |
+ | |---Project[bag][1] - scope-54
+ |
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-57
+# Plan on vertex
+b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-22
+|
+|---b: New For Each(false,false)[bag] - scope-21
+ | |
+ | Project[int][0] - scope-15
+ | |
+ | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-19
+ | |
+ | |---Project[bag][1] - scope-55
+ |
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-12
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4-OPTOFF.gld?rev=1586670&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4-OPTOFF.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4-OPTOFF.gld Fri Apr 11 14:57:15 2014
@@ -0,0 +1,91 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-128
+#--------------------------------------------------
+Tez vertex scope-118 -> Tez vertex scope-120,
+Tez vertex scope-119 -> Tez vertex scope-120,
+Tez vertex scope-120 -> Tez vertex scope-122,Tez vertex scope-124,Tez vertex scope-126,
+Tez vertex scope-122
+Tez vertex scope-124
+Tez vertex scope-126
+
+Tez vertex scope-118
+# Plan on vertex
+c: Local Rearrange[tuple]{int}(false) - scope-85 -> scope-120
+| |
+| Project[int][0] - scope-86
+|
+|---a: New For Each(false,false)[bag] - scope-72
+ | |
+ | Cast[int] - scope-67
+ | |
+ | |---Project[bytearray][0] - scope-66
+ | |
+ | Cast[int] - scope-70
+ | |
+ | |---Project[bytearray][1] - scope-69
+ |
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-65
+Tez vertex scope-119
+# Plan on vertex
+c: Local Rearrange[tuple]{int}(false) - scope-87 -> scope-120
+| |
+| Project[int][0] - scope-88
+|
+|---b: New For Each(false,false)[bag] - scope-80
+ | |
+ | Cast[int] - scope-75
+ | |
+ | |---Project[bytearray][0] - scope-74
+ | |
+ | Cast[int] - scope-78
+ | |
+ | |---Project[bytearray][1] - scope-77
+ |
+ |---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-73
+Tez vertex scope-120
+# Plan on vertex
+POValueOutputTez - scope-121 -> [scope-122, scope-124, scope-126]
+|
+|---c: New For Each(true,true)[tuple] - scope-91
+ | |
+ | Project[bag][1] - scope-89
+ | |
+ | Project[bag][2] - scope-90
+ |
+ |---c: Package(Packager)[tuple]{int} - scope-84
+Tez vertex scope-122
+# Plan on vertex
+c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-95
+|
+|---POValueInputTez - scope-123 <- scope-120
+Tez vertex scope-124
+# Plan on vertex
+d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-105
+|
+|---d: New For Each(false,false,false)[bag] - scope-104
+ | |
+ | Project[int][0] - scope-98
+ | |
+ | Project[int][1] - scope-100
+ | |
+ | Project[int][3] - scope-102
+ |
+ |---POValueInputTez - scope-125 <- scope-120
+Tez vertex scope-126
+# Plan on vertex
+e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-117
+|
+|---e: New For Each(false,false,false,false)[bag] - scope-116
+ | |
+ | Project[int][0] - scope-108
+ | |
+ | Project[int][1] - scope-110
+ | |
+ | Project[int][2] - scope-112
+ | |
+ | Project[int][3] - scope-114
+ |
+ |---POValueInputTez - scope-127 <- scope-120
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld?rev=1586670&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld Fri Apr 11 14:57:15 2014
@@ -0,0 +1,79 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-64
+#--------------------------------------------------
+Tez vertex scope-53 -> Tez vertex scope-55,
+Tez vertex scope-54 -> Tez vertex scope-55,
+Tez vertex scope-55
+
+Tez vertex scope-53
+# Plan on vertex
+c: Local Rearrange[tuple]{int}(false) - scope-20 -> scope-55
+| |
+| Project[int][0] - scope-21
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-54
+# Plan on vertex
+c: Local Rearrange[tuple]{int}(false) - scope-22 -> scope-55
+| |
+| Project[int][0] - scope-23
+|
+|---b: New For Each(false,false)[bag] - scope-15
+ | |
+ | Cast[int] - scope-10
+ | |
+ | |---Project[bytearray][0] - scope-9
+ | |
+ | Cast[int] - scope-13
+ | |
+ | |---Project[bytearray][1] - scope-12
+ |
+ |---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-55
+# Plan on vertex
+c: Split - scope-63
+| |
+| c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-30
+| |
+| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-40
+| |
+| |---d: New For Each(false,false,false)[bag] - scope-39
+| | |
+| | Project[int][0] - scope-33
+| | |
+| | Project[int][1] - scope-35
+| | |
+| | Project[int][3] - scope-37
+| |
+| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-52
+| |
+| |---e: New For Each(false,false,false,false)[bag] - scope-51
+| | |
+| | Project[int][0] - scope-43
+| | |
+| | Project[int][1] - scope-45
+| | |
+| | Project[int][2] - scope-47
+| | |
+| | Project[int][3] - scope-49
+|
+|---c: New For Each(true,true)[tuple] - scope-26
+ | |
+ | Project[bag][1] - scope-24
+ | |
+ | Project[bag][2] - scope-25
+ |
+ |---c: Package(Packager)[tuple]{int} - scope-19
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5-OPTOFF.gld?rev=1586670&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5-OPTOFF.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5-OPTOFF.gld Fri Apr 11 14:57:15 2014
@@ -0,0 +1,81 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-117
+#--------------------------------------------------
+Tez vertex scope-103 -> Tez vertex scope-104,
+Tez vertex scope-104 -> Tez vertex scope-106,Tez vertex scope-108,Tez vertex scope-110,
+Tez vertex scope-110 -> Tez vertex scope-113,Tez vertex scope-115,
+Tez vertex scope-113
+Tez vertex scope-115
+Tez vertex scope-106
+Tez vertex scope-108
+
+Tez vertex scope-103
+# Plan on vertex
+b: Local Rearrange[tuple]{int}(false) - scope-70 -> scope-104
+| |
+| Project[int][0] - scope-71
+|
+|---a: New For Each(false,false)[bag] - scope-67
+ | |
+ | Cast[int] - scope-62
+ | |
+ | |---Project[bytearray][0] - scope-61
+ | |
+ | Cast[int] - scope-65
+ | |
+ | |---Project[bytearray][1] - scope-64
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-60
+Tez vertex scope-104
+# Plan on vertex
+POValueOutputTez - scope-105 -> [scope-106, scope-108, scope-110]
+|
+|---b: Package(Packager)[tuple]{int} - scope-69
+Tez vertex scope-110
+# Plan on vertex
+POValueOutputTez - scope-112 -> [scope-113, scope-115]
+|
+|---d: New For Each(true)[bag] - scope-90
+ | |
+ | Project[bag][1] - scope-88
+ |
+ |---POValueInputTez - scope-111 <- scope-104
+Tez vertex scope-113
+# Plan on vertex
+d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-94
+|
+|---POValueInputTez - scope-114 <- scope-110
+Tez vertex scope-115
+# Plan on vertex
+e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-102
+|
+|---e: New For Each(false,false)[bag] - scope-101
+ | |
+ | Project[int][0] - scope-97
+ | |
+ | Project[int][1] - scope-99
+ |
+ |---POValueInputTez - scope-116 <- scope-110
+Tez vertex scope-106
+# Plan on vertex
+b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-75
+|
+|---POValueInputTez - scope-107 <- scope-104
+Tez vertex scope-108
+# Plan on vertex
+c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-85
+|
+|---c: New For Each(false,false)[bag] - scope-84
+ | |
+ | Project[bag][0] - scope-79
+ | |
+ | |---Project[bag][1] - scope-78
+ | |
+ | Project[bag][1] - scope-82
+ | |
+ | |---Project[bag][1] - scope-81
+ |
+ |---POValueInputTez - scope-109 <- scope-104
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld?rev=1586670&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld Fri Apr 11 14:57:15 2014
@@ -0,0 +1,61 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-59
+#--------------------------------------------------
+Tez vertex scope-43 -> Tez vertex scope-44,
+Tez vertex scope-44
+
+Tez vertex scope-43
+# Plan on vertex
+b: Local Rearrange[tuple]{int}(false) - scope-10 -> scope-44
+| |
+| Project[int][0] - scope-11
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-44
+# Plan on vertex
+b: Split - scope-58
+| |
+| b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-15
+| |
+| c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-25
+| |
+| |---c: New For Each(false,false)[bag] - scope-24
+| | |
+| | Project[bag][0] - scope-19
+| | |
+| | |---Project[bag][1] - scope-18
+| | |
+| | Project[bag][1] - scope-22
+| | |
+| | |---Project[bag][1] - scope-21
+| |
+| d: Split - scope-57
+| | |
+| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-34
+| | |
+| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-42
+| | |
+| | |---e: New For Each(false,false)[bag] - scope-41
+| | | |
+| | | Project[int][0] - scope-37
+| | | |
+| | | Project[int][1] - scope-39
+| |
+| |---d: New For Each(true)[bag] - scope-30
+| | |
+| | Project[bag][1] - scope-28
+|
+|---b: Package(Packager)[tuple]{int} - scope-9
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-1.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-1.gld?rev=1586670&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-1.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-1.gld Fri Apr 11 14:57:15 2014
@@ -0,0 +1,33 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-15
+#--------------------------------------------------
+Tez vertex scope-11 -> Tez vertex scope-14,Tez vertex scope-12,
+Tez vertex scope-12 -> Tez vertex scope-14,
+Tez vertex scope-14
+
+Tez vertex scope-11
+# Plan on vertex
+POCounterTez - scope-8 -> scope-14,scope-12
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-12
+# Plan on vertex
+PORankStatsTez - scope-13 <- scope-11 -> scope-14
+Tez vertex scope-14
+# Plan on vertex
+b: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-10
+|
+|---PORankTez - scope-9 <- scope-11,scope-12
Added: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld?rev=1586670&view=auto
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld (added)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld Fri Apr 11 14:57:15 2014
@@ -0,0 +1,103 @@
+#--------------------------------------------------
+# There are 1 DAGs in the session
+#--------------------------------------------------
+#--------------------------------------------------
+# TEZ DAG plan: scope-51
+#--------------------------------------------------
+Tez vertex scope-23 -> Tez vertex scope-24,
+Tez vertex scope-24 -> Tez vertex scope-41,Tez vertex scope-31,
+Tez vertex scope-31 -> Tez vertex scope-41,
+Tez vertex scope-41 -> Tez vertex scope-43,
+Tez vertex scope-43 -> Tez vertex scope-49,Tez vertex scope-47,
+Tez vertex scope-47 -> Tez vertex scope-49,
+Tez vertex scope-49
+
+Tez vertex scope-23
+# Plan on vertex
+b: Local Rearrange[tuple]{int}(false) - scope-10 -> scope-24
+| |
+| Project[int][0] - scope-11
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-24
+# Plan on vertex
+Local Rearrange[tuple]{tuple}(false) - scope-27 -> scope-31
+| |
+| Constant(DummyVal) - scope-26
+|
+|---ReservoirSample - scope-30
+ |
+ |---New For Each(false)[tuple] - scope-29
+ | |
+ | Project[int][0] - scope-28
+ |
+ |---b: Local Rearrange[tuple]{int}(false) - scope-25 -> scope-41
+ | |
+ | Project[int][0] - scope-15
+ |
+ |---New For Each(true,false)[tuple] - scope-14
+ | |
+ | Project[int][0] - scope-12
+ | |
+ | Project[bag][1] - scope-13
+ |
+ |---b: Package(Packager)[tuple]{int} - scope-9
+Tez vertex scope-31
+# Plan on vertex
+POValueOutputTez - scope-40 -> [scope-41]
+|
+|---New For Each(false)[tuple] - scope-39
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-38
+ | |
+ | |---Project[tuple][*] - scope-37
+ |
+ |---New For Each(false,false)[tuple] - scope-36
+ | |
+ | Constant(1) - scope-35
+ | |
+ | Project[bag][1] - scope-33
+ |
+ |---Package(Packager)[tuple]{bytearray} - scope-32
+Tez vertex scope-41
+# Plan on vertex
+POIdentityInOutTez - scope-42 <- scope-24 -> scope-43
+| |
+| Project[int][0] - scope-15
+Tez vertex scope-43
+# Plan on vertex
+POCounterTez - scope-17 -> scope-49,scope-47
+| |
+| Project[int][0] - scope-15
+|
+|---New For Each(true)[tuple] - scope-46
+ | |
+ | Project[bag][1] - scope-45
+ |
+ |---Package(LitePackager)[tuple]{int} - scope-44
+Tez vertex scope-47
+# Plan on vertex
+PORankStatsTez - scope-48 <- scope-43 -> scope-49
+Tez vertex scope-49
+# Plan on vertex
+b: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-22
+|
+|---New For Each(false,true)[tuple] - scope-21
+ | |
+ | Project[long][0] - scope-19
+ | |
+ | Project[bag][2] - scope-20
+ |
+ |---PORankTez - scope-18 <- scope-43,scope-47
+ | |
+ | Project[int][0] - scope-15
Modified: pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld?rev=1586670&r1=1586669&r2=1586670&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld (original)
+++ pig/branches/tez/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld Fri Apr 11 14:57:15 2014
@@ -84,7 +84,21 @@ d: Local Rearrange[tuple]{int}(false) -
| |---Project[bag][1] - scope-44
|
|---d: Package(CombinerPackager)[tuple]{int} - scope-47
-# Combine plan on edge <scope-31>
+# Combine plan on edge <scope-30>
+d: Local Rearrange[tuple]{int}(false) - scope-52 -> scope-35
+| |
+| Project[int][0] - scope-54
+|
+|---e: New For Each(false,false)[bag] - scope-41
+ | |
+ | Project[int][0] - scope-42
+ | |
+ | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-43
+ | |
+ | |---Project[bag][1] - scope-44
+ |
+ |---d: Package(CombinerPackager)[tuple]{int} - scope-47
+# Combine plan on edge <scope-29>
d: Local Rearrange[tuple]{int}(false) - scope-52 -> scope-35
| |
| Project[int][0] - scope-54
Modified: pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java?rev=1586670&r1=1586669&r2=1586670&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/tez/TestTezCompiler.java Fri Apr 11 14:57:15 2014
@@ -74,6 +74,9 @@ public class TestTezCompiler {
public void setUp() throws ExecException {
NodeIdGenerator.reset();
PigServer.resetScope();
+ pc.getProperties().remove(PigConfiguration.OPT_MULTIQUERY);
+ pc.getProperties().remove(PigConfiguration.TEZ_OPT_UNION);
+ pc.getProperties().remove(PigConfiguration.PIG_EXEC_NO_SECONDARY_KEY);
pigServer = new PigServer(pc);
}
@@ -156,79 +159,7 @@ public class TestTezCompiler {
run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC13.gld");
}
- @Test
- public void testSplitSingleVertex() throws Exception {
- String query =
- "a = load 'file:///tmp/input' as (x:int, y:int);" +
- "split a into b if x <= 5, c if x <= 10, d if x >10;" +
- "store b into 'file:///tmp/output/b';" +
- "store c into 'file:///tmp/output/c';" +
- "store d into 'file:///tmp/output/d';";
-
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC6.gld");
- }
-
- @Test
- public void testSplitMultiVertex() throws Exception {
- String query =
- "a = load 'file:///tmp/input' as (x:int, y:int);" +
- "split a into b if x <= 5, c if x <= 10, d if x >10;" +
- "split b into e if x < 3, f if x >= 3;" +
- // No Combiner on the edge to b1/b2 vertex as both b1 and b2 are stored
- "b1 = group b by x;" +
- "b2 = foreach b1 generate group, SUM(b.x);" +
- // Case of two outputs within a split going to same edge as input
- "c1 = join c by x, b by x;" +
- "c2 = group c by x;" +
- // Combiner on the edge to c3 vertex
- "c3 = foreach c2 generate group, SUM(c.x);" +
- "d1 = filter d by x == 5;" +
- "e1 = order e by x;" +
- // TODO: Physical plan has extra split for f1 - 1-2: Split - scope-80
- // POSplit has only 1 sub plan. Optimized and removed in MR plan.
- // Needs to be removed in Tez plan as well.
- "f1 = limit f 1;" +
- "f2 = union d1, f1;" +
- "store b1 into 'file:///tmp/output/b1';" +
- "store b2 into 'file:///tmp/output/b2';" +
- "store c1 into 'file:///tmp/output/c1';" +
- "store c3 into 'file:///tmp/output/c1';" +
- "store d1 into 'file:///tmp/output/d1';" +
- "store e1 into 'file:///tmp/output/e1';" +
- "store f1 into 'file:///tmp/output/f1';" +
- "store f2 into 'file:///tmp/output/f2';";
-
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC7.gld");
- }
-
- @Test
- public void testMultipleGroupBySplit() throws Exception {
- String query =
- "a = load 'file:///tmp/input' as (x:int, y:int);" +
- "b = group a by x;" +
- "b = foreach b generate group, COUNT(a.x);" +
- "c = group a by (x,y);" +
- "c = foreach c generate group, COUNT(a.y);" +
- "store b into 'file:///tmp/output/b';" +
- "store c into 'file:///tmp/output/c';";
-
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC8.gld");
- }
-
- @Test
- public void testJoinWithSplit() throws Exception {
- String query =
- "a = load 'file:///tmp/input1' as (x:int, y:int);" +
- "b = load 'file:///tmp/input2' as (x:int, z:int);" +
- "c = join a by x, b by x;" +
- "d = foreach c generate $0, $1, $3;" +
- "e = foreach c generate $0, $1, $2, $3;" +
- "store c into 'file:///tmp/output/c';" +
- "store d into 'file:///tmp/output/d';" +
- "store e into 'file:///tmp/output/e';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC9.gld");
- }
@Test
public void testReplicatedJoinInMapper() throws Exception {
@@ -276,17 +207,8 @@ public class TestTezCompiler {
run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC14.gld");
// With optimization turned off
- pigServer.getPigContext().getProperties()
- .setProperty(PigConfiguration.PIG_EXEC_NO_SECONDARY_KEY, "true");
-
- try {
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC15.gld");
- } finally {
- pigServer.getPigContext().getProperties()
- .setProperty(PigConfiguration.PIG_EXEC_NO_SECONDARY_KEY, "false");
- }
-
-
+ setProperty(PigConfiguration.PIG_EXEC_NO_SECONDARY_KEY, "true");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC15.gld");
}
@Test
@@ -314,157 +236,233 @@ public class TestTezCompiler {
}
@Test
+ public void testMulitQueryWithSplitSingleVertex() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "split a into b if x <= 5, c if x <= 10, d if x >10;" +
+ "store b into 'file:///tmp/output/b';" +
+ "store c into 'file:///tmp/output/c';" +
+ "store d into 'file:///tmp/output/d';";
+
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld");
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld");
+ }
+
+ @Test
+ public void testMulitQueryWithSplitMultiVertex() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "split a into b if x <= 5, c if x <= 10, d if x >10;" +
+ "split b into e if x < 3, f if x >= 3;" +
+ // No Combiner on the edge to b1/b2 vertex as both b1 and b2 are stored
+ "b1 = group b by x;" +
+ "b2 = foreach b1 generate group, SUM(b.x);" +
+ // Case of two outputs within a split going to same edge as input
+ "c1 = join c by x, b by x;" +
+ "c2 = group c by x;" +
+ // Combiner on the edge to c3 vertex
+ "c3 = foreach c2 generate group, SUM(c.x);" +
+ "d1 = filter d by x == 5;" +
+ "e1 = order e by x;" +
+ // TODO: Physical plan has extra split for f1 - 1-2: Split - scope-80
+ // POSplit has only 1 sub plan. Optimized and removed in MR plan.
+ // Needs to be removed in Tez plan as well.
+ "f1 = limit f 1;" +
+ "f2 = union d1, f1;" +
+ "store b1 into 'file:///tmp/output/b1';" +
+ "store b2 into 'file:///tmp/output/b2';" +
+ "store c1 into 'file:///tmp/output/c1';" +
+ "store c3 into 'file:///tmp/output/c1';" +
+ "store d1 into 'file:///tmp/output/d1';" +
+ "store e1 into 'file:///tmp/output/e1';" +
+ "store f1 into 'file:///tmp/output/f1';" +
+ "store f2 into 'file:///tmp/output/f2';";
+
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld");
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld");
+ }
+
+ @Test
+ public void testMultiQueryWithGroupBy() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = group a by x;" +
+ "b = foreach b generate group, COUNT(a.x);" +
+ "c = group a by (x,y);" +
+ "c = foreach c generate group, COUNT(a.y);" +
+ "store b into 'file:///tmp/output/b';" +
+ "store c into 'file:///tmp/output/c';";
+
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld");
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld");
+ }
+
+ @Test
+ public void testMultiQueryWithJoin() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input1' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input2' as (x:int, z:int);" +
+ "c = join a by x, b by x;" +
+ "d = foreach c generate $0, $1, $3;" +
+ "e = foreach c generate $0, $1, $2, $3;" +
+ "store c into 'file:///tmp/output/c';" +
+ "store d into 'file:///tmp/output/d';" +
+ "store e into 'file:///tmp/output/e';";
+
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4.gld");
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-4-OPTOFF.gld");
+ }
+
+ @Test
+ public void testMultiQueryWithNestedSplit() throws Exception {
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = group a by x;" + //b: {group: int,a: {(x: int,y: int)}}
+ "store b into 'file:///tmp/output/b';" +
+ "c = foreach b generate a.x, a.y;" + //c: {{(x: int)},{(y: int)}}
+ "store c into 'file:///tmp/output/c';" +
+ "d = foreach b GENERATE FLATTEN(a);" + //d: {a::x: int,a::y: int}
+ "store d into 'file:///tmp/output/d';" +
+ "e = foreach d GENERATE a::x, a::y;" +
+ "store e into 'file:///tmp/output/e';";
+
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5.gld");
+ setProperty(PigConfiguration.OPT_MULTIQUERY, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-5-OPTOFF.gld");
+ }
+
+ @Test
public void testUnionStore() throws Exception {
- try {
- String query =
- "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
- "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
- "c = union onschema a, b;" +
- "store c into 'file:///tmp/output';";
-
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld");
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld");
- } finally {
- removeProperty(PigConfiguration.TEZ_OPT_UNION);
- }
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "store c into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld");
}
@Test
public void testUnionGroupBy() throws Exception {
- try {
- String query =
- "a = load 'file:///tmp/input' as (x:int, y:int);" +
- "b = load 'file:///tmp/input' as (y:int, x:int);" +
- "c = union onschema a, b;" +
- "d = group c by x;" +
- "e = foreach d generate group, SUM(c.y);" +
- "store e into 'file:///tmp/output';";
-
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld");
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld");
- } finally {
- removeProperty(PigConfiguration.TEZ_OPT_UNION);
- }
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:int);" +
+ "b = load 'file:///tmp/input' as (y:int, x:int);" +
+ "c = union onschema a, b;" +
+ "d = group c by x;" +
+ "e = foreach d generate group, SUM(c.y);" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld");
}
@Test
public void testUnionJoin() throws Exception {
- try {
- String query =
- "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
- "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
- "c = union onschema a, b;" +
- "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
- "e = join c by x, d by x;" +
- "store e into 'file:///tmp/output';";
-
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-3.gld");
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-3-OPTOFF.gld");
- } finally {
- removeProperty(PigConfiguration.TEZ_OPT_UNION);
- }
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join c by x, d by x;" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-3.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-3-OPTOFF.gld");
}
@Test
public void testUnionReplicateJoin() throws Exception {
- try {
- String query =
- "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
- "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
- "c = union onschema a, b;" +
- "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
- "e = join c by x, d by x using 'replicated';" +
- "store e into 'file:///tmp/output';";
-
- //TODO: PIG-3856 Not optimized
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-4.gld");
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-4-OPTOFF.gld");
-
- query =
- "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
- "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
- "c = union onschema a, b;" +
- "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
- "e = join d by x, c by x using 'replicated';" +
- "store e into 'file:///tmp/output';";
-
- // Optimized
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-5.gld");
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-5-OPTOFF.gld");
-
- } finally {
- removeProperty(PigConfiguration.TEZ_OPT_UNION);
- }
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join c by x, d by x using 'replicated';" +
+ "store e into 'file:///tmp/output';";
+
+ //TODO: PIG-3856 Not optimized
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-4.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-4-OPTOFF.gld");
+
+ query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join d by x, c by x using 'replicated';" +
+ "store e into 'file:///tmp/output';";
+
+ // Optimized
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-5.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-5-OPTOFF.gld");
}
@Test
public void testUnionSkewedJoin() throws Exception {
- try {
- String query =
- "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
- "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
- "c = union onschema a, b;" +
- "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
- "e = join c by x, d by x using 'skewed';" +
- "store e into 'file:///tmp/output';";
-
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld");
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld");
- } finally {
- removeProperty(PigConfiguration.TEZ_OPT_UNION);
- }
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = load 'file:///tmp/input1' as (x:int, z:chararray);" +
+ "e = join c by x, d by x using 'skewed';" +
+ "store e into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld");
}
@Test
public void testUnionOrderby() throws Exception {
- try {
- String query =
- "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
- "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
- "c = union onschema a, b;" +
- "d = order c by x;" +
- "store d into 'file:///tmp/output';";
-
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld");
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld");
- } finally {
- removeProperty(PigConfiguration.TEZ_OPT_UNION);
- }
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = order c by x;" +
+ "store d into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld");
}
//TODO: PIG-3854 Limit is too convoluted and can be simplified.
@Test
public void testUnionLimit() throws Exception {
- try {
- String query =
- "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
- "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
- "c = union onschema a, b;" +
- "d = limit c 1;" +
- "store d into 'file:///tmp/output';";
-
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8.gld");
- setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
- run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8-OPTOFF.gld");
- } finally {
- removeProperty(PigConfiguration.TEZ_OPT_UNION);
- }
+ String query =
+ "a = load 'file:///tmp/input' as (x:int, y:chararray);" +
+ "b = load 'file:///tmp/input' as (y:chararray, x:int);" +
+ "c = union onschema a, b;" +
+ "d = limit c 1;" +
+ "store d into 'file:///tmp/output';";
+
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + true);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8.gld");
+ setProperty(PigConfiguration.TEZ_OPT_UNION, "" + false);
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-8-OPTOFF.gld");
}
@Test
@@ -474,7 +472,7 @@ public class TestTezCompiler {
"b = rank a;" +
"store b into 'file:///tmp/output/d';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC20.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-1.gld");
}
@Test
@@ -485,17 +483,13 @@ public class TestTezCompiler {
"b = rank a by x;" +
"store b into 'file:///tmp/output/d';";
- run(query, "test/org/apache/pig/test/data/GoldenFiles/TEZC21.gld");
+ run(query, "test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld");
}
private void setProperty(String property, String value) {
pigServer.getPigContext().getProperties().setProperty(property, value);
}
- private void removeProperty(String property) {
- pigServer.getPigContext().getProperties().remove(property);
- }
-
private void run(String query, String expectedFile) throws Exception {
PhysicalPlan pp = Util.buildPp(pigServer, query);
TezLauncher launcher = new TezLauncher();