You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/06/17 20:15:27 UTC
svn commit: r1603243 [2/2] - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/
src/org/apache/pig/backend/hadoop/executionengine/tez/
src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/...
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld Tue Jun 17 18:15:26 2014
@@ -2,26 +2,30 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-35
+# TEZ DAG plan: scope-37
#--------------------------------------------------
-Tez vertex scope-11 -> Tez vertex scope-28,Tez vertex scope-18,
-Tez vertex scope-18 -> Tez vertex scope-28,
-Tez vertex scope-28 -> Tez vertex scope-30,
-Tez vertex scope-30
+Tez vertex scope-11 -> Tez vertex scope-30,Tez vertex scope-20,
+Tez vertex scope-20 -> Tez vertex scope-30,
+Tez vertex scope-30 -> Tez vertex scope-32,
+Tez vertex scope-32
Tez vertex scope-11
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-14 -> scope-18
+Local Rearrange[tuple]{tuple}(false) - scope-14 -> scope-20
| |
| Constant(DummyVal) - scope-13
|
-|---ReservoirSample - scope-17
+|---New For Each(true,true)[tuple] - scope-19
+ | |
+ | Project[int][0] - scope-8
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-18
+ | |
+ | |---Project[tuple][*] - scope-17
|
- |---New For Each(false)[tuple] - scope-16
- | |
- | Project[int][0] - scope-15
+ |---ReservoirSample - scope-16
|
- |---b: Local Rearrange[tuple]{int}(false) - scope-12 -> scope-28
+ |---b: Local Rearrange[tuple]{int}(false) - scope-12 -> scope-30
| |
| Project[int][0] - scope-8
|
@@ -36,34 +40,34 @@ Local Rearrange[tuple]{tuple}(false) - s
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:PigStorage(',')) - scope-0
-Tez vertex scope-18
+Tez vertex scope-20
# Plan on vertex
-POValueOutputTez - scope-27 -> [scope-28]
+POValueOutputTez - scope-29 -> [scope-30]
|
-|---New For Each(false)[tuple] - scope-26
+|---New For Each(false)[tuple] - scope-28
| |
- | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-25
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-27
| |
- | |---Project[tuple][*] - scope-24
+ | |---Project[tuple][*] - scope-26
|
- |---New For Each(false,false)[tuple] - scope-23
+ |---New For Each(false,false)[tuple] - scope-25
| |
- | Constant(1) - scope-22
+ | Constant(-1) - scope-24
| |
- | Project[bag][1] - scope-20
+ | Project[bag][1] - scope-22
|
- |---Package(Packager)[tuple]{bytearray} - scope-19
-Tez vertex scope-28
+ |---Package(Packager)[tuple]{bytearray} - scope-21
+Tez vertex scope-30
# Plan on vertex
-POIdentityInOutTez - scope-29 <- scope-11 -> scope-30
+POIdentityInOutTez - scope-31 <- scope-11 -> scope-32
| |
| Project[int][0] - scope-8
-Tez vertex scope-30
+Tez vertex scope-32
# Plan on vertex
b: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-10
|
-|---New For Each(true)[tuple] - scope-33
+|---New For Each(true)[tuple] - scope-35
| |
- | Project[bag][1] - scope-32
+ | Project[bag][1] - scope-34
|
- |---Package(LitePackager)[tuple]{int} - scope-31
+ |---Package(LitePackager)[tuple]{int} - scope-33
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld Tue Jun 17 18:15:26 2014
@@ -47,13 +47,13 @@ POValueOutputTez - scope-45 -> [scope-4
|
|---New For Each(false)[tuple] - scope-44
| |
- | POUserFunc(org.apache.pig.impl.builtin.PartitionSkewedKeys)[tuple] - scope-43
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.PartitionSkewedKeysTez)[tuple] - scope-43
| |
| |---Project[tuple][*] - scope-42
|
|---New For Each(false,false)[tuple] - scope-41
| |
- | Constant(1) - scope-40
+ | Constant(-1) - scope-40
| |
| Project[bag][1] - scope-38
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld Tue Jun 17 18:15:26 2014
@@ -2,307 +2,311 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-418
+# TEZ DAG plan: scope-422
#--------------------------------------------------
-Tez vertex scope-319 -> Tez vertex scope-321,Tez vertex scope-332,Tez vertex scope-343,
-Tez vertex scope-332 -> Tez vertex scope-335,Tez vertex scope-340,
-Tez vertex scope-335 -> Tez vertex scope-339,
-Tez vertex scope-321 -> Tez vertex scope-324,Tez vertex scope-337,Tez vertex scope-348,
-Tez vertex scope-337 -> Tez vertex scope-339,
-Tez vertex scope-339
-Tez vertex scope-348 -> Tez vertex scope-351,Tez vertex scope-375,
-Tez vertex scope-375 -> Tez vertex scope-379,
-Tez vertex scope-379 -> Tez vertex scope-385,Tez vertex scope-389,
-Tez vertex scope-385
-Tez vertex scope-343 -> Tez vertex scope-346,Tez vertex scope-387,
-Tez vertex scope-387 -> Tez vertex scope-415,
-Tez vertex scope-389 -> Tez vertex scope-415,
-Tez vertex scope-415
-Tez vertex scope-346
-Tez vertex scope-340 -> Tez vertex scope-342,
-Tez vertex scope-342
-Tez vertex scope-324 -> Tez vertex scope-326,
-Tez vertex scope-326 -> Tez vertex scope-328,Tez vertex scope-330,
-Tez vertex scope-328
-Tez vertex scope-351 -> Tez vertex scope-369,Tez vertex scope-359,
-Tez vertex scope-359 -> Tez vertex scope-369,
-Tez vertex scope-369 -> Tez vertex scope-371,
-Tez vertex scope-371
+Tez vertex scope-321 -> Tez vertex scope-323,Tez vertex scope-334,Tez vertex scope-345,
+Tez vertex scope-323 -> Tez vertex scope-326,Tez vertex scope-339,Tez vertex scope-350,
+Tez vertex scope-350 -> Tez vertex scope-353,Tez vertex scope-379,
+Tez vertex scope-379 -> Tez vertex scope-383,
+Tez vertex scope-383 -> Tez vertex scope-389,Tez vertex scope-393,
+Tez vertex scope-389
+Tez vertex scope-334 -> Tez vertex scope-337,Tez vertex scope-342,
+Tez vertex scope-337 -> Tez vertex scope-341,
+Tez vertex scope-339 -> Tez vertex scope-341,
+Tez vertex scope-341
+Tez vertex scope-345 -> Tez vertex scope-348,Tez vertex scope-391,
+Tez vertex scope-348
+Tez vertex scope-342 -> Tez vertex scope-344,
+Tez vertex scope-344
+Tez vertex scope-353 -> Tez vertex scope-373,Tez vertex scope-363,
+Tez vertex scope-363 -> Tez vertex scope-373,
+Tez vertex scope-373 -> Tez vertex scope-375,
+Tez vertex scope-375
+Tez vertex scope-326 -> Tez vertex scope-328,
+Tez vertex scope-328 -> Tez vertex scope-330,Tez vertex scope-332,
Tez vertex scope-330
+Tez vertex scope-391 -> Tez vertex scope-419,
+Tez vertex scope-393 -> Tez vertex scope-419,
+Tez vertex scope-419
+Tez vertex scope-332
-Tez vertex scope-319
+Tez vertex scope-321
# Plan on vertex
-POValueOutputTez - scope-320 -> [scope-321, scope-332, scope-343]
+POValueOutputTez - scope-322 -> [scope-323, scope-345, scope-334]
|
-|---a: New For Each(false,false)[bag] - scope-220
+|---a: New For Each(false,false)[bag] - scope-222
| |
- | Cast[int] - scope-215
+ | Cast[int] - scope-217
| |
- | |---Project[bytearray][0] - scope-214
+ | |---Project[bytearray][0] - scope-216
| |
- | Cast[int] - scope-218
+ | Cast[int] - scope-220
| |
- | |---Project[bytearray][1] - scope-217
+ | |---Project[bytearray][1] - scope-219
|
- |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-213
-Tez vertex scope-332
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-215
+Tez vertex scope-323
# Plan on vertex
-POValueOutputTez - scope-334 -> [scope-340, scope-335]
+POValueOutputTez - scope-325 -> [scope-339, scope-326, scope-350]
|
-|---c: Filter[bag] - scope-247
+|---b: Filter[bag] - scope-224
| |
- | Less Than or Equal[boolean] - scope-250
+ | Less Than or Equal[boolean] - scope-227
| |
- | |---Project[int][0] - scope-248
+ | |---Project[int][0] - scope-225
| |
- | |---Constant(10) - scope-249
+ | |---Constant(5) - scope-226
|
- |---POValueInputTez - scope-333 <- scope-319
-Tez vertex scope-335
+ |---POValueInputTez - scope-324 <- scope-321
+Tez vertex scope-350
+# Plan on vertex
+POValueOutputTez - scope-352 -> [scope-379, scope-353]
+|
+|---POValueInputTez - scope-351 <- scope-323
+Tez vertex scope-379
# Plan on vertex
-c1: Local Rearrange[tuple]{int}(false) - scope-260 -> scope-339
+f1: Local Rearrange[tuple]{tuple}(false) - scope-382 -> scope-383
| |
-| Project[int][0] - scope-261
+| Project[tuple][*] - scope-381
|
-|---POValueInputTez - scope-336 <- scope-332
-Tez vertex scope-321
+|---f1: Limit - scope-310
+ |
+ |---f: Filter[bag] - scope-306
+ | |
+ | Greater Than or Equal[boolean] - scope-309
+ | |
+ | |---Project[int][0] - scope-307
+ | |
+ | |---Constant(3) - scope-308
+ |
+ |---POValueInputTez - scope-380 <- scope-350
+Tez vertex scope-383
# Plan on vertex
-POValueOutputTez - scope-323 -> [scope-348, scope-337, scope-324]
+POValueOutputTez - scope-388 -> [scope-393, scope-389]
|
-|---b: Filter[bag] - scope-222
- | |
- | Less Than or Equal[boolean] - scope-225
- | |
- | |---Project[int][0] - scope-223
- | |
- | |---Constant(5) - scope-224
+|---f1: Limit - scope-387
|
- |---POValueInputTez - scope-322 <- scope-319
-Tez vertex scope-337
+ |---f1: New For Each(true)[bag] - scope-386
+ | |
+ | Project[tuple][1] - scope-385
+ |
+ |---f1: Package(Packager)[tuple]{tuple} - scope-384
+Tez vertex scope-389
# Plan on vertex
-c1: Local Rearrange[tuple]{int}(false) - scope-262 -> scope-339
-| |
-| Project[int][0] - scope-263
+f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-314
|
-|---POValueInputTez - scope-338 <- scope-321
-Tez vertex scope-339
+|---POValueInputTez - scope-390 <- scope-383
+Tez vertex scope-334
# Plan on vertex
-c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-267
+POValueOutputTez - scope-336 -> [scope-337, scope-342]
|
-|---c1: New For Each(true,true)[tuple] - scope-266
+|---c: Filter[bag] - scope-249
| |
- | Project[bag][1] - scope-264
+ | Less Than or Equal[boolean] - scope-252
| |
- | Project[bag][2] - scope-265
+ | |---Project[int][0] - scope-250
+ | |
+ | |---Constant(10) - scope-251
|
- |---c1: Package(Packager)[tuple]{int} - scope-259
-Tez vertex scope-348
+ |---POValueInputTez - scope-335 <- scope-321
+Tez vertex scope-337
# Plan on vertex
-POValueOutputTez - scope-350 -> [scope-375, scope-351]
+c1: Local Rearrange[tuple]{int}(false) - scope-262 -> scope-341
+| |
+| Project[int][0] - scope-263
|
-|---POValueInputTez - scope-349 <- scope-321
-Tez vertex scope-375
+|---POValueInputTez - scope-338 <- scope-334
+Tez vertex scope-339
# Plan on vertex
-f1: Local Rearrange[tuple]{tuple}(false) - scope-378 -> scope-379
+c1: Local Rearrange[tuple]{int}(false) - scope-264 -> scope-341
| |
-| Project[tuple][*] - scope-377
+| Project[int][0] - scope-265
|
-|---f1: Limit - scope-308
- |
- |---f: Filter[bag] - scope-304
- | |
- | Greater Than or Equal[boolean] - scope-307
- | |
- | |---Project[int][0] - scope-305
- | |
- | |---Constant(3) - scope-306
- |
- |---POValueInputTez - scope-376 <- scope-348
-Tez vertex scope-379
+|---POValueInputTez - scope-340 <- scope-323
+Tez vertex scope-341
# Plan on vertex
-POValueOutputTez - scope-384 -> [scope-389, scope-385]
+c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-269
|
-|---f1: Limit - scope-383
+|---c1: New For Each(true,true)[tuple] - scope-268
+ | |
+ | Project[bag][1] - scope-266
+ | |
+ | Project[bag][2] - scope-267
|
- |---f1: New For Each(true)[bag] - scope-382
- | |
- | Project[tuple][1] - scope-381
- |
- |---f1: Package(Packager)[tuple]{tuple} - scope-380
-Tez vertex scope-385
-# Plan on vertex
-f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-312
-|
-|---POValueInputTez - scope-386 <- scope-379
-Tez vertex scope-343
+ |---c1: Package(Packager)[tuple]{int} - scope-261
+Tez vertex scope-345
# Plan on vertex
-POValueOutputTez - scope-345 -> [scope-387, scope-346]
+POValueOutputTez - scope-347 -> [scope-391, scope-348]
|
-|---d1: Filter[bag] - scope-286
+|---d1: Filter[bag] - scope-288
| |
- | Equal To[boolean] - scope-289
+ | Equal To[boolean] - scope-291
| |
- | |---Project[int][0] - scope-287
+ | |---Project[int][0] - scope-289
| |
- | |---Constant(5) - scope-288
+ | |---Constant(5) - scope-290
|
- |---d: Filter[bag] - scope-282
+ |---d: Filter[bag] - scope-284
| |
- | Greater Than[boolean] - scope-285
+ | Greater Than[boolean] - scope-287
| |
- | |---Project[int][0] - scope-283
+ | |---Project[int][0] - scope-285
| |
- | |---Constant(10) - scope-284
+ | |---Constant(10) - scope-286
|
- |---POValueInputTez - scope-344 <- scope-319
-Tez vertex scope-387
-# Plan on vertex
-f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-416
-|
-|---POValueInputTez - scope-388 <- scope-343
-Tez vertex scope-389
-# Plan on vertex
-f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-417
-|
-|---POValueInputTez - scope-390 <- scope-379
-Tez vertex group scope-415 <- [scope-387, scope-389] -> null
-# No plan on vertex group
-Tez vertex scope-346
+ |---POValueInputTez - scope-346 <- scope-321
+Tez vertex scope-348
# Plan on vertex
-d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-293
+d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-295
|
-|---POValueInputTez - scope-347 <- scope-343
-Tez vertex scope-340
+|---POValueInputTez - scope-349 <- scope-345
+Tez vertex scope-342
# Plan on vertex
-c2: Local Rearrange[tuple]{int}(false) - scope-407 -> scope-342
+c2: Local Rearrange[tuple]{int}(false) - scope-411 -> scope-344
| |
-| Project[int][0] - scope-409
+| Project[int][0] - scope-413
|
-|---c3: New For Each(false,false)[bag] - scope-395
+|---c3: New For Each(false,false)[bag] - scope-399
| |
- | Project[int][0] - scope-396
+ | Project[int][0] - scope-400
| |
- | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-397
+ | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-401
| |
- | |---Project[bag][0] - scope-398
+ | |---Project[bag][0] - scope-402
| |
- | |---Project[bag][1] - scope-399
+ | |---Project[bag][1] - scope-403
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-410
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-414
|
- |---POValueInputTez - scope-341 <- scope-332
-Tez vertex scope-342
-# Combine plan on edge <scope-340>
-c2: Local Rearrange[tuple]{int}(false) - scope-411 -> scope-342
+ |---POValueInputTez - scope-343 <- scope-334
+Tez vertex scope-344
+# Combine plan on edge <scope-342>
+c2: Local Rearrange[tuple]{int}(false) - scope-415 -> scope-344
| |
-| Project[int][0] - scope-413
+| Project[int][0] - scope-417
|
-|---c3: New For Each(false,false)[bag] - scope-400
+|---c3: New For Each(false,false)[bag] - scope-404
| |
- | Project[int][0] - scope-401
+ | Project[int][0] - scope-405
| |
- | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-402
+ | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-406
| |
- | |---Project[bag][1] - scope-403
+ | |---Project[bag][1] - scope-407
|
- |---c2: Package(CombinerPackager)[tuple]{int} - scope-406
+ |---c2: Package(CombinerPackager)[tuple]{int} - scope-410
# Plan on vertex
-c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-281
+c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-283
|
-|---c3: New For Each(false,false)[bag] - scope-280
+|---c3: New For Each(false,false)[bag] - scope-282
| |
- | Project[int][0] - scope-274
+ | Project[int][0] - scope-276
| |
- | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-278
+ | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-280
| |
- | |---Project[bag][1] - scope-404
+ | |---Project[bag][1] - scope-408
|
- |---c2: Package(CombinerPackager)[tuple]{int} - scope-271
-Tez vertex scope-324
+ |---c2: Package(CombinerPackager)[tuple]{int} - scope-273
+Tez vertex scope-353
# Plan on vertex
-b1: Local Rearrange[tuple]{int}(false) - scope-231 -> scope-326
+Local Rearrange[tuple]{tuple}(false) - scope-357 -> scope-363
| |
-| Project[int][0] - scope-232
-|
-|---POValueInputTez - scope-325 <- scope-321
-Tez vertex scope-326
-# Plan on vertex
-POValueOutputTez - scope-327 -> [scope-330, scope-328]
+| Constant(DummyVal) - scope-356
|
-|---b1: Package(Packager)[tuple]{int} - scope-230
-Tez vertex scope-328
-# Plan on vertex
-b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-236
-|
-|---POValueInputTez - scope-329 <- scope-326
-Tez vertex scope-351
-# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-355 -> scope-359
-| |
-| Constant(DummyVal) - scope-354
-|
-|---ReservoirSample - scope-358
+|---New For Each(true,true)[tuple] - scope-362
+ | |
+ | Project[int][0] - scope-303
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-361
+ | |
+ | |---Project[tuple][*] - scope-360
|
- |---New For Each(false)[tuple] - scope-357
- | |
- | Project[int][0] - scope-356
+ |---ReservoirSample - scope-359
|
- |---e1: Local Rearrange[tuple]{int}(false) - scope-353 -> scope-369
+ |---e1: Local Rearrange[tuple]{int}(false) - scope-355 -> scope-373
| |
- | Project[int][0] - scope-301
+ | Project[int][0] - scope-303
|
- |---e: Filter[bag] - scope-297
+ |---e: Filter[bag] - scope-299
| |
- | Less Than[boolean] - scope-300
+ | Less Than[boolean] - scope-302
| |
- | |---Project[int][0] - scope-298
+ | |---Project[int][0] - scope-300
| |
- | |---Constant(3) - scope-299
+ | |---Constant(3) - scope-301
|
- |---POValueInputTez - scope-352 <- scope-348
-Tez vertex scope-359
+ |---POValueInputTez - scope-354 <- scope-350
+Tez vertex scope-363
# Plan on vertex
-POValueOutputTez - scope-368 -> [scope-369]
+POValueOutputTez - scope-372 -> [scope-373]
|
-|---New For Each(false)[tuple] - scope-367
+|---New For Each(false)[tuple] - scope-371
| |
- | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-366
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-370
| |
- | |---Project[tuple][*] - scope-365
+ | |---Project[tuple][*] - scope-369
|
- |---New For Each(false,false)[tuple] - scope-364
+ |---New For Each(false,false)[tuple] - scope-368
| |
- | Constant(1) - scope-363
+ | Constant(-1) - scope-367
| |
- | Project[bag][1] - scope-361
+ | Project[bag][1] - scope-365
|
- |---Package(Packager)[tuple]{bytearray} - scope-360
-Tez vertex scope-369
+ |---Package(Packager)[tuple]{bytearray} - scope-364
+Tez vertex scope-373
# Plan on vertex
-POIdentityInOutTez - scope-370 <- scope-351 -> scope-371
+POIdentityInOutTez - scope-374 <- scope-353 -> scope-375
| |
-| Project[int][0] - scope-301
-Tez vertex scope-371
+| Project[int][0] - scope-303
+Tez vertex scope-375
# Plan on vertex
-e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-303
+e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-305
|
-|---New For Each(true)[tuple] - scope-374
+|---New For Each(true)[tuple] - scope-378
| |
- | Project[bag][1] - scope-373
+ | Project[bag][1] - scope-377
|
- |---Package(LitePackager)[tuple]{int} - scope-372
+ |---Package(LitePackager)[tuple]{int} - scope-376
+Tez vertex scope-326
+# Plan on vertex
+b1: Local Rearrange[tuple]{int}(false) - scope-233 -> scope-328
+| |
+| Project[int][0] - scope-234
+|
+|---POValueInputTez - scope-327 <- scope-323
+Tez vertex scope-328
+# Plan on vertex
+POValueOutputTez - scope-329 -> [scope-330, scope-332]
+|
+|---b1: Package(Packager)[tuple]{int} - scope-232
Tez vertex scope-330
# Plan on vertex
-b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-246
+b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-238
+|
+|---POValueInputTez - scope-331 <- scope-328
+Tez vertex scope-391
+# Plan on vertex
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-420
+|
+|---POValueInputTez - scope-392 <- scope-345
+Tez vertex scope-393
+# Plan on vertex
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-421
+|
+|---POValueInputTez - scope-394 <- scope-383
+Tez vertex group scope-419 <- [scope-391, scope-393] -> null
+# No plan on vertex group
+Tez vertex scope-332
+# Plan on vertex
+b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-248
|
-|---b2: New For Each(false,false)[bag] - scope-245
+|---b2: New For Each(false,false)[bag] - scope-247
| |
- | Project[int][0] - scope-239
+ | Project[int][0] - scope-241
| |
- | POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-243
+ | POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-245
| |
- | |---Project[bag][0] - scope-242
+ | |---Project[bag][0] - scope-244
| |
- | |---Project[bag][1] - scope-241
+ | |---Project[bag][1] - scope-243
|
- |---POValueInputTez - scope-331 <- scope-326
+ |---POValueInputTez - scope-333 <- scope-328
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Tue Jun 17 18:15:26 2014
@@ -2,24 +2,24 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-212
+# TEZ DAG plan: scope-214
#--------------------------------------------------
-Tez vertex scope-106 -> Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-156,Tez vertex scope-146,Tez vertex scope-166,Tez vertex scope-209,
-Tez vertex scope-166 -> Tez vertex scope-209,
-Tez vertex scope-209
-Tez vertex scope-113
+Tez vertex scope-106 -> Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-158,Tez vertex scope-148,Tez vertex scope-168,Tez vertex scope-211,
+Tez vertex scope-148 -> Tez vertex scope-158,
+Tez vertex scope-158 -> Tez vertex scope-160,
+Tez vertex scope-160
Tez vertex scope-119 -> Tez vertex scope-126,Tez vertex scope-129,
Tez vertex scope-126
Tez vertex scope-129
-Tez vertex scope-146 -> Tez vertex scope-156,
-Tez vertex scope-156 -> Tez vertex scope-158,
-Tez vertex scope-158
+Tez vertex scope-113
+Tez vertex scope-168 -> Tez vertex scope-211,
+Tez vertex scope-211
Tez vertex scope-106
# Plan on vertex
-1-1: Split - scope-208
+1-1: Split - scope-210
| |
-| b: Split - scope-205
+| b: Split - scope-207
| | |
| | b1: Local Rearrange[tuple]{int}(false) - scope-18 -> scope-113
| | | |
@@ -29,19 +29,23 @@ Tez vertex scope-106
| | | |
| | | Project[int][0] - scope-50
| | |
-| | 1-2: Split - scope-204
+| | 1-2: Split - scope-206
| | | |
-| | | Local Rearrange[tuple]{tuple}(false) - scope-142 -> scope-146
+| | | Local Rearrange[tuple]{tuple}(false) - scope-142 -> scope-148
| | | | |
| | | | Constant(DummyVal) - scope-141
| | | |
-| | | |---ReservoirSample - scope-145
+| | | |---New For Each(true,true)[tuple] - scope-147
+| | | | |
+| | | | Project[int][0] - scope-88
+| | | | |
+| | | | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-146
+| | | | |
+| | | | |---Project[tuple][*] - scope-145
| | | |
-| | | |---New For Each(false)[tuple] - scope-144
-| | | | |
-| | | | Project[int][0] - scope-143
+| | | |---ReservoirSample - scope-144
| | | |
-| | | |---e1: Local Rearrange[tuple]{int}(false) - scope-140 -> scope-156
+| | | |---e1: Local Rearrange[tuple]{int}(false) - scope-140 -> scope-158
| | | | |
| | | | Project[int][0] - scope-88
| | | |
@@ -53,9 +57,9 @@ Tez vertex scope-106
| | | | |
| | | | |---Constant(3) - scope-86
| | | |
-| | | f1: Local Rearrange[tuple]{tuple}(false) - scope-165 -> scope-166
+| | | f1: Local Rearrange[tuple]{tuple}(false) - scope-167 -> scope-168
| | | | |
-| | | | Project[tuple][*] - scope-164
+| | | | Project[tuple][*] - scope-166
| | | |
| | | |---f1: Limit - scope-95
| | | |
@@ -75,11 +79,11 @@ Tez vertex scope-106
| | |
| | |---Constant(5) - scope-11
| |
-| d1: Split - scope-207
+| d1: Split - scope-209
| | |
| | d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-80
| | |
-| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210
+| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-212
| |
| |---d1: Filter[bag] - scope-73
| | |
@@ -110,65 +114,60 @@ Tez vertex scope-106
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-166
+Tez vertex scope-148
# Plan on vertex
-f1: Split - scope-203
-| |
-| f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99
-| |
-| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-211
+POValueOutputTez - scope-157 -> [scope-158]
|
-|---f1: Limit - scope-170
+|---New For Each(false)[tuple] - scope-156
+ | |
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-155
+ | |
+ | |---Project[tuple][*] - scope-154
|
- |---f1: New For Each(true)[bag] - scope-169
+ |---New For Each(false,false)[tuple] - scope-153
+ | |
+ | Constant(-1) - scope-152
| |
- | Project[tuple][1] - scope-168
+ | Project[bag][1] - scope-150
|
- |---f1: Package(Packager)[tuple]{tuple} - scope-167
-Tez vertex group scope-209 <- [scope-106, scope-166] -> null
-# No plan on vertex group
-Tez vertex scope-113
+ |---Package(Packager)[tuple]{bytearray} - scope-149
+Tez vertex scope-158
# Plan on vertex
-b1: Split - scope-202
-| |
-| b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-23
-| |
-| b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-33
+POIdentityInOutTez - scope-159 <- scope-106 -> scope-160
| |
-| |---b2: New For Each(false,false)[bag] - scope-32
-| | |
-| | Project[int][0] - scope-26
-| | |
-| | POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-30
-| | |
-| | |---Project[bag][0] - scope-29
-| | |
-| | |---Project[bag][1] - scope-28
+| Project[int][0] - scope-88
+Tez vertex scope-160
+# Plan on vertex
+e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-90
|
-|---b1: Package(Packager)[tuple]{int} - scope-17
+|---New For Each(true)[tuple] - scope-163
+ | |
+ | Project[bag][1] - scope-162
+ |
+ |---Package(LitePackager)[tuple]{int} - scope-161
Tez vertex scope-119
# Plan on vertex
-c: Split - scope-206
+c: Split - scope-208
| |
| c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126
| | |
| | Project[int][0] - scope-48
| |
-| c2: Local Rearrange[tuple]{int}(false) - scope-194 -> scope-129
+| c2: Local Rearrange[tuple]{int}(false) - scope-196 -> scope-129
| | |
-| | Project[int][0] - scope-196
+| | Project[int][0] - scope-198
| |
-| |---c3: New For Each(false,false)[bag] - scope-182
+| |---c3: New For Each(false,false)[bag] - scope-184
| | |
-| | Project[int][0] - scope-183
+| | Project[int][0] - scope-185
| | |
-| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-184
+| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-186
| | |
-| | |---Project[bag][0] - scope-185
+| | |---Project[bag][0] - scope-187
| | |
-| | |---Project[bag][1] - scope-186
+| | |---Project[bag][1] - scope-188
| |
-| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-197
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-199
|
|---c: Filter[bag] - scope-34
| |
@@ -192,19 +191,19 @@ c1: Store(file:///tmp/output/c1:org.apac
|---c1: Package(Packager)[tuple]{int} - scope-46
Tez vertex scope-129
# Combine plan on edge <scope-119>
-c2: Local Rearrange[tuple]{int}(false) - scope-198 -> scope-129
+c2: Local Rearrange[tuple]{int}(false) - scope-200 -> scope-129
| |
-| Project[int][0] - scope-200
+| Project[int][0] - scope-202
|
-|---c3: New For Each(false,false)[bag] - scope-187
+|---c3: New For Each(false,false)[bag] - scope-189
| |
- | Project[int][0] - scope-188
+ | Project[int][0] - scope-190
| |
- | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-189
+ | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-191
| |
- | |---Project[bag][1] - scope-190
+ | |---Project[bag][1] - scope-192
|
- |---c2: Package(CombinerPackager)[tuple]{int} - scope-193
+ |---c2: Package(CombinerPackager)[tuple]{int} - scope-195
# Plan on vertex
c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-68
|
@@ -214,37 +213,42 @@ c3: Store(file:///tmp/output/c1:org.apac
| |
| POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-65
| |
- | |---Project[bag][1] - scope-191
+ | |---Project[bag][1] - scope-193
|
|---c2: Package(CombinerPackager)[tuple]{int} - scope-58
-Tez vertex scope-146
+Tez vertex scope-113
# Plan on vertex
-POValueOutputTez - scope-155 -> [scope-156]
+b1: Split - scope-204
+| |
+| b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-23
+| |
+| b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-33
+| |
+| |---b2: New For Each(false,false)[bag] - scope-32
+| | |
+| | Project[int][0] - scope-26
+| | |
+| | POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-30
+| | |
+| | |---Project[bag][0] - scope-29
+| | |
+| | |---Project[bag][1] - scope-28
|
-|---New For Each(false)[tuple] - scope-154
- | |
- | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-153
- | |
- | |---Project[tuple][*] - scope-152
- |
- |---New For Each(false,false)[tuple] - scope-151
- | |
- | Constant(1) - scope-150
- | |
- | Project[bag][1] - scope-148
- |
- |---Package(Packager)[tuple]{bytearray} - scope-147
-Tez vertex scope-156
+|---b1: Package(Packager)[tuple]{int} - scope-17
+Tez vertex scope-168
# Plan on vertex
-POIdentityInOutTez - scope-157 <- scope-106 -> scope-158
+f1: Split - scope-205
| |
-| Project[int][0] - scope-88
-Tez vertex scope-158
-# Plan on vertex
-e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-90
+| f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99
+| |
+| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-213
|
-|---New For Each(true)[tuple] - scope-161
- | |
- | Project[bag][1] - scope-160
+|---f1: Limit - scope-172
|
- |---Package(LitePackager)[tuple]{int} - scope-159
+ |---f1: New For Each(true)[bag] - scope-171
+ | |
+ | Project[tuple][1] - scope-170
+ |
+ |---f1: Package(Packager)[tuple]{tuple} - scope-169
+Tez vertex group scope-211 <- [scope-106, scope-168] -> null
+# No plan on vertex group
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld Tue Jun 17 18:15:26 2014
@@ -2,15 +2,15 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-51
+# TEZ DAG plan: scope-53
#--------------------------------------------------
Tez vertex scope-23 -> Tez vertex scope-24,
-Tez vertex scope-24 -> Tez vertex scope-41,Tez vertex scope-31,
-Tez vertex scope-31 -> Tez vertex scope-41,
-Tez vertex scope-41 -> Tez vertex scope-43,
-Tez vertex scope-43 -> Tez vertex scope-49,Tez vertex scope-47,
-Tez vertex scope-47 -> Tez vertex scope-49,
-Tez vertex scope-49
+Tez vertex scope-24 -> Tez vertex scope-43,Tez vertex scope-33,
+Tez vertex scope-33 -> Tez vertex scope-43,
+Tez vertex scope-43 -> Tez vertex scope-45,
+Tez vertex scope-45 -> Tez vertex scope-51,Tez vertex scope-49,
+Tez vertex scope-49 -> Tez vertex scope-51,
+Tez vertex scope-51
Tez vertex scope-23
# Plan on vertex
@@ -31,17 +31,21 @@ b: Local Rearrange[tuple]{int}(false) -
|---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-24
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-27 -> scope-31
+Local Rearrange[tuple]{tuple}(false) - scope-27 -> scope-33
| |
| Constant(DummyVal) - scope-26
|
-|---ReservoirSample - scope-30
+|---New For Each(true,true)[tuple] - scope-32
+ | |
+ | Project[int][0] - scope-15
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-31
+ | |
+ | |---Project[tuple][*] - scope-30
|
- |---New For Each(false)[tuple] - scope-29
- | |
- | Project[int][0] - scope-28
+ |---ReservoirSample - scope-29
|
- |---b: Local Rearrange[tuple]{int}(false) - scope-25 -> scope-41
+ |---b: Local Rearrange[tuple]{int}(false) - scope-25 -> scope-43
| |
| Project[int][0] - scope-15
|
@@ -52,44 +56,44 @@ Local Rearrange[tuple]{tuple}(false) - s
| Project[bag][1] - scope-13
|
|---b: Package(Packager)[tuple]{int} - scope-9
-Tez vertex scope-31
+Tez vertex scope-33
# Plan on vertex
-POValueOutputTez - scope-40 -> [scope-41]
+POValueOutputTez - scope-42 -> [scope-43]
|
-|---New For Each(false)[tuple] - scope-39
+|---New For Each(false)[tuple] - scope-41
| |
- | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-38
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-40
| |
- | |---Project[tuple][*] - scope-37
+ | |---Project[tuple][*] - scope-39
|
- |---New For Each(false,false)[tuple] - scope-36
+ |---New For Each(false,false)[tuple] - scope-38
| |
- | Constant(1) - scope-35
+ | Constant(-1) - scope-37
| |
- | Project[bag][1] - scope-33
+ | Project[bag][1] - scope-35
|
- |---Package(Packager)[tuple]{bytearray} - scope-32
-Tez vertex scope-41
+ |---Package(Packager)[tuple]{bytearray} - scope-34
+Tez vertex scope-43
# Plan on vertex
-POIdentityInOutTez - scope-42 <- scope-24 -> scope-43
+POIdentityInOutTez - scope-44 <- scope-24 -> scope-45
| |
| Project[int][0] - scope-15
-Tez vertex scope-43
+Tez vertex scope-45
# Plan on vertex
-POCounterTez - scope-17 -> scope-49,scope-47
+POCounterTez - scope-17 -> scope-51,scope-49
| |
| Project[int][0] - scope-15
|
-|---New For Each(true)[tuple] - scope-46
+|---New For Each(true)[tuple] - scope-48
| |
- | Project[bag][1] - scope-45
+ | Project[bag][1] - scope-47
|
- |---Package(LitePackager)[tuple]{int} - scope-44
-Tez vertex scope-47
-# Plan on vertex
-PORankStatsTez - scope-48 <- scope-43 -> scope-49
+ |---Package(LitePackager)[tuple]{int} - scope-46
Tez vertex scope-49
# Plan on vertex
+PORankStatsTez - scope-50 <- scope-45 -> scope-51
+Tez vertex scope-51
+# Plan on vertex
b: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-22
|
|---New For Each(false,true)[tuple] - scope-21
@@ -98,6 +102,6 @@ b: Store(file:///tmp/output/d:org.apache
| |
| Project[bag][2] - scope-20
|
- |---PORankTez - scope-18 <- scope-43,scope-47
+ |---PORankTez - scope-18 <- scope-45,scope-49
| |
| Project[int][0] - scope-15
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld Tue Jun 17 18:15:26 2014
@@ -69,13 +69,13 @@ POValueOutputTez - scope-138 -> [scope-
|
|---New For Each(false)[tuple] - scope-137
| |
- | POUserFunc(org.apache.pig.impl.builtin.PartitionSkewedKeys)[tuple] - scope-136
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.PartitionSkewedKeysTez)[tuple] - scope-136
| |
| |---Project[tuple][*] - scope-135
|
|---New For Each(false,false)[tuple] - scope-134
| |
- | Constant(1) - scope-133
+ | Constant(-1) - scope-133
| |
| Project[bag][1] - scope-131
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld Tue Jun 17 18:15:26 2014
@@ -83,13 +83,13 @@ POValueOutputTez - scope-52 -> [scope-5
|
|---New For Each(false)[tuple] - scope-51
| |
- | POUserFunc(org.apache.pig.impl.builtin.PartitionSkewedKeys)[tuple] - scope-50
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.PartitionSkewedKeysTez)[tuple] - scope-50
| |
| |---Project[tuple][*] - scope-49
|
|---New For Each(false,false)[tuple] - scope-48
| |
- | Constant(1) - scope-47
+ | Constant(-1) - scope-47
| |
| Project[bag][1] - scope-45
|
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld Tue Jun 17 18:15:26 2014
@@ -2,90 +2,94 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-117
+# TEZ DAG plan: scope-125
#--------------------------------------------------
-Tez vertex scope-88 -> Tez vertex scope-90,
-Tez vertex scope-89 -> Tez vertex scope-90,
-Tez vertex scope-90 -> Tez vertex scope-110,Tez vertex scope-100,
-Tez vertex scope-100 -> Tez vertex scope-110,
-Tez vertex scope-110 -> Tez vertex scope-112,
-Tez vertex scope-112
+Tez vertex scope-94 -> Tez vertex scope-96,
+Tez vertex scope-95 -> Tez vertex scope-96,
+Tez vertex scope-96 -> Tez vertex scope-118,Tez vertex scope-108,
+Tez vertex scope-108 -> Tez vertex scope-118,
+Tez vertex scope-118 -> Tez vertex scope-120,
+Tez vertex scope-120
-Tez vertex scope-88
+Tez vertex scope-94
# Plan on vertex
-POValueOutputTez - scope-92 -> [scope-90]
+POValueOutputTez - scope-98 -> [scope-96]
|
-|---a: New For Each(false,false)[bag] - scope-75
+|---a: New For Each(false,false)[bag] - scope-81
| |
- | Cast[int] - scope-70
+ | Cast[int] - scope-76
| |
- | |---Project[bytearray][0] - scope-69
+ | |---Project[bytearray][0] - scope-75
| |
- | Cast[chararray] - scope-73
+ | Cast[chararray] - scope-79
| |
- | |---Project[bytearray][1] - scope-72
+ | |---Project[bytearray][1] - scope-78
|
- |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-68
-Tez vertex scope-89
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-74
+Tez vertex scope-95
# Plan on vertex
-POValueOutputTez - scope-93 -> [scope-90]
+POValueOutputTez - scope-99 -> [scope-96]
|
-|---c: New For Each(false,false)[bag] - scope-83
+|---c: New For Each(false,false)[bag] - scope-89
| |
- | Cast[int] - scope-78
+ | Cast[int] - scope-84
| |
- | |---Project[bytearray][1] - scope-77
+ | |---Project[bytearray][1] - scope-83
| |
- | Cast[chararray] - scope-81
+ | Cast[chararray] - scope-87
| |
- | |---Project[bytearray][0] - scope-80
+ | |---Project[bytearray][0] - scope-86
|
- |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-76
-Tez vertex scope-90
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-82
+Tez vertex scope-96
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-96 -> scope-100
+Local Rearrange[tuple]{tuple}(false) - scope-102 -> scope-108
| |
-| Constant(DummyVal) - scope-95
+| Constant(DummyVal) - scope-101
|
-|---ReservoirSample - scope-99
+|---New For Each(true,true)[tuple] - scope-107
+ | |
+ | Project[int][0] - scope-91
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-106
+ | |
+ | |---Project[tuple][*] - scope-105
|
- |---New For Each(false)[tuple] - scope-98
- | |
- | Project[int][0] - scope-97
+ |---ReservoirSample - scope-104
|
- |---d: Local Rearrange[tuple]{int}(false) - scope-94 -> scope-110
+ |---d: Local Rearrange[tuple]{int}(false) - scope-100 -> scope-118
| |
- | Project[int][0] - scope-85
+ | Project[int][0] - scope-91
|
- |---POShuffledValueInputTez - scope-91 <- [scope-88, scope-89]
-Tez vertex scope-100
+ |---POShuffledValueInputTez - scope-97 <- [scope-94, scope-95]
+Tez vertex scope-108
# Plan on vertex
-POValueOutputTez - scope-109 -> [scope-110]
+POValueOutputTez - scope-117 -> [scope-118]
|
-|---New For Each(false)[tuple] - scope-108
+|---New For Each(false)[tuple] - scope-116
| |
- | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-107
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-115
| |
- | |---Project[tuple][*] - scope-106
+ | |---Project[tuple][*] - scope-114
|
- |---New For Each(false,false)[tuple] - scope-105
+ |---New For Each(false,false)[tuple] - scope-113
| |
- | Constant(1) - scope-104
+ | Constant(-1) - scope-112
| |
- | Project[bag][1] - scope-102
+ | Project[bag][1] - scope-110
|
- |---Package(Packager)[tuple]{bytearray} - scope-101
-Tez vertex scope-110
+ |---Package(Packager)[tuple]{bytearray} - scope-109
+Tez vertex scope-118
# Plan on vertex
-POIdentityInOutTez - scope-111 <- scope-90 -> scope-112
+POIdentityInOutTez - scope-119 <- scope-96 -> scope-120
| |
-| Project[int][0] - scope-85
-Tez vertex scope-112
+| Project[int][0] - scope-91
+Tez vertex scope-120
# Plan on vertex
-d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-87
+d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-93
|
-|---New For Each(true)[tuple] - scope-115
+|---New For Each(true)[tuple] - scope-123
| |
- | Project[bag][1] - scope-114
+ | Project[bag][1] - scope-122
|
- |---Package(LitePackager)[tuple]{int} - scope-113
+ |---Package(LitePackager)[tuple]{int} - scope-121
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld Tue Jun 17 18:15:26 2014
@@ -2,29 +2,33 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-67
+# TEZ DAG plan: scope-73
#--------------------------------------------------
-Tez vertex scope-20 -> Tez vertex scope-49,Tez vertex scope-50,
-Tez vertex scope-21 -> Tez vertex scope-49,Tez vertex scope-50,
-Tez vertex scope-50 -> Tez vertex scope-32,
-Tez vertex scope-32 -> Tez vertex scope-42,
-Tez vertex scope-49 -> Tez vertex scope-42,
-Tez vertex scope-42 -> Tez vertex scope-44,
-Tez vertex scope-44
+Tez vertex scope-20 -> Tez vertex scope-51,Tez vertex scope-52,
+Tez vertex scope-21 -> Tez vertex scope-51,Tez vertex scope-52,
+Tez vertex scope-52 -> Tez vertex scope-34,
+Tez vertex scope-34 -> Tez vertex scope-44,
+Tez vertex scope-51 -> Tez vertex scope-44,
+Tez vertex scope-44 -> Tez vertex scope-46,
+Tez vertex scope-46
Tez vertex scope-20
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-54 -> scope-32
+Local Rearrange[tuple]{tuple}(false) - scope-54 -> scope-34
| |
| Constant(DummyVal) - scope-55
|
-|---ReservoirSample - scope-53
+|---New For Each(true,true)[tuple] - scope-62
+ | |
+ | Project[int][0] - scope-59
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-61
+ | |
+ | |---Project[tuple][*] - scope-60
|
- |---New For Each(false)[tuple] - scope-52
- | |
- | Project[int][0] - scope-51
+ |---ReservoirSample - scope-53
|
- |---d: Local Rearrange[tuple]{int}(false) - scope-57 -> scope-42
+ |---d: Local Rearrange[tuple]{int}(false) - scope-57 -> scope-44
| |
| Project[int][0] - scope-58
|
@@ -41,19 +45,23 @@ Local Rearrange[tuple]{tuple}(false) - s
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-21
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-62 -> scope-32
+Local Rearrange[tuple]{tuple}(false) - scope-64 -> scope-34
| |
-| Constant(DummyVal) - scope-63
+| Constant(DummyVal) - scope-65
|
-|---ReservoirSample - scope-61
+|---New For Each(true,true)[tuple] - scope-72
+ | |
+ | Project[int][0] - scope-69
+ | |
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-71
+ | |
+ | |---Project[tuple][*] - scope-70
|
- |---New For Each(false)[tuple] - scope-60
- | |
- | Project[int][0] - scope-59
+ |---ReservoirSample - scope-63
|
- |---d: Local Rearrange[tuple]{int}(false) - scope-65 -> scope-42
+ |---d: Local Rearrange[tuple]{int}(false) - scope-67 -> scope-44
| |
- | Project[int][0] - scope-66
+ | Project[int][0] - scope-68
|
|---c: New For Each(false,false)[bag] - scope-15
| |
@@ -66,38 +74,38 @@ Local Rearrange[tuple]{tuple}(false) - s
| |---Project[bytearray][0] - scope-12
|
|---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
-Tez vertex group scope-50 <- [scope-20, scope-21] -> scope-32
+Tez vertex group scope-52 <- [scope-20, scope-21] -> scope-34
# No plan on vertex group
-Tez vertex scope-32
+Tez vertex scope-34
# Plan on vertex
-POValueOutputTez - scope-41 -> [scope-42]
+POValueOutputTez - scope-43 -> [scope-44]
|
-|---New For Each(false)[tuple] - scope-40
+|---New For Each(false)[tuple] - scope-42
| |
- | POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-39
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-41
| |
- | |---Project[tuple][*] - scope-38
+ | |---Project[tuple][*] - scope-40
|
- |---New For Each(false,false)[tuple] - scope-37
+ |---New For Each(false,false)[tuple] - scope-39
| |
- | Constant(1) - scope-36
+ | Constant(-1) - scope-38
| |
- | Project[bag][1] - scope-34
+ | Project[bag][1] - scope-36
|
- |---Package(Packager)[tuple]{bytearray} - scope-33
-Tez vertex group scope-49 <- [scope-20, scope-21] -> scope-42
+ |---Package(Packager)[tuple]{bytearray} - scope-35
+Tez vertex group scope-51 <- [scope-20, scope-21] -> scope-44
# No plan on vertex group
-Tez vertex scope-42
+Tez vertex scope-44
# Plan on vertex
-POIdentityInOutTez - scope-43 <- scope-49 -> scope-44
+POIdentityInOutTez - scope-45 <- scope-51 -> scope-46
| |
| Project[int][0] - scope-17
-Tez vertex scope-44
+Tez vertex scope-46
# Plan on vertex
d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-19
|
-|---New For Each(true)[tuple] - scope-47
+|---New For Each(true)[tuple] - scope-49
| |
- | Project[bag][1] - scope-46
+ | Project[bag][1] - scope-48
|
- |---Package(LitePackager)[tuple]{int} - scope-45
+ |---Package(LitePackager)[tuple]{int} - scope-47
Added: pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java?rev=1603243&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java (added)
+++ pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java Tue Jun 17 18:15:26 2014
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.tez;
+
+import static org.junit.Assert.assertEquals;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.pig.PigConfiguration;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.InputSizeReducerEstimator;
+import org.apache.pig.test.MiniGenericCluster;
+import org.apache.pig.test.Util;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestTezAutoParallelism {
+ private static final String INPUT_FILE1 = TestTezAutoParallelism.class.getName() + "_1";
+ private static final String INPUT_FILE2 = TestTezAutoParallelism.class.getName() + "_2";
+ private static final String INPUT_DIR = "build/test/data";
+
+ private static PigServer pigServer;
+ private static Properties properties;
+ private static MiniGenericCluster cluster;
+
+ @BeforeClass
+ public static void oneTimeSetUp() throws Exception {
+ cluster = MiniGenericCluster.buildCluster(MiniGenericCluster.EXECTYPE_TEZ);
+ properties = cluster.getProperties();
+ createFiles();
+ }
+
+ @AfterClass
+ public static void oneTimeTearDown() throws Exception {
+ deleteFiles();
+ cluster.shutDown();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ pigServer = new PigServer(MiniGenericCluster.EXECTYPE_TEZ, properties);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ pigServer.shutdown();
+ pigServer = null;
+ }
+
+ private static void createFiles() throws IOException {
+ new File(INPUT_DIR).mkdir();
+
+ PrintWriter w = new PrintWriter(new FileWriter(INPUT_DIR + "/" + INPUT_FILE1));
+
+ String boyNames[] = {"Noah", "Liam", "Jacob", "Mason", "William",
+ "Ethan", "Michael", "Alexander", "Jayden", "Daniel"};
+ String girlNames[] = {"Sophia", "Emma", "Olivia", "Isabella", "Ava",
+ "Mia", "Emily", "Abigail", "Madison", "Elizabeth"};
+
+ String names[] = new String[boyNames.length + girlNames.length];
+ for (int i=0;i<boyNames.length;i++) {
+ names[i] = boyNames[i];
+ }
+ for (int i=0;i<girlNames.length;i++) {
+ names[boyNames.length+i] = girlNames[i];
+ }
+
+ Random rand = new Random(1);
+ for (int i=0;i<1000;i++) {
+ w.println(names[rand.nextInt(names.length)] + "\t" + rand.nextInt(18));
+ }
+ w.close();
+ Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE1, INPUT_FILE1);
+
+ w = new PrintWriter(new FileWriter(INPUT_DIR + "/" + INPUT_FILE2));
+ for (String name : boyNames) {
+ w.println(name + "\t" + "M");
+ }
+ for (String name : girlNames) {
+ w.println(name + "\t" + "F");
+ }
+ w.close();
+ Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE2, INPUT_FILE2);
+ }
+
+ private static void deleteFiles() {
+ Util.deleteDirectory(new File(INPUT_DIR));
+ }
+
+ @Test
+ public void testGroupBy() throws IOException{
+ // parallelism is 3 originally, reduce to 1
+ pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+ pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "3000");
+ pigServer.getPigContext().getProperties().setProperty(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM,
+ Long.toString(InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER));
+ pigServer.registerQuery("A = load '" + INPUT_FILE1 + "' as (name:chararray, age:int);");
+ pigServer.registerQuery("B = group A by name;");
+ pigServer.store("B", "output1");
+ FileSystem fs = cluster.getFileSystem();
+ FileStatus[] files = fs.listStatus(new Path("output1"), new PathFilter(){
+ public boolean accept(Path path) {
+ if (path.getName().startsWith("part")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ assertEquals(files.length, 1);
+ }
+
+ @Test
+ public void testOrderbyDecreaseParallelism() throws IOException{
+ // order by parallelism is 3 originally, reduce to 1
+ pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+ pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "3000");
+ pigServer.getPigContext().getProperties().setProperty(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM,
+ Long.toString(InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER));
+ pigServer.registerQuery("A = load '" + INPUT_FILE1 + "' as (name:chararray, age:int);");
+ pigServer.registerQuery("B = group A by name parallel 3;");
+ pigServer.registerQuery("C = foreach B generate group as name, AVG(A.age) as age;");
+ pigServer.registerQuery("D = order C by age;");
+ pigServer.store("D", "output2");
+ FileSystem fs = cluster.getFileSystem();
+ FileStatus[] files = fs.listStatus(new Path("output2"), new PathFilter(){
+ public boolean accept(Path path) {
+ if (path.getName().startsWith("part")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ assertEquals(files.length, 1);
+ }
+
+ @Test
+ public void testOrderbyIncreaseParallelism() throws IOException{
+ // order by parallelism is 3 originally, increase to 4
+ pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+ pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "3000");
+ pigServer.getPigContext().getProperties().setProperty(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM, "1000");
+ pigServer.registerQuery("A = load '" + INPUT_FILE1 + "' as (name:chararray, age:int);");
+ pigServer.registerQuery("B = group A by name parallel 3;");
+ pigServer.registerQuery("C = foreach B generate group as name, AVG(A.age) as age;");
+ pigServer.registerQuery("D = order C by age;");
+ pigServer.store("D", "output3");
+ FileSystem fs = cluster.getFileSystem();
+ FileStatus[] files = fs.listStatus(new Path("output3"), new PathFilter(){
+ public boolean accept(Path path) {
+ if (path.getName().startsWith("part")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ assertEquals(files.length, 4);
+ }
+
+ @Test
+ public void testSkewedJoinDecreaseParallelism() throws IOException{
+ // skewed join parallelism is 4 originally, reduce to 1
+ pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+ pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "3000");
+ pigServer.getPigContext().getProperties().setProperty(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM,
+ Long.toString(InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER));
+ pigServer.registerQuery("A = load '" + INPUT_FILE1 + "' as (name:chararray, age:int);");
+ pigServer.registerQuery("B = load '" + INPUT_FILE2 + "' as (name:chararray, gender:chararray);");
+ pigServer.registerQuery("C = join A by name, B by name using 'skewed';");
+ pigServer.store("C", "output4");
+ FileSystem fs = cluster.getFileSystem();
+ FileStatus[] files = fs.listStatus(new Path("output4"), new PathFilter(){
+ public boolean accept(Path path) {
+ if (path.getName().startsWith("part")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ assertEquals(files.length, 1);
+ }
+
+ @Test
+ public void testSkewedJoinIncreaseParallelism() throws IOException{
+ // skewed join parallelism is 3 originally, increase to 5
+ pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+ pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "3000");
+ pigServer.getPigContext().getProperties().setProperty(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM, "80000");
+ pigServer.registerQuery("A = load '" + INPUT_FILE1 + "' as (name:chararray, age:int);");
+ pigServer.registerQuery("B = load '" + INPUT_FILE2 + "' as (name:chararray, gender:chararray);");
+ pigServer.registerQuery("C = join A by name, B by name using 'skewed';");
+ pigServer.store("C", "output5");
+ FileSystem fs = cluster.getFileSystem();
+ FileStatus[] files = fs.listStatus(new Path("output5"), new PathFilter(){
+ public boolean accept(Path path) {
+ if (path.getName().startsWith("part")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ assertEquals(files.length, 5);
+ }
+}
Modified: pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java Tue Jun 17 18:15:26 2014
@@ -21,21 +21,33 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import java.io.File;
+import java.io.IOException;
import java.net.URI;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.MultiQueryOptimizerTez;
import org.apache.pig.backend.hadoop.executionengine.tez.TezCompiler;
import org.apache.pig.backend.hadoop.executionengine.tez.TezJobControlCompiler;
import org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType;
import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
+import org.apache.pig.builtin.PigStorage;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.util.Pair;
import org.apache.pig.test.Util;
@@ -56,7 +68,12 @@ import org.junit.runner.RunWith;
@TestOrder({
"testRun1",
"testRun2",
- "testRun3"
+ "testRun3",
+ "testTezParallelismEstimatorOrderBy",
+ "testTezParallelismEstimatorFilterFlatten",
+ "testTezParallelismEstimatorHashJoin",
+ "testTezParallelismEstimatorSplitBranch",
+ "testTezParallelismDefaultParallelism"
})
public class TestTezJobControlCompiler {
private static PigContext pc;
@@ -66,7 +83,6 @@ public class TestTezJobControlCompiler {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- pc = new PigContext(new TezLocalExecType(), new Properties());
input1 = Util.createTempFileDelOnExit("input1", "txt").toURI();
input2 = Util.createTempFileDelOnExit("input2", "txt").toURI();
}
@@ -77,6 +93,7 @@ public class TestTezJobControlCompiler {
@Before
public void setUp() throws ExecException {
+ pc = new PigContext(new TezLocalExecType(), new Properties());
pigServer = new PigServer(pc);
}
@@ -167,11 +184,112 @@ public class TestTezJobControlCompiler {
assertTrue(leaf.getInputVertices().contains(root1));
}
+ static public class ArbitarySplitsInputformat extends TextInputFormat {
+ @Override
+ public List<InputSplit> getSplits(JobContext job) throws IOException {
+ String inputDir = job.getConfiguration().get(INPUT_DIR, "");
+ String numSplitString = inputDir.substring(inputDir.lastIndexOf(File.separator)+1);
+ int numSplit = Integer.parseInt(numSplitString);
+ List<InputSplit> splits = new ArrayList<InputSplit>();
+ for (int i=0;i<numSplit;i++) {
+ splits.add(new FileSplit(new Path("dummy"), 0, 0, null));
+ }
+ return splits;
+ }
+ }
+
+ static public class ArbitarySplitsLoader extends PigStorage {
+ public ArbitarySplitsLoader() {}
+
+ @Override
+ public InputFormat getInputFormat() {
+ return new ArbitarySplitsInputformat();
+ }
+ }
+
+ @Test
+ public void testTezParallelismEstimatorOrderBy() throws Exception{
+ pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+ String query = "a = load '2' using " + ArbitarySplitsLoader.class.getName()
+ + "() as (name:chararray, age:int, gpa:double);"
+ + "b = group a by name parallel 3;"
+ + "c = foreach b generate group as name, AVG(a.age) as age;"
+ + "d = order c by age;"
+ + "store d into 'output';";
+ Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+ TezOperator sortOper = compiledPlan.first.getLeaves().get(0);
+ Vertex sortVertex = compiledPlan.second.getVertex(sortOper.getOperatorKey().toString());
+ assertEquals(sortVertex.getParallelism(), -1);
+ }
+
+ @Test
+ public void testTezParallelismEstimatorFilterFlatten() throws Exception{
+ pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+ String query = "a = load '10' using " + ArbitarySplitsLoader.class.getName()
+ + "() as (name:chararray, age:int, gpa:double);"
+ + "b = filter a by age>20;"
+ + "c = group b by name;"
+ + "d = foreach c generate group, flatten(b.gpa);"
+ + "e = group d by group;"
+ + "store e into 'output';";
+ Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+ TezOperator leafOper = compiledPlan.first.getLeaves().get(0);
+ Vertex leafVertex = compiledPlan.second.getVertex(leafOper.getOperatorKey().toString());
+ assertEquals(leafVertex.getParallelism(), 70);
+ }
+
+ @Test
+ public void testTezParallelismEstimatorHashJoin() throws Exception{
+ pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+ String query = "a = load '10' using " + ArbitarySplitsLoader.class.getName()
+ + "() as (name:chararray, age:int, gpa:double);"
+ + "b = load '5' using " + ArbitarySplitsLoader.class.getName()
+ + "() as (name:chararray, course:chararray);"
+ + "c = join a by name, b by name;"
+ + "store c into 'output';";
+ Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+ TezOperator leafOper = compiledPlan.first.getLeaves().get(0);
+ Vertex leafVertex = compiledPlan.second.getVertex(leafOper.getOperatorKey().toString());
+ assertEquals(leafVertex.getParallelism(), 15);
+ }
+
+ @Test
+ public void testTezParallelismEstimatorSplitBranch() throws Exception{
+ pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+ String query = "a = load '10' using " + ArbitarySplitsLoader.class.getName()
+ + "() as (name:chararray, age:int, gpa:double);"
+ + "b = filter a by age>20;"
+ + "c = filter a by age>50;"
+ + "store b into 'o1';"
+ + "d = group c by name;"
+ + "store d into 'o2';";
+ Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+ TezOperator leafOper = compiledPlan.first.getLeaves().get(0);
+ Vertex leafVertex = compiledPlan.second.getVertex(leafOper.getOperatorKey().toString());
+ assertEquals(leafVertex.getParallelism(), 7);
+ }
+
+ @Test
+ public void testTezParallelismDefaultParallelism() throws Exception{
+ pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+ pc.defaultParallel = 5;
+ String query = "a = load '10' using " + ArbitarySplitsLoader.class.getName()
+ + "() as (name:chararray, age:int, gpa:double);"
+ + "b = group a by name;"
+ + "store b into 'output';";
+ Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+ TezOperator leafOper = compiledPlan.first.getLeaves().get(0);
+ Vertex leafVertex = compiledPlan.second.getVertex(leafOper.getOperatorKey().toString());
+ assertEquals(leafVertex.getParallelism(), 5);
+ }
+
private Pair<TezOperPlan, DAG> compile(String query) throws Exception {
PhysicalPlan pp = Util.buildPp(pigServer, query);
TezCompiler comp = new TezCompiler(pp, pc);
TezOperPlan tezPlan = comp.compile();
TezJobControlCompiler jobComp = new TezJobControlCompiler(pc, new Configuration());
+ MultiQueryOptimizerTez mqOptimizer = new MultiQueryOptimizerTez(tezPlan);
+ mqOptimizer.visit();
DAG dag = jobComp.buildDAG(tezPlan, new HashMap<String, LocalResource>());
return new Pair<TezOperPlan, DAG>(tezPlan, dag);
}
Modified: pig/trunk/test/tez-tests
URL: http://svn.apache.org/viewvc/pig/trunk/test/tez-tests?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/tez-tests (original)
+++ pig/trunk/test/tez-tests Tue Jun 17 18:15:26 2014
@@ -46,4 +46,5 @@
**/TestSecondarySortTez.java
**/TestTezCompiler.java
**/TestTezJobControlCompiler.java
-**/TestTezLauncher.java
\ No newline at end of file
+**/TestTezLauncher.java
+**/TestTezAutoParallelism.java