You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2014/12/16 20:22:36 UTC
svn commit: r1646050 - in /hive/trunk: itests/src/test/resources/
ql/src/java/org/apache/hadoop/hive/ql/parse/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/tez/
Author: vikram
Date: Tue Dec 16 19:22:35 2014
New Revision: 1646050
URL: http://svn.apache.org/r1646050
Log:
HIVE-9055: Tez: union all followed by group by followed by another union all gives error (Vikram Dixit K, reviewed by Prasanth J)
Added:
hive/trunk/ql/src/test/queries/clientpositive/tez_multi_union.q
hive/trunk/ql/src/test/results/clientpositive/tez/tez_multi_union.q.out
Modified:
hive/trunk/itests/src/test/resources/testconfiguration.properties
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1646050&r1=1646049&r2=1646050&view=diff
==============================================================================
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Tue Dec 16 19:22:35 2014
@@ -268,6 +268,7 @@ minitez.query.files.shared=alter_merge_2
auto_sortmerge_join_8.q,\
auto_sortmerge_join_9.q
+
minitez.query.files=bucket_map_join_tez1.q,\
bucket_map_join_tez2.q,\
dynamic_partition_pruning.q,\
@@ -288,7 +289,8 @@ minitez.query.files=bucket_map_join_tez1
tez_union_group_by.q,\
tez_smb_main.q,\
tez_smb_1.q,\
- vectorized_dynamic_partition_pruning.q
+ vectorized_dynamic_partition_pruning.q,\
+ tez_multi_union.q
beeline.positive.exclude=add_part_exist.q,\
alter1.q,\
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java?rev=1646050&r1=1646049&r2=1646050&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java Tue Dec 16 19:22:35 2014
@@ -111,7 +111,7 @@ public class GenTezWork implements NodeP
// will result into a vertex with multiple FS or RS operators.
if (context.childToWorkMap.containsKey(operator)) {
// if we've seen both root and child, we can bail.
-
+
// clear out the mapjoin set. we don't need it anymore.
context.currentMapJoinOperators.clear();
@@ -349,17 +349,20 @@ public class GenTezWork implements NodeP
} else if (followingWork instanceof UnionWork) {
// this can only be possible if there is merge work followed by the union
UnionWork unionWork = (UnionWork) followingWork;
- int index = getMergeIndex(tezWork, unionWork, rs);
- // guaranteed to be instance of MergeJoinWork if index is valid
- BaseWork baseWork = tezWork.getChildren(unionWork).get(index);
- if (baseWork instanceof MergeJoinWork) {
- MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
- // disconnect the connection to union work and connect to merge work
- followingWork = mergeJoinWork;
- rWork = (ReduceWork) mergeJoinWork.getMainWork();
+ int index = getFollowingWorkIndex(tezWork, unionWork, rs);
+ if (index != -1) {
+ BaseWork baseWork = tezWork.getChildren(unionWork).get(index);
+ if (baseWork instanceof MergeJoinWork) {
+ MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
+ // disconnect the connection to union work and connect to merge work
+ followingWork = mergeJoinWork;
+ rWork = (ReduceWork) mergeJoinWork.getMainWork();
+ } else {
+ rWork = (ReduceWork) baseWork;
+ }
} else {
- throw new SemanticException("Unknown work type found: "
- + baseWork.getClass().getCanonicalName());
+ throw new SemanticException("Following work not found for the reduce sink: "
+ + rs.getName());
}
} else {
rWork = (ReduceWork) followingWork;
@@ -403,19 +406,13 @@ public class GenTezWork implements NodeP
return null;
}
- private int getMergeIndex(TezWork tezWork, UnionWork unionWork, ReduceSinkOperator rs) {
+ private int getFollowingWorkIndex(TezWork tezWork, UnionWork unionWork, ReduceSinkOperator rs) {
int index = 0;
for (BaseWork baseWork : tezWork.getChildren(unionWork)) {
- if (baseWork instanceof MergeJoinWork) {
- MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
- int tag = mergeJoinWork.getMergeJoinOperator().getTagForOperator(rs);
- if (tag != -1) {
- return index;
- } else {
- index++;
- }
- } else {
+ if (tezWork.getEdgeProperty(unionWork, baseWork).equals(TezEdgeProperty.EdgeType.CONTAINS)) {
index++;
+ } else {
+ return index;
}
}
Added: hive/trunk/ql/src/test/queries/clientpositive/tez_multi_union.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/tez_multi_union.q?rev=1646050&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/tez_multi_union.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/tez_multi_union.q Tue Dec 16 19:22:35 2014
@@ -0,0 +1,8 @@
+select key from
+(
+select key from src
+union all
+select key from src
+) tab group by key
+union all
+select key from src;
Added: hive/trunk/ql/src/test/results/clientpositive/tez/tez_multi_union.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/tez_multi_union.q.out?rev=1646050&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/tez_multi_union.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/tez_multi_union.q.out Tue Dec 16 19:22:35 2014
@@ -0,0 +1,831 @@
+PREHOOK: query: select key from
+(
+select key from src
+union all
+select key from src
+) tab group by key
+union all
+select key from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select key from
+(
+select key from src
+union all
+select key from src
+) tab group by key
+union all
+select key from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+0
+10
+100
+103
+104
+105
+11
+111
+113
+114
+116
+118
+119
+12
+120
+125
+126
+128
+129
+131
+133
+134
+136
+137
+138
+143
+145
+146
+149
+15
+150
+152
+153
+155
+156
+157
+158
+160
+162
+163
+164
+165
+166
+167
+168
+169
+17
+170
+172
+174
+175
+176
+177
+178
+179
+18
+180
+181
+183
+186
+187
+189
+19
+190
+191
+192
+193
+194
+195
+196
+197
+199
+2
+20
+200
+201
+202
+203
+205
+207
+208
+209
+213
+214
+216
+217
+218
+219
+221
+222
+223
+224
+226
+228
+229
+230
+233
+235
+237
+238
+239
+24
+241
+242
+244
+247
+248
+249
+252
+255
+256
+257
+258
+26
+260
+262
+263
+265
+266
+27
+272
+273
+274
+275
+277
+278
+28
+280
+281
+282
+283
+284
+285
+286
+287
+288
+289
+291
+292
+296
+298
+30
+302
+305
+306
+307
+308
+309
+310
+311
+315
+316
+317
+318
+321
+322
+323
+325
+327
+33
+331
+332
+333
+335
+336
+338
+339
+34
+341
+342
+344
+345
+348
+35
+351
+353
+356
+360
+362
+364
+365
+366
+367
+368
+369
+37
+373
+374
+375
+377
+378
+379
+382
+384
+386
+389
+392
+393
+394
+395
+396
+397
+399
+4
+400
+401
+402
+403
+404
+406
+407
+409
+41
+411
+413
+414
+417
+418
+419
+42
+421
+424
+427
+429
+43
+430
+431
+432
+435
+436
+437
+438
+439
+44
+443
+444
+446
+448
+449
+452
+453
+454
+455
+457
+458
+459
+460
+462
+463
+466
+467
+468
+469
+47
+470
+472
+475
+477
+478
+479
+480
+481
+482
+483
+484
+485
+487
+489
+490
+491
+492
+493
+494
+495
+496
+497
+498
+5
+51
+53
+54
+57
+58
+64
+65
+66
+67
+69
+70
+72
+74
+76
+77
+78
+8
+80
+82
+83
+84
+85
+86
+87
+9
+90
+92
+95
+96
+97
+98
+238
+86
+311
+27
+165
+409
+255
+278
+98
+484
+265
+193
+401
+150
+273
+224
+369
+66
+128
+213
+146
+406
+429
+374
+152
+469
+145
+495
+37
+327
+281
+277
+209
+15
+82
+403
+166
+417
+430
+252
+292
+219
+287
+153
+193
+338
+446
+459
+394
+237
+482
+174
+413
+494
+207
+199
+466
+208
+174
+399
+396
+247
+417
+489
+162
+377
+397
+309
+365
+266
+439
+342
+367
+325
+167
+195
+475
+17
+113
+155
+203
+339
+0
+455
+128
+311
+316
+57
+302
+205
+149
+438
+345
+129
+170
+20
+489
+157
+378
+221
+92
+111
+47
+72
+4
+280
+35
+427
+277
+208
+356
+399
+169
+382
+498
+125
+386
+437
+469
+192
+286
+187
+176
+54
+459
+51
+138
+103
+239
+213
+216
+430
+278
+176
+289
+221
+65
+318
+332
+311
+275
+137
+241
+83
+333
+180
+284
+12
+230
+181
+67
+260
+404
+384
+489
+353
+373
+272
+138
+217
+84
+348
+466
+58
+8
+411
+230
+208
+348
+24
+463
+431
+179
+172
+42
+129
+158
+119
+496
+0
+322
+197
+468
+393
+454
+100
+298
+199
+191
+418
+96
+26
+165
+327
+230
+205
+120
+131
+51
+404
+43
+436
+156
+469
+468
+308
+95
+196
+288
+481
+457
+98
+282
+197
+187
+318
+318
+409
+470
+137
+369
+316
+169
+413
+85
+77
+0
+490
+87
+364
+179
+118
+134
+395
+282
+138
+238
+419
+15
+118
+72
+90
+307
+19
+435
+10
+277
+273
+306
+224
+309
+389
+327
+242
+369
+392
+272
+331
+401
+242
+452
+177
+226
+5
+497
+402
+396
+317
+395
+58
+35
+336
+95
+11
+168
+34
+229
+233
+143
+472
+322
+498
+160
+195
+42
+321
+430
+119
+489
+458
+78
+76
+41
+223
+492
+149
+449
+218
+228
+138
+453
+30
+209
+64
+468
+76
+74
+342
+69
+230
+33
+368
+103
+296
+113
+216
+367
+344
+167
+274
+219
+239
+485
+116
+223
+256
+263
+70
+487
+480
+401
+288
+191
+5
+244
+438
+128
+467
+432
+202
+316
+229
+469
+463
+280
+2
+35
+283
+331
+235
+80
+44
+193
+321
+335
+104
+466
+366
+175
+403
+483
+53
+105
+257
+406
+409
+190
+406
+401
+114
+258
+90
+203
+262
+348
+424
+12
+396
+201
+217
+164
+431
+454
+478
+298
+125
+431
+164
+424
+187
+382
+5
+70
+397
+480
+291
+24
+351
+255
+104
+70
+163
+438
+119
+414
+200
+491
+237
+439
+360
+248
+479
+305
+417
+199
+444
+120
+429
+169
+443
+323
+325
+277
+230
+478
+178
+468
+310
+317
+333
+493
+460
+207
+249
+265
+480
+83
+136
+353
+172
+214
+462
+233
+406
+133
+175
+189
+454
+375
+401
+421
+407
+384
+256
+26
+134
+67
+384
+379
+18
+462
+492
+100
+298
+9
+341
+498
+146
+458
+362
+186
+285
+348
+167
+18
+273
+183
+281
+344
+97
+469
+315
+84
+28
+37
+448
+152
+348
+307
+194
+414
+477
+222
+126
+90
+169
+403
+400
+200
+97