You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/12/24 03:55:42 UTC
hive git commit: HIVE-12735: Constant folding for WHEN/CASE
expression does not set return type correctly (Prasanth Jayachandran reviewed
by Ashutosh Chauhan, Gopal V)
Repository: hive
Updated Branches:
refs/heads/branch-2.0 c51e1ea02 -> 1b1892f83
HIVE-12735: Constant folding for WHEN/CASE expression does not set return type correctly (Prasanth Jayachandran reviewed by Ashutosh Chauhan, Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1b1892f8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1b1892f8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1b1892f8
Branch: refs/heads/branch-2.0
Commit: 1b1892f83bfdb84a3f39199ba3c0c8813a379bf5
Parents: c51e1ea
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Wed Dec 23 20:46:01 2015 -0600
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Wed Dec 23 20:55:27 2015 -0600
----------------------------------------------------------------------
.../optimizer/ConstantPropagateProcFactory.java | 12 +-
.../clientpositive/constprog_when_case.q | 12 +
.../clientpositive/constprog_when_case.q.out | 1074 ++++++++++++++++++
.../clientpositive/fold_eq_with_case_when.q.out | 4 +-
4 files changed, 1098 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1b1892f8/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
index b18b5af..d182d80 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
@@ -568,7 +568,11 @@ public final class ConstantPropagateProcFactory {
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
}
- return caseOrWhenexpr;
+ // after constant folding of child expression the return type of UDFWhen might have changed,
+ // so recreate the expression
+ ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF,
+ caseOrWhenexpr.getFuncText(), children);
+ return newCaseOrWhenExpr;
} else if (childUDF instanceof GenericUDFCase) {
for (i = 2; i < children.size(); i+=2) {
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
@@ -579,7 +583,11 @@ public final class ConstantPropagateProcFactory {
children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
}
- return caseOrWhenexpr;
+ // after constant folding of child expression the return type of UDFCase might have changed,
+ // so recreate the expression
+ ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF,
+ caseOrWhenexpr.getFuncText(), children);
+ return newCaseOrWhenExpr;
} else {
// cant happen
return null;
http://git-wip-us.apache.org/repos/asf/hive/blob/1b1892f8/ql/src/test/queries/clientpositive/constprog_when_case.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/constprog_when_case.q b/ql/src/test/queries/clientpositive/constprog_when_case.q
new file mode 100644
index 0000000..b2a10f3
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/constprog_when_case.q
@@ -0,0 +1,12 @@
+set hive.fetch.task.conversion=none;
+set hive.explain.user=false;
+set hive.vectorized.execution.enabled=true;
+
+create table src_orc (bool0 boolean, key0 string, key1 string, key2 string) stored as orc;
+insert overwrite table src_orc select true, key, value, key from src;
+insert into table src_orc select true, key, value, key from src;
+
+explain SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN TRUE THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2 ) ) FROM src_orc;
+
+SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN TRUE THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2 ) ) FROM src_orc;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/1b1892f8/ql/src/test/results/clientpositive/constprog_when_case.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constprog_when_case.q.out b/ql/src/test/results/clientpositive/constprog_when_case.q.out
new file mode 100644
index 0000000..7f8a504
--- /dev/null
+++ b/ql/src/test/results/clientpositive/constprog_when_case.q.out
@@ -0,0 +1,1074 @@
+PREHOOK: query: create table src_orc (bool0 boolean, key0 string, key1 string, key2 string) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_orc
+POSTHOOK: query: create table src_orc (bool0 boolean, key0 string, key1 string, key2 string) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_orc
+PREHOOK: query: insert overwrite table src_orc select true, key, value, key from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@src_orc
+POSTHOOK: query: insert overwrite table src_orc select true, key, value, key from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@src_orc
+POSTHOOK: Lineage: src_orc.bool0 SIMPLE []
+POSTHOOK: Lineage: src_orc.key0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_orc.key1 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_orc.key2 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: insert into table src_orc select true, key, value, key from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@src_orc
+POSTHOOK: query: insert into table src_orc select true, key, value, key from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@src_orc
+POSTHOOK: Lineage: src_orc.bool0 SIMPLE []
+POSTHOOK: Lineage: src_orc.key0 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_orc.key1 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_orc.key2 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+PREHOOK: query: explain SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN TRUE THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2 ) ) FROM src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: explain SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN TRUE THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2 ) ) FROM src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src_orc
+ Statistics: Num rows: 1000 Data size: 266000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: if(CASE WHEN (bool0) THEN (true) WHEN ((not bool0)) THEN (false) END, key0, if(CASE WHEN (bool0) THEN (false) WHEN ((not bool0)) THEN (true) END, key1, key2)) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1000 Data size: 266000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1000 Data size: 266000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN TRUE THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2 ) ) FROM src_orc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_orc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN TRUE THEN 1 WHEN NOT TRUE THEN 0 END) ), key0, IF ( ( (CASE WHEN bool0 THEN 1 WHEN NOT bool0 THEN 0 END) = (CASE WHEN FALSE THEN 1 WHEN NOT FALSE THEN 0 END) ), key1, key2 ) ) FROM src_orc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_orc
+#### A masked pattern was here ####
+238
+86
+311
+27
+165
+409
+255
+278
+98
+484
+265
+193
+401
+150
+273
+224
+369
+66
+128
+213
+146
+406
+429
+374
+152
+469
+145
+495
+37
+327
+281
+277
+209
+15
+82
+403
+166
+417
+430
+252
+292
+219
+287
+153
+193
+338
+446
+459
+394
+237
+482
+174
+413
+494
+207
+199
+466
+208
+174
+399
+396
+247
+417
+489
+162
+377
+397
+309
+365
+266
+439
+342
+367
+325
+167
+195
+475
+17
+113
+155
+203
+339
+0
+455
+128
+311
+316
+57
+302
+205
+149
+438
+345
+129
+170
+20
+489
+157
+378
+221
+92
+111
+47
+72
+4
+280
+35
+427
+277
+208
+356
+399
+169
+382
+498
+125
+386
+437
+469
+192
+286
+187
+176
+54
+459
+51
+138
+103
+239
+213
+216
+430
+278
+176
+289
+221
+65
+318
+332
+311
+275
+137
+241
+83
+333
+180
+284
+12
+230
+181
+67
+260
+404
+384
+489
+353
+373
+272
+138
+217
+84
+348
+466
+58
+8
+411
+230
+208
+348
+24
+463
+431
+179
+172
+42
+129
+158
+119
+496
+0
+322
+197
+468
+393
+454
+100
+298
+199
+191
+418
+96
+26
+165
+327
+230
+205
+120
+131
+51
+404
+43
+436
+156
+469
+468
+308
+95
+196
+288
+481
+457
+98
+282
+197
+187
+318
+318
+409
+470
+137
+369
+316
+169
+413
+85
+77
+0
+490
+87
+364
+179
+118
+134
+395
+282
+138
+238
+419
+15
+118
+72
+90
+307
+19
+435
+10
+277
+273
+306
+224
+309
+389
+327
+242
+369
+392
+272
+331
+401
+242
+452
+177
+226
+5
+497
+402
+396
+317
+395
+58
+35
+336
+95
+11
+168
+34
+229
+233
+143
+472
+322
+498
+160
+195
+42
+321
+430
+119
+489
+458
+78
+76
+41
+223
+492
+149
+449
+218
+228
+138
+453
+30
+209
+64
+468
+76
+74
+342
+69
+230
+33
+368
+103
+296
+113
+216
+367
+344
+167
+274
+219
+239
+485
+116
+223
+256
+263
+70
+487
+480
+401
+288
+191
+5
+244
+438
+128
+467
+432
+202
+316
+229
+469
+463
+280
+2
+35
+283
+331
+235
+80
+44
+193
+321
+335
+104
+466
+366
+175
+403
+483
+53
+105
+257
+406
+409
+190
+406
+401
+114
+258
+90
+203
+262
+348
+424
+12
+396
+201
+217
+164
+431
+454
+478
+298
+125
+431
+164
+424
+187
+382
+5
+70
+397
+480
+291
+24
+351
+255
+104
+70
+163
+438
+119
+414
+200
+491
+237
+439
+360
+248
+479
+305
+417
+199
+444
+120
+429
+169
+443
+323
+325
+277
+230
+478
+178
+468
+310
+317
+333
+493
+460
+207
+249
+265
+480
+83
+136
+353
+172
+214
+462
+233
+406
+133
+175
+189
+454
+375
+401
+421
+407
+384
+256
+26
+134
+67
+384
+379
+18
+462
+492
+100
+298
+9
+341
+498
+146
+458
+362
+186
+285
+348
+167
+18
+273
+183
+281
+344
+97
+469
+315
+84
+28
+37
+448
+152
+348
+307
+194
+414
+477
+222
+126
+90
+169
+403
+400
+200
+97
+238
+86
+311
+27
+165
+409
+255
+278
+98
+484
+265
+193
+401
+150
+273
+224
+369
+66
+128
+213
+146
+406
+429
+374
+152
+469
+145
+495
+37
+327
+281
+277
+209
+15
+82
+403
+166
+417
+430
+252
+292
+219
+287
+153
+193
+338
+446
+459
+394
+237
+482
+174
+413
+494
+207
+199
+466
+208
+174
+399
+396
+247
+417
+489
+162
+377
+397
+309
+365
+266
+439
+342
+367
+325
+167
+195
+475
+17
+113
+155
+203
+339
+0
+455
+128
+311
+316
+57
+302
+205
+149
+438
+345
+129
+170
+20
+489
+157
+378
+221
+92
+111
+47
+72
+4
+280
+35
+427
+277
+208
+356
+399
+169
+382
+498
+125
+386
+437
+469
+192
+286
+187
+176
+54
+459
+51
+138
+103
+239
+213
+216
+430
+278
+176
+289
+221
+65
+318
+332
+311
+275
+137
+241
+83
+333
+180
+284
+12
+230
+181
+67
+260
+404
+384
+489
+353
+373
+272
+138
+217
+84
+348
+466
+58
+8
+411
+230
+208
+348
+24
+463
+431
+179
+172
+42
+129
+158
+119
+496
+0
+322
+197
+468
+393
+454
+100
+298
+199
+191
+418
+96
+26
+165
+327
+230
+205
+120
+131
+51
+404
+43
+436
+156
+469
+468
+308
+95
+196
+288
+481
+457
+98
+282
+197
+187
+318
+318
+409
+470
+137
+369
+316
+169
+413
+85
+77
+0
+490
+87
+364
+179
+118
+134
+395
+282
+138
+238
+419
+15
+118
+72
+90
+307
+19
+435
+10
+277
+273
+306
+224
+309
+389
+327
+242
+369
+392
+272
+331
+401
+242
+452
+177
+226
+5
+497
+402
+396
+317
+395
+58
+35
+336
+95
+11
+168
+34
+229
+233
+143
+472
+322
+498
+160
+195
+42
+321
+430
+119
+489
+458
+78
+76
+41
+223
+492
+149
+449
+218
+228
+138
+453
+30
+209
+64
+468
+76
+74
+342
+69
+230
+33
+368
+103
+296
+113
+216
+367
+344
+167
+274
+219
+239
+485
+116
+223
+256
+263
+70
+487
+480
+401
+288
+191
+5
+244
+438
+128
+467
+432
+202
+316
+229
+469
+463
+280
+2
+35
+283
+331
+235
+80
+44
+193
+321
+335
+104
+466
+366
+175
+403
+483
+53
+105
+257
+406
+409
+190
+406
+401
+114
+258
+90
+203
+262
+348
+424
+12
+396
+201
+217
+164
+431
+454
+478
+298
+125
+431
+164
+424
+187
+382
+5
+70
+397
+480
+291
+24
+351
+255
+104
+70
+163
+438
+119
+414
+200
+491
+237
+439
+360
+248
+479
+305
+417
+199
+444
+120
+429
+169
+443
+323
+325
+277
+230
+478
+178
+468
+310
+317
+333
+493
+460
+207
+249
+265
+480
+83
+136
+353
+172
+214
+462
+233
+406
+133
+175
+189
+454
+375
+401
+421
+407
+384
+256
+26
+134
+67
+384
+379
+18
+462
+492
+100
+298
+9
+341
+498
+146
+458
+362
+186
+285
+348
+167
+18
+273
+183
+281
+344
+97
+469
+315
+84
+28
+37
+448
+152
+348
+307
+194
+414
+477
+222
+126
+90
+169
+403
+400
+200
+97
http://git-wip-us.apache.org/repos/asf/hive/blob/1b1892f8/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out b/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
index 231d98f..69dd050 100644
--- a/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
+++ b/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
@@ -137,7 +137,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: CASE (key) WHEN ('238') THEN (true) WHEN ('94') THEN (true) ELSE (false) END (type: int)
+ predicate: CASE (key) WHEN ('238') THEN (true) WHEN ('94') THEN (true) ELSE (false) END (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
@@ -209,7 +209,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: CASE (key) WHEN ('238') THEN (CASE WHEN ((key <> '238')) THEN (true) WHEN ((key = '23')) THEN (true) END) END (type: int)
+ predicate: CASE (key) WHEN ('238') THEN (CASE WHEN ((key <> '238')) THEN (true) WHEN ((key = '23')) THEN (true) END) END (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)