You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dj...@apache.org on 2018/05/08 18:42:52 UTC
[47/58] [abbrv] hive git commit: HIVE-18910 : Migrate to Murmur hash
for shuffle and bucketing (Deepak Jaiswal, reviewed by Jasone Dere)
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/llap/bucket5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket5.q.out b/ql/src/test/results/clientpositive/llap/bucket5.q.out
index 6649344..953353a 100644
--- a/ql/src/test/results/clientpositive/llap/bucket5.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket5.q.out
@@ -93,6 +93,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments 'default','default'
@@ -115,6 +116,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments 'default','default'
@@ -158,6 +160,7 @@ STAGE PLANS:
SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -246,6 +249,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -330,6 +334,7 @@ STAGE PLANS:
SORTBUCKETCOLSPREFIX TRUE
bucket_count 2
bucket_field_name key
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -369,6 +374,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -426,6 +432,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -457,6 +464,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -479,6 +487,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -519,6 +528,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -550,6 +560,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -572,6 +583,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -637,6 +649,7 @@ Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}}
SORTBUCKETCOLSPREFIX TRUE
+ bucketing_version 2
numFiles 2
numRows 500
rawDataSize 5312
@@ -661,16 +674,16 @@ POSTHOOK: query: SELECT * FROM bucketed_table TABLESAMPLE (BUCKET 1 OUT OF 2) s
POSTHOOK: type: QUERY
POSTHOOK: Input: default@bucketed_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-0 val_0
-0 val_0
-0 val_0
2 val_2
-4 val_4
-8 val_8
10 val_10
-12 val_12
-12 val_12
-18 val_18
+17 val_17
+19 val_19
+20 val_20
+27 val_27
+30 val_30
+41 val_41
+43 val_43
+51 val_51
PREHOOK: query: SELECT * FROM bucketed_table TABLESAMPLE (BUCKET 2 OUT OF 2) s LIMIT 10
PREHOOK: type: QUERY
PREHOOK: Input: default@bucketed_table
@@ -679,16 +692,16 @@ POSTHOOK: query: SELECT * FROM bucketed_table TABLESAMPLE (BUCKET 2 OUT OF 2) s
POSTHOOK: type: QUERY
POSTHOOK: Input: default@bucketed_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
+0 val_0
+0 val_0
+0 val_0
+4 val_4
5 val_5
5 val_5
5 val_5
+8 val_8
9 val_9
11 val_11
-15 val_15
-15 val_15
-17 val_17
-19 val_19
-27 val_27
PREHOOK: query: SELECT COUNT(DISTINCT INPUT__FILE__NAME) FROM bucketed_table
PREHOOK: type: QUERY
PREHOOK: Input: default@bucketed_table
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/llap/bucket6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket6.q.out b/ql/src/test/results/clientpositive/llap/bucket6.q.out
index 1994aeb..fe74066 100644
--- a/ql/src/test/results/clientpositive/llap/bucket6.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket6.q.out
@@ -135,42 +135,18 @@ POSTHOOK: query: select * from src_bucket limit 100
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src_bucket
POSTHOOK: Output: hdfs://### HDFS PATH ###
-0 val_0
-0 val_0
-0 val_0
-0 val_0
-0 val_0
-0 val_0
-0 val_0
-0 val_0
-0 val_0
-0 val_0
-0 val_0
-0 val_0
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-103 val_103
-105 val_105
-105 val_105
-105 val_105
-105 val_105
-11 val_11
-11 val_11
-11 val_11
-11 val_11
-114 val_114
-114 val_114
-114 val_114
-114 val_114
-116 val_116
-116 val_116
-116 val_116
-116 val_116
+10 val_10
+10 val_10
+10 val_10
+10 val_10
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
118 val_118
118 val_118
118 val_118
@@ -179,34 +155,50 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
118 val_118
118 val_118
118 val_118
-125 val_125
-125 val_125
-125 val_125
-125 val_125
-125 val_125
-125 val_125
-125 val_125
-125 val_125
-129 val_129
-129 val_129
-129 val_129
-129 val_129
-129 val_129
-129 val_129
-129 val_129
-129 val_129
-134 val_134
-134 val_134
-134 val_134
-134 val_134
-134 val_134
-134 val_134
-134 val_134
-134 val_134
-136 val_136
-136 val_136
-136 val_136
-136 val_136
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+120 val_120
+120 val_120
+120 val_120
+120 val_120
+120 val_120
+120 val_120
+120 val_120
+120 val_120
+126 val_126
+126 val_126
+126 val_126
+126 val_126
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+131 val_131
+131 val_131
+131 val_131
+131 val_131
+133 val_133
+133 val_133
+133 val_133
+133 val_133
138 val_138
138 val_138
138 val_138
@@ -223,10 +215,6 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
138 val_138
138 val_138
138 val_138
-143 val_143
-143 val_143
-143 val_143
-143 val_143
145 val_145
145 val_145
145 val_145
@@ -235,3 +223,15 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
149 val_149
149 val_149
149 val_149
+149 val_149
+149 val_149
+149 val_149
+149 val_149
+15 val_15
+15 val_15
+15 val_15
+15 val_15
+15 val_15
+15 val_15
+15 val_15
+15 val_15
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
index 26024b0..3a7d744 100644
--- a/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket_groupby.q.out
@@ -347,8 +347,8 @@ POSTHOOK: Input: default@clustergroupby
POSTHOOK: Input: default@clustergroupby@ds=101
#### A masked pattern was here ####
1 10
-2 74
3 416
+2 74
PREHOOK: query: explain
select abs(length(key)), count(1) from clustergroupby where ds='101' group by abs(length(key)) limit 10
PREHOOK: type: QUERY
@@ -428,8 +428,8 @@ POSTHOOK: Input: default@clustergroupby
POSTHOOK: Input: default@clustergroupby@ds=101
#### A masked pattern was here ####
1 10
-2 74
3 416
+2 74
PREHOOK: query: explain
select key, count(1) from clustergroupby where ds='101' group by key,'a' order by key,'a' limit 10
PREHOOK: type: QUERY
@@ -712,312 +712,312 @@ POSTHOOK: Input: default@clustergroupby@ds=101
#### A masked pattern was here ####
10 2
100 4
+103 4
104 4
-111 2
-114 2
-116 2
+105 2
+11 2
126 2
128 6
-131 2
+129 4
+133 2
134 4
+136 2
+143 2
+149 4
152 4
-155 2
+153 2
+157 2
+162 2
+166 2
169 8
-17 2
170 2
-172 4
-178 2
-179 4
-189 2
-19 2
-192 2
-194 2
-195 4
-199 6
-20 2
-207 4
+174 4
+176 4
+193 6
+196 2
+201 2
+205 4
+209 4
213 4
218 2
+221 4
223 4
-229 4
233 4
-235 2
+239 4
24 4
-256 4
-257 2
+241 2
+242 4
+247 2
+248 2
26 4
-265 4
-266 2
-27 2
-278 4
-280 4
-284 2
-291 2
-30 2
+263 2
+272 4
+275 2
+285 2
+296 2
+298 6
302 2
-305 2
-316 6
-321 4
+307 4
+310 2
323 2
-33 2
+325 4
333 4
+335 2
338 2
-344 4
-353 4
-356 2
-364 2
-368 2
-369 6
-37 4
+360 2
+362 2
+367 4
+375 2
377 2
-378 2
-386 2
+384 6
392 2
-397 4
-404 4
-407 2
-411 2
-418 2
+399 4
+401 10
+402 2
+403 6
+413 4
+417 6
419 2
-424 4
-430 6
-432 2
-436 2
-437 2
-448 2
-454 6
-457 2
-468 8
-477 2
+42 4
+427 2
+438 6
+44 2
+458 4
+459 4
+466 6
+469 10
+472 2
+475 2
+478 4
479 2
-490 2
-492 4
-493 2
+483 2
+489 8
495 2
496 2
-497 2
+5 6
57 2
-67 4
-8 2
-80 2
-85 2
+58 4
+64 2
+66 2
+69 2
+72 4
+76 4
+77 2
9 2
-92 2
-97 4
-105 2
-11 2
+90 6
+0 6
113 4
-136 2
+114 2
+116 2
+118 4
137 4
138 8
-143 2
-150 2
+145 2
+146 4
+155 2
158 2
-160 2
-162 2
+164 4
165 4
-166 2
-174 4
-177 2
-18 4
-180 2
+178 2
+181 2
+183 2
186 2
-196 2
-197 4
-217 4
-219 4
-226 2
+189 2
+190 2
+191 4
+194 2
+195 4
+199 6
+2 2
+207 4
+208 6
+214 2
+216 4
+222 2
+224 4
+229 4
230 10
238 4
-242 4
-248 2
-252 2
+244 2
+258 2
260 2
-263 2
-274 2
-277 8
-296 2
-307 4
-308 2
-325 4
-327 6
+262 2
+266 2
+27 2
+273 6
+28 2
+284 2
+288 4
+30 2
+315 2
+321 4
+33 2
342 4
-348 10
35 6
-351 2
-367 4
+364 2
+365 2
+366 2
+368 2
+386 2
389 2
-393 2
394 2
+396 6
+397 4
4 2
-403 6
-409 6
+41 2
414 4
421 2
-427 2
-429 4
-435 2
-438 6
-439 4
+444 2
+446 2
+448 2
+452 2
+454 6
455 2
-458 4
460 2
-462 4
-475 2
-478 4
-480 6
-484 2
-487 2
-53 2
-58 4
-66 2
-69 2
-77 2
+463 4
+482 2
+492 4
+493 2
+65 2
78 2
-83 4
-90 6
-95 4
-103 4
+80 2
+97 4
+111 2
+12 4
120 4
-129 4
-133 2
-145 2
-146 4
-149 4
-15 4
-157 2
-181 2
-187 6
-191 4
-2 2
-200 4
-201 2
-202 2
+131 2
+150 2
+156 2
+160 2
+163 2
+167 6
+168 2
+17 2
+175 4
+180 2
+20 2
203 4
-208 6
-214 2
-216 4
-222 2
-224 4
-228 2
+226 2
237 4
-239 4
-241 2
-244 2
255 4
-258 2
-262 2
-273 6
-275 2
-28 2
+256 4
+274 2
+277 8
+281 4
282 4
-285 2
286 2
287 2
-288 4
+289 2
292 2
-298 6
-310 2
-311 6
+308 2
+309 4
+318 6
+327 6
332 2
-335 2
-336 2
339 2
-360 2
-362 2
-373 2
-384 6
-395 4
-396 6
-402 2
-417 6
-42 4
+34 2
+341 2
+348 10
+356 2
+37 4
+374 2
+379 2
+382 4
+409 6
+429 4
43 2
-444 2
-449 2
-453 2
-459 4
-47 2
+430 6
+431 6
+432 2
+435 2
+437 2
+439 4
+443 2
+457 2
+462 4
+467 2
+470 2
481 2
-482 2
-483 2
-485 2
-489 8
+487 2
494 2
-65 2
-72 4
+51 4
+70 6
74 2
-76 4
82 2
-86 2
-87 2
-96 2
-0 6
-118 4
+83 4
+84 4
119 6
-12 4
125 4
-153 2
-156 2
-163 2
-164 4
-167 6
-168 2
-175 4
-176 4
-183 2
-190 2
-193 6
-205 4
-209 4
-221 4
-247 2
+15 4
+172 4
+177 2
+179 4
+18 4
+187 6
+19 2
+192 2
+197 4
+200 4
+202 2
+217 4
+219 4
+228 2
+235 2
249 2
-272 4
-281 4
+252 2
+257 2
+265 4
+278 4
+280 4
283 2
-289 2
+291 2
+305 2
306 2
-309 4
-315 2
+311 6
+316 6
317 4
-318 6
322 4
331 4
-34 2
-341 2
+336 2
+344 4
345 2
-365 2
-366 2
-374 2
-375 2
-379 2
-382 4
-399 4
+351 2
+353 4
+369 6
+373 2
+378 2
+393 2
+395 4
400 2
-401 10
+404 4
406 8
-41 2
-413 4
-431 6
-44 2
-443 2
-446 2
-452 2
-463 4
-466 6
-467 2
-469 10
-470 2
-472 2
+407 2
+411 2
+418 2
+424 4
+436 2
+449 2
+453 2
+468 8
+47 2
+477 2
+480 6
+484 2
+485 2
+490 2
491 2
+497 2
498 6
-5 6
-51 4
+53 2
54 2
-64 2
-70 6
-84 4
+67 4
+8 2
+85 2
+86 2
+87 2
+92 2
+95 4
+96 2
98 4
PREHOOK: query: explain
select key, count(1) from clustergroupby group by key, 3
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/llap/bucket_many.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucket_many.q.out b/ql/src/test/results/clientpositive/llap/bucket_many.q.out
index 2202745..d1fb481 100644
--- a/ql/src/test/results/clientpositive/llap/bucket_many.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucket_many.q.out
@@ -60,6 +60,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments 'default','default'
@@ -81,6 +82,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments 'default','default'
@@ -122,6 +124,7 @@ STAGE PLANS:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 256
bucket_field_name key
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -204,6 +207,7 @@ STAGE PLANS:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count 256
bucket_field_name key
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments
@@ -244,10 +248,10 @@ POSTHOOK: Output: default@bucket_many
POSTHOOK: Lineage: bucket_many.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: bucket_many.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: explain
-select * from bucket_many tablesample (bucket 1 out of 256) s
+select * from bucket_many tablesample (bucket 2 out of 256) s
PREHOOK: type: QUERY
POSTHOOK: query: explain
-select * from bucket_many tablesample (bucket 1 out of 256) s
+select * from bucket_many tablesample (bucket 2 out of 256) s
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-0 is a root stage
@@ -260,22 +264,25 @@ STAGE PLANS:
TableScan
alias: s
Filter Operator
- predicate: (((hash(key) & 2147483647) % 256) = 0) (type: boolean)
+ predicate: (((hash(key) & 2147483647) % 256) = 1) (type: boolean)
Select Operator
expressions: key (type: int), value (type: string)
outputColumnNames: _col0, _col1
ListSink
-PREHOOK: query: select * from bucket_many tablesample (bucket 1 out of 256) s
+PREHOOK: query: select * from bucket_many tablesample (bucket 2 out of 256) s
PREHOOK: type: QUERY
PREHOOK: Input: default@bucket_many
#### A masked pattern was here ####
-POSTHOOK: query: select * from bucket_many tablesample (bucket 1 out of 256) s
+POSTHOOK: query: select * from bucket_many tablesample (bucket 2 out of 256) s
POSTHOOK: type: QUERY
POSTHOOK: Input: default@bucket_many
#### A masked pattern was here ####
-256 val_256
-256 val_256
-0 val_0
-0 val_0
-0 val_0
+462 val_462
+462 val_462
+309 val_309
+282 val_282
+282 val_282
+176 val_176
+176 val_176
+309 val_309