You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dj...@apache.org on 2018/05/08 18:42:20 UTC
[15/58] [abbrv] hive git commit: HIVE-18910 : Migrate to Murmur hash
for shuffle and bucketing (Deepak Jaiswal, reviewed by Jasone Dere)
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union22.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union22.q.out b/ql/src/test/results/clientpositive/union22.q.out
index 50d06c0..9e329f3 100644
--- a/ql/src/test/results/clientpositive/union22.q.out
+++ b/ql/src/test/results/clientpositive/union22.q.out
@@ -121,6 +121,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns k0,k1,k2,k3,k4,k5
columns.comments
@@ -249,6 +250,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns k1,k2,k3,k4
columns.comments
@@ -296,6 +298,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns k0,k1,k2,k3,k4,k5
columns.comments
@@ -344,6 +347,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns k1,k2,k3,k4
columns.comments
@@ -397,6 +401,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns k1,k2,k3,k4
columns.comments
@@ -490,6 +495,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns k0,k1,k2,k3,k4,k5
columns.comments
@@ -555,6 +561,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns k1,k2,k3,k4
columns.comments
@@ -660,6 +667,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns k1,k2,k3,k4
columns.comments
@@ -707,6 +715,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns k0,k1,k2,k3,k4,k5
columns.comments
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union24.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union24.q.out b/ql/src/test/results/clientpositive/union24.q.out
index dfa614e..81dba3e 100644
--- a/ql/src/test/results/clientpositive/union24.q.out
+++ b/ql/src/test/results/clientpositive/union24.q.out
@@ -111,6 +111,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -132,6 +133,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -348,6 +350,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -369,6 +372,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -394,6 +398,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -415,6 +420,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -440,6 +446,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -461,6 +468,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -623,6 +631,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -644,6 +653,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -669,6 +679,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -690,6 +701,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -877,6 +889,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -898,6 +911,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -923,6 +937,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -944,6 +959,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -1095,6 +1111,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -1116,6 +1133,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -1141,6 +1159,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -1162,6 +1181,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -1419,6 +1439,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -1440,6 +1461,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -1465,6 +1487,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
@@ -1486,6 +1509,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,count
columns.comments
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_pos_alias.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_pos_alias.q.out b/ql/src/test/results/clientpositive/union_pos_alias.q.out
index 493316b..fc49174 100644
--- a/ql/src/test/results/clientpositive/union_pos_alias.q.out
+++ b/ql/src/test/results/clientpositive/union_pos_alias.q.out
@@ -336,6 +336,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_ppr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_ppr.q.out b/ql/src/test/results/clientpositive/union_ppr.q.out
index 8633c12..1be1e81 100644
--- a/ql/src/test/results/clientpositive/union_ppr.q.out
+++ b/ql/src/test/results/clientpositive/union_ppr.q.out
@@ -110,6 +110,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments 'default','default'
@@ -158,6 +159,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments 'default','default'
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_1.q.out b/ql/src/test/results/clientpositive/union_remove_1.q.out
index c5ef152..7c09071 100644
--- a/ql/src/test/results/clientpositive/union_remove_1.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_1.q.out
@@ -171,6 +171,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 40
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_10.q.out b/ql/src/test/results/clientpositive/union_remove_10.q.out
index 500c123..39cb313 100644
--- a/ql/src/test/results/clientpositive/union_remove_10.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_10.q.out
@@ -222,6 +222,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 3
totalSize 271
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_11.q.out b/ql/src/test/results/clientpositive/union_remove_11.q.out
index 39de6a2..4fc7659 100644
--- a/ql/src/test/results/clientpositive/union_remove_11.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_11.q.out
@@ -211,6 +211,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 1
totalSize 115
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_12.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_12.q.out b/ql/src/test/results/clientpositive/union_remove_12.q.out
index 29141e1..4c2ac88 100644
--- a/ql/src/test/results/clientpositive/union_remove_12.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_12.q.out
@@ -212,6 +212,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 194
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_13.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_13.q.out b/ql/src/test/results/clientpositive/union_remove_13.q.out
index 5e3983a..819028b 100644
--- a/ql/src/test/results/clientpositive/union_remove_13.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_13.q.out
@@ -231,6 +231,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 192
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_14.q.out b/ql/src/test/results/clientpositive/union_remove_14.q.out
index 29141e1..4c2ac88 100644
--- a/ql/src/test/results/clientpositive/union_remove_14.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_14.q.out
@@ -212,6 +212,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 194
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_15.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_15.q.out b/ql/src/test/results/clientpositive/union_remove_15.q.out
index 2f34a37..f03b701 100644
--- a/ql/src/test/results/clientpositive/union_remove_15.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_15.q.out
@@ -188,6 +188,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
numPartitions 2
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_16.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_16.q.out b/ql/src/test/results/clientpositive/union_remove_16.q.out
index 8c0ef93..3aa2be1 100644
--- a/ql/src/test/results/clientpositive/union_remove_16.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_16.q.out
@@ -222,6 +222,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
numPartitions 2
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_17.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_17.q.out b/ql/src/test/results/clientpositive/union_remove_17.q.out
index 4e50b23..460d085 100644
--- a/ql/src/test/results/clientpositive/union_remove_17.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_17.q.out
@@ -149,6 +149,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
numPartitions 2
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_18.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_18.q.out b/ql/src/test/results/clientpositive/union_remove_18.q.out
index 9195eff..cb7e99c 100644
--- a/ql/src/test/results/clientpositive/union_remove_18.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_18.q.out
@@ -200,6 +200,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 12
numPartitions 6
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_19.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_19.q.out b/ql/src/test/results/clientpositive/union_remove_19.q.out
index 8f9be65..808a351 100644
--- a/ql/src/test/results/clientpositive/union_remove_19.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_19.q.out
@@ -171,6 +171,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 40
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_2.q.out b/ql/src/test/results/clientpositive/union_remove_2.q.out
index cd1f243..b8a8d7e 100644
--- a/ql/src/test/results/clientpositive/union_remove_2.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_2.q.out
@@ -180,6 +180,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 3
totalSize 68
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_20.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_20.q.out b/ql/src/test/results/clientpositive/union_remove_20.q.out
index 02fe296..9b986b3 100644
--- a/ql/src/test/results/clientpositive/union_remove_20.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_20.q.out
@@ -179,6 +179,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 40
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_21.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_21.q.out b/ql/src/test/results/clientpositive/union_remove_21.q.out
index 9af1bac..bc21e22 100644
--- a/ql/src/test/results/clientpositive/union_remove_21.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_21.q.out
@@ -163,6 +163,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 20
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_22.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_22.q.out b/ql/src/test/results/clientpositive/union_remove_22.q.out
index 12c0b18..3167be4 100644
--- a/ql/src/test/results/clientpositive/union_remove_22.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_22.q.out
@@ -181,6 +181,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 60
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_23.q.out b/ql/src/test/results/clientpositive/union_remove_23.q.out
index bc8edc0..e19d6cd 100644
--- a/ql/src/test/results/clientpositive/union_remove_23.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_23.q.out
@@ -219,6 +219,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 40
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_24.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_24.q.out b/ql/src/test/results/clientpositive/union_remove_24.q.out
index a0a7d38..6bbfa96 100644
--- a/ql/src/test/results/clientpositive/union_remove_24.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_24.q.out
@@ -179,6 +179,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 60
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_3.q.out b/ql/src/test/results/clientpositive/union_remove_3.q.out
index b3630a7..28fb92b 100644
--- a/ql/src/test/results/clientpositive/union_remove_3.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_3.q.out
@@ -169,6 +169,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 1
totalSize 72
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_4.q.out b/ql/src/test/results/clientpositive/union_remove_4.q.out
index e5d0807..f0be721 100644
--- a/ql/src/test/results/clientpositive/union_remove_4.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_4.q.out
@@ -215,6 +215,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 40
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_5.q.out b/ql/src/test/results/clientpositive/union_remove_5.q.out
index f60b88b..e9e1173 100644
--- a/ql/src/test/results/clientpositive/union_remove_5.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_5.q.out
@@ -224,6 +224,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 3
totalSize 68
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_7.q.out b/ql/src/test/results/clientpositive/union_remove_7.q.out
index b64b3b0..fb8fb42 100644
--- a/ql/src/test/results/clientpositive/union_remove_7.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_7.q.out
@@ -171,6 +171,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 178
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_8.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_8.q.out b/ql/src/test/results/clientpositive/union_remove_8.q.out
index f1ae75a..23821d3 100644
--- a/ql/src/test/results/clientpositive/union_remove_8.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_8.q.out
@@ -180,6 +180,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 3
totalSize 271
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_remove_9.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_remove_9.q.out b/ql/src/test/results/clientpositive/union_remove_9.q.out
index 6baecd5..2b4ee4c 100644
--- a/ql/src/test/results/clientpositive/union_remove_9.q.out
+++ b/ql/src/test/results/clientpositive/union_remove_9.q.out
@@ -229,6 +229,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
numFiles 2
totalSize 192
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/union_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/union_stats.q.out b/ql/src/test/results/clientpositive/union_stats.q.out
index 5b6ada7..8bd3f44 100644
--- a/ql/src/test/results/clientpositive/union_stats.q.out
+++ b/ql/src/test/results/clientpositive/union_stats.q.out
@@ -90,6 +90,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments 'default','default'
@@ -111,6 +112,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key,value
columns.comments 'default','default'
@@ -313,6 +315,7 @@ Retention: 0
Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ bucketing_version 2
numFiles 1
numRows 1000
rawDataSize 10624
@@ -361,6 +364,7 @@ Retention: 0
Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ bucketing_version 2
numFiles 1
numRows 1500
rawDataSize 15936
@@ -417,6 +421,7 @@ Retention: 0
Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ bucketing_version 2
numFiles 1
numRows 1500
rawDataSize 15936
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/unset_table_view_property.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/unset_table_view_property.q.out b/ql/src/test/results/clientpositive/unset_table_view_property.q.out
index 7097963..5887971 100644
--- a/ql/src/test/results/clientpositive/unset_table_view_property.q.out
+++ b/ql/src/test/results/clientpositive/unset_table_view_property.q.out
@@ -17,6 +17,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
+bucketing_version 2
numFiles 0
numRows 0
rawDataSize 0
@@ -36,6 +37,7 @@ POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
+bucketing_version 2
c 3
#### A masked pattern was here ####
numFiles 0
@@ -56,6 +58,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
+bucketing_version 2
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -76,6 +79,7 @@ POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
+bucketing_version 2
c 3
d 4
#### A masked pattern was here ####
@@ -97,6 +101,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
+bucketing_version 2
c 3
#### A masked pattern was here ####
numFiles 0
@@ -117,6 +122,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
+bucketing_version 2
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -138,6 +144,7 @@ POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
b 2
+bucketing_version 2
c 3
d 4
#### A masked pattern was here ####
@@ -160,6 +167,7 @@ POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
+bucketing_version 2
c 3
#### A masked pattern was here ####
numFiles 0
@@ -181,6 +189,7 @@ POSTHOOK: query: SHOW TBLPROPERTIES vt.testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"col1":"true","col2":"true"}}
a 1
+bucketing_version 2
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -218,6 +227,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testView
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testView
POSTHOOK: type: SHOW_TBLPROPERTIES
+bucketing_version 2
#### A masked pattern was here ####
propA 100
propB 200
@@ -234,6 +244,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testView
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testView
POSTHOOK: type: SHOW_TBLPROPERTIES
+bucketing_version 2
#### A masked pattern was here ####
PREHOOK: query: ALTER VIEW vt.testView SET TBLPROPERTIES ('propA'='100', 'propC'='300', 'propD'='400')
PREHOOK: type: ALTERVIEW_PROPERTIES
@@ -247,6 +258,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testView
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testView
POSTHOOK: type: SHOW_TBLPROPERTIES
+bucketing_version 2
#### A masked pattern was here ####
propA 100
propC 300
@@ -264,6 +276,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testView
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testView
POSTHOOK: type: SHOW_TBLPROPERTIES
+bucketing_version 2
#### A masked pattern was here ####
propD 400
#### A masked pattern was here ####
@@ -279,6 +292,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testView
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testView
POSTHOOK: type: SHOW_TBLPROPERTIES
+bucketing_version 2
#### A masked pattern was here ####
PREHOOK: query: ALTER VIEW vt.testView SET TBLPROPERTIES ('propA'='100', 'propB' = '200', 'propC'='300', 'propD'='400')
PREHOOK: type: ALTERVIEW_PROPERTIES
@@ -292,6 +306,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testView
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testView
POSTHOOK: type: SHOW_TBLPROPERTIES
+bucketing_version 2
#### A masked pattern was here ####
propA 100
propB 200
@@ -310,6 +325,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testView
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testView
POSTHOOK: type: SHOW_TBLPROPERTIES
+bucketing_version 2
#### A masked pattern was here ####
propA 100
propB 200
@@ -326,6 +342,7 @@ PREHOOK: query: SHOW TBLPROPERTIES vt.testView
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES vt.testView
POSTHOOK: type: SHOW_TBLPROPERTIES
+bucketing_version 2
#### A masked pattern was here ####
propA 100
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/view_alias.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/view_alias.q.out b/ql/src/test/results/clientpositive/view_alias.q.out
index 6e3c2b5..bbd721a 100644
--- a/ql/src/test/results/clientpositive/view_alias.q.out
+++ b/ql/src/test/results/clientpositive/view_alias.q.out
@@ -30,6 +30,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -96,6 +97,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -164,6 +166,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -232,6 +235,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -298,6 +302,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -412,6 +417,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/test/results/clientpositive/view_cbo.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/view_cbo.q.out b/ql/src/test/results/clientpositive/view_cbo.q.out
index 3983e5b..747c4c2 100644
--- a/ql/src/test/results/clientpositive/view_cbo.q.out
+++ b/ql/src/test/results/clientpositive/view_cbo.q.out
@@ -121,6 +121,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -190,6 +191,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -269,6 +271,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -330,6 +333,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -379,6 +383,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -428,6 +433,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -479,6 +485,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -530,6 +537,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -581,6 +589,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
@@ -632,6 +641,7 @@ Database: default
Retention: 0
Table Type: VIRTUAL_VIEW
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
# Storage Information
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
index 635ea04..a442cb1 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.serde2.objectinspector;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.lang.reflect.Type;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@@ -28,9 +29,12 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampLocalTZObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector;
+import org.apache.hive.common.util.Murmur3;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.serde.serdeConstants;
@@ -154,7 +158,7 @@ public final class ObjectInspectorUtils {
@Override
public int hashCode() {
- return ObjectInspectorUtils.getBucketHashCode(objects, oi);
+ return ObjectInspectorUtils.getBucketHashCodeOld(objects, oi);
}
}
@@ -618,13 +622,25 @@ public final class ObjectInspectorUtils {
* @param bucketFields the bucketed fields of the row
* @param bucketFieldInspectors the ObjectInpsectors for each of the bucketed fields
* @param totalBuckets the number of buckets in the table
- * @return the bucket number
+ * @return the bucket number using Murmur hash
*/
public static int getBucketNumber(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors, int totalBuckets) {
return getBucketNumber(getBucketHashCode(bucketFields, bucketFieldInspectors), totalBuckets);
}
/**
+ * Computes the bucket number to which the bucketFields belong to
+ * @param bucketFields the bucketed fields of the row
+ * @param bucketFieldInspectors the ObjectInpsectors for each of the bucketed fields
+ * @param totalBuckets the number of buckets in the table
+ * @return the bucket number
+ */
+ @Deprecated
+ public static int getBucketNumberOld(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors, int totalBuckets) {
+ return getBucketNumber(getBucketHashCodeOld(bucketFields, bucketFieldInspectors), totalBuckets);
+ }
+
+ /**
* https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL+BucketedTables
* @param hashCode as produced by {@link #getBucketHashCode(Object[], ObjectInspector[])}
*/
@@ -636,13 +652,15 @@ public final class ObjectInspectorUtils {
}
return (hashCode & Integer.MAX_VALUE) % numberOfBuckets;
}
+
/**
* Computes the hash code for the given bucketed fields
* @param bucketFields
* @param bucketFieldInspectors
* @return
*/
- public static int getBucketHashCode(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors) {
+ @Deprecated
+ public static int getBucketHashCodeOld(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors) {
int hashCode = 0;
for (int i = 0; i < bucketFields.length; i++) {
int fieldHash = ObjectInspectorUtils.hashCode(bucketFields[i], bucketFieldInspectors[i]);
@@ -651,7 +669,6 @@ public final class ObjectInspectorUtils {
return hashCode;
}
-
public static int hashCode(Object o, ObjectInspector objIns) {
if (o == null) {
return 0;
@@ -769,6 +786,144 @@ public final class ObjectInspectorUtils {
}
}
+ public static int getBucketHashCode(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors) {
+ int hashCode = 0;
+ ByteBuffer b = ByteBuffer.allocate(8); // To be used with primitive types
+ for (int i = 0; i < bucketFields.length; i++) {
+ int fieldHash = ObjectInspectorUtils.hashCodeMurmur(
+ bucketFields[i], bucketFieldInspectors[i], b);
+ hashCode = 31 * hashCode + fieldHash;
+ }
+ return hashCode;
+ }
+
+ public static int hashCodeMurmur(Object o, ObjectInspector objIns, ByteBuffer byteBuffer) {
+ if (o == null) {
+ return 0;
+ }
+ // Reset the bytebuffer
+ byteBuffer.clear();
+ switch (objIns.getCategory()) {
+ case PRIMITIVE: {
+ PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) objIns);
+ switch (poi.getPrimitiveCategory()) {
+ case VOID:
+ return 0;
+ case BOOLEAN:
+ return (((BooleanObjectInspector) poi).get(o) ? 1 : 0);
+ case BYTE:
+ return ((ByteObjectInspector) poi).get(o);
+ case SHORT: {
+ byteBuffer.putShort(((ShortObjectInspector) poi).get(o));
+ return Murmur3.hash32(byteBuffer.array(), 2);
+ }
+ case INT: {
+ byteBuffer.putInt(((IntObjectInspector) poi).get(o));
+ return Murmur3.hash32(byteBuffer.array(), 4);
+ }
+ case LONG: {
+ byteBuffer.putLong(((LongObjectInspector) poi).get(o));
+ return Murmur3.hash32(byteBuffer.array(), 8);
+ }
+ case FLOAT: {
+ byteBuffer.putFloat(Float.floatToIntBits(((FloatObjectInspector) poi).get(o)));
+ return Murmur3.hash32(byteBuffer.array(), 4);
+ }
+ case DOUBLE: {
+ // This hash function returns the same result as Double.hashCode()
+ // while DoubleWritable.hashCode returns a different result.
+ byteBuffer.putDouble(Double.doubleToLongBits(((DoubleObjectInspector) poi).get(o)));
+ return Murmur3.hash32(byteBuffer.array(), 8);
+ }
+ case STRING: {
+ // This hash function returns the same result as String.hashCode() when
+ // all characters are ASCII, while Text.hashCode() always returns a
+ // different result.
+ Text text = ((StringObjectInspector) poi).getPrimitiveWritableObject(o);
+ return Murmur3.hash32(text.getBytes(), text.getLength());
+ }
+ case CHAR: {
+ Text text = ((HiveCharObjectInspector) poi).getPrimitiveWritableObject(o).getStrippedValue();
+ return Murmur3.hash32(text.getBytes(), text.getLength());
+ }
+ case VARCHAR: {
+ Text text = ((HiveVarcharObjectInspector)poi).getPrimitiveWritableObject(o).getTextValue();
+ return Murmur3.hash32(text.getBytes(), text.getLength());
+ }
+ case BINARY:
+ return Murmur3.hash32(((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).getBytes());
+
+ case DATE:
+ byteBuffer.putInt(((DateObjectInspector) poi).getPrimitiveWritableObject(o).getDays());
+ return Murmur3.hash32(byteBuffer.array(), 4);
+ case TIMESTAMP: {
+ TimestampWritable t = ((TimestampObjectInspector) poi)
+ .getPrimitiveWritableObject(o);
+ return Murmur3.hash32(t.getBytes());
+ }
+ case TIMESTAMPLOCALTZ:
+ return Murmur3.hash32((((TimestampLocalTZObjectInspector) poi).getPrimitiveWritableObject(o)).getBytes());
+ case INTERVAL_YEAR_MONTH:
+ byteBuffer.putInt(((HiveIntervalYearMonthObjectInspector) poi)
+ .getPrimitiveWritableObject(o).hashCode());
+ return Murmur3.hash32(byteBuffer.array(), 4);
+ case INTERVAL_DAY_TIME:
+ byteBuffer.putInt(((HiveIntervalDayTimeObjectInspector) poi)
+ .getPrimitiveWritableObject(o).hashCode());
+ return Murmur3.hash32(byteBuffer.array(), 4);
+ case DECIMAL:
+ // Since getBucketHashCode uses this, HiveDecimal return the old (much slower) but
+ // compatible hash code.
+ return Murmur3.hash32(((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).getInternalStorage());
+
+ default: {
+ throw new RuntimeException("Unknown type: "
+ + poi.getPrimitiveCategory());
+ }
+ }
+ }
+ case LIST: {
+ int r = 0;
+ ListObjectInspector listOI = (ListObjectInspector)objIns;
+ ObjectInspector elemOI = listOI.getListElementObjectInspector();
+ for (int ii = 0; ii < listOI.getListLength(o); ++ii) {
+ //r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI);
+ r = 31 * r + hashCodeMurmur(listOI.getListElement(o, ii), elemOI, byteBuffer);
+ }
+ return r;
+ }
+ case MAP: {
+ int r = 0;
+ MapObjectInspector mapOI = (MapObjectInspector)objIns;
+ ObjectInspector keyOI = mapOI.getMapKeyObjectInspector();
+ ObjectInspector valueOI = mapOI.getMapValueObjectInspector();
+ Map<?, ?> map = mapOI.getMap(o);
+ for (Map.Entry<?,?> entry : map.entrySet()) {
+ r += hashCodeMurmur(entry.getKey(), keyOI, byteBuffer) ^
+ hashCode(entry.getValue(), valueOI);
+ }
+ return r;
+ }
+ case STRUCT:
+ int r = 0;
+ StructObjectInspector structOI = (StructObjectInspector)objIns;
+ List<? extends StructField> fields = structOI.getAllStructFieldRefs();
+ for (StructField field : fields) {
+ r = 31 * r + hashCodeMurmur(structOI.getStructFieldData(o, field),
+ field.getFieldObjectInspector(), byteBuffer);
+ }
+ return r;
+
+ case UNION:
+ UnionObjectInspector uOI = (UnionObjectInspector)objIns;
+ byte tag = uOI.getTag(o);
+ return hashCodeMurmur(uOI.getField(o), uOI.getObjectInspectors().get(tag), byteBuffer);
+
+ default:
+ throw new RuntimeException("Unknown type: "+ objIns.getTypeName());
+ }
+ }
+
/**
* Compare two arrays of objects with their respective arrays of
* ObjectInspectors.
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java
index eca7405..23ca194 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java
@@ -150,9 +150,9 @@ public class TestObjectInspectorUtils extends TestCase {
struct.add("two");
struct.add(true);
- int hashCode = ObjectInspectorUtils.getBucketHashCode(struct.toArray(), fieldObjectInspectors.toArray(new ObjectInspector[fieldObjectInspectors.size()]));
+ int hashCode = ObjectInspectorUtils.getBucketHashCodeOld(struct.toArray(), fieldObjectInspectors.toArray(new ObjectInspector[fieldObjectInspectors.size()]));
assertEquals("", 3574518, hashCode);
- int bucketId = ObjectInspectorUtils.getBucketNumber(struct.toArray(), fieldObjectInspectors.toArray(new ObjectInspector[fieldObjectInspectors.size()]), 16);
+ int bucketId = ObjectInspectorUtils.getBucketNumberOld(struct.toArray(), fieldObjectInspectors.toArray(new ObjectInspector[fieldObjectInspectors.size()]), 16);
assertEquals("", 6, bucketId);
assertEquals("", bucketId, ObjectInspectorUtils.getBucketNumber(hashCode, 16));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
index 2b35e6f..c9d1051 100644
--- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
+++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
@@ -87,4 +87,5 @@ public class hive_metastoreConstants {
public static final String TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties";
+ public static final String TABLE_BUCKETING_VERSION = "bucketing_version";
}
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hive/common/util/Murmur3.java b/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
index 6848a59..c896fa7 100644
--- a/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
+++ b/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
@@ -61,7 +61,18 @@ public class Murmur3 {
* @return - hashcode
*/
public static int hash32(byte[] data) {
- return hash32(data, data.length, DEFAULT_SEED);
+ return hash32(data, 0, data.length, DEFAULT_SEED);
+ }
+
+ /**
+ * Murmur3 32-bit variant.
+ *
+ * @param data - input byte array
+ * @param length - length of array
+ * @return - hashcode
+ */
+ public static int hash32(byte[] data, int length) {
+ return hash32(data, 0, length, DEFAULT_SEED);
}
/**
@@ -73,16 +84,29 @@ public class Murmur3 {
* @return - hashcode
*/
public static int hash32(byte[] data, int length, int seed) {
+ return hash32(data, 0, length, seed);
+ }
+
+ /**
+ * Murmur3 32-bit variant.
+ *
+ * @param data - input byte array
+ * @param offset - offset of data
+ * @param length - length of array
+ * @param seed - seed. (default 0)
+ * @return - hashcode
+ */
+ public static int hash32(byte[] data, int offset, int length, int seed) {
int hash = seed;
final int nblocks = length >> 2;
// body
for (int i = 0; i < nblocks; i++) {
int i_4 = i << 2;
- int k = (data[i_4] & 0xff)
- | ((data[i_4 + 1] & 0xff) << 8)
- | ((data[i_4 + 2] & 0xff) << 16)
- | ((data[i_4 + 3] & 0xff) << 24);
+ int k = (data[offset + i_4] & 0xff)
+ | ((data[offset + i_4 + 1] & 0xff) << 8)
+ | ((data[offset + i_4 + 2] & 0xff) << 16)
+ | ((data[offset + i_4 + 3] & 0xff) << 24);
// mix functions
k *= C1_32;
@@ -97,11 +121,11 @@ public class Murmur3 {
int k1 = 0;
switch (length - idx) {
case 3:
- k1 ^= data[idx + 2] << 16;
+ k1 ^= data[offset + idx + 2] << 16;
case 2:
- k1 ^= data[idx + 1] << 8;
+ k1 ^= data[offset + idx + 1] << 8;
case 1:
- k1 ^= data[idx];
+ k1 ^= data[offset + idx];
// mix functions
k1 *= C1_32;
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
----------------------------------------------------------------------
diff --git a/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java b/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
index 9a01255..f20366b 100644
--- a/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
+++ b/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
@@ -229,7 +229,7 @@ public class TestMurmur3 {
final int seed = 123, arraySize = 1023;
byte[] bytes = new byte[arraySize];
new Random(seed).nextBytes(bytes);
- int expected = Murmur3.hash32(bytes);
+ int expected = Murmur3.hash32(bytes, arraySize);
Murmur3.IncrementalHash32 same = new IncrementalHash32(), diff = new IncrementalHash32();
for (int blockSize = 1; blockSize <= arraySize; ++blockSize) {
byte[] block = new byte[blockSize];
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/streaming/src/test/org/apache/hive/streaming/TestStreaming.java
----------------------------------------------------------------------
diff --git a/streaming/src/test/org/apache/hive/streaming/TestStreaming.java b/streaming/src/test/org/apache/hive/streaming/TestStreaming.java
index 3343d10..0ec3048 100644
--- a/streaming/src/test/org/apache/hive/streaming/TestStreaming.java
+++ b/streaming/src/test/org/apache/hive/streaming/TestStreaming.java
@@ -1491,9 +1491,9 @@ public class TestStreaming {
// assert bucket listing is as expected
Assert.assertEquals("number of buckets does not match expectation", actual1.values().size(), 3);
- Assert.assertEquals("records in bucket does not match expectation", actual1.get(0).size(), 2);
+ Assert.assertTrue("bucket 0 shouldn't have been created", actual1.get(0) == null);
Assert.assertEquals("records in bucket does not match expectation", actual1.get(1).size(), 1);
- Assert.assertTrue("bucket 2 shouldn't have been created", actual1.get(2) == null);
+ Assert.assertEquals("records in bucket does not match expectation", actual1.get(2).size(), 2);
Assert.assertEquals("records in bucket does not match expectation", actual1.get(3).size(), 1);
}
private void runCmdOnDriver(String cmd) throws QueryFailedException {
@@ -1696,7 +1696,7 @@ public class TestStreaming {
} else if (file.contains("bucket_00001")) {
corruptDataFile(file, conf, -1);
} else if (file.contains("bucket_00002")) {
- Assert.assertFalse("bucket 2 shouldn't have been created", true);
+ corruptDataFile(file, conf, 100);
} else if (file.contains("bucket_00003")) {
corruptDataFile(file, conf, 100);
}
@@ -1726,9 +1726,9 @@ public class TestStreaming {
System.setErr(origErr);
errDump = new String(myErr.toByteArray());
- Assert.assertEquals(true, errDump.contains("bucket_00000 recovered successfully!"));
- Assert.assertEquals(true, errDump.contains("No readable footers found. Creating empty orc file."));
Assert.assertEquals(true, errDump.contains("bucket_00001 recovered successfully!"));
+ Assert.assertEquals(true, errDump.contains("No readable footers found. Creating empty orc file."));
+ Assert.assertEquals(true, errDump.contains("bucket_00002 recovered successfully!"));
Assert.assertEquals(true, errDump.contains("bucket_00003 recovered successfully!"));
Assert.assertEquals(false, errDump.contains("Exception"));
Assert.assertEquals(false, errDump.contains("is still open for writes."));