You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/01/09 22:14:12 UTC
hive git commit: HIVE-15558 : fix char whitespace handling for
vectorization (Sergey Shelukhin, reviewed by Matt McCline)
Repository: hive
Updated Branches:
refs/heads/master 2f501a8a0 -> b4ac43db4
HIVE-15558 : fix char whitespace handling for vectorization (Sergey Shelukhin, reviewed by Matt McCline)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4ac43db
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4ac43db
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4ac43db
Branch: refs/heads/master
Commit: b4ac43db41e5af0fb867a3a5139c4dff2e36e865
Parents: 2f501a8
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Jan 9 13:35:05 2017 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Jan 9 13:35:05 2017 -0800
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/vector/VectorExtractRow.java | 3 ++-
.../schema_evol_text_vec_part_all_primitive.q.out | 10 +++++-----
.../llap/schema_evol_text_vec_table.q.out | 14 +++++++-------
.../llap/schema_evol_text_vecrow_part.q.out | 10 +++++-----
.../schema_evol_text_vecrow_part_all_primitive.q.out | 10 +++++-----
.../llap/schema_evol_text_vecrow_table.q.out | 14 +++++++-------
6 files changed, 31 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b4ac43db/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
index 631dcb2..94eaf56 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
@@ -306,7 +306,8 @@ public class VectorExtractRow {
maxLengths[logicalColumnIndex]);
HiveCharWritable hiveCharWritable = (HiveCharWritable) primitiveWritable;
- hiveCharWritable.set(new String(bytes, start, adjustedLength, Charsets.UTF_8), -1);
+ hiveCharWritable.set(new String(bytes, start, adjustedLength, Charsets.UTF_8),
+ maxLengths[logicalColumnIndex]);
return primitiveWritable;
}
case DECIMAL:
http://git-wip-us.apache.org/repos/asf/hive/blob/b4ac43db/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out
index 46da4f0..456f83d 100644
--- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out
+++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out
@@ -299,7 +299,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=10 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"]
- TableScan [TS_0] (rows=10 width=470)
+ TableScan [TS_0] (rows=10 width=507)
default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint
@@ -514,7 +514,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=6 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"]
- TableScan [TS_0] (rows=6 width=425)
+ TableScan [TS_0] (rows=6 width=453)
default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double
@@ -645,7 +645,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=6 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"]
- TableScan [TS_0] (rows=6 width=145)
+ TableScan [TS_0] (rows=6 width=150)
default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp
@@ -760,7 +760,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=6 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- TableScan [TS_0] (rows=6 width=62)
+ TableScan [TS_0] (rows=6 width=76)
default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date
@@ -956,7 +956,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=13 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
- TableScan [TS_0] (rows=13 width=100)
+ TableScan [TS_0] (rows=13 width=109)
default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params
http://git-wip-us.apache.org/repos/asf/hive/blob/b4ac43db/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out
index fa6c8c0..5206770 100644
--- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out
+++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vec_table.q.out
@@ -416,9 +416,9 @@ Stage-0
Stage-1
Map 1 vectorized, llap
File Output Operator [FS_4]
- Select Operator [SEL_3] (rows=5 width=52)
+ Select Operator [SEL_3] (rows=5 width=90)
Output:["_col0","_col1","_col2","_col3","_col4"]
- TableScan [TS_0] (rows=5 width=52)
+ TableScan [TS_0] (rows=5 width=90)
default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"]
PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double
@@ -1024,11 +1024,11 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@table_change_string_group_string_group_string
#### A masked pattern was here ####
insert_num c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 b
-101 freckled freckled freckled freckled freckled freckled freckled freckled freckled freckled original
-102 ox ox ox ox ox ox ox ox ox ox original
-103 original
-104 I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked original
-105 200 200 200 200 200 200 200 200 200 200 original
+101 freckled freckled freckled freckled freckled freckled freckled freckled freckled freckled original
+102 ox ox ox ox ox ox ox ox ox ox original
+103 original
+104 I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked original
+105 200 200 200 200 200 200 200 200 200 200 original
111 filler filler filler filler filler filler filler filler filler filler new
PREHOOK: query: drop table table_change_string_group_string_group_string
PREHOOK: type: DROPTABLE
http://git-wip-us.apache.org/repos/asf/hive/blob/b4ac43db/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out
index 4a35b93..0298a29 100644
--- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out
+++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part.q.out
@@ -396,7 +396,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=5 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
- TableScan [TS_0] (rows=5 width=56)
+ TableScan [TS_0] (rows=5 width=94)
default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double
@@ -512,7 +512,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=6 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"]
- TableScan [TS_0] (rows=6 width=154)
+ TableScan [TS_0] (rows=6 width=172)
default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp
@@ -705,7 +705,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=6 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"]
- TableScan [TS_0] (rows=6 width=153)
+ TableScan [TS_0] (rows=6 width=182)
default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group
@@ -880,7 +880,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=6 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
- TableScan [TS_0] (rows=6 width=231)
+ TableScan [TS_0] (rows=6 width=253)
default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group
@@ -1043,7 +1043,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=6 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"]
- TableScan [TS_0] (rows=6 width=70)
+ TableScan [TS_0] (rows=6 width=200)
default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string
http://git-wip-us.apache.org/repos/asf/hive/blob/b4ac43db/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out
index 41a61a8..4787841 100644
--- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out
+++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_part_all_primitive.q.out
@@ -299,7 +299,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=10 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49","_col50","_col51","_col52","_col53","_col54","_col55"]
- TableScan [TS_0] (rows=10 width=475)
+ TableScan [TS_0] (rows=10 width=512)
default@part_change_various_various_boolean_to_bigint,part_change_various_various_boolean_to_bigint,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","c34","c35","c36","c37","c38","c39","c40","c41","c42","c43","c44","c45","c46","c47","c48","c49","c50","c51","c52","c53","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,c34,c35,c36,c37,c38,c39,c40,c41,c42,c43,c44,c45,c46,c47,c48,c49,c50,c51,c52,c53,b from part_change_various_various_boolean_to_bigint
@@ -514,7 +514,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=6 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35"]
- TableScan [TS_0] (rows=6 width=427)
+ TableScan [TS_0] (rows=6 width=455)
default@part_change_various_various_decimal_to_double,part_change_various_various_decimal_to_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","c21","c22","c23","c24","c25","c26","c27","c28","c29","c30","c31","c32","c33","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28,c29,c30,c31,c32,c33,b from part_change_various_various_decimal_to_double
@@ -645,7 +645,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=6 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14"]
- TableScan [TS_0] (rows=6 width=145)
+ TableScan [TS_0] (rows=6 width=151)
default@part_change_various_various_timestamp,part_change_various_various_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,b from part_change_various_various_timestamp
@@ -760,7 +760,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=6 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"]
- TableScan [TS_0] (rows=6 width=62)
+ TableScan [TS_0] (rows=6 width=76)
default@part_change_various_various_date,part_change_various_various_date,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,b from part_change_various_various_date
@@ -956,7 +956,7 @@ Stage-0
File Output Operator [FS_4]
Select Operator [SEL_3] (rows=13 width=4)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8"]
- TableScan [TS_0] (rows=13 width=100)
+ TableScan [TS_0] (rows=13 width=109)
default@part_change_same_type_different_params,part_change_same_type_different_params,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","b"]
PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,b from part_change_same_type_different_params
http://git-wip-us.apache.org/repos/asf/hive/blob/b4ac43db/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out
index 7c9524e..5c59620 100644
--- a/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out
+++ b/ql/src/test/results/clientpositive/llap/schema_evol_text_vecrow_table.q.out
@@ -416,9 +416,9 @@ Stage-0
Stage-1
Map 1 vectorized, llap
File Output Operator [FS_4]
- Select Operator [SEL_3] (rows=5 width=52)
+ Select Operator [SEL_3] (rows=5 width=90)
Output:["_col0","_col1","_col2","_col3","_col4"]
- TableScan [TS_0] (rows=5 width=52)
+ TableScan [TS_0] (rows=5 width=90)
default@table_change_string_group_double,table_change_string_group_double,Tbl:COMPLETE,Col:NONE,Output:["insert_num","c1","c2","c3","b"]
PREHOOK: query: select insert_num,c1,c2,c3,b from table_change_string_group_double
@@ -1024,11 +1024,11 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@table_change_string_group_string_group_string
#### A masked pattern was here ####
insert_num c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 b
-101 freckled freckled freckled freckled freckled freckled freckled freckled freckled freckled original
-102 ox ox ox ox ox ox ox ox ox ox original
-103 original
-104 I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked original
-105 200 200 200 200 200 200 200 200 200 200 original
+101 freckled freckled freckled freckled freckled freckled freckled freckled freckled freckled original
+102 ox ox ox ox ox ox ox ox ox ox original
+103 original
+104 I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked I cooked original
+105 200 200 200 200 200 200 200 200 200 200 original
111 filler filler filler filler filler filler filler filler filler filler new
PREHOOK: query: drop table table_change_string_group_string_group_string
PREHOOK: type: DROPTABLE