You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/07/09 11:07:43 UTC
svn commit: r1501145 [12/13] - in /hive/branches/vectorization: ./
beeline/src/java/org/apache/hive/beeline/
beeline/src/test/org/apache/hive/beeline/src/test/
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/common/metri...
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoin_union_remove_1.q.out Tue Jul 9 09:07:35 2013
@@ -124,14 +124,12 @@ STAGE PLANS:
expr: _col5
type: string
outputColumnNames: _col0, _col1, _col2, _col3
- Select Operator
- SELECT * : (no compute)
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Stage: Stage-3
Map Reduce
@@ -198,14 +196,12 @@ STAGE PLANS:
expr: _col5
type: string
outputColumnNames: _col0, _col1, _col2, _col3
- Select Operator
- SELECT * : (no compute)
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Stage: Stage-0
Fetch Operator
@@ -314,14 +310,12 @@ STAGE PLANS:
expr: _col5
type: string
outputColumnNames: _col0, _col1, _col2, _col3
- Select Operator
- SELECT * : (no compute)
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Stage: Stage-3
Map Reduce
@@ -388,14 +382,12 @@ STAGE PLANS:
expr: _col5
type: string
outputColumnNames: _col0, _col1, _col2, _col3
- Select Operator
- SELECT * : (no compute)
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Stage: Stage-0
Fetch Operator
@@ -509,16 +501,14 @@ STAGE PLANS:
expr: _col5
type: string
outputColumnNames: _col0, _col1, _col2, _col3
- Select Operator
- SELECT * : (no compute)
- File Output Operator
- compressed: false
- GlobalTableId: 1
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.dest1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
Stage: Stage-0
Move Operator
@@ -595,16 +585,14 @@ STAGE PLANS:
expr: _col5
type: string
outputColumnNames: _col0, _col1, _col2, _col3
- Select Operator
- SELECT * : (no compute)
- File Output Operator
- compressed: false
- GlobalTableId: 1
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.dest1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
PREHOOK: query: INSERT OVERWRITE TABLE DEST1
@@ -729,16 +717,14 @@ STAGE PLANS:
expr: _col5
type: string
outputColumnNames: _col0, _col1, _col2, _col3
- Select Operator
- SELECT * : (no compute)
- File Output Operator
- compressed: false
- GlobalTableId: 1
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.dest1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
Stage: Stage-0
Move Operator
@@ -815,16 +801,14 @@ STAGE PLANS:
expr: _col5
type: string
outputColumnNames: _col0, _col1, _col2, _col3
- Select Operator
- SELECT * : (no compute)
- File Output Operator
- compressed: false
- GlobalTableId: 1
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.dest1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
PREHOOK: query: INSERT OVERWRITE TABLE DEST1
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/skewjoin_union_remove_2.q.out Tue Jul 9 09:07:35 2013
@@ -158,14 +158,12 @@ STAGE PLANS:
expr: _col9
type: string
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Select Operator
- SELECT * : (no compute)
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Stage: Stage-4
Map Reduce
@@ -259,14 +257,12 @@ STAGE PLANS:
expr: _col9
type: string
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Select Operator
- SELECT * : (no compute)
- File Output Operator
- compressed: false
- GlobalTableId: 0
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Stage: Stage-0
Fetch Operator
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin9.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin9.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin9.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin9.q.out Tue Jul 9 09:07:35 2013
@@ -78,6 +78,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3
columns.types int:string:string:int
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -173,6 +174,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3
columns.types int:string:string:int
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_13.q.out Tue Jul 9 09:07:35 2013
@@ -191,6 +191,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3
columns.types int:string:int:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -392,6 +393,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3
columns.types int:string:int:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/smb_mapjoin_15.q.out Tue Jul 9 09:07:35 2013
@@ -165,6 +165,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3
columns.types int:string:int:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -414,6 +415,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3,_col4,_col5
columns.types int:int:string:int:int:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -603,6 +605,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3,_col4,_col5
columns.types int:int:string:int:int:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -824,6 +827,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3,_col4,_col5
columns.types int:int:string:int:int:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out Tue Jul 9 09:07:35 2013
@@ -182,6 +182,7 @@ STAGE PLANS:
columns _col0
columns.types bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out Tue Jul 9 09:07:35 2013
@@ -214,6 +214,7 @@ STAGE PLANS:
columns _col0
columns.types bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out Tue Jul 9 09:07:35 2013
@@ -316,6 +316,7 @@ STAGE PLANS:
columns _col0
columns.types bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/transform_ppr1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/transform_ppr1.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/transform_ppr1.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/transform_ppr1.q.out Tue Jul 9 09:07:35 2013
@@ -281,6 +281,7 @@ STAGE PLANS:
columns _col0,_col1
columns.types string:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/transform_ppr2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/transform_ppr2.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/transform_ppr2.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/transform_ppr2.q.out Tue Jul 9 09:07:35 2013
@@ -191,6 +191,7 @@ STAGE PLANS:
columns _col0,_col1
columns.types string:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column.q.out Tue Jul 9 09:07:35 2013
@@ -55,26 +55,26 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: SELECT * FROM test_tab
+PREHOOK: query: SELECT * FROM test_tab ORDER BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@test_tab
#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM test_tab
+POSTHOOK: query: SELECT * FROM test_tab ORDER BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_tab
#### A masked pattern was here ####
POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-238 val_238
-86 val_86
-311 val_311
-27 val_27
165 val_165
-409 val_409
+238 val_238
255 val_255
+27 val_27
278 val_278
-98 val_98
+311 val_311
+409 val_409
484 val_484
+86 val_86
+98 val_98
PREHOOK: query: -- Truncate 1 column
TRUNCATE TABLE test_tab COLUMNS (key)
PREHOOK: type: TRUNCATETABLE
@@ -124,27 +124,27 @@ Sort Columns: []
Storage Desc Params:
serialization.format 1
PREHOOK: query: -- First column should be null
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@test_tab
#### A masked pattern was here ####
POSTHOOK: query: -- First column should be null
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_tab
#### A masked pattern was here ####
POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-NULL val_238
-NULL val_86
-NULL val_311
-NULL val_27
NULL val_165
-NULL val_409
+NULL val_238
NULL val_255
+NULL val_27
NULL val_278
-NULL val_98
+NULL val_311
+NULL val_409
NULL val_484
+NULL val_86
+NULL val_98
PREHOOK: query: -- Truncate multiple columns
INSERT OVERWRITE TABLE test_tab SELECT * FROM src LIMIT 10
PREHOOK: type: QUERY
@@ -210,12 +210,12 @@ Sort Columns: []
Storage Desc Params:
serialization.format 1
PREHOOK: query: -- Both columns should be null
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@test_tab
#### A masked pattern was here ####
POSTHOOK: query: -- Both columns should be null
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_tab
#### A masked pattern was here ####
@@ -286,12 +286,12 @@ Sort Columns: []
Storage Desc Params:
serialization.format 1
PREHOOK: query: -- Both columns should be null
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@test_tab
#### A masked pattern was here ####
POSTHOOK: query: -- Both columns should be null
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_tab
#### A masked pattern was here ####
@@ -378,11 +378,11 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: SELECT * FROM test_tab
+PREHOOK: query: SELECT * FROM test_tab ORDER BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@test_tab
#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM test_tab
+POSTHOOK: query: SELECT * FROM test_tab ORDER BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_tab
#### A masked pattern was here ####
@@ -392,16 +392,16 @@ POSTHOOK: Lineage: test_tab.key SIMPLE [
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-238 val_238
-86 val_86
-311 val_311
-27 val_27
165 val_165
-409 val_409
+238 val_238
255 val_255
+27 val_27
278 val_278
-98 val_98
+311 val_311
+409 val_409
484 val_484
+86 val_86
+98 val_98
PREHOOK: query: -- Truncate 1 column
TRUNCATE TABLE test_tab COLUMNS (key)
PREHOOK: type: TRUNCATETABLE
@@ -460,12 +460,12 @@ Sort Columns: []
Storage Desc Params:
serialization.format 1
PREHOOK: query: -- First column should be null
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@test_tab
#### A masked pattern was here ####
POSTHOOK: query: -- First column should be null
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_tab
#### A masked pattern was here ####
@@ -475,16 +475,16 @@ POSTHOOK: Lineage: test_tab.key SIMPLE [
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-NULL val_238
-NULL val_86
-NULL val_311
-NULL val_27
NULL val_165
-NULL val_409
+NULL val_238
NULL val_255
+NULL val_27
NULL val_278
-NULL val_98
+NULL val_311
+NULL val_409
NULL val_484
+NULL val_86
+NULL val_98
PREHOOK: query: -- Truncate 2 columns
TRUNCATE TABLE test_tab COLUMNS (key, value)
PREHOOK: type: TRUNCATETABLE
@@ -543,12 +543,12 @@ Sort Columns: []
Storage Desc Params:
serialization.format 1
PREHOOK: query: -- Both columns should be null
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@test_tab
#### A masked pattern was here ####
POSTHOOK: query: -- Both columns should be null
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_tab
#### A masked pattern was here ####
@@ -643,12 +643,12 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
-PREHOOK: query: SELECT * FROM test_tab_part WHERE part = '1'
+PREHOOK: query: SELECT * FROM test_tab_part WHERE part = '1' ORDER BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@test_tab_part
PREHOOK: Input: default@test_tab_part@part=1
#### A masked pattern was here ####
-POSTHOOK: query: SELECT * FROM test_tab_part WHERE part = '1'
+POSTHOOK: query: SELECT * FROM test_tab_part WHERE part = '1' ORDER BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_tab_part
POSTHOOK: Input: default@test_tab_part@part=1
@@ -661,16 +661,16 @@ POSTHOOK: Lineage: test_tab.value SIMPLE
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab_part PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab_part PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-238 val_238 1
-86 val_86 1
-311 val_311 1
-27 val_27 1
165 val_165 1
-409 val_409 1
+238 val_238 1
255 val_255 1
+27 val_27 1
278 val_278 1
-98 val_98 1
+311 val_311 1
+409 val_409 1
484 val_484 1
+86 val_86 1
+98 val_98 1
PREHOOK: query: TRUNCATE TABLE test_tab_part PARTITION (part = '1') COLUMNS (key)
PREHOOK: type: TRUNCATETABLE
PREHOOK: Input: default@test_tab_part
@@ -734,13 +734,13 @@ Sort Columns: []
Storage Desc Params:
serialization.format 1
PREHOOK: query: -- First column should be null
-SELECT * FROM test_tab_part WHERE part = '1'
+SELECT * FROM test_tab_part WHERE part = '1' ORDER BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@test_tab_part
PREHOOK: Input: default@test_tab_part@part=1
#### A masked pattern was here ####
POSTHOOK: query: -- First column should be null
-SELECT * FROM test_tab_part WHERE part = '1'
+SELECT * FROM test_tab_part WHERE part = '1' ORDER BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_tab_part
POSTHOOK: Input: default@test_tab_part@part=1
@@ -753,13 +753,13 @@ POSTHOOK: Lineage: test_tab.value SIMPLE
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab_part PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab_part PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-NULL val_238 1
-NULL val_86 1
-NULL val_311 1
-NULL val_27 1
NULL val_165 1
-NULL val_409 1
+NULL val_238 1
NULL val_255 1
+NULL val_27 1
NULL val_278 1
-NULL val_98 1
+NULL val_311 1
+NULL val_409 1
NULL val_484 1
+NULL val_86 1
+NULL val_98 1
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column_list_bucket.q.out Tue Jul 9 09:07:35 2013
@@ -112,6 +112,7 @@ STAGE PLANS:
columns _col0,_col1,_col2
columns.types string:string:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -234,6 +235,7 @@ STAGE PLANS:
columns _col0,_col1,_col2
columns.types string:string:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column_merge.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column_merge.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column_merge.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/truncate_column_merge.q.out Tue Jul 9 09:07:35 2013
@@ -69,12 +69,12 @@ POSTHOOK: Lineage: test_tab.key SIMPLE [
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: -- The first column (key) should be null for all 10 rows
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
PREHOOK: type: QUERY
PREHOOK: Input: default@test_tab
#### A masked pattern was here ####
POSTHOOK: query: -- The first column (key) should be null for all 10 rows
-SELECT * FROM test_tab
+SELECT * FROM test_tab ORDER BY value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@test_tab
#### A masked pattern was here ####
@@ -82,16 +82,16 @@ POSTHOOK: Lineage: test_tab.key SIMPLE [
POSTHOOK: Lineage: test_tab.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: test_tab.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-NULL val_238
-NULL val_86
-NULL val_311
-NULL val_27
+NULL val_165
NULL val_165
NULL val_238
-NULL val_86
-NULL val_311
+NULL val_238
NULL val_27
-NULL val_165
+NULL val_27
+NULL val_311
+NULL val_311
+NULL val_86
+NULL val_86
PREHOOK: query: -- The value should be 1 indicating the table has 1 file
SELECT COUNT(DISTINCT INPUT__FILE__NAME) FROM test_tab
PREHOOK: type: QUERY
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/udf_explode.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/udf_explode.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/udf_explode.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/udf_explode.q.out Tue Jul 9 09:07:35 2013
@@ -48,6 +48,7 @@ STAGE PLANS:
columns col
columns.types int
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -287,6 +288,7 @@ STAGE PLANS:
columns _col0,_col1
columns.types int:bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -372,6 +374,7 @@ STAGE PLANS:
columns key,value
columns.types int:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -625,6 +628,7 @@ STAGE PLANS:
columns _col0,_col1,_col2
columns.types int:string:bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/udf_inline.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/udf_inline.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/udf_inline.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/udf_inline.q.out Tue Jul 9 09:07:35 2013
@@ -75,3 +75,28 @@ POSTHOOK: Input: default@src
#### A masked pattern was here ####
1 dude!
2 Wheres
+PREHOOK: query: -- HIVE-3475 INLINE UDTF doesn't convert types properly
+select * from (SELECT
+ ARRAY(
+ STRUCT (1,'dude!'),
+ STRUCT (2,'Wheres'),
+ STRUCT (3,'my car?')
+ ) as value FROM SRC limit 1) input
+ LATERAL VIEW inline(value) myTable AS id, text
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: -- HIVE-3475 INLINE UDTF doesn't convert types properly
+select * from (SELECT
+ ARRAY(
+ STRUCT (1,'dude!'),
+ STRUCT (2,'Wheres'),
+ STRUCT (3,'my car?')
+ ) as value FROM SRC limit 1) input
+ LATERAL VIEW inline(value) myTable AS id, text
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+[{"col1":1,"col2":"dude!"},{"col1":2,"col2":"Wheres"},{"col1":3,"col2":"my car?"}] 1 dude!
+[{"col1":1,"col2":"dude!"},{"col1":2,"col2":"Wheres"},{"col1":3,"col2":"my car?"}] 2 Wheres
+[{"col1":1,"col2":"dude!"},{"col1":2,"col2":"Wheres"},{"col1":3,"col2":"my car?"}] 3 my car?
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/udf_java_method.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/udf_java_method.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/udf_java_method.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/udf_java_method.q.out Tue Jul 9 09:07:35 2013
@@ -81,6 +81,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3,_col4,_col5,_col6
columns.types string:string:string:string:string:string:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/udf_reflect.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/udf_reflect.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/udf_reflect.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/udf_reflect.q.out Tue Jul 9 09:07:35 2013
@@ -81,6 +81,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7
columns.types string:string:string:string:string:string:string:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/udf_reflect2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/udf_reflect2.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/udf_reflect2.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/udf_reflect2.q.out Tue Jul 9 09:07:35 2013
@@ -181,6 +181,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13,_col14,_col15,_col16,_col17,_col18,_col19,_col20,_col21,_col22,_col23,_col24,_col25,_col26,_col27,_col28,_col29,_col30
columns.types int:tinyint:smallint:int:bigint:float:double:string:string:string:boolean:boolean:boolean:boolean:boolean:binary:int:int:string:string:string:string:string:timestamp:int:int:int:int:int:int:bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/udtf_explode.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/udtf_explode.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/udtf_explode.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/udtf_explode.q.out Tue Jul 9 09:07:35 2013
@@ -48,6 +48,7 @@ STAGE PLANS:
columns col
columns.types int
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -287,6 +288,7 @@ STAGE PLANS:
columns _col0,_col1
columns.types int:bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -565,6 +567,7 @@ STAGE PLANS:
columns _col0,_col1,_col2
columns.types int:string:bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/union24.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/union24.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/union24.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/union24.q.out Tue Jul 9 09:07:35 2013
@@ -477,6 +477,7 @@ STAGE PLANS:
columns _col0,_col1
columns.types string:bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -965,6 +966,7 @@ STAGE PLANS:
columns _col0,_col1
columns.types string:bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
@@ -1518,6 +1520,7 @@ STAGE PLANS:
columns _col0,_col1
columns.types string:bigint
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/union31.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/union31.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/union31.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/union31.q.out Tue Jul 9 09:07:35 2013
@@ -965,7 +965,7 @@ STAGE PLANS:
Forward
Group By Operator
aggregations:
- expr: count(VALUE._col0)
+ expr: count(1)
bucketGroup: false
keys:
expr: KEY._col0
@@ -989,7 +989,7 @@ STAGE PLANS:
name: default.t7
Group By Operator
aggregations:
- expr: count(VALUE._col0)
+ expr: count(1)
bucketGroup: false
keys:
expr: KEY._col0
Modified: hive/branches/vectorization/ql/src/test/results/clientpositive/union_ppr.q.out
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/clientpositive/union_ppr.q.out?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/clientpositive/union_ppr.q.out (original)
+++ hive/branches/vectorization/ql/src/test/results/clientpositive/union_ppr.q.out Tue Jul 9 09:07:35 2013
@@ -244,6 +244,7 @@ STAGE PLANS:
columns _col0,_col1,_col2,_col3
columns.types string:string:string:string
escape.delim \
+ hive.serialization.extend.nesting.levels true
serialization.format 1
TotalFiles: 1
GatherStats: false
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/cast1.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/cast1.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/cast1.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/cast1.q.xml Tue Jul 9 09:07:35 2013
@@ -201,6 +201,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3,_col4,_col5,_col6</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby2.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby2.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby2.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby2.q.xml Tue Jul 9 09:07:35 2013
@@ -1341,6 +1341,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby3.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby3.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby3.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby3.q.xml Tue Jul 9 09:07:35 2013
@@ -1540,6 +1540,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3,_col4</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby4.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby4.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby4.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby4.q.xml Tue Jul 9 09:07:35 2013
@@ -987,6 +987,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby5.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby5.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby5.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby5.q.xml Tue Jul 9 09:07:35 2013
@@ -1080,6 +1080,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby6.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby6.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby6.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/groupby6.q.xml Tue Jul 9 09:07:35 2013
@@ -987,6 +987,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/input20.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/input20.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/input20.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/input20.q.xml Tue Jul 9 09:07:35 2013
@@ -1099,6 +1099,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/input8.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/input8.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/input8.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/input8.q.xml Tue Jul 9 09:07:35 2013
@@ -197,6 +197,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/input_part1.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/input_part1.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/input_part1.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/input_part1.q.xml Tue Jul 9 09:07:35 2013
@@ -238,6 +238,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/input_testxpath.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/input_testxpath.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/input_testxpath.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/input_testxpath.q.xml Tue Jul 9 09:07:35 2013
@@ -205,6 +205,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/input_testxpath2.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/input_testxpath2.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/input_testxpath2.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/input_testxpath2.q.xml Tue Jul 9 09:07:35 2013
@@ -209,6 +209,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/join4.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/join4.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/join4.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/join4.q.xml Tue Jul 9 09:07:35 2013
@@ -1734,6 +1734,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/join5.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/join5.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/join5.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/join5.q.xml Tue Jul 9 09:07:35 2013
@@ -1734,6 +1734,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/join6.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/join6.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/join6.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/join6.q.xml Tue Jul 9 09:07:35 2013
@@ -1734,6 +1734,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/join7.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/join7.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/join7.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/join7.q.xml Tue Jul 9 09:07:35 2013
@@ -2468,6 +2468,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3,_col4,_col5</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/join8.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/join8.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/join8.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/join8.q.xml Tue Jul 9 09:07:35 2013
@@ -1820,6 +1820,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/sample1.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/sample1.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/sample1.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/sample1.q.xml Tue Jul 9 09:07:35 2013
@@ -238,6 +238,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/udf1.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/udf1.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/udf1.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/udf1.q.xml Tue Jul 9 09:07:35 2013
@@ -201,6 +201,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13,_col14,_col15,_col16</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/udf4.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/udf4.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/udf4.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/udf4.q.xml Tue Jul 9 09:07:35 2013
@@ -177,6 +177,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1,_col2,_col3,_col4,_col5,_col6,_col7,_col8,_col9,_col10,_col11,_col12,_col13,_col14,_col15,_col16,_col17,_col18</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/udf6.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/udf6.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/udf6.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/udf6.q.xml Tue Jul 9 09:07:35 2013
@@ -197,6 +197,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/udf_case.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/udf_case.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/udf_case.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/udf_case.q.xml Tue Jul 9 09:07:35 2013
@@ -201,6 +201,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/results/compiler/plan/udf_when.q.xml
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/results/compiler/plan/udf_when.q.xml?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/results/compiler/plan/udf_when.q.xml (original)
+++ hive/branches/vectorization/ql/src/test/results/compiler/plan/udf_when.q.xml Tue Jul 9 09:07:35 2013
@@ -201,6 +201,10 @@
<void property="properties">
<object class="java.util.Properties">
<void method="put">
+ <string>hive.serialization.extend.nesting.levels</string>
+ <string>true</string>
+ </void>
+ <void method="put">
<string>columns</string>
<string>_col0,_col1</string>
</void>
Modified: hive/branches/vectorization/ql/src/test/templates/TestNegativeCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/templates/TestNegativeCliDriver.vm?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/templates/TestNegativeCliDriver.vm (original)
+++ hive/branches/vectorization/ql/src/test/templates/TestNegativeCliDriver.vm Tue Jul 9 09:07:35 2013
@@ -129,7 +129,7 @@ public class $className extends TestCase
qt.cliInit(fname, false);
int ecode = qt.executeClient(fname);
if (ecode == 0) {
- fail("Client Execution failed with error code = " + ecode
+ fail("Client Execution was expected to fail, but succeeded with error code 0 "
+ debugHint);
}
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritable.java Tue Jul 9 09:07:35 2013
@@ -62,8 +62,13 @@ public class TimestampWritable implement
private static final int NO_DECIMAL_MASK = 0x7FFFFFFF;
private static final int HAS_DECIMAL_MASK = 0x80000000;
- private static final DateFormat dateFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ private static final ThreadLocal<DateFormat> threadLocalDateFormat =
+ new ThreadLocal<DateFormat>() {
+ @Override
+ protected synchronized DateFormat initialValue() {
+ return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ }
+ };
private Timestamp timestamp = new Timestamp(0);
@@ -325,13 +330,13 @@ public class TimestampWritable implement
if (timestampString.length() > 19) {
if (timestampString.length() == 21) {
if (timestampString.substring(19).compareTo(".0") == 0) {
- return dateFormat.format(timestamp);
+ return threadLocalDateFormat.get().format(timestamp);
}
}
- return dateFormat.format(timestamp) + timestampString.substring(19);
+ return threadLocalDateFormat.get().format(timestamp) + timestampString.substring(19);
}
- return dateFormat.format(timestamp);
+ return threadLocalDateFormat.get().format(timestamp);
}
@Override
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java Tue Jul 9 09:07:35 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.serde2.la
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyListObjectInspector;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory;
@@ -38,6 +39,7 @@ import org.apache.hadoop.hive.serde2.laz
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyShortObjectInspector;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyTimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyVoidObjectInspector;
import org.apache.hadoop.hive.serde2.lazydio.LazyDioBoolean;
import org.apache.hadoop.hive.serde2.lazydio.LazyDioByte;
import org.apache.hadoop.hive.serde2.lazydio.LazyDioDouble;
@@ -46,9 +48,9 @@ import org.apache.hadoop.hive.serde2.laz
import org.apache.hadoop.hive.serde2.lazydio.LazyDioLong;
import org.apache.hadoop.hive.serde2.lazydio.LazyDioShort;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
@@ -115,6 +117,8 @@ public final class LazyFactory {
return new LazyBinary((LazyBinaryObjectInspector) oi);
case DECIMAL:
return new LazyHiveDecimal((LazyHiveDecimalObjectInspector) oi);
+ case VOID:
+ return new LazyVoid((LazyVoidObjectInspector) oi);
default:
throw new RuntimeException("Internal error: no LazyObject for " + p);
}
@@ -200,10 +204,11 @@ public final class LazyFactory {
* @param nullSequence
* The sequence of bytes representing NULL.
* @return The ObjectInspector
+ * @throws SerDeException
*/
public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo,
byte[] separator, int separatorIndex, Text nullSequence, boolean escaped,
- byte escapeChar) {
+ byte escapeChar) throws SerDeException {
ObjectInspector.Category c = typeInfo.getCategory();
switch (c) {
case PRIMITIVE:
@@ -217,13 +222,14 @@ public final class LazyFactory {
nullSequence, escaped, escapeChar), createLazyObjectInspector(
((MapTypeInfo) typeInfo).getMapValueTypeInfo(), separator,
separatorIndex + 2, nullSequence, escaped, escapeChar),
- separator[separatorIndex], separator[separatorIndex + 1],
+ LazyUtils.getSeparator(separator, separatorIndex),
+ LazyUtils.getSeparator(separator, separatorIndex+1),
nullSequence, escaped, escapeChar);
case LIST:
return LazyObjectInspectorFactory.getLazySimpleListObjectInspector(
createLazyObjectInspector(((ListTypeInfo) typeInfo)
.getListElementTypeInfo(), separator, separatorIndex + 1,
- nullSequence, escaped, escapeChar), separator[separatorIndex],
+ nullSequence, escaped, escapeChar), LazyUtils.getSeparator(separator, separatorIndex),
nullSequence, escaped, escapeChar);
case STRUCT:
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
@@ -238,7 +244,8 @@ public final class LazyFactory {
escapeChar));
}
return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
- fieldNames, fieldObjectInspectors, separator[separatorIndex],
+ fieldNames, fieldObjectInspectors,
+ LazyUtils.getSeparator(separator, separatorIndex),
nullSequence, false, escaped, escapeChar);
case UNION:
UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
@@ -249,7 +256,8 @@ public final class LazyFactory {
escapeChar));
}
return LazyObjectInspectorFactory.getLazyUnionObjectInspector(lazyOIs,
- separator[separatorIndex], nullSequence, escaped, escapeChar);
+ LazyUtils.getSeparator(separator, separatorIndex),
+ nullSequence, escaped, escapeChar);
}
throw new RuntimeException("Hive LazySerDe Internal error.");
@@ -262,13 +270,14 @@ public final class LazyFactory {
* @param lastColumnTakesRest
* whether the last column of the struct should take the rest of the
* row if there are extra fields.
+ * @throws SerDeException
* @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text,
* boolean, byte)
*/
public static ObjectInspector createLazyStructInspector(
List<String> columnNames, List<TypeInfo> typeInfos, byte[] separators,
Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
- byte escapeChar) {
+ byte escapeChar) throws SerDeException {
ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>(
typeInfos.size());
for (int i = 0; i < typeInfos.size(); i++) {
@@ -283,13 +292,14 @@ public final class LazyFactory {
/**
* Create a hierarchical ObjectInspector for ColumnarStruct with the given
* columnNames and columnTypeInfos.
+ * @throws SerDeException
*
* @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text,
* boolean, byte)
*/
public static ObjectInspector createColumnarStructInspector(
List<String> columnNames, List<TypeInfo> columnTypes, byte[] separators,
- Text nullSequence, boolean escaped, byte escapeChar) {
+ Text nullSequence, boolean escaped, byte escapeChar) throws SerDeException {
ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>(
columnTypes.size());
for (int i = 0; i < columnTypes.size(); i++) {
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java Tue Jul 9 09:07:35 2013
@@ -65,6 +65,9 @@ public class LazySimpleSerDe extends Abs
public static final Log LOG = LogFactory.getLog(LazySimpleSerDe.class
.getName());
+ public static final String SERIALIZATION_EXTEND_NESTING_LEVELS
+ = "hive.serialization.extend.nesting.levels";
+
public static final byte[] DefaultSeparators = {(byte) 1, (byte) 2, (byte) 3};
private ObjectInspector cachedObjectInspector;
@@ -177,6 +180,7 @@ public class LazySimpleSerDe extends Abs
*
* @see SerDe#initialize(Configuration, Properties)
*/
+ @Override
public void initialize(Configuration job, Properties tbl)
throws SerDeException {
@@ -209,9 +213,13 @@ public class LazySimpleSerDe extends Abs
public static SerDeParameters initSerdeParams(Configuration job,
Properties tbl, String serdeName) throws SerDeException {
SerDeParameters serdeParams = new SerDeParameters();
- // Read the separators: We use 8 levels of separators by default, but we
- // should change this when we allow users to specify more than 10 levels
- // of separators through DDL.
+ // Read the separators: We use 8 levels of separators by default,
+ // and 24 if SERIALIZATION_EXTEND_NESTING_LEVELS is set to true
+ // The levels possible are the set of control chars that we can use as
+ // special delimiters, ie they should absent in the data or escaped.
+ // To increase this level further, we need to stop relying
+ // on single control chars delimiters
+
serdeParams.separators = new byte[8];
serdeParams.separators[0] = getByte(tbl.getProperty(serdeConstants.FIELD_DELIM,
tbl.getProperty(serdeConstants.SERIALIZATION_FORMAT)), DefaultSeparators[0]);
@@ -219,8 +227,53 @@ public class LazySimpleSerDe extends Abs
.getProperty(serdeConstants.COLLECTION_DELIM), DefaultSeparators[1]);
serdeParams.separators[2] = getByte(
tbl.getProperty(serdeConstants.MAPKEY_DELIM), DefaultSeparators[2]);
- for (int i = 3; i < serdeParams.separators.length; i++) {
- serdeParams.separators[i] = (byte) (i + 1);
+ String extendedNesting =
+ tbl.getProperty(SERIALIZATION_EXTEND_NESTING_LEVELS);
+ if(extendedNesting == null || !extendedNesting.equalsIgnoreCase("true")){
+ //use the default smaller set of separators for backward compatibility
+ for (int i = 3; i < serdeParams.separators.length; i++) {
+ serdeParams.separators[i] = (byte) (i + 1);
+ }
+ }
+ else{
+ //If extended nesting is enabled, set the extended set of separator chars
+
+ final int MAX_CTRL_CHARS = 29;
+ byte[] extendedSeparators = new byte[MAX_CTRL_CHARS];
+ int extendedSeparatorsIdx = 0;
+
+ //get the first 3 separators that have already been set (defaults to 1,2,3)
+ for(int i = 0; i < 3; i++){
+ extendedSeparators[extendedSeparatorsIdx++] = serdeParams.separators[i];
+ }
+
+ for (byte asciival = 4; asciival <= MAX_CTRL_CHARS; asciival++) {
+
+ //use only control chars that are very unlikely to be part of the string
+ // the following might/likely to be used in text files for strings
+ // 9 (horizontal tab, HT, \t, ^I)
+ // 10 (line feed, LF, \n, ^J),
+ // 12 (form feed, FF, \f, ^L),
+ // 13 (carriage return, CR, \r, ^M),
+ // 27 (escape, ESC, \e [GCC only], ^[).
+
+ //reserving the following values for future dynamic level impl
+ // 30
+ // 31
+
+ switch(asciival){
+ case 9:
+ case 10:
+ case 12:
+ case 13:
+ case 27:
+ continue;
+ }
+ extendedSeparators[extendedSeparatorsIdx++] = asciival;
+ }
+
+ serdeParams.separators =
+ Arrays.copyOfRange(extendedSeparators, 0, extendedSeparatorsIdx);
}
serdeParams.nullString = tbl.getProperty(
@@ -272,6 +325,7 @@ public class LazySimpleSerDe extends Abs
* @return The deserialized row Object.
* @see SerDe#deserialize(Writable)
*/
+ @Override
public Object deserialize(Writable field) throws SerDeException {
if (byteArrayRef == null) {
byteArrayRef = new ByteArrayRef();
@@ -297,6 +351,7 @@ public class LazySimpleSerDe extends Abs
/**
* Returns the ObjectInspector for the row.
*/
+ @Override
public ObjectInspector getObjectInspector() throws SerDeException {
return cachedObjectInspector;
}
@@ -306,6 +361,7 @@ public class LazySimpleSerDe extends Abs
*
* @see SerDe#getSerializedClass()
*/
+ @Override
public Class<? extends Writable> getSerializedClass() {
return Text.class;
}
@@ -324,6 +380,7 @@ public class LazySimpleSerDe extends Abs
* @throws IOException
* @see SerDe#serialize(Object, ObjectInspector)
*/
+ @Override
public Writable serialize(Object obj, ObjectInspector objInspector)
throws SerDeException {
@@ -410,11 +467,12 @@ public class LazySimpleSerDe extends Abs
* 128. Negative byte values (or byte values >= 128) are never
* escaped.
* @throws IOException
+ * @throws SerDeException
*/
public static void serialize(ByteStream.Output out, Object obj,
ObjectInspector objInspector, byte[] separators, int level,
Text nullSequence, boolean escaped, byte escapeChar, boolean[] needsEscape)
- throws IOException {
+ throws IOException, SerDeException {
if (obj == null) {
out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
@@ -430,7 +488,7 @@ public class LazySimpleSerDe extends Abs
needsEscape);
return;
case LIST:
- separator = (char) separators[level];
+ separator = (char) LazyUtils.getSeparator(separators, level);
ListObjectInspector loi = (ListObjectInspector) objInspector;
list = loi.getList(obj);
ObjectInspector eoi = loi.getListElementObjectInspector();
@@ -447,8 +505,10 @@ public class LazySimpleSerDe extends Abs
}
return;
case MAP:
- separator = (char) separators[level];
- char keyValueSeparator = (char) separators[level + 1];
+ separator = (char) LazyUtils.getSeparator(separators, level);
+ char keyValueSeparator =
+ (char) LazyUtils.getSeparator(separators, level + 1);
+
MapObjectInspector moi = (MapObjectInspector) objInspector;
ObjectInspector koi = moi.getMapKeyObjectInspector();
ObjectInspector voi = moi.getMapValueObjectInspector();
@@ -472,7 +532,7 @@ public class LazySimpleSerDe extends Abs
}
return;
case STRUCT:
- separator = (char) separators[level];
+ separator = (char) LazyUtils.getSeparator(separators, level);
StructObjectInspector soi = (StructObjectInspector) objInspector;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
list = soi.getStructFieldsDataAsList(obj);
@@ -490,7 +550,7 @@ public class LazySimpleSerDe extends Abs
}
return;
case UNION:
- separator = (char) separators[level];
+ separator = (char) LazyUtils.getSeparator(separators, level);
UnionObjectInspector uoi = (UnionObjectInspector) objInspector;
List<? extends ObjectInspector> ois = uoi.getObjectInspectors();
if (ois == null) {
@@ -517,6 +577,7 @@ public class LazySimpleSerDe extends Abs
* Returns the statistics after (de)serialization)
*/
+ @Override
public SerDeStats getSerDeStats() {
// must be different
assert (lastOperationSerialize != lastOperationDeserialize);
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java Tue Jul 9 09:07:35 2013
@@ -73,6 +73,7 @@ public class LazyTimestamp extends LazyP
} else {
try {
t = Timestamp.valueOf(s);
+ isNull = false;
} catch (IllegalArgumentException e) {
isNull = true;
logExceptionMessage(bytes, start, length, "TIMESTAMP");
Modified: hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (original)
+++ hive/branches/vectorization/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java Tue Jul 9 09:07:35 2013
@@ -367,6 +367,30 @@ public final class LazyUtils {
return Arrays.copyOf(sourceBw.getBytes(), sourceBw.getLength());
}
+ /**
+ * Utility function to get separator for current level used in serialization.
+ * Used to get a better log message when out of bound lookup happens
+ * @param separators - array of separators byte, byte at index x indicates
+ * separator used at that level
+ * @param level - nesting level
+ * @return separator at given level
+ * @throws SerDeException
+ */
+ static byte getSeparator(byte[] separators, int level) throws SerDeException {
+ try{
+ return separators[level];
+ }catch(ArrayIndexOutOfBoundsException e){
+ String msg = "Number of levels of nesting supported for " +
+ "LazySimpleSerde is " + (separators.length - 1) +
+ " Unable to work with level " + level;
+ if(separators.length < 9){
+ msg += ". Use " + LazySimpleSerDe.SERIALIZATION_EXTEND_NESTING_LEVELS +
+ " serde property for tables using LazySimpleSerde.";
+ }
+ throw new SerDeException(msg, e);
+ }
+ }
+
private LazyUtils() {
// prevent instantiation
}