You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2017/09/27 23:54:33 UTC
hive git commit: HIVE-17611: Add new LazyBinary SerDe for faster
writes (Jason Dere, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master 3cec82eaa -> f6b4f9e89
HIVE-17611: Add new LazyBinary SerDe for faster writes (Jason Dere, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f6b4f9e8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f6b4f9e8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f6b4f9e8
Branch: refs/heads/master
Commit: f6b4f9e89224f80678b9d98aca34da3a42aed9f3
Parents: 3cec82e
Author: Jason Dere <jd...@hortonworks.com>
Authored: Wed Sep 27 16:53:46 2017 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Wed Sep 27 16:53:46 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 3 +-
.../queries/clientpositive/input_lazyserde2.q | 53 ++
.../clientpositive/input_lazyserde2.q.out | 844 +++++++++++++++++++
.../hive/serde2/lazybinary/LazyBinarySerDe.java | 14 +-
.../serde2/lazybinary/LazyBinarySerDe2.java | 623 ++++++++++++++
.../serde2/lazybinary/TestLazyBinaryFast.java | 7 +-
.../serde2/lazybinary/TestLazyBinarySerDe.java | 2 +-
.../serde2/lazybinary/TestLazyBinarySerDe2.java | 56 ++
8 files changed, 1590 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f6b4f9e8/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 4814fcd..551aafc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -221,6 +221,7 @@ import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2;
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
@@ -7170,7 +7171,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
fileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
Class<? extends Deserializer> serdeClass = LazySimpleSerDe.class;
if (fileFormat.equals(PlanUtils.LLAP_OUTPUT_FORMAT_KEY)) {
- serdeClass = LazyBinarySerDe.class;
+ serdeClass = LazyBinarySerDe2.class;
}
table_desc =
PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, fileFormat,
http://git-wip-us.apache.org/repos/asf/hive/blob/f6b4f9e8/ql/src/test/queries/clientpositive/input_lazyserde2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/input_lazyserde2.q b/ql/src/test/queries/clientpositive/input_lazyserde2.q
new file mode 100644
index 0000000..0f58907
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/input_lazyserde2.q
@@ -0,0 +1,53 @@
+-- SORT_QUERY_RESULTS
+
+DROP TABLE dest1;
+CREATE TABLE dest1(a array<int>, b array<string>, c map<string,string>, d int, e string)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '1'
+COLLECTION ITEMS TERMINATED BY '2'
+MAP KEYS TERMINATED BY '3'
+LINES TERMINATED BY '10'
+STORED AS TEXTFILE;
+
+EXPLAIN
+FROM src_thrift
+INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1;
+
+FROM src_thrift
+INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1;
+
+SELECT dest1.* FROM dest1 CLUSTER BY 1;
+
+SELECT dest1.a[0], dest1.b[0], dest1.c['key2'], dest1.d, dest1.e FROM dest1 CLUSTER BY 1;
+
+DROP TABLE dest1;
+
+CREATE TABLE dest1(a array<int>) ROW FORMAT DELIMITED FIELDS TERMINATED BY '1' ESCAPED BY '\\';
+INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint FROM src_thrift DISTRIBUTE BY 1;
+SELECT * from dest1;
+DROP TABLE dest1;
+
+CREATE TABLE dest1(a map<string,string>) ROW FORMAT DELIMITED FIELDS TERMINATED BY '1' ESCAPED BY '\\';
+INSERT OVERWRITE TABLE dest1 SELECT src_thrift.mstringstring FROM src_thrift DISTRIBUTE BY 1;
+SELECT * from dest1;
+
+CREATE TABLE destBin(a UNIONTYPE<int, double, array<string>, struct<col1:int,col2:string>>) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2' STORED AS SEQUENCEFILE;
+INSERT OVERWRITE TABLE destBin SELECT create_union( CASE WHEN key < 100 THEN 0 WHEN key < 200 THEN 1 WHEN key < 300 THEN 2 WHEN key < 400 THEN 3 ELSE 0 END, key, 2.0D, array("one","two"), struct(5,"five")) FROM srcbucket2;
+SELECT * from destBin;
+DROP TABLE destBin;
+
+DROP TABLE dest2;
+DROP TABLE dest3;
+
+CREATE TABLE dest2 (a map<string,map<string,map<string,uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>>>>)
+ ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2' STORED AS SEQUENCEFILE;
+INSERT OVERWRITE TABLE dest2 SELECT src_thrift.attributes FROM src_thrift;
+SELECT a from dest2 limit 10;
+
+CREATE TABLE dest3 (
+unionfield1 uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>,
+unionfield2 uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>,
+unionfield3 uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>
+) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2' STORED AS SEQUENCEFILE;
+INSERT OVERWRITE TABLE dest3 SELECT src_thrift.unionField1,src_thrift.unionField2,src_thrift.unionField3 from src_thrift;
+SELECT unionfield1, unionField2, unionfield3 from dest3 limit 10;
http://git-wip-us.apache.org/repos/asf/hive/blob/f6b4f9e8/ql/src/test/results/clientpositive/input_lazyserde2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input_lazyserde2.q.out b/ql/src/test/results/clientpositive/input_lazyserde2.q.out
new file mode 100644
index 0000000..82c72db
--- /dev/null
+++ b/ql/src/test/results/clientpositive/input_lazyserde2.q.out
@@ -0,0 +1,844 @@
+PREHOOK: query: DROP TABLE dest1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE dest1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE dest1(a array<int>, b array<string>, c map<string,string>, d int, e string)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '1'
+COLLECTION ITEMS TERMINATED BY '2'
+MAP KEYS TERMINATED BY '3'
+LINES TERMINATED BY '10'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest1
+POSTHOOK: query: CREATE TABLE dest1(a array<int>, b array<string>, c map<string,string>, d int, e string)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '1'
+COLLECTION ITEMS TERMINATED BY '2'
+MAP KEYS TERMINATED BY '3'
+LINES TERMINATED BY '10'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest1
+PREHOOK: query: EXPLAIN
+FROM src_thrift
+INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+FROM src_thrift
+INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src_thrift
+ Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: lint (type: array<int>), lstring (type: array<string>), mstringstring (type: map<string,string>), aint (type: int), astring (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Map-reduce partition columns: 1 (type: int)
+ Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: array<int>), _col1 (type: array<string>), _col2 (type: map<string,string>), _col3 (type: int), _col4 (type: string)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: array<int>), VALUE._col1 (type: array<string>), VALUE._col2 (type: map<string,string>), VALUE._col3 (type: int), VALUE._col4 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: FROM src_thrift
+INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_thrift
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM src_thrift
+INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_thrift
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>, comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array<string>, comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map<string,string>, comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ]
+POSTHOOK: Lineage: dest1.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ]
+PREHOOK: query: SELECT dest1.* FROM dest1 CLUSTER BY 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT dest1.* FROM dest1 CLUSTER BY 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+NULL NULL NULL 0 NULL
+[0,0,0] ["0","0","0"] {"key_0":"value_0"} 1712634731 record_0
+[1,2,3] ["10","100","1000"] {"key_1":"value_1"} 465985200 record_1
+[2,4,6] ["20","200","2000"] {"key_2":"value_2"} -751827638 record_2
+[3,6,9] ["30","300","3000"] {"key_3":"value_3"} 477111222 record_3
+[4,8,12] ["40","400","4000"] {"key_4":"value_4"} -734328909 record_4
+[5,10,15] ["50","500","5000"] {"key_5":"value_5"} -1952710710 record_5
+[6,12,18] ["60","600","6000"] {"key_6":"value_6"} 1244525190 record_6
+[7,14,21] ["70","700","7000"] {"key_7":"value_7"} -1461153973 record_7
+[8,16,24] ["80","800","8000"] {"key_8":"value_8"} 1638581578 record_8
+[9,18,27] ["90","900","9000"] {"key_9":"value_9"} 336964413 record_9
+PREHOOK: query: SELECT dest1.a[0], dest1.b[0], dest1.c['key2'], dest1.d, dest1.e FROM dest1 CLUSTER BY 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT dest1.a[0], dest1.b[0], dest1.c['key2'], dest1.d, dest1.e FROM dest1 CLUSTER BY 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+0 0 NULL 1712634731 record_0
+1 10 NULL 465985200 record_1
+2 20 NULL -751827638 record_2
+3 30 NULL 477111222 record_3
+4 40 NULL -734328909 record_4
+5 50 NULL -1952710710 record_5
+6 60 NULL 1244525190 record_6
+7 70 NULL -1461153973 record_7
+8 80 NULL 1638581578 record_8
+9 90 NULL 336964413 record_9
+NULL NULL NULL 0 NULL
+PREHOOK: query: DROP TABLE dest1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@dest1
+PREHOOK: Output: default@dest1
+POSTHOOK: query: DROP TABLE dest1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: default@dest1
+PREHOOK: query: CREATE TABLE dest1(a array<int>) ROW FORMAT DELIMITED FIELDS TERMINATED BY '1' ESCAPED BY '\\'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest1
+POSTHOOK: query: CREATE TABLE dest1(a array<int>) ROW FORMAT DELIMITED FIELDS TERMINATED BY '1' ESCAPED BY '\\'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest1
+PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint FROM src_thrift DISTRIBUTE BY 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_thrift
+PREHOOK: Output: default@dest1
+POSTHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint FROM src_thrift DISTRIBUTE BY 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_thrift
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array<int>, comment:from deserializer), ]
+PREHOOK: query: SELECT * from dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+NULL
+[0,0,0]
+[1,2,3]
+[2,4,6]
+[3,6,9]
+[4,8,12]
+[5,10,15]
+[6,12,18]
+[7,14,21]
+[8,16,24]
+[9,18,27]
+PREHOOK: query: DROP TABLE dest1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@dest1
+PREHOOK: Output: default@dest1
+POSTHOOK: query: DROP TABLE dest1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: default@dest1
+PREHOOK: query: CREATE TABLE dest1(a map<string,string>) ROW FORMAT DELIMITED FIELDS TERMINATED BY '1' ESCAPED BY '\\'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest1
+POSTHOOK: query: CREATE TABLE dest1(a map<string,string>) ROW FORMAT DELIMITED FIELDS TERMINATED BY '1' ESCAPED BY '\\'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest1
+PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src_thrift.mstringstring FROM src_thrift DISTRIBUTE BY 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_thrift
+PREHOOK: Output: default@dest1
+POSTHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT src_thrift.mstringstring FROM src_thrift DISTRIBUTE BY 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_thrift
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map<string,string>, comment:from deserializer), ]
+PREHOOK: query: SELECT * from dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+NULL
+{"key_0":"value_0"}
+{"key_1":"value_1"}
+{"key_2":"value_2"}
+{"key_3":"value_3"}
+{"key_4":"value_4"}
+{"key_5":"value_5"}
+{"key_6":"value_6"}
+{"key_7":"value_7"}
+{"key_8":"value_8"}
+{"key_9":"value_9"}
+PREHOOK: query: CREATE TABLE destBin(a UNIONTYPE<int, double, array<string>, struct<col1:int,col2:string>>) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2' STORED AS SEQUENCEFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@destBin
+POSTHOOK: query: CREATE TABLE destBin(a UNIONTYPE<int, double, array<string>, struct<col1:int,col2:string>>) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2' STORED AS SEQUENCEFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@destBin
+PREHOOK: query: INSERT OVERWRITE TABLE destBin SELECT create_union( CASE WHEN key < 100 THEN 0 WHEN key < 200 THEN 1 WHEN key < 300 THEN 2 WHEN key < 400 THEN 3 ELSE 0 END, key, 2.0D, array("one","two"), struct(5,"five")) FROM srcbucket2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket2
+PREHOOK: Output: default@destbin
+POSTHOOK: query: INSERT OVERWRITE TABLE destBin SELECT create_union( CASE WHEN key < 100 THEN 0 WHEN key < 200 THEN 1 WHEN key < 300 THEN 2 WHEN key < 400 THEN 3 ELSE 0 END, key, 2.0D, array("one","two"), struct(5,"five")) FROM srcbucket2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket2
+POSTHOOK: Output: default@destbin
+POSTHOOK: Lineage: destbin.a EXPRESSION [(srcbucket2)srcbucket2.FieldSchema(name:key, type:int, comment:null), ]
+PREHOOK: query: SELECT * from destBin
+PREHOOK: type: QUERY
+PREHOOK: Input: default@destbin
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from destBin
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@destbin
+#### A masked pattern was here ####
+{0:0}
+{0:0}
+{0:0}
+{0:10}
+{0:11}
+{0:12}
+{0:12}
+{0:15}
+{0:15}
+{0:17}
+{0:18}
+{0:18}
+{0:19}
+{0:20}
+{0:24}
+{0:24}
+{0:26}
+{0:26}
+{0:27}
+{0:28}
+{0:2}
+{0:30}
+{0:33}
+{0:34}
+{0:35}
+{0:35}
+{0:35}
+{0:37}
+{0:37}
+{0:400}
+{0:401}
+{0:401}
+{0:401}
+{0:401}
+{0:401}
+{0:402}
+{0:403}
+{0:403}
+{0:403}
+{0:404}
+{0:404}
+{0:406}
+{0:406}
+{0:406}
+{0:406}
+{0:407}
+{0:409}
+{0:409}
+{0:409}
+{0:411}
+{0:413}
+{0:413}
+{0:414}
+{0:414}
+{0:417}
+{0:417}
+{0:417}
+{0:418}
+{0:419}
+{0:41}
+{0:421}
+{0:424}
+{0:424}
+{0:427}
+{0:429}
+{0:429}
+{0:42}
+{0:42}
+{0:430}
+{0:430}
+{0:430}
+{0:431}
+{0:431}
+{0:431}
+{0:432}
+{0:435}
+{0:436}
+{0:437}
+{0:438}
+{0:438}
+{0:438}
+{0:439}
+{0:439}
+{0:43}
+{0:443}
+{0:444}
+{0:446}
+{0:448}
+{0:449}
+{0:44}
+{0:452}
+{0:453}
+{0:454}
+{0:454}
+{0:454}
+{0:455}
+{0:457}
+{0:458}
+{0:458}
+{0:459}
+{0:459}
+{0:460}
+{0:462}
+{0:462}
+{0:463}
+{0:463}
+{0:466}
+{0:466}
+{0:466}
+{0:467}
+{0:468}
+{0:468}
+{0:468}
+{0:468}
+{0:469}
+{0:469}
+{0:469}
+{0:469}
+{0:469}
+{0:470}
+{0:472}
+{0:475}
+{0:477}
+{0:478}
+{0:478}
+{0:479}
+{0:47}
+{0:480}
+{0:480}
+{0:480}
+{0:481}
+{0:482}
+{0:483}
+{0:484}
+{0:485}
+{0:487}
+{0:489}
+{0:489}
+{0:489}
+{0:489}
+{0:490}
+{0:491}
+{0:492}
+{0:492}
+{0:493}
+{0:494}
+{0:495}
+{0:496}
+{0:497}
+{0:498}
+{0:498}
+{0:498}
+{0:4}
+{0:51}
+{0:51}
+{0:53}
+{0:54}
+{0:57}
+{0:58}
+{0:58}
+{0:5}
+{0:5}
+{0:5}
+{0:64}
+{0:65}
+{0:66}
+{0:67}
+{0:67}
+{0:69}
+{0:70}
+{0:70}
+{0:70}
+{0:72}
+{0:72}
+{0:74}
+{0:76}
+{0:76}
+{0:77}
+{0:78}
+{0:80}
+{0:82}
+{0:83}
+{0:83}
+{0:84}
+{0:84}
+{0:85}
+{0:86}
+{0:87}
+{0:8}
+{0:90}
+{0:90}
+{0:90}
+{0:92}
+{0:95}
+{0:95}
+{0:96}
+{0:97}
+{0:97}
+{0:98}
+{0:98}
+{0:9}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{1:2.0}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{2:["one","two"]}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+{3:{"col1":5,"col2":"five"}}
+PREHOOK: query: DROP TABLE destBin
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@destbin
+PREHOOK: Output: default@destbin
+POSTHOOK: query: DROP TABLE destBin
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@destbin
+POSTHOOK: Output: default@destbin
+PREHOOK: query: DROP TABLE dest2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE dest2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE dest3
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE dest3
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE dest2 (a map<string,map<string,map<string,uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>>>>)
+ ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2' STORED AS SEQUENCEFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest2
+POSTHOOK: query: CREATE TABLE dest2 (a map<string,map<string,map<string,uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>>>>)
+ ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2' STORED AS SEQUENCEFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest2
+PREHOOK: query: INSERT OVERWRITE TABLE dest2 SELECT src_thrift.attributes FROM src_thrift
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_thrift
+PREHOOK: Output: default@dest2
+POSTHOOK: query: INSERT OVERWRITE TABLE dest2 SELECT src_thrift.attributes FROM src_thrift
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_thrift
+POSTHOOK: Output: default@dest2
+POSTHOOK: Lineage: dest2.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:attributes, type:map<string,map<string,map<string,uniontype<int,bigint,string,double,boolean,array<string>,map<string,string>>>>>, comment:from deserializer), ]
+PREHOOK: query: SELECT a from dest2 limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest2
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT a from dest2 limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest2
+#### A masked pattern was here ####
+{"key_0":{"erVal0":{"value_0":{3:1.0}}}}
+{"key_1":{"erVal1":{"value_1":{3:1.0}}}}
+{"key_2":{"erVal2":{"value_2":{3:1.0}}}}
+{"key_3":{"erVal3":{"value_3":{3:1.0}}}}
+{"key_4":{"erVal4":{"value_4":{3:1.0}}}}
+{"key_5":{"erVal5":{"value_5":{3:1.0}}}}
+{"key_6":{"erVal6":{"value_6":{3:1.0}}}}
+{"key_7":{"erVal7":{"value_7":{3:1.0}}}}
+{"key_8":{"erVal8":{"value_8":{3:1.0}}}}
+{"key_9":{"erVal9":{"value_9":{3:1.0}}}}
+PREHOOK: query: CREATE TABLE dest3 (
+unionfield1 uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>,
+unionfield2 uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>,
+unionfield3 uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>
+) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2' STORED AS SEQUENCEFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest3
+POSTHOOK: query: CREATE TABLE dest3 (
+unionfield1 uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>,
+unionfield2 uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>,
+unionfield3 uniontype<int, bigint, string, double, boolean, array<string>, map<string,string>>
+) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2' STORED AS SEQUENCEFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest3
+PREHOOK: query: INSERT OVERWRITE TABLE dest3 SELECT src_thrift.unionField1,src_thrift.unionField2,src_thrift.unionField3 from src_thrift
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_thrift
+PREHOOK: Output: default@dest3
+POSTHOOK: query: INSERT OVERWRITE TABLE dest3 SELECT src_thrift.unionField1,src_thrift.unionField2,src_thrift.unionField3 from src_thrift
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_thrift
+POSTHOOK: Output: default@dest3
+POSTHOOK: Lineage: dest3.unionfield1 SIMPLE [(src_thrift)src_thrift.FieldSchema(name:unionfield1, type:uniontype<int,bigint,string,double,boolean,array<string>,map<string,string>>, comment:from deserializer), ]
+POSTHOOK: Lineage: dest3.unionfield2 SIMPLE [(src_thrift)src_thrift.FieldSchema(name:unionfield2, type:uniontype<int,bigint,string,double,boolean,array<string>,map<string,string>>, comment:from deserializer), ]
+POSTHOOK: Lineage: dest3.unionfield3 SIMPLE [(src_thrift)src_thrift.FieldSchema(name:unionfield3, type:uniontype<int,bigint,string,double,boolean,array<string>,map<string,string>>, comment:from deserializer), ]
+PREHOOK: query: SELECT unionfield1, unionField2, unionfield3 from dest3 limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest3
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT unionfield1, unionField2, unionfield3 from dest3 limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest3
+#### A masked pattern was here ####
+{2:"test0"} {6:{"key_0":"value_0"}} {5:["0","0","0"]}
+{2:"test1"} {6:{"key_1":"value_1"}} {5:["10","100","1000"]}
+{2:"test2"} {6:{"key_2":"value_2"}} {5:["20","200","2000"]}
+{2:"test3"} {6:{"key_3":"value_3"}} {5:["30","300","3000"]}
+{2:"test4"} {6:{"key_4":"value_4"}} {5:["40","400","4000"]}
+{2:"test5"} {6:{"key_5":"value_5"}} {5:["50","500","5000"]}
+{2:"test6"} {6:{"key_6":"value_6"}} {5:["60","600","6000"]}
+{2:"test7"} {6:{"key_7":"value_7"}} {5:["70","700","7000"]}
+{2:"test8"} {6:{"key_8":"value_8"}} {5:["80","800","8000"]}
+{2:"test9"} {6:{"key_9":"value_9"}} {5:["90","900","9000"]}
http://git-wip-us.apache.org/repos/asf/hive/blob/f6b4f9e8/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
index 7ab2083..a07d704 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
@@ -98,10 +98,10 @@ public class LazyBinarySerDe extends AbstractSerDe {
// The object for storing row data
LazyBinaryStruct cachedLazyBinaryStruct;
- private int serializedSize;
- private SerDeStats stats;
- private boolean lastOperationSerialize;
- private boolean lastOperationDeserialize;
+ int serializedSize;
+ SerDeStats stats;
+ boolean lastOperationSerialize;
+ boolean lastOperationDeserialize;
/**
* Initialize the SerDe with configuration and table information.
@@ -297,7 +297,7 @@ public class LazyBinarySerDe extends AbstractSerDe {
serialize(byteStream, uoi.getField(obj), uoi.getObjectInspectors().get(tag), false, warnedOnceNullMapKey);
}
- private static void serializeText(
+ protected static void serializeText(
RandomAccessOutput byteStream, Text t, boolean skipLengthPrefix) {
/* write byte size of the string which is a vint */
int length = t.getLength();
@@ -317,7 +317,7 @@ public class LazyBinarySerDe extends AbstractSerDe {
public boolean value;
}
- private static void writeDateToByteStream(RandomAccessOutput byteStream,
+ public static void writeDateToByteStream(RandomAccessOutput byteStream,
DateWritable date) {
LazyBinaryUtils.writeVInt(byteStream, date.getDays());
}
@@ -692,7 +692,7 @@ public class LazyBinarySerDe extends AbstractSerDe {
}
}
- private static void writeSizeAtOffset(
+ protected static void writeSizeAtOffset(
RandomAccessOutput byteStream, int byteSizeStart, int size) {
byteStream.writeInt(byteSizeStart, size);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f6b4f9e8/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe2.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe2.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe2.java
new file mode 100644
index 0000000..b0452d8
--- /dev/null
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe2.java
@@ -0,0 +1,623 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.lazybinary;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeSpec;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable;
+import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Subclass of LazyBinarySerDe with faster serialization, initializing a serializer based on the
+ * row columns rather than checking the ObjectInspector category/primitiveType for every value.
+ * This appears to be around 3x faster than the LazyBinarSerDe serialization.
+ */
+@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES})
+public class LazyBinarySerDe2 extends LazyBinarySerDe {
+ LBSerializer rowSerializer;
+
+ public LazyBinarySerDe2() throws SerDeException {
+ super();
+ }
+
+ @Override
+ public void initialize(Configuration conf, Properties tbl) throws SerDeException {
+ super.initialize(conf, tbl);
+ ObjectInspector oi = getObjectInspector();
+
+ rowSerializer = createLBSerializer(oi);
+ }
+
+ @Override
+ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
+ if (objInspector.getCategory() != Category.STRUCT) {
+ throw new SerDeException(getClass().toString()
+ + " can only serialize struct types, but we got: "
+ + objInspector.getTypeName());
+ }
+
+ // serialize the row as BytesWritable
+ serializeByteStream.reset();
+ rowSerializer.serializeValue(serializeByteStream, obj, objInspector, true, nullMapKey);
+ serializeBytesWritable.set(serializeByteStream.getData(), 0, serializeByteStream.getLength());
+
+ // Stats bookkeeping
+ serializedSize = serializeByteStream.getLength();
+ lastOperationSerialize = true;
+ lastOperationDeserialize = false;
+
+ return serializeBytesWritable;
+ }
+
+ /**
+ * Generate a LBSerializer for the given primitive ObjectInspector
+ * @param poi
+ * @return
+ */
+ LBSerializer createPrimitiveLBSerializer(PrimitiveObjectInspector poi) {
+ switch (poi.getPrimitiveCategory()) {
+ case VOID:
+ return new LBVoidSerializer();
+ case BOOLEAN:
+ return new LBBooleanSerializer();
+ case BYTE:
+ return new LBByteSerializer();
+ case SHORT:
+ return new LBShortSerializer();
+ case INT:
+ return new LBIntSerializer();
+ case LONG:
+ return new LBLongSerializer();
+ case FLOAT:
+ return new LBFloatSerializer();
+ case DOUBLE:
+ return new LBDoubleSerializer();
+ case STRING:
+ return new LBStringSerializer();
+ case CHAR:
+ return new LBHiveCharSerializer();
+ case VARCHAR:
+ return new LBHiveVarcharSerializer();
+ case BINARY:
+ return new LBBinarySerializer();
+ case DATE:
+ return new LBDateSerializer();
+ case TIMESTAMP:
+ return new LBTimestampSerializer();
+ case INTERVAL_YEAR_MONTH:
+ return new LBHiveIntervalYearMonthSerializer();
+ case INTERVAL_DAY_TIME:
+ return new LBHiveIntervalDayTimeSerializer();
+ case DECIMAL:
+ return new LBHiveDecimalSerializer();
+ default:
+ throw new IllegalArgumentException("Unsupported primitive category " + poi.getPrimitiveCategory());
+ }
+ }
+
+ /**
+ * Generate a LBSerializer for the given ObjectInspector
+ * @param oi
+ * @return
+ */
+ LBSerializer createLBSerializer(ObjectInspector oi) {
+ switch (oi.getCategory()) {
+ case PRIMITIVE:
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
+ return createPrimitiveLBSerializer(poi);
+ case LIST:
+ ListObjectInspector loi = (ListObjectInspector) oi;
+ ObjectInspector eoi = loi.getListElementObjectInspector();
+ return new LBListSerializer(createLBSerializer(eoi));
+ case MAP:
+ MapObjectInspector moi = (MapObjectInspector) oi;
+ ObjectInspector koi = moi.getMapKeyObjectInspector();
+ ObjectInspector voi = moi.getMapValueObjectInspector();
+ return new LBMapSerializer(createLBSerializer(koi), createLBSerializer(voi));
+ case STRUCT:
+ StructObjectInspector soi = (StructObjectInspector) oi;
+ List<? extends StructField> fields = soi.getAllStructFieldRefs();
+ LBSerializer[] fieldSerializers = new LBSerializer[fields.size()];
+ for (int idx = 0; idx < fieldSerializers.length; ++idx) {
+ fieldSerializers[idx] = createLBSerializer(fields.get(idx).getFieldObjectInspector());
+ }
+ return new LBStructSerializer(fieldSerializers);
+ case UNION:
+ UnionObjectInspector uoi = (UnionObjectInspector) oi;
+ List<ObjectInspector> unionFields = uoi.getObjectInspectors();
+ LBSerializer[] unionFieldSerializers = new LBSerializer[unionFields.size()];
+ for (int idx = 0; idx < unionFieldSerializers.length; ++idx) {
+ unionFieldSerializers[idx] = createLBSerializer(unionFields.get(idx));
+ }
+ return new LBUnionSerializer(unionFieldSerializers);
+ default:
+ throw new IllegalArgumentException("Unsupported category " + oi.getCategory());
+ }
+ }
+
+ /**
+ * Abstract serializer class for serializing to LazyBinary format.
+ */
+ abstract static class LBSerializer {
+ public void serializeValue(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ if (obj != null) {
+ serialize(byteStream, obj, objInspector, skipLengthPrefix, warnedOnceNullMapKey);
+ }
+ }
+
+ abstract void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey);
+ }
+
+ static class LBVoidSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ return;
+ }
+ }
+
+ static class LBBooleanSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ boolean v = ((BooleanObjectInspector) objInspector).get(obj);
+ byteStream.write((byte) (v ? 1 : 0));
+ }
+ }
+
+ static class LBByteSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ ByteObjectInspector boi = (ByteObjectInspector) objInspector;
+ byte v = boi.get(obj);
+ byteStream.write(v);
+ }
+ }
+
+ static class LBShortSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ ShortObjectInspector spoi = (ShortObjectInspector) objInspector;
+ short v = spoi.get(obj);
+ byteStream.write((byte) (v >> 8));
+ byteStream.write((byte) (v));
+ }
+ }
+
+ static class LBIntSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ IntObjectInspector ioi = (IntObjectInspector) objInspector;
+ int v = ioi.get(obj);
+ LazyBinaryUtils.writeVInt(byteStream, v);
+ }
+ }
+
+ static class LBLongSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ LongObjectInspector loi = (LongObjectInspector) objInspector;
+ long v = loi.get(obj);
+ LazyBinaryUtils.writeVLong(byteStream, v);
+ return;
+ }
+ }
+
+ static class LBFloatSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ FloatObjectInspector foi = (FloatObjectInspector) objInspector;
+ int v = Float.floatToIntBits(foi.get(obj));
+ byteStream.write((byte) (v >> 24));
+ byteStream.write((byte) (v >> 16));
+ byteStream.write((byte) (v >> 8));
+ byteStream.write((byte) (v));
+ }
+ }
+
+ static class LBDoubleSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ DoubleObjectInspector doi = (DoubleObjectInspector) objInspector;
+ LazyBinaryUtils.writeDouble(byteStream, doi.get(obj));
+ }
+ }
+
+ static class LBStringSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ StringObjectInspector soi = (StringObjectInspector) objInspector;
+ Text t = soi.getPrimitiveWritableObject(obj);
+ serializeText(byteStream, t, skipLengthPrefix);
+ }
+ }
+
+ static class LBHiveCharSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ HiveCharObjectInspector hcoi = (HiveCharObjectInspector) objInspector;
+ Text t = hcoi.getPrimitiveWritableObject(obj).getTextValue();
+ serializeText(byteStream, t, skipLengthPrefix);
+ }
+ }
+
+ static class LBHiveVarcharSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ HiveVarcharObjectInspector hcoi = (HiveVarcharObjectInspector) objInspector;
+ Text t = hcoi.getPrimitiveWritableObject(obj).getTextValue();
+ serializeText(byteStream, t, skipLengthPrefix);
+ }
+ }
+
+ static class LBBinarySerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ BinaryObjectInspector baoi = (BinaryObjectInspector) objInspector;
+ BytesWritable bw = baoi.getPrimitiveWritableObject(obj);
+ int length = bw.getLength();
+ if(!skipLengthPrefix){
+ LazyBinaryUtils.writeVInt(byteStream, length);
+ } else {
+ if (length == 0){
+ throw new RuntimeException("LazyBinaryColumnarSerde cannot serialize a non-null zero "
+ + "length binary field. Consider using either LazyBinarySerde or ColumnarSerde.");
+ }
+ }
+ byteStream.write(bw.getBytes(),0,length);
+ }
+ }
+
+ static class LBDateSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ DateWritable d = ((DateObjectInspector) objInspector).getPrimitiveWritableObject(obj);
+ LazyBinarySerDe.writeDateToByteStream(byteStream, d);
+ }
+ }
+
+ static class LBTimestampSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ TimestampObjectInspector toi = (TimestampObjectInspector) objInspector;
+ TimestampWritable t = toi.getPrimitiveWritableObject(obj);
+ t.writeToByteStream(byteStream);
+ }
+ }
+
+ static class LBHiveIntervalYearMonthSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ HiveIntervalYearMonthWritable intervalYearMonth =
+ ((HiveIntervalYearMonthObjectInspector) objInspector).getPrimitiveWritableObject(obj);
+ intervalYearMonth.writeToByteStream(byteStream);
+ }
+ }
+
+ static class LBHiveIntervalDayTimeSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ HiveIntervalDayTimeWritable intervalDayTime =
+ ((HiveIntervalDayTimeObjectInspector) objInspector).getPrimitiveWritableObject(obj);
+ intervalDayTime.writeToByteStream(byteStream);
+ }
+ }
+
+ static class LBHiveDecimalSerializer extends LBSerializer {
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ HiveDecimalObjectInspector bdoi = (HiveDecimalObjectInspector) objInspector;
+ HiveDecimalWritable t = bdoi.getPrimitiveWritableObject(obj);
+ if (t == null) {
+ return;
+ }
+ writeToByteStream(byteStream, t);
+ }
+ }
+
+ static class LBListSerializer extends LBSerializer {
+ LBSerializer elementSerializer;
+
+ public LBListSerializer(LBSerializer elementSerializer) {
+ super();
+ this.elementSerializer = elementSerializer;
+ }
+
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ ListObjectInspector loi = (ListObjectInspector) objInspector;
+ ObjectInspector eoi = loi.getListElementObjectInspector();
+
+ int byteSizeStart = 0;
+ int listStart = 0;
+ if (!skipLengthPrefix) {
+ // 1/ reserve spaces for the byte size of the list
+ // which is a integer and takes four bytes
+ byteSizeStart = byteStream.getLength();
+ byteStream.reserve(4);
+ listStart = byteStream.getLength();
+ }
+ // 2/ write the size of the list as a VInt
+ int size = loi.getListLength(obj);
+ LazyBinaryUtils.writeVInt(byteStream, size);
+
+ // 3/ write the null bytes
+ byte nullByte = 0;
+ for (int eid = 0; eid < size; eid++) {
+ // set the bit to 1 if an element is not null
+ if (null != loi.getListElement(obj, eid)) {
+ nullByte |= 1 << (eid % 8);
+ }
+ // store the byte every eight elements or
+ // if this is the last element
+ if (7 == eid % 8 || eid == size - 1) {
+ byteStream.write(nullByte);
+ nullByte = 0;
+ }
+ }
+
+ // 4/ write element by element from the list
+ for (int eid = 0; eid < size; eid++) {
+ elementSerializer.serializeValue(
+ byteStream, loi.getListElement(obj, eid), eoi,
+ false, warnedOnceNullMapKey);
+ }
+
+ if (!skipLengthPrefix) {
+ // 5/ update the list byte size
+ int listEnd = byteStream.getLength();
+ int listSize = listEnd - listStart;
+ writeSizeAtOffset(byteStream, byteSizeStart, listSize);
+ }
+ return;
+ }
+ }
+
+ static class LBMapSerializer extends LBSerializer {
+ LBSerializer keySerializer;
+ LBSerializer valSerializer;
+
+ public LBMapSerializer(LBSerializer keySerializer,
+ LBSerializer valSerializer) {
+ super();
+ this.keySerializer = keySerializer;
+ this.valSerializer = valSerializer;
+ }
+
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ MapObjectInspector moi = (MapObjectInspector) objInspector;
+ ObjectInspector koi = moi.getMapKeyObjectInspector();
+ ObjectInspector voi = moi.getMapValueObjectInspector();
+ Map<?, ?> map = moi.getMap(obj);
+
+ int byteSizeStart = 0;
+ int mapStart = 0;
+ if (!skipLengthPrefix) {
+ // 1/ reserve spaces for the byte size of the map
+ // which is a integer and takes four bytes
+ byteSizeStart = byteStream.getLength();
+ byteStream.reserve(4);
+ mapStart = byteStream.getLength();
+ }
+
+ // 2/ write the size of the map which is a VInt
+ int size = map.size();
+ LazyBinaryUtils.writeVInt(byteStream, size);
+
+ // 3/ write the null bytes
+ int b = 0;
+ byte nullByte = 0;
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ // set the bit to 1 if a key is not null
+ if (null != entry.getKey()) {
+ nullByte |= 1 << (b % 8);
+ } else if (warnedOnceNullMapKey != null) {
+ if (!warnedOnceNullMapKey.value) {
+ LOG.warn("Null map key encountered! Ignoring similar problems.");
+ }
+ warnedOnceNullMapKey.value = true;
+ }
+ b++;
+ // set the bit to 1 if a value is not null
+ if (null != entry.getValue()) {
+ nullByte |= 1 << (b % 8);
+ }
+ b++;
+ // write the byte to stream every 4 key-value pairs
+ // or if this is the last key-value pair
+ if (0 == b % 8 || b == size * 2) {
+ byteStream.write(nullByte);
+ nullByte = 0;
+ }
+ }
+
+ // 4/ write key-value pairs one by one
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ keySerializer.serializeValue(byteStream, entry.getKey(), koi, false, warnedOnceNullMapKey);
+ valSerializer.serializeValue(byteStream, entry.getValue(), voi, false, warnedOnceNullMapKey);
+ }
+
+ if (!skipLengthPrefix) {
+ // 5/ update the byte size of the map
+ int mapEnd = byteStream.getLength();
+ int mapSize = mapEnd - mapStart;
+ writeSizeAtOffset(byteStream, byteSizeStart, mapSize);
+ }
+ return;
+ }
+ }
+
+ static class LBStructSerializer extends LBSerializer {
+ LBSerializer[] serializers;
+ Object[] fieldData;
+
+ public LBStructSerializer(LBSerializer[] serializers) {
+ super();
+ this.serializers = serializers;
+ this.fieldData = new Object[serializers.length];
+ }
+
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj, ObjectInspector objInspector,
+ boolean skipLengthPrefix, BooleanRef warnedOnceNullMapKey) {
+ int lasti = 0;
+ byte nullByte = 0;
+ int size = serializers.length;
+
+ int byteSizeStart = 0;
+ int typeStart = 0;
+ if (!skipLengthPrefix) {
+ // 1/ reserve spaces for the byte size of the struct
+ // which is a integer and takes four bytes
+ byteSizeStart = byteStream.getLength();
+ byteStream.reserve(4);
+ typeStart = byteStream.getLength();
+ }
+
+ StructObjectInspector soi = (StructObjectInspector) objInspector;
+ List<? extends StructField> fields = soi.getAllStructFieldRefs();
+ for (int i = 0; i < size; ++i) {
+ StructField structField = fields.get(i);
+ fieldData[i] = soi.getStructFieldData(obj, structField);
+ }
+
+ for (int i = 0; i < size; ++i) {
+ // set bit to 1 if a field is not null
+ if (null != fieldData[i]) {
+ nullByte |= 1 << (i % 8);
+ }
+ // write the null byte every eight elements or
+ // if this is the last element and serialize the
+ // corresponding 8 struct fields at the same time
+ if (7 == i % 8 || i == size - 1) {
+ byteStream.write(nullByte);
+ for (int j = lasti; j <= i; j++) {
+ serializers[j].serializeValue(
+ byteStream, fieldData[j], fields.get(j).getFieldObjectInspector(),
+ false, warnedOnceNullMapKey);
+ }
+ lasti = i + 1;
+ nullByte = 0;
+ }
+ }
+
+ if (!skipLengthPrefix) {
+ // 3/ update the byte size of the struct
+ int typeEnd = byteStream.getLength();
+ int typeSize = typeEnd - typeStart;
+ writeSizeAtOffset(byteStream, byteSizeStart, typeSize);
+ }
+ }
+ }
+
+ static class LBUnionSerializer extends LBSerializer {
+ LBSerializer[] unionFieldSerializers;
+
+ public LBUnionSerializer(LBSerializer[] unionFieldSerializers) {
+ this.unionFieldSerializers = unionFieldSerializers;
+ }
+
+ @Override
+ void serialize(RandomAccessOutput byteStream, Object obj,
+ ObjectInspector objInspector, boolean skipLengthPrefix,
+ BooleanRef warnedOnceNullMapKey) {
+ int byteSizeStart = 0;
+ int typeStart = 0;
+ if (!skipLengthPrefix) {
+ // 1/ reserve spaces for the byte size of the struct
+ // which is a integer and takes four bytes
+ byteSizeStart = byteStream.getLength();
+ byteStream.reserve(4);
+ typeStart = byteStream.getLength();
+ }
+
+ // 2/ serialize the union - tag/value
+ UnionObjectInspector uoi = (UnionObjectInspector) objInspector;
+ byte tag = uoi.getTag(obj);
+ byteStream.write(tag);
+ unionFieldSerializers[tag].serializeValue(
+ byteStream, uoi.getField(obj), uoi.getObjectInspectors().get(tag),
+ false, warnedOnceNullMapKey);
+
+ if (!skipLengthPrefix) {
+ // 3/ update the byte size of the struct
+ int typeEnd = byteStream.getLength();
+ int typeSize = typeEnd - typeStart;
+ writeSizeAtOffset(byteStream, byteSizeStart, typeSize);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f6b4f9e8/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
index 13c73be..3e20676 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
@@ -274,14 +274,15 @@ public class TestLazyBinaryFast extends TestCase {
String fieldNames = ObjectInspectorUtils.getFieldNames(rowStructObjectInspector);
String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowStructObjectInspector);
- AbstractSerDe serde = TestLazyBinarySerDe.getSerDe(fieldNames, fieldTypes);
+ TestLazyBinarySerDe testLazyBinarySerDe = new TestLazyBinarySerDe();
+ AbstractSerDe serde = testLazyBinarySerDe.getSerDe(fieldNames, fieldTypes);
AbstractSerDe serde_fewer = null;
if (doWriteFewerColumns) {
String partialFieldNames = ObjectInspectorUtils.getFieldNames(writeRowStructObjectInspector);
String partialFieldTypes = ObjectInspectorUtils.getFieldTypes(writeRowStructObjectInspector);
- serde_fewer = TestLazyBinarySerDe.getSerDe(partialFieldNames, partialFieldTypes);;
+ serde_fewer = testLazyBinarySerDe.getSerDe(partialFieldNames, partialFieldTypes);;
}
testLazyBinaryFast(
@@ -345,4 +346,4 @@ public class TestLazyBinaryFast extends TestCase {
public void testLazyBinaryFastComplexDepthFour() throws Throwable {
testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes.ALL, 4);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f6b4f9e8/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
index 0cd5736..70a5dd0 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
@@ -93,7 +93,7 @@ public class TestLazyBinarySerDe extends TestCase {
* @return the initialized LazyBinarySerDe
* @throws Throwable
*/
- protected static AbstractSerDe getSerDe(String fieldNames, String fieldTypes) throws Throwable {
+ protected AbstractSerDe getSerDe(String fieldNames, String fieldTypes) throws Throwable {
Properties schema = new Properties();
schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
http://git-wip-us.apache.org/repos/asf/hive/blob/f6b4f9e8/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe2.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe2.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe2.java
new file mode 100644
index 0000000..1a93769
--- /dev/null
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe2.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazybinary;
+
+import java.util.Properties;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+
+
+/**
+ * TestLazyBinarySerDe2.
+ *
+ */
+public class TestLazyBinarySerDe2 extends TestLazyBinarySerDe {
+
+ /**
+ * Initialize the LazyBinarySerDe.
+ *
+ * @param fieldNames
+ * table field names
+ * @param fieldTypes
+ * table field types
+ * @return the initialized LazyBinarySerDe
+ * @throws Throwable
+ */
+ @Override
+ protected AbstractSerDe getSerDe(String fieldNames, String fieldTypes) throws Throwable {
+ Properties schema = new Properties();
+ schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
+ schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
+
+ LazyBinarySerDe2 serde = new LazyBinarySerDe2();
+ SerDeUtils.initializeSerDe(serde, new Configuration(), schema, null);
+ return serde;
+ }
+}