You are viewing a plain text version of this content. The canonical link for it is here.

Posted to commits@hive.apache.org by se...@apache.org on 2017/10/13 00:15:40 UTC

[01/50] [abbrv] hive git commit: HIVE-14671 : Merge branch 'master' into hive-14535 (Wei Zheng)

Repository: hive
Updated Branches:
  refs/heads/master 9375cf3c0 -> c1a1d5960


http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/test/results/clientpositive/mm_all.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/mm_all.q.out
index 6bed878,0000000..e08f1a8
mode 100644,000000..100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@@ -1,2412 -1,0 +1,2443 @@@
 +PREHOOK: query: drop table intermediate
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table intermediate
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@intermediate
 +POSTHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@intermediate
 +PREHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=455
 +POSTHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=455
 +POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=456
 +POSTHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=456
 +POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=457
 +POSTHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=457
 +POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: drop table part_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table part_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@part_mm
 +POSTHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@part_mm
 +PREHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
 +PREHOOK: type: QUERY
 +POSTHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
 +POSTHOOK: type: QUERY
 +STAGE DEPENDENCIES:
 +  Stage-1 is a root stage
 +  Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
 +  Stage-4
 +  Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
 +  Stage-2 depends on stages: Stage-0
 +  Stage-3
 +  Stage-5
 +  Stage-6 depends on stages: Stage-5
 +
 +STAGE PLANS:
 +  Stage: Stage-1
 +    Map Reduce
 +      Map Operator Tree:
 +          TableScan
 +            alias: intermediate
 +            Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
 +            Select Operator
 +              expressions: key (type: int)
 +              outputColumnNames: _col0
 +              Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
 +              File Output Operator
 +                compressed: false
 +                Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
 +                table:
 +                    input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +                    output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
 +                    serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
 +                    name: default.part_mm
 +                Write Type: INSERT
 +
 +  Stage: Stage-7
 +    Conditional Operator
 +
 +  Stage: Stage-4
 +    Move Operator
 +      files:
 +          hdfs directory: true
 +#### A masked pattern was here ####
 +
 +  Stage: Stage-0
 +    Move Operator
 +      tables:
 +          partition:
 +            key_mm 455
 +          replace: false
 +          table:
 +              input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +              output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
 +              serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
 +              name: default.part_mm
 +          micromanaged table: true
 +
 +  Stage: Stage-2
 +    Stats-Aggr Operator
 +
 +  Stage: Stage-3
 +    Merge File Operator
 +      Map Operator Tree:
 +          ORC File Merge Operator
 +      merge level: stripe
 +      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +
 +  Stage: Stage-5
 +    Merge File Operator
 +      Map Operator Tree:
 +          ORC File Merge Operator
 +      merge level: stripe
 +      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +
 +  Stage: Stage-6
 +    Move Operator
 +      files:
 +          hdfs directory: true
 +#### A masked pattern was here ####
 +
 +PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@part_mm@key_mm=456
 +POSTHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@part_mm@key_mm=456
 +POSTHOOK: Lineage: part_mm PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from part_mm order by key, key_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Input: default@part_mm@key_mm=455
 +PREHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from part_mm order by key, key_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Input: default@part_mm@key_mm=455
 +POSTHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +0	455
 +0	455
 +0	456
 +10	455
 +10	455
 +10	456
 +97	455
 +97	455
 +97	456
 +98	455
 +98	455
 +98	456
 +100	455
 +100	455
 +100	456
 +103	455
 +103	455
 +103	456
 +PREHOOK: query: select * from part_mm order by key, key_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Input: default@part_mm@key_mm=455
 +PREHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from part_mm order by key, key_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Input: default@part_mm@key_mm=455
 +POSTHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +0	455
 +0	455
 +0	456
 +10	455
 +10	455
 +10	456
 +97	455
 +97	455
 +97	456
 +98	455
 +98	455
 +98	456
 +100	455
 +100	455
 +100	456
 +103	455
 +103	455
 +103	456
 +PREHOOK: query: truncate table part_mm
 +PREHOOK: type: TRUNCATETABLE
 +PREHOOK: Output: default@part_mm@key_mm=455
 +PREHOOK: Output: default@part_mm@key_mm=456
 +POSTHOOK: query: truncate table part_mm
 +POSTHOOK: type: TRUNCATETABLE
 +POSTHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: Output: default@part_mm@key_mm=456
 +PREHOOK: query: select * from part_mm order by key, key_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Input: default@part_mm@key_mm=455
 +PREHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from part_mm order by key, key_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Input: default@part_mm@key_mm=455
 +POSTHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +PREHOOK: query: drop table part_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Output: default@part_mm
 +POSTHOOK: query: drop table part_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Output: default@part_mm
 +PREHOOK: query: drop table simple_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table simple_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@simple_mm
 +PREHOOK: query: insert into table simple_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: insert into table simple_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@simple_mm
 +POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from simple_mm order by key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from simple_mm order by key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +0
 +10
 +97
 +98
 +100
 +103
 +PREHOOK: query: insert into table simple_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: insert into table simple_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@simple_mm
 +POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from simple_mm order by key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from simple_mm order by key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +0
 +0
 +10
 +10
 +97
 +97
 +98
 +98
 +100
 +100
 +103
 +103
 +PREHOOK: query: truncate table simple_mm
 +PREHOOK: type: TRUNCATETABLE
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: truncate table simple_mm
 +POSTHOOK: type: TRUNCATETABLE
 +POSTHOOK: Output: default@simple_mm
 +PREHOOK: query: select * from simple_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from simple_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +PREHOOK: query: drop table simple_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@simple_mm
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: drop table simple_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@simple_mm
 +POSTHOOK: Output: default@simple_mm
 +PREHOOK: query: drop table dp_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table dp_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
 +  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@dp_mm
 +POSTHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
 +  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@dp_mm
 +PREHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@dp_mm@key1=123
 +POSTHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=0
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=10
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=100
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=103
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=97
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=98
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=0).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=100).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=103).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=10).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=97).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=98).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from dp_mm order by key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@dp_mm
 +PREHOOK: Input: default@dp_mm@key1=123/key2=0
 +PREHOOK: Input: default@dp_mm@key1=123/key2=10
 +PREHOOK: Input: default@dp_mm@key1=123/key2=100
 +PREHOOK: Input: default@dp_mm@key1=123/key2=103
 +PREHOOK: Input: default@dp_mm@key1=123/key2=97
 +PREHOOK: Input: default@dp_mm@key1=123/key2=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from dp_mm order by key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@dp_mm
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=0
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=10
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=100
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=103
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=97
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=98
 +#### A masked pattern was here ####
 +0	123	0
 +10	123	10
 +97	123	97
 +98	123	98
 +100	123	100
 +103	123	103
 +PREHOOK: query: drop table dp_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@dp_mm
 +PREHOOK: Output: default@dp_mm
 +POSTHOOK: query: drop table dp_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@dp_mm
 +POSTHOOK: Output: default@dp_mm
 +PREHOOK: query: create table union_mm(id int)  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: create table union_mm(id int)  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@union_mm
 +PREHOOK: query: insert into table union_mm 
 +select temps.p from (
 +select key as p from intermediate 
 +union all 
 +select key + 1 as p from intermediate ) temps
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: insert into table union_mm 
 +select temps.p from (
 +select key as p from intermediate 
 +union all 
 +select key + 1 as p from intermediate ) temps
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@union_mm
 +POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from union_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from union_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +0
 +1
 +10
 +11
 +97
 +98
 +98
 +99
 +100
 +101
 +103
 +104
 +PREHOOK: query: insert into table union_mm 
 +select p from
 +(
 +select key + 1 as p from intermediate
 +union all
 +select key from intermediate
 +) tab group by p
 +union all
 +select key + 2 as p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: insert into table union_mm 
 +select p from
 +(
 +select key + 1 as p from intermediate
 +union all
 +select key from intermediate
 +) tab group by p
 +union all
 +select key + 2 as p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@union_mm
 +POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from union_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from union_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +0
 +0
 +1
 +1
 +2
 +10
 +10
 +11
 +11
 +12
 +97
 +97
 +98
 +98
 +98
 +99
 +99
 +99
 +100
 +100
 +100
 +101
 +101
 +102
 +103
 +103
 +104
 +104
 +105
 +PREHOOK: query: insert into table union_mm
 +SELECT p FROM
 +(
 +  SELECT key + 1 as p FROM intermediate
 +  UNION ALL
 +  SELECT key as p FROM ( 
 +    SELECT distinct key FROM (
 +      SELECT key FROM (
 +        SELECT key + 2 as key FROM intermediate
 +        UNION ALL
 +        SELECT key FROM intermediate
 +      )t1 
 +    group by key)t2
 +  )t3
 +)t4
 +group by p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: insert into table union_mm
 +SELECT p FROM
 +(
 +  SELECT key + 1 as p FROM intermediate
 +  UNION ALL
 +  SELECT key as p FROM ( 
 +    SELECT distinct key FROM (
 +      SELECT key FROM (
 +        SELECT key + 2 as key FROM intermediate
 +        UNION ALL
 +        SELECT key FROM intermediate
 +      )t1 
 +    group by key)t2
 +  )t3
 +)t4
 +group by p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@union_mm
 +POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from union_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from union_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +0
 +0
 +0
 +1
 +1
 +1
 +2
 +2
 +10
 +10
 +10
 +11
 +11
 +11
 +12
 +12
 +97
 +97
 +97
 +98
 +98
 +98
 +98
 +99
 +99
 +99
 +99
 +100
 +100
 +100
 +100
 +101
 +101
 +101
 +102
 +102
 +103
 +103
 +103
 +104
 +104
 +104
 +105
 +105
 +PREHOOK: query: drop table union_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@union_mm
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: drop table union_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@union_mm
 +POSTHOOK: Output: default@union_mm
 +PREHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@partunion_mm
 +POSTHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@partunion_mm
 +PREHOOK: query: insert into table partunion_mm partition(key)
 +select temps.* from (
 +select key as p, key from intermediate 
 +union all 
 +select key + 1 as p, key + 1 from intermediate ) temps
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@partunion_mm
 +POSTHOOK: query: insert into table partunion_mm partition(key)
 +select temps.* from (
 +select key as p, key from intermediate 
 +union all 
 +select key + 1 as p, key + 1 from intermediate ) temps
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@partunion_mm@key=0
 +POSTHOOK: Output: default@partunion_mm@key=1
 +POSTHOOK: Output: default@partunion_mm@key=10
 +POSTHOOK: Output: default@partunion_mm@key=100
 +POSTHOOK: Output: default@partunion_mm@key=101
 +POSTHOOK: Output: default@partunion_mm@key=103
 +POSTHOOK: Output: default@partunion_mm@key=104
 +POSTHOOK: Output: default@partunion_mm@key=11
 +POSTHOOK: Output: default@partunion_mm@key=97
 +POSTHOOK: Output: default@partunion_mm@key=98
 +POSTHOOK: Output: default@partunion_mm@key=99
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=0).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=100).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=101).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=103).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=104).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=10).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=11).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=1).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=97).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=98).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=99).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from partunion_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@partunion_mm
 +PREHOOK: Input: default@partunion_mm@key=0
 +PREHOOK: Input: default@partunion_mm@key=1
 +PREHOOK: Input: default@partunion_mm@key=10
 +PREHOOK: Input: default@partunion_mm@key=100
 +PREHOOK: Input: default@partunion_mm@key=101
 +PREHOOK: Input: default@partunion_mm@key=103
 +PREHOOK: Input: default@partunion_mm@key=104
 +PREHOOK: Input: default@partunion_mm@key=11
 +PREHOOK: Input: default@partunion_mm@key=97
 +PREHOOK: Input: default@partunion_mm@key=98
 +PREHOOK: Input: default@partunion_mm@key=99
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from partunion_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@partunion_mm
 +POSTHOOK: Input: default@partunion_mm@key=0
 +POSTHOOK: Input: default@partunion_mm@key=1
 +POSTHOOK: Input: default@partunion_mm@key=10
 +POSTHOOK: Input: default@partunion_mm@key=100
 +POSTHOOK: Input: default@partunion_mm@key=101
 +POSTHOOK: Input: default@partunion_mm@key=103
 +POSTHOOK: Input: default@partunion_mm@key=104
 +POSTHOOK: Input: default@partunion_mm@key=11
 +POSTHOOK: Input: default@partunion_mm@key=97
 +POSTHOOK: Input: default@partunion_mm@key=98
 +POSTHOOK: Input: default@partunion_mm@key=99
 +#### A masked pattern was here ####
 +0	0
 +1	1
 +10	10
 +11	11
 +97	97
 +98	98
 +98	98
 +99	99
 +100	100
 +101	101
 +103	103
 +104	104
 +PREHOOK: query: drop table partunion_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@partunion_mm
 +PREHOOK: Output: default@partunion_mm
 +POSTHOOK: query: drop table partunion_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@partunion_mm
 +POSTHOOK: Output: default@partunion_mm
 +PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
 + stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@skew_mm
 +POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
 + stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@skew_mm
 +PREHOOK: query: insert into table skew_mm 
 +select key, key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@skew_mm
 +POSTHOOK: query: insert into table skew_mm 
 +select key, key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@skew_mm
 +POSTHOOK: Lineage: skew_mm.k1 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_mm.k2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_mm.k4 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from skew_mm order by k2, k1, k4
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@skew_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from skew_mm order by k2, k1, k4
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@skew_mm
 +#### A masked pattern was here ####
 +0	0	0
 +10	10	10
 +97	97	97
 +98	98	98
 +100	100	100
 +103	103	103
 +PREHOOK: query: drop table skew_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@skew_mm
 +PREHOOK: Output: default@skew_mm
 +POSTHOOK: query: drop table skew_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@skew_mm
 +POSTHOOK: Output: default@skew_mm
 +PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) 
 +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@skew_dp_union_mm
 +POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) 
 +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@skew_dp_union_mm
 +PREHOOK: query: insert into table skew_dp_union_mm partition (k3)
 +select key as i, key as j, key as k, key as l from intermediate
 +union all 
 +select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@skew_dp_union_mm
 +POSTHOOK: query: insert into table skew_dp_union_mm partition (k3)
 +select key as i, key as j, key as k, key as l from intermediate
 +union all 
 +select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=0
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=10
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=100
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=101
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=102
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=103
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=104
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=107
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=14
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=4
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=97
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=98
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@skew_dp_union_mm
 +PREHOOK: Input: default@skew_dp_union_mm@k3=0
 +PREHOOK: Input: default@skew_dp_union_mm@k3=10
 +PREHOOK: Input: default@skew_dp_union_mm@k3=100
 +PREHOOK: Input: default@skew_dp_union_mm@k3=101
 +PREHOOK: Input: default@skew_dp_union_mm@k3=102
 +PREHOOK: Input: default@skew_dp_union_mm@k3=103
 +PREHOOK: Input: default@skew_dp_union_mm@k3=104
 +PREHOOK: Input: default@skew_dp_union_mm@k3=107
 +PREHOOK: Input: default@skew_dp_union_mm@k3=14
 +PREHOOK: Input: default@skew_dp_union_mm@k3=4
 +PREHOOK: Input: default@skew_dp_union_mm@k3=97
 +PREHOOK: Input: default@skew_dp_union_mm@k3=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@skew_dp_union_mm
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=0
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=10
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=100
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=101
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=102
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=103
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=104
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=107
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=14
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=4
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=97
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=98
 +#### A masked pattern was here ####
 +0	0	0	0
 +1	2	3	4
 +10	10	10	10
 +11	12	13	14
 +97	97	97	97
 +98	98	98	98
 +98	99	100	101
 +99	100	101	102
 +100	100	100	100
 +101	102	103	104
 +103	103	103	103
 +104	105	106	107
 +PREHOOK: query: drop table skew_dp_union_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@skew_dp_union_mm
 +PREHOOK: Output: default@skew_dp_union_mm
 +POSTHOOK: query: drop table skew_dp_union_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@skew_dp_union_mm
 +POSTHOOK: Output: default@skew_dp_union_mm
 +PREHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@merge0_mm
 +PREHOOK: query: insert into table merge0_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: insert into table merge0_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge0_mm
 +POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: insert into table merge0_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: insert into table merge0_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge0_mm
 +POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: drop table merge0_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@merge0_mm
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: drop table merge0_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@merge0_mm
 +POSTHOOK: Output: default@merge0_mm
 +PREHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@merge2_mm
 +PREHOOK: query: insert into table merge2_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: insert into table merge2_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge2_mm
 +POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge2_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge2_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: insert into table merge2_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: insert into table merge2_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge2_mm
 +POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge2_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge2_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: drop table merge2_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@merge2_mm
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: drop table merge2_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@merge2_mm
 +POSTHOOK: Output: default@merge2_mm
 +PREHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@merge1_mm
 +PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge1_mm@key=0
 +POSTHOOK: Output: default@merge1_mm@key=10
 +POSTHOOK: Output: default@merge1_mm@key=100
 +POSTHOOK: Output: default@merge1_mm@key=103
 +POSTHOOK: Output: default@merge1_mm@key=97
 +POSTHOOK: Output: default@merge1_mm@key=98
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge1_mm order by id, key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge1_mm
 +PREHOOK: Input: default@merge1_mm@key=0
 +PREHOOK: Input: default@merge1_mm@key=10
 +PREHOOK: Input: default@merge1_mm@key=100
 +PREHOOK: Input: default@merge1_mm@key=103
 +PREHOOK: Input: default@merge1_mm@key=97
 +PREHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge1_mm order by id, key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge1_mm
 +POSTHOOK: Input: default@merge1_mm@key=0
 +POSTHOOK: Input: default@merge1_mm@key=10
 +POSTHOOK: Input: default@merge1_mm@key=100
 +POSTHOOK: Input: default@merge1_mm@key=103
 +POSTHOOK: Input: default@merge1_mm@key=97
 +POSTHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +0	0
 +10	10
 +97	97
 +98	98
 +100	100
 +103	103
 +PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge1_mm@key=0
 +POSTHOOK: Output: default@merge1_mm@key=10
 +POSTHOOK: Output: default@merge1_mm@key=100
 +POSTHOOK: Output: default@merge1_mm@key=103
 +POSTHOOK: Output: default@merge1_mm@key=97
 +POSTHOOK: Output: default@merge1_mm@key=98
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge1_mm order by id, key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge1_mm
 +PREHOOK: Input: default@merge1_mm@key=0
 +PREHOOK: Input: default@merge1_mm@key=10
 +PREHOOK: Input: default@merge1_mm@key=100
 +PREHOOK: Input: default@merge1_mm@key=103
 +PREHOOK: Input: default@merge1_mm@key=97
 +PREHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge1_mm order by id, key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge1_mm
 +POSTHOOK: Input: default@merge1_mm@key=0
 +POSTHOOK: Input: default@merge1_mm@key=10
 +POSTHOOK: Input: default@merge1_mm@key=100
 +POSTHOOK: Input: default@merge1_mm@key=103
 +POSTHOOK: Input: default@merge1_mm@key=97
 +POSTHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +0	0
 +0	0
 +10	10
 +10	10
 +97	97
 +97	97
 +98	98
 +98	98
 +100	100
 +100	100
 +103	103
 +103	103
 +PREHOOK: query: drop table merge1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@merge1_mm
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: drop table merge1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@merge1_mm
 +POSTHOOK: Output: default@merge1_mm
 +PREHOOK: query: drop table ctas0_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table ctas0_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate
 +PREHOOK: type: CREATETABLE_AS_SELECT
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@ctas0_mm
 +POSTHOOK: query: create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate
 +POSTHOOK: type: CREATETABLE_AS_SELECT
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@ctas0_mm
 +POSTHOOK: Lineage: ctas0_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: ctas0_mm.p SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from ctas0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@ctas0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from ctas0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@ctas0_mm
 +#### A masked pattern was here ####
 +98	455
 +97	455
 +0	456
 +10	456
 +100	457
 +103	457
 +PREHOOK: query: drop table ctas0_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@ctas0_mm
 +PREHOOK: Output: default@ctas0_mm
 +POSTHOOK: query: drop table ctas0_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@ctas0_mm
 +POSTHOOK: Output: default@ctas0_mm
 +PREHOOK: query: drop table ctas1_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table ctas1_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
 +  select * from intermediate union all select * from intermediate
 +PREHOOK: type: CREATETABLE_AS_SELECT
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@ctas1_mm
 +POSTHOOK: query: create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
 +  select * from intermediate union all select * from intermediate
 +POSTHOOK: type: CREATETABLE_AS_SELECT
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@ctas1_mm
 +POSTHOOK: Lineage: ctas1_mm.key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: ctas1_mm.p EXPRESSION [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from ctas1_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@ctas1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from ctas1_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@ctas1_mm
 +#### A masked pattern was here ####
 +98	455
 +98	455
 +97	455
 +97	455
 +0	456
 +0	456
 +10	456
 +10	456
 +100	457
 +100	457
 +103	457
 +103	457
 +PREHOOK: query: drop table ctas1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@ctas1_mm
 +PREHOOK: Output: default@ctas1_mm
 +POSTHOOK: query: drop table ctas1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@ctas1_mm
 +POSTHOOK: Output: default@ctas1_mm
 +PREHOOK: query: drop table load0_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table load0_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@load0_mm
 +POSTHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@load0_mm
 +PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@load0_mm
 +POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@load0_mm
 +PREHOOK: query: select count(1) from load0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@load0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select count(1) from load0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@load0_mm
 +#### A masked pattern was here ####
 +500
 +PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@load0_mm
 +POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@load0_mm
 +PREHOOK: query: select count(1) from load0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@load0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select count(1) from load0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@load0_mm
 +#### A masked pattern was here ####
 +1000
 +PREHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@load0_mm
 +POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@load0_mm
 +PREHOOK: query: select count(1) from load0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@load0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select count(1) from load0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@load0_mm
 +#### A masked pattern was here ####
 +500
 +PREHOOK: query: drop table load0_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@load0_mm
 +PREHOOK: Output: default@load0_mm
 +POSTHOOK: query: drop table load0_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@load0_mm
 +POSTHOOK: Output: default@load0_mm
 +PREHOOK: query: drop table intermediate2
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table intermediate2
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table intermediate2 (key string, value string) stored as textfile
 +#### A masked pattern was here ####
 +PREHOOK: type: CREATETABLE
 +#### A masked pattern was here ####
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: create table intermediate2 (key string, value string) stored as textfile
 +#### A masked pattern was here ####
 +POSTHOOK: type: CREATETABLE
 +#### A masked pattern was here ####
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@intermediate2
 +PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@intermediate2
 +PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@intermediate2
 +PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@intermediate2
 +PREHOOK: query: drop table load1_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table load1_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@load1_mm
 +POSTHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@load1_mm
 +#### A masked pattern was here ####
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@load1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@load1_mm
 +#### A masked pattern was here ####
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@load1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@load1_mm
 +PREHOOK: query: select count(1) from load1_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@load1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select count(1) from load1_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@load1_mm
 +#### A masked pattern was here ####
 +1000
 +PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@intermediate2
 +PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@intermediate2
 +PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@intermediate2
 +#### A masked pattern was here ####
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@load1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@load1_mm
 +PREHOOK: query: select count(1) from load1_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@load1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select count(1) from load1_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@load1_mm
 +#### A masked pattern was here ####
 +1050
 +PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@intermediate2
 +#### A masked pattern was here ####
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@load1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@load1_mm
 +PREHOOK: query: select count(1) from load1_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@load1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select count(1) from load1_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@load1_mm
 +#### A masked pattern was here ####
 +500
 +PREHOOK: query: drop table load1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@load1_mm
 +PREHOOK: Output: default@load1_mm
 +POSTHOOK: query: drop table load1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@load1_mm
 +POSTHOOK: Output: default@load1_mm
 +PREHOOK: query: drop table load2_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table load2_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table load2_mm (key string, value string)
 +  partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@load2_mm
 +POSTHOOK: query: create table load2_mm (key string, value string)
 +  partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@load2_mm
 +PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@intermediate2
 +PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@intermediate2
 +PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@intermediate2
 +#### A masked pattern was here ####
 +PREHOOK: type: LOAD
 +#### A masked pattern was here ####
 +PREHOOK: Output: default@load2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: type: LOAD
 +#### A masked pattern was here ####
 +POSTHOOK: Output: default@load2_mm
 +POSTHOOK: Output: default@load2_mm@k=5/l=5
 +PREHOOK: query: select count(1) from load2_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@load2_mm
 +PREHOOK: Input: default@load2_mm@k=5/l=5
 +#### A masked pattern was here ####
 +POSTHOOK: query: select count(1) from load2_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@load2_mm
 +POSTHOOK: Input: default@load2_mm@k=5/l=5
 +#### A masked pattern was here ####
 +1025
 +PREHOOK: query: drop table load2_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@load2_mm
 +PREHOOK: Output: default@load2_mm
 +POSTHOOK: query: drop table load2_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@load2_mm
 +POSTHOOK: Output: default@load2_mm
 +PREHOOK: query: drop table intermediate2
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@intermediate2
 +PREHOOK: Output: default@intermediate2
 +POSTHOOK: query: drop table intermediate2
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@intermediate2
 +POSTHOOK: Output: default@intermediate2
 +PREHOOK: query: drop table multi0_1_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table multi0_1_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: drop table multi0_2_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table multi0_2_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table multi0_1_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: create table multi0_1_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@multi0_1_mm
 +PREHOOK: query: create table multi0_2_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: create table multi0_2_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@multi0_2_mm
 +PREHOOK: query: insert into table multi0_1_mm select key, p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: insert into table multi0_1_mm select key, p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_1_mm
 +POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: insert into table multi0_2_mm select p, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: insert into table multi0_2_mm select p, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_2_mm
 +POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from multi0_1_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_1_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +0	456
 +10	456
 +97	455
 +98	455
 +100	457
 +103	457
 +PREHOOK: query: select * from multi0_2_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_2_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +455	97
 +455	98
 +456	0
 +456	10
 +457	100
 +457	103
 +PREHOOK: query: insert into table multi0_1_mm select p, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: insert into table multi0_1_mm select p, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_1_mm
 +POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table multi0_2_mm select key, p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: insert into table multi0_2_mm select key, p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_2_mm
 +POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from multi0_1_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_1_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +0	456
 +10	456
 +97	455
 +98	455
 +100	457
 +103	457
 +455	97
 +455	98
 +456	0
 +456	10
 +457	100
 +457	103
 +PREHOOK: query: select * from multi0_2_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_2_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +0	456
 +10	456
 +97	455
 +98	455
 +100	457
 +103	457
 +455	97
 +455	98
 +456	0
 +456	10
 +457	100
 +457	103
 +PREHOOK: query: drop table multi0_1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@multi0_1_mm
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: drop table multi0_1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@multi0_1_mm
 +POSTHOOK: Output: default@multi0_1_mm
 +PREHOOK: query: drop table multi0_2_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@multi0_2_mm
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: drop table multi0_2_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@multi0_2_mm
 +POSTHOOK: Output: default@multi0_2_mm
 +PREHOOK: query: drop table multi1_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table multi1_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@multi1_mm
 +POSTHOOK: query: create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@multi1_mm
 +PREHOOK: query: from intermediate
 +insert into table multi1_mm partition(p=1) select p, key
 +insert into table multi1_mm partition(p=2) select key, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm@p=1
 +PREHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: query: from intermediate
 +insert into table multi1_mm partition(p=1) select p, key
 +insert into table multi1_mm partition(p=2) select key, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +0	456	2
 +10	456	2
 +97	455	2
 +98	455	2
 +100	457	2
 +103	457	2
 +455	97	1
 +455	98	1
 +456	0	1
 +456	10	1
 +457	100	1
 +457	103	1
 +PREHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +0	456	1
 +0	456	2
 +10	456	1
 +10	456	2
 +97	455	1
 +97	455	2
 +98	455	1
 +98	455	2
 +100	457	1
 +100	457	2
 +103	457	1
 +103	457	2
 +455	97	1
 +455	97	2
 +455	98	1
 +455	98	2
 +456	0	1
 +456	0	2
 +456	10	1
 +456	10	2
 +457	100	1
 +457	100	2
 +457	103	1
 +457	103	2
 +PREHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, p
 +insert into table multi1_mm partition(p=1) select key, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm
 +PREHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, p
 +insert into table multi1_mm partition(p=1) select key, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
++POSTHOOK: Output: default@multi1_mm@p=455
++POSTHOOK: Output: default@multi1_mm@p=456
++POSTHOOK: Output: default@multi1_mm@p=457
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
- POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
- POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
++PREHOOK: Input: default@multi1_mm@p=455
++PREHOOK: Input: default@multi1_mm@p=456
++PREHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
++POSTHOOK: Input: default@multi1_mm@p=455
++POSTHOOK: Input: default@multi1_mm@p=456
++POSTHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +0	456	1
 +0	456	1
 +0	456	2
 +10	456	1
 +10	456	1
 +10	456	2
 +97	455	1
 +97	455	1
 +97	455	2
 +98	455	1
 +98	455	1
 +98	455	2
 +100	457	1
 +100	457	1
 +100	457	2
 +103	457	1
 +103	457	1
 +103	457	2
 +455	97	1
 +455	97	2
++455	97	455
 +455	98	1
 +455	98	2
++455	98	455
 +456	0	1
 +456	0	2
++456	0	456
 +456	10	1
 +456	10	2
++456	10	456
 +457	100	1
 +457	100	2
++457	100	457
 +457	103	1
 +457	103	2
++457	103	457
 +PREHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, 1
 +insert into table multi1_mm partition(p=1) select key, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm
 +PREHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, 1
 +insert into table multi1_mm partition(p=1) select key, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
++PREHOOK: Input: default@multi1_mm@p=455
++PREHOOK: Input: default@multi1_mm@p=456
++PREHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
++POSTHOOK: Input: default@multi1_mm@p=455
++POSTHOOK: Input: default@multi1_mm@p=456
++POSTHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +0	456	1
 +0	456	1
 +0	456	1
 +0	456	2
 +10	456	1
 +10	456	1
 +10	456	1
 +10	456	2
 +97	455	1
 +97	455	1
 +97	455	1
 +97	455	2
 +98	455	1
 +98	455	1
 +98	455	1
 +98	455	2
 +100	457	1
 +100	457	1
 +100	457	1
 +100	457	2
 +103	457	1
 +103	457	1
 +103	457	1
 +103	457	2
 +455	97	1
 +455	97	1
 +455	97	2
++455	97	455
 +455	98	1
 +455	98	1
 +455	98	2
++455	98	455
 +456	0	1
 +456	0	1
 +456	0	2
++456	0	456
 +456	10	1
 +456	10	1
 +456	10	2
++456	10	456
 +457	100	1
 +457	100	1
 +457	100	2
++457	100	457
 +457	103	1
 +457	103	1
 +457	103	2
++457	103	457
 +PREHOOK: query: drop table multi1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Output: default@multi1_mm
 +POSTHOOK: query: drop table multi1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Output: default@multi1_mm
 +PREHOOK: query: drop table stats_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table stats_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table stats_mm(key int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@stats_mm
 +POSTHOOK: query: create table stats_mm(key int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@stats_mm
 +PREHOOK: query: insert into table stats_mm  select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@stats_mm
 +POSTHOOK: query: insert into table stats_mm  select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@stats_mm
 +POSTHOOK: Lineage: stats_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: desc formatted stats_mm
 +PREHOOK: type: DESCTABLE
 +PREHOOK: Input: default@stats_mm
 +POSTHOOK: query: desc formatted stats_mm
 +POSTHOOK: type: DESCTABLE
 +POSTHOOK: I

<TRUNCATED>

[47/50] [abbrv] hive git commit: HIVE-15212 : merge branch into master - fix some tests (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-15212 : merge branch into master - fix some tests (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/98c5b637
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/98c5b637
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/98c5b637

Branch: refs/heads/master
Commit: 98c5b637df2df2ee8ada89a2862ce928ee074ebc
Parents: ae4ac4b
Author: sergey <se...@apache.org>
Authored: Tue Oct 10 15:21:27 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Tue Oct 10 15:21:27 2017 -0700

----------------------------------------------------------------------
 .../test/resources/testconfiguration.properties |  3 +-
 .../queries/clientnegative/mm_bucket_convert.q  | 18 ---------
 ql/src/test/queries/clientnegative/mm_convert.q | 10 +++++
 .../clientnegative/mm_bucket_convert.q.out      | 41 --------------------
 .../results/clientnegative/mm_convert.q.out     | 17 ++++++++
 ql/src/test/results/clientpositive/mm_all.q.out | 13 +------
 6 files changed, 31 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/98c5b637/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index a72f66d..2fa5abd 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -34,7 +34,8 @@ disabled.query.files=ql_rewrite_gbtoidx.q,\
   ql_rewrite_gbtoidx_cbo_2.q,\
   rcfile_merge1.q,\
   smb_mapjoin_8.q,\
-  stats_filemetadata.q
+  stats_filemetadata.q,\
+  mm_exim.q
 
 # NOTE: Add tests to minitez only if it is very
 # specific to tez and cannot be added to minillap.

http://git-wip-us.apache.org/repos/asf/hive/blob/98c5b637/ql/src/test/queries/clientnegative/mm_bucket_convert.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/mm_bucket_convert.q b/ql/src/test/queries/clientnegative/mm_bucket_convert.q
deleted file mode 100644
index 2ded047..0000000
--- a/ql/src/test/queries/clientnegative/mm_bucket_convert.q
+++ /dev/null
@@ -1,18 +0,0 @@
-set hive.mapred.mode=nonstrict;
-set hive.explain.user=false;
-set hive.fetch.task.conversion=none;
-set hive.exec.dynamic.partition.mode=nonstrict;
-
-drop table bucket0_mm;
-drop table bucket1_mm;
-create table bucket0_mm(key int, id int) clustered by (key) into 2 buckets
-  tblproperties("transactional"="true", "transactional_properties"="insert_only");
-create table bucket1_mm(key int, id int) clustered by (key) into 2 buckets
-  tblproperties("transactional"="true", "transactional_properties"="insert_only");
-
-set hive.strict.checks.bucketing=false;
-alter table bucket0_mm unset tblproperties('transactional_properties', 'transactional');
-set hive.strict.checks.bucketing=true;
-alter table bucket1_mm unset tblproperties('transactional_properties', 'transactional');
-
-

http://git-wip-us.apache.org/repos/asf/hive/blob/98c5b637/ql/src/test/queries/clientnegative/mm_convert.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/mm_convert.q b/ql/src/test/queries/clientnegative/mm_convert.q
new file mode 100644
index 0000000..acbf4bb
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/mm_convert.q
@@ -0,0 +1,10 @@
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+set hive.fetch.task.conversion=none;
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+drop table convert_mm;
+create table convert_mm(key int, id int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
+alter table convert_mm unset tblproperties('transactional_properties', 'transactional');
+
+

http://git-wip-us.apache.org/repos/asf/hive/blob/98c5b637/ql/src/test/results/clientnegative/mm_bucket_convert.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/mm_bucket_convert.q.out b/ql/src/test/results/clientnegative/mm_bucket_convert.q.out
deleted file mode 100644
index b732d3e..0000000
--- a/ql/src/test/results/clientnegative/mm_bucket_convert.q.out
+++ /dev/null
@@ -1,41 +0,0 @@
-PREHOOK: query: drop table bucket0_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table bucket0_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: drop table bucket1_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table bucket1_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table bucket0_mm(key int, id int) clustered by (key) into 2 buckets
-  tblproperties("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@bucket0_mm
-POSTHOOK: query: create table bucket0_mm(key int, id int) clustered by (key) into 2 buckets
-  tblproperties("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@bucket0_mm
-PREHOOK: query: create table bucket1_mm(key int, id int) clustered by (key) into 2 buckets
-  tblproperties("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@bucket1_mm
-POSTHOOK: query: create table bucket1_mm(key int, id int) clustered by (key) into 2 buckets
-  tblproperties("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@bucket1_mm
-PREHOOK: query: alter table bucket0_mm unset tblproperties('transactional_properties', 'transactional')
-PREHOOK: type: ALTERTABLE_PROPERTIES
-PREHOOK: Input: default@bucket0_mm
-PREHOOK: Output: default@bucket0_mm
-POSTHOOK: query: alter table bucket0_mm unset tblproperties('transactional_properties', 'transactional')
-POSTHOOK: type: ALTERTABLE_PROPERTIES
-POSTHOOK: Input: default@bucket0_mm
-POSTHOOK: Output: default@bucket0_mm
-PREHOOK: query: alter table bucket1_mm unset tblproperties('transactional_properties', 'transactional')
-PREHOOK: type: ALTERTABLE_PROPERTIES
-PREHOOK: Input: default@bucket1_mm
-PREHOOK: Output: default@bucket1_mm
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Converting bucketed tables from MM is not supported by default; copying files from multiple MM directories may potentially break the buckets. You can set hive.strict.checks.bucketing to false for this query if you want to force the conversion.

http://git-wip-us.apache.org/repos/asf/hive/blob/98c5b637/ql/src/test/results/clientnegative/mm_convert.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/mm_convert.q.out b/ql/src/test/results/clientnegative/mm_convert.q.out
new file mode 100644
index 0000000..496f8b5
--- /dev/null
+++ b/ql/src/test/results/clientnegative/mm_convert.q.out
@@ -0,0 +1,17 @@
+PREHOOK: query: drop table convert_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table convert_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table convert_mm(key int, id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@convert_mm
+POSTHOOK: query: create table convert_mm(key int, id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@convert_mm
+PREHOOK: query: alter table convert_mm unset tblproperties('transactional_properties', 'transactional')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@convert_mm
+PREHOOK: Output: default@convert_mm
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Cannot convert an ACID table to non-ACID

http://git-wip-us.apache.org/repos/asf/hive/blob/98c5b637/ql/src/test/results/clientpositive/mm_all.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/mm_all.q.out b/ql/src/test/results/clientpositive/mm_all.q.out
index fa9e4a7..c943f01 100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@ -88,10 +88,7 @@ STAGE PLANS:
     Conditional Operator
 
   Stage: Stage-4
-    Move Operator
-      files:
-          hdfs directory: true
-#### A masked pattern was here ####
+    Dependency Collection
 
   Stage: Stage-0
     Move Operator
@@ -125,10 +122,7 @@ STAGE PLANS:
       input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 
   Stage: Stage-6
-    Move Operator
-      files:
-          hdfs directory: true
-#### A masked pattern was here ####
+    Dependency Collection
 
 PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 PREHOOK: type: QUERY
@@ -1792,7 +1786,6 @@ POSTHOOK: query: desc formatted stats_mm
 POSTHOOK: type: DESCTABLE
 POSTHOOK: Input: default@stats_mm
 # col_name            	data_type           	comment             
-	 	 
 key                 	int                 	                    
 	 	 
 # Detailed Table Information	 	 
@@ -1843,7 +1836,6 @@ POSTHOOK: query: desc formatted stats_mm
 POSTHOOK: type: DESCTABLE
 POSTHOOK: Input: default@stats_mm
 # col_name            	data_type           	comment             
-	 	 
 key                 	int                 	                    
 	 	 
 # Detailed Table Information	 	 
@@ -1902,7 +1894,6 @@ POSTHOOK: query: desc formatted stats2_mm
 POSTHOOK: type: DESCTABLE
 POSTHOOK: Input: default@stats2_mm
 # col_name            	data_type           	comment             
-	 	 
 _c0                 	array<string>       	                    
 	 	 
 # Detailed Table Information

[19/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42a38577
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42a38577
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42a38577

Branch: refs/heads/master
Commit: 42a38577bc9d242bed040c503b0d1d46f74e0301
Parents: 81853c1 b82d38a
Author: Wei Zheng <we...@apache.org>
Authored: Wed Sep 6 21:24:02 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Wed Sep 6 21:24:02 2017 -0700

----------------------------------------------------------------------
 .gitignore                                      |      1 +
 LICENSE                                         |      2 +-
 .../accumulo/AccumuloConnectionParameters.java  |     57 +-
 .../hive/accumulo/AccumuloStorageHandler.java   |     77 +-
 .../hive/accumulo/HiveAccumuloHelper.java       |    396 +-
 .../org/apache/hadoop/hive/accumulo/Utils.java  |      3 +-
 .../mr/HiveAccumuloTableInputFormat.java        |    244 +-
 .../mr/HiveAccumuloTableOutputFormat.java       |     84 +-
 .../serde/CompositeAccumuloRowIdFactory.java    |      3 +-
 .../serde/DefaultAccumuloRowIdFactory.java      |      5 +-
 .../accumulo/TestAccumuloStorageHandler.java    |      6 +-
 .../hive/accumulo/TestHiveAccumuloHelper.java   |    184 +-
 .../mr/TestHiveAccumuloTableInputFormat.java    |     37 +-
 .../mr/TestHiveAccumuloTableOutputFormat.java   |     29 +-
 .../positive/accumulo_predicate_pushdown.q.out  |      3 +-
 .../results/positive/accumulo_queries.q.out     |      2 +-
 .../java/org/apache/hive/beeline/BeeLine.java   |      4 +-
 .../org/apache/hive/beeline/BeeLineOpts.java    |      9 +
 .../java/org/apache/hive/beeline/Commands.java  |     32 +-
 .../org/apache/hive/beeline/HiveSchemaTool.java |     58 +-
 .../src/java/org/apache/hive/beeline/Rows.java  |     21 +-
 beeline/src/main/resources/BeeLine.properties   |      1 +
 .../apache/hive/beeline/TestBufferedRows.java   |      2 +-
 .../org/apache/hive/beeline/TestCommands.java   |     29 +-
 .../hive/beeline/TestIncrementalRows.java       |    136 +
 .../TestIncrementalRowsWithNormalization.java   |     90 -
 .../hive/beeline/TestTableOutputFormat.java     |      2 +-
 bin/ext/hbaseimport.sh                          |     27 -
 bin/ext/hbaseschematool.sh                      |     27 -
 bin/ext/metastore.sh                            |      2 +-
 binary-package-licenses/org.json-LICENSE        |     22 -
 .../org/apache/hadoop/hive/cli/CliDriver.java   |     10 +-
 common/pom.xml                                  |      8 +-
 .../apache/hadoop/hive/common/FileUtils.java    |     80 +-
 .../org/apache/hadoop/hive/common/LogUtils.java |     41 +
 .../hadoop/hive/common/jsonexplain/Vertex.java  |      2 +-
 .../common/jsonexplain/tez/TezJsonParser.java   |      2 +
 .../common/metrics/common/MetricsConstant.java  |     14 -
 .../common/ndv/NumDistinctValueEstimator.java   |     51 +
 .../ndv/NumDistinctValueEstimatorFactory.java   |     74 +
 .../hadoop/hive/common/ndv/fm/FMSketch.java     |    359 +
 .../hive/common/ndv/fm/FMSketchUtils.java       |    132 +
 .../hive/common/ndv/hll/HLLConstants.java       |    929 +
 .../hive/common/ndv/hll/HLLDenseRegister.java   |    192 +
 .../hadoop/hive/common/ndv/hll/HLLRegister.java |     50 +
 .../hive/common/ndv/hll/HLLSparseRegister.java  |    260 +
 .../hadoop/hive/common/ndv/hll/HyperLogLog.java |    634 +
 .../hive/common/ndv/hll/HyperLogLogUtils.java   |    392 +
 .../hadoop/hive/common/type/TimestampTZ.java    |    122 +-
 .../hive/common/type/TimestampTZUtil.java       |    150 +
 .../org/apache/hadoop/hive/conf/Constants.java  |      2 +
 .../org/apache/hadoop/hive/conf/HiveConf.java   |    259 +-
 .../apache/hadoop/hive/conf/HiveConfUtil.java   |     20 +-
 .../hive/common/util/HiveStringUtils.java       |     92 +-
 .../java/org/apache/hive/http/HttpServer.java   |      1 +
 .../java/org/apache/hive/http/StackServlet.java |      2 +-
 .../src/main/resources/hive-log4j2.properties   |      8 +-
 .../hadoop/hive/common/TestFileUtils.java       |     17 +-
 .../ndv/fm/TestFMSketchSerialization.java       |     98 +
 .../hive/common/ndv/hll/TestHLLNoBias.java      |    114 +
 .../common/ndv/hll/TestHLLSerialization.java    |    267 +
 .../hive/common/ndv/hll/TestHyperLogLog.java    |    227 +
 .../common/ndv/hll/TestHyperLogLogDense.java    |     82 +
 .../common/ndv/hll/TestHyperLogLogSparse.java   |     81 +
 .../common/ndv/hll/TestSparseEncodeHash.java    |     56 +
 .../hive/common/type/TestTimestampTZ.java       |     63 +-
 .../apache/hadoop/hive/conf/TestHiveConf.java   |     11 +
 .../hive/conf/TestHiveConfRestrictList.java     |     12 +-
 .../hadoop/hive/conf/TestHiveConfUtil.java      |     68 +
 .../hive/common/util/TestHiveStringUtils.java   |     51 +
 .../clientpositive/udf_row_sequence.q.out       |      6 +-
 data/conf/hive-log4j2.properties                |     11 +-
 data/conf/hive-site.xml                         |      4 +
 data/conf/llap/hive-site.xml                    |      9 +
 data/conf/spark/standalone/hive-site.xml        |     17 +-
 data/conf/spark/yarn-client/hive-site.xml       |     10 +
 data/conf/tez/hive-site.xml                     |     10 +
 data/files/having_data.txt                      |      8 +
 data/files/impala_int96_timestamp.parq          |    Bin 241 -> 0 bytes
 data/scripts/input20_script.py                  |     18 +-
 dev-support/checkstyle_YETUS-484.sh             |    406 +
 dev-support/findbugs_YETUS-471.sh               |    488 +
 dev-support/hive-personality.sh                 |    100 +
 dev-support/maven_YETUS-506.sh                  |    687 +
 dev-support/smart-apply-patch.sh                |     18 +
 dev-support/test-patch.sh                       |     18 +
 dev-support/yetus-wrapper.sh                    |    185 +
 druid-handler/pom.xml                           |      4 +
 .../hadoop/hive/druid/DruidStorageHandler.java  |    189 +-
 .../hive/druid/DruidStorageHandlerUtils.java    |    378 +-
 .../hadoop/hive/druid/io/DruidOutputFormat.java |     21 +-
 .../hive/druid/security/DruidKerberosUtil.java  |    126 +
 .../hive/druid/security/KerberosHttpClient.java |    163 +
 .../druid/security/ResponseCookieHandler.java   |     92 +
 .../RetryIfUnauthorizedResponseHandler.java     |    104 +
 .../druid/security/RetryResponseHolder.java     |     46 +
 .../serde/DruidGroupByQueryRecordReader.java    |     44 +-
 .../hadoop/hive/druid/serde/DruidSerDe.java     |     42 +-
 .../hive/druid/serde/DruidSerDeUtils.java       |     26 +-
 .../hadoop/hive/druid/TestDruidSerDe.java       |     50 +-
 .../hive/druid/TestDruidStorageHandler.java     |    384 +-
 .../hive/ql/io/TestDruidRecordWriter.java       |      6 +-
 errata.txt                                      |      1 +
 .../results/positive/hbase_ppd_key_range.q.out  |      3 +-
 .../test/results/positive/hbase_queries.q.out   |      2 +-
 .../test/results/positive/ppd_key_ranges.q.out  |     34 +-
 .../cli/SemanticAnalysis/HCatAuthUtil.java      |      3 +-
 .../mapreduce/FileRecordWriterContainer.java    |      7 +-
 .../hcatalog/mapreduce/HCatBaseInputFormat.java |      3 +
 .../mapreduce/HCatBaseOutputFormat.java         |     15 +-
 .../hive/hcatalog/mapreduce/SpecialCases.java   |     27 +
 .../apache/hive/hcatalog/cli/TestPermsGrp.java  |      4 +-
 .../mapreduce/TestHCatOutputFormat.java         |     43 +-
 .../mapreduce/TestHCatPartitionPublish.java     |      6 +-
 .../apache/hive/hcatalog/pig/HCatLoader.java    |      2 +-
 .../pig/TestHCatLoaderComplexSchema.java        |      3 -
 .../hcatalog/pig/TestHCatLoaderEncryption.java  |      9 +-
 .../hive/hcatalog/pig/TestHCatStorerMulti.java  |     10 +-
 .../hcatalog/pig/TestParquetHCatLoader.java     |     49 -
 .../hcatalog/pig/TestParquetHCatStorer.java     |    167 -
 .../listener/DbNotificationListener.java        |    101 +-
 .../MetaStoreEventListenerConstants.java        |     33 -
 .../hcatalog/messaging/AddPartitionMessage.java |      2 +
 .../messaging/AlterPartitionMessage.java        |      2 +
 .../hcatalog/messaging/AlterTableMessage.java   |      1 +
 .../hcatalog/messaging/CreateTableMessage.java  |      2 +
 .../messaging/DropPartitionMessage.java         |      1 +
 .../hcatalog/messaging/DropTableMessage.java    |      1 +
 .../hive/hcatalog/messaging/InsertMessage.java  |      1 +
 .../hive/hcatalog/messaging/MessageFactory.java |     12 +
 .../messaging/json/JSONAddPartitionMessage.java |     15 +-
 .../json/JSONAlterPartitionMessage.java         |     19 +-
 .../messaging/json/JSONAlterTableMessage.java   |     18 +-
 .../messaging/json/JSONCreateTableMessage.java  |     16 +-
 .../json/JSONDropPartitionMessage.java          |     15 +-
 .../messaging/json/JSONDropTableMessage.java    |     11 +-
 .../messaging/json/JSONInsertMessage.java       |     14 +-
 .../messaging/json/JSONMessageFactory.java      |     28 +-
 .../listener/TestNotificationListener.java      |      7 +
 hcatalog/src/test/e2e/hcatalog/tests/hcat.conf  |      2 +-
 .../streaming/AbstractRecordWriter.java         |     44 +-
 .../hive/hcatalog/streaming/HiveEndPoint.java   |      4 +-
 .../mutate/worker/BucketIdResolverImpl.java     |      6 +-
 .../mutate/worker/MutatorCoordinator.java       |      7 +-
 .../streaming/mutate/worker/MutatorImpl.java    |     16 +-
 .../apache/hive/hcatalog/streaming/package.html |     21 +-
 .../hive/hcatalog/streaming/TestStreaming.java  |    229 +-
 .../streaming/mutate/StreamingAssert.java       |     46 +-
 .../streaming/mutate/TestMutations.java         |     56 +-
 .../mutate/worker/TestBucketIdResolverImpl.java |      6 +-
 .../mutate/worker/TestMutatorImpl.java          |      2 +-
 .../hive/hcatalog/api/TestHCatClient.java       |    118 +-
 .../api/TestHCatClientNotification.java         |      5 +
 .../listener/DummyRawStoreFailEvent.java        |     27 +-
 .../listener/TestDbNotificationListener.java    |     19 +-
 .../src/test/queries/clientpositive/having.q    |     12 +
 ...rt_overwrite_dynamic_partitions_merge_move.q |     37 +
 ...rt_overwrite_dynamic_partitions_merge_only.q |     37 +
 ...ert_overwrite_dynamic_partitions_move_only.q |     37 +
 .../test/results/clientpositive/having.q.out    |     59 +
 .../insert_blobstore_to_blobstore.q.out         |      6 +
 .../insert_empty_into_blobstore.q.out           |      3 +
 .../clientpositive/insert_into_table.q.out      |      2 +
 ...verwrite_dynamic_partitions_merge_move.q.out |    138 +
 ...verwrite_dynamic_partitions_merge_only.q.out |    128 +
 ...overwrite_dynamic_partitions_move_only.q.out |    158 +
 .../clientpositive/insert_overwrite_table.q.out |      2 +
 .../clientpositive/orc_format_nonpart.q.out     |      6 +
 .../clientpositive/orc_format_part.q.out        |     16 +
 .../orc_nonstd_partitions_loc.q.out             |     24 +
 .../clientpositive/rcfile_format_nonpart.q.out  |      6 +
 .../clientpositive/rcfile_format_part.q.out     |     16 +
 .../rcfile_nonstd_partitions_loc.q.out          |     24 +
 .../clientpositive/zero_rows_blobstore.q.out    |      2 +
 itests/hive-jmh/pom.xml                         |      6 +
 .../benchmark/storage/ColumnarStorageBench.java |      3 +-
 .../vectorization/mapjoin/AbstractMapJoin.java  |    179 +
 .../mapjoin/MapJoinMultiKeyBench.java           |    313 +
 .../mapjoin/MapJoinMultiKeyBenchBase.java       |     68 +
 .../mapjoin/MapJoinOneLongKeyBench.java         |    313 +
 .../mapjoin/MapJoinOneLongKeyBenchBase.java     |     66 +
 .../mapjoin/MapJoinOneStringKeyBench.java       |    313 +
 .../mapjoin/MapJoinOneStringKeyBenchBase.java   |     66 +
 .../hive/minikdc/TestHs2HooksWithMiniKdc.java   |     28 +-
 .../security/TestHadoopAuthBridge23.java        |    416 +
 .../hive/thrift/TestHadoopAuthBridge23.java     |    417 -
 itests/hive-unit/pom.xml                        |      6 +
 .../apache/hadoop/hive/hooks/TestHs2Hooks.java  |     27 +-
 .../AbstractTestAuthorizationApiAuthorizer.java |      4 +-
 .../hadoop/hive/metastore/TestFilterHooks.java  |      4 +-
 .../hive/metastore/TestHiveMetaStore.java       |    144 +-
 .../metastore/TestHiveMetaStoreStatsMerge.java  |    199 -
 ...TestHiveMetaStoreWithEnvironmentContext.java |      4 +-
 .../metastore/TestMetaStoreAuthorization.java   |      4 +-
 .../TestMetaStoreEndFunctionListener.java       |      5 +-
 .../metastore/TestMetaStoreEventListener.java   |     72 +-
 .../TestMetaStoreEventListenerOnlyOnCommit.java |      4 +-
 .../metastore/TestMetaStoreInitListener.java    |      5 +-
 .../metastore/TestMetaStoreListenersError.java  |      6 +-
 .../hive/metastore/TestMetaStoreMetrics.java    |     82 +-
 .../hive/metastore/TestRemoteHiveMetaStore.java |      4 +-
 .../TestRemoteHiveMetaStoreIpAddress.java       |      4 +-
 .../hive/metastore/TestReplChangeManager.java   |     37 +-
 .../hive/metastore/TestRetryingHMSHandler.java  |      4 +-
 .../metastore/hbase/HBaseIntegrationTests.java  |    112 -
 .../TestHBaseAggrStatsCacheIntegration.java     |    691 -
 .../hive/metastore/hbase/TestHBaseImport.java   |    775 -
 .../hbase/TestHBaseMetastoreMetrics.java        |    130 -
 .../metastore/hbase/TestHBaseMetastoreSql.java  |    223 -
 .../metastore/hbase/TestHBaseSchemaTool.java    |    582 -
 .../metastore/hbase/TestHBaseSchemaTool2.java   |     59 -
 .../hbase/TestHBaseStoreIntegration.java        |   1801 -
 .../hbase/TestStorageDescriptorSharing.java     |    191 -
 .../apache/hadoop/hive/ql/TestAcidOnTez.java    |    367 +-
 .../hive/ql/TestAcidOnTezWithSplitUpdate.java   |     28 -
 ...estDDLWithRemoteMetastoreSecondNamenode.java |      4 +-
 .../hadoop/hive/ql/TestLocationQueries.java     |      2 +-
 .../hadoop/hive/ql/parse/TestExportImport.java  |    100 +
 .../hive/ql/parse/TestReplicationScenarios.java |   2772 +-
 ...TestReplicationScenariosAcrossInstances.java |     91 +-
 .../hadoop/hive/ql/parse/WarehouseInstance.java |     89 +-
 .../security/StorageBasedMetastoreTestBase.java |      5 +-
 .../TestAuthorizationPreEventListener.java      |     23 +-
 .../TestClientSideAuthorizationProvider.java    |      4 +-
 .../TestMetastoreAuthorizationProvider.java     |      4 +-
 .../TestMultiAuthorizationPreEventListener.java |      4 +-
 .../TestHiveAuthorizerCheckInvocation.java      |     38 +
 .../hive/ql/txn/compactor/TestCompactor.java    |      4 +-
 .../org/apache/hive/beeline/TestSchemaTool.java |     22 +-
 .../apache/hive/jdbc/TestJdbcWithMiniHS2.java   |     36 +-
 .../hive/service/TestDFSErrorHandling.java      |    152 +
 .../operation/TestOperationLoggingLayout.java   |     52 +
 .../service/cli/session/TestQueryDisplay.java   |      3 +-
 itests/pom.xml                                  |      2 +-
 itests/qtest-accumulo/pom.xml                   |      3 +-
 .../test/resources/testconfiguration.properties |     38 +-
 .../hadoop/hive/accumulo/AccumuloQTestUtil.java |      2 +-
 .../hive/cli/control/AbstractCliConfig.java     |      4 +-
 .../control/AbstractCoreBlobstoreCliDriver.java |      3 +-
 .../hadoop/hive/cli/control/CliConfigs.java     |      4 +-
 .../hadoop/hive/cli/control/CoreCliDriver.java  |      4 +-
 .../hive/cli/control/CoreCompareCliDriver.java  |      2 +-
 .../hive/cli/control/CoreNegativeCliDriver.java |      2 +-
 .../hive/cli/control/CorePerfCliDriver.java     |      2 +-
 .../hadoop/hive/hbase/HBaseQTestUtil.java       |      2 +-
 .../metastore/hbase/HBaseStoreTestUtil.java     |     45 -
 .../org/apache/hadoop/hive/ql/QTestUtil.java    |     87 +-
 .../hadoop/hive/ql/parse/CoreParseNegative.java |      3 +-
 .../org/apache/hive/beeline/Parallelized.java   |      9 +-
 .../org/apache/hive/jdbc/miniHS2/MiniHS2.java   |      4 +-
 .../hive/storage/jdbc/JdbcRecordReader.java     |      7 +-
 .../org/apache/hive/storage/jdbc/JdbcSerDe.java |     18 +-
 .../storage/jdbc/dao/JdbcRecordIterator.java    |      8 +-
 .../src/test/java/org/apache/TestSuite.java     |     29 -
 .../org/apache/hive/storage/jdbc/TestSuite.java |     29 +
 .../dao/GenericJdbcDatabaseAccessorTest.java    |      8 +-
 jdbc/pom.xml                                    |      2 +-
 .../org/apache/hive/jdbc/HiveBaseResultSet.java |      6 +-
 .../org/apache/hive/jdbc/HiveConnection.java    |      5 +-
 .../java/org/apache/hive/jdbc/JdbcColumn.java   |      8 +-
 .../hadoop/hive/llap/LlapBaseRecordReader.java  |    103 +-
 .../apache/hadoop/hive/llap/SubmitWorkInfo.java |     16 +-
 .../ext/LlapTaskUmbilicalExternalClient.java    |    504 +-
 .../apache/hadoop/hive/llap/io/api/LlapIo.java  |      4 +-
 .../hive/llap/registry/ServiceInstance.java     |      2 +-
 .../hive/llap/tez/LlapProtocolClientProxy.java  |    423 +-
 .../helpers/LlapTaskUmbilicalServer.java        |     44 +-
 .../hadoop/hive/llap/TestAsyncPbRpcProxy.java   |    145 +
 .../tez/TestLlapDaemonProtocolClientProxy.java  |    144 -
 .../daemon/rpc/LlapDaemonProtocolProtos.java    |   1854 +-
 .../hadoop/hive/llap/AsyncPbRpcProxy.java       |    435 +
 .../hive/llap/impl/LlapProtocolClientImpl.java  |     20 +-
 .../hadoop/hive/llap/io/ChunkedInputStream.java |     16 +-
 .../hive/llap/io/ChunkedOutputStream.java       |      9 +-
 .../src/protobuf/LlapDaemonProtocol.proto       |     16 +-
 .../hive/llap/io/TestChunkedInputStream.java    |      6 +-
 .../hadoop/hive/llap/LlapBaseInputFormat.java   |      2 -
 llap-server/bin/llapDaemon.sh                   |      3 +
 llap-server/pom.xml                             |      7 +-
 .../hadoop/hive/llap/cache/BuddyAllocator.java  |   1496 +-
 .../hive/llap/cache/EvictionDispatcher.java     |     23 +-
 .../hive/llap/cache/LlapAllocatorBuffer.java    |    414 +
 .../hive/llap/cache/LlapCacheableBuffer.java    |      6 +-
 .../hadoop/hive/llap/cache/LlapDataBuffer.java  |    119 +-
 .../hive/llap/cache/LowLevelCacheImpl.java      |     70 +-
 .../llap/cache/LowLevelCacheMemoryManager.java  |      7 -
 .../hive/llap/cache/LowLevelCachePolicy.java    |      2 -
 .../llap/cache/LowLevelFifoCachePolicy.java     |     16 +-
 .../llap/cache/LowLevelLrfuCachePolicy.java     |     95 +-
 .../hadoop/hive/llap/cache/MemoryManager.java   |      2 -
 .../hive/llap/cache/SerDeLowLevelCacheImpl.java |    115 +-
 .../hadoop/hive/llap/cache/SimpleAllocator.java |     22 +-
 .../hive/llap/cache/SimpleBufferManager.java    |     12 +-
 .../hadoop/hive/llap/cli/LlapServiceDriver.java |      6 +-
 .../configuration/LlapDaemonConfiguration.java  |      9 +-
 .../hive/llap/daemon/ContainerRunner.java       |      7 +-
 .../llap/daemon/impl/ContainerRunnerImpl.java   |     26 +-
 .../impl/EvictingPriorityBlockingQueue.java     |     29 +-
 .../hive/llap/daemon/impl/LlapDaemon.java       |      8 +
 .../daemon/impl/LlapProtocolServerImpl.java     |     12 +
 .../llap/daemon/impl/QueryFragmentInfo.java     |     24 +-
 .../hadoop/hive/llap/daemon/impl/QueryInfo.java |      7 +-
 .../hadoop/hive/llap/daemon/impl/Scheduler.java |      6 +-
 .../impl/StaticPermanentFunctionChecker.java    |      3 +-
 .../llap/daemon/impl/TaskExecutorService.java   |    451 +-
 .../llap/daemon/impl/TaskRunnerCallable.java    |     22 +-
 .../comparator/FirstInFirstOutComparator.java   |     14 +-
 .../comparator/LlapQueueComparatorBase.java     |     48 +
 .../comparator/ShortestJobFirstComparator.java  |     17 +-
 .../daemon/services/impl/LlapWebServices.java   |      1 +
 .../impl/SystemConfigurationServlet.java        |    196 +
 .../hive/llap/io/api/impl/LlapInputFormat.java  |     63 +-
 .../hive/llap/io/api/impl/LlapIoImpl.java       |    108 +-
 .../hive/llap/io/api/impl/LlapRecordReader.java |     88 +-
 .../llap/io/decode/EncodedDataConsumer.java     |      1 +
 .../io/decode/GenericColumnVectorProducer.java  |      9 +-
 .../llap/io/decode/OrcColumnVectorProducer.java |     15 +-
 .../llap/io/decode/OrcEncodedDataConsumer.java  |     12 +
 .../llap/io/encoded/OrcEncodedDataReader.java   |    122 +-
 .../llap/io/encoded/SerDeEncodedDataReader.java |     75 +-
 .../io/encoded/VectorDeserializeOrcWriter.java  |      3 +
 .../llap/io/metadata/OrcFileEstimateErrors.java |      4 +-
 .../hive/llap/io/metadata/OrcFileMetadata.java  |      4 +-
 .../llap/io/metadata/OrcStripeMetadata.java     |      4 +-
 .../io/metadata/ParquetMetadataCacheImpl.java   |    353 +
 .../hive/llap/metrics/LlapDaemonJvmInfo.java    |      3 +-
 .../hive/llap/metrics/LlapDaemonJvmMetrics.java |      7 +-
 .../llap/shufflehandler/FadvisedFileRegion.java |     14 +-
 .../llap/shufflehandler/ShuffleHandler.java     |     23 +-
 llap-server/src/main/resources/templates.py     |      2 +-
 .../hive/llap/cache/TestBuddyAllocator.java     |     73 +-
 .../cache/TestBuddyAllocatorForceEvict.java     |    470 +
 .../hive/llap/cache/TestLowLevelCacheImpl.java  |     29 +-
 .../llap/cache/TestLowLevelLrfuCachePolicy.java |     56 -
 .../hive/llap/cache/TestOrcMetadataCache.java   |     12 -
 .../daemon/impl/TaskExecutorTestHelpers.java    |     97 +-
 .../impl/TestEvictingPriorityBlockingQueue.java |     12 +-
 .../daemon/impl/TestTaskExecutorService.java    |    165 +-
 .../TestFirstInFirstOutComparator.java          |     36 +-
 .../TestShortestJobFirstComparator.java         |     35 +-
 metastore/if/hive_metastore.thrift              |   1593 -
 metastore/pom.xml                               |     90 +-
 .../upgrade/derby/022-HIVE-11107.derby.sql      |      2 +-
 .../upgrade/derby/043-HIVE-16922.derby.sql      |      3 +
 .../upgrade/derby/044-HIVE-16997.derby.sql      |      2 +
 .../upgrade/derby/hive-schema-3.0.0.derby.sql   |      2 +-
 .../derby/upgrade-2.3.0-to-3.0.0.derby.sql      |      2 +
 .../upgrade/hive/hive-schema-3.0.0.hive.sql     |     24 +
 .../upgrade/mssql/028-HIVE-16922.mssql.sql      |      3 +
 .../upgrade/mssql/029-HIVE-16997.mssql.sql      |      2 +
 .../upgrade/mssql/hive-schema-3.0.0.mssql.sql   |      2 +
 .../mssql/upgrade-2.3.0-to-3.0.0.mssql.sql      |      2 +
 .../upgrade/mysql/043-HIVE-16922.mysql.sql      |      3 +
 .../upgrade/mysql/044-HIVE-16997.mysql.sql      |      2 +
 .../upgrade/mysql/hive-schema-3.0.0.mysql.sql   |      4 +-
 .../mysql/upgrade-2.3.0-to-3.0.0.mysql.sql      |      2 +
 .../upgrade/oracle/043-HIVE-16922.oracle.sql    |      3 +
 .../upgrade/oracle/044-HIVE-16997.oracle.sql    |      2 +
 .../upgrade/oracle/hive-schema-3.0.0.oracle.sql |      2 +
 .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql    |      2 +
 .../postgres/042-HIVE-16922.postgres.sql        |      3 +
 .../postgres/043-HIVE-16997.postgres.sql        |      2 +
 .../postgres/hive-schema-3.0.0.postgres.sql     |      2 +
 .../upgrade-2.3.0-to-3.0.0.postgres.sql         |      2 +
 .../metastore/hbase/HbaseMetastoreProto.java    |  46709 ----
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp  |  73240 ------
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.h    |  23202 --
 .../ThriftHiveMetastore_server.skeleton.cpp     |    834 -
 .../thrift/gen-cpp/hive_metastore_constants.cpp |     65 -
 .../thrift/gen-cpp/hive_metastore_constants.h   |     48 -
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp |  21453 --
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |   8861 -
 .../hive/metastore/api/AbortTxnRequest.java     |    387 -
 .../hive/metastore/api/AbortTxnsRequest.java    |    438 -
 .../metastore/api/AddDynamicPartitions.java     |    864 -
 .../metastore/api/AddForeignKeyRequest.java     |    443 -
 .../api/AddNotNullConstraintRequest.java        |    443 -
 .../metastore/api/AddPartitionsRequest.java     |    849 -
 .../hive/metastore/api/AddPartitionsResult.java |    447 -
 .../metastore/api/AddPrimaryKeyRequest.java     |    443 -
 .../api/AddUniqueConstraintRequest.java         |    443 -
 .../hadoop/hive/metastore/api/AggrStats.java    |    542 -
 .../metastore/api/AlreadyExistsException.java   |    395 -
 .../metastore/api/BinaryColumnStatsData.java    |    687 -
 .../metastore/api/BooleanColumnStatsData.java   |    687 -
 .../metastore/api/CacheFileMetadataRequest.java |    703 -
 .../metastore/api/CacheFileMetadataResult.java  |    387 -
 .../hive/metastore/api/CheckLockRequest.java    |    589 -
 .../metastore/api/ClearFileMetadataRequest.java |    438 -
 .../metastore/api/ClearFileMetadataResult.java  |    283 -
 .../hive/metastore/api/ClientCapabilities.java  |    441 -
 .../hive/metastore/api/ClientCapability.java    |     45 -
 .../hive/metastore/api/ColumnStatistics.java    |    549 -
 .../metastore/api/ColumnStatisticsData.java     |    675 -
 .../metastore/api/ColumnStatisticsDesc.java     |    798 -
 .../hive/metastore/api/ColumnStatisticsObj.java |    593 -
 .../hive/metastore/api/CommitTxnRequest.java    |    387 -
 .../hive/metastore/api/CompactionRequest.java   |    977 -
 .../hive/metastore/api/CompactionResponse.java  |    583 -
 .../hive/metastore/api/CompactionType.java      |     45 -
 .../api/ConfigValSecurityException.java         |    395 -
 .../api/CurrentNotificationEventId.java         |    387 -
 .../hive/metastore/api/DataOperationType.java   |     57 -
 .../hadoop/hive/metastore/api/Database.java     |   1095 -
 .../apache/hadoop/hive/metastore/api/Date.java  |    387 -
 .../hive/metastore/api/DateColumnStatsData.java |    814 -
 .../hadoop/hive/metastore/api/Decimal.java      |    497 -
 .../metastore/api/DecimalColumnStatsData.java   |    814 -
 .../metastore/api/DoubleColumnStatsData.java    |    790 -
 .../metastore/api/DropConstraintRequest.java    |    591 -
 .../hive/metastore/api/DropPartitionsExpr.java  |    505 -
 .../metastore/api/DropPartitionsRequest.java    |   1112 -
 .../metastore/api/DropPartitionsResult.java     |    447 -
 .../hive/metastore/api/EnvironmentContext.java  |    447 -
 .../hive/metastore/api/EventRequestType.java    |     48 -
 .../hadoop/hive/metastore/api/FieldSchema.java  |    603 -
 .../metastore/api/FileMetadataExprType.java     |     42 -
 .../hive/metastore/api/FireEventRequest.java    |    861 -
 .../metastore/api/FireEventRequestData.java     |    309 -
 .../hive/metastore/api/FireEventResponse.java   |    283 -
 .../hive/metastore/api/ForeignKeysRequest.java  |    707 -
 .../hive/metastore/api/ForeignKeysResponse.java |    443 -
 .../hadoop/hive/metastore/api/Function.java     |   1199 -
 .../hadoop/hive/metastore/api/FunctionType.java |     42 -
 .../metastore/api/GetAllFunctionsResponse.java  |    447 -
 .../api/GetFileMetadataByExprRequest.java       |    773 -
 .../api/GetFileMetadataByExprResult.java        |    553 -
 .../metastore/api/GetFileMetadataRequest.java   |    438 -
 .../metastore/api/GetFileMetadataResult.java    |    540 -
 .../metastore/api/GetOpenTxnsInfoResponse.java  |    542 -
 .../hive/metastore/api/GetOpenTxnsResponse.java |    750 -
 .../api/GetPrincipalsInRoleRequest.java         |    389 -
 .../api/GetPrincipalsInRoleResponse.java        |    443 -
 .../api/GetRoleGrantsForPrincipalRequest.java   |    502 -
 .../api/GetRoleGrantsForPrincipalResponse.java  |    443 -
 .../hive/metastore/api/GetTableRequest.java     |    605 -
 .../hive/metastore/api/GetTableResult.java      |    394 -
 .../hive/metastore/api/GetTablesRequest.java    |    659 -
 .../hive/metastore/api/GetTablesResult.java     |    443 -
 .../api/GrantRevokePrivilegeRequest.java        |    620 -
 .../api/GrantRevokePrivilegeResponse.java       |    390 -
 .../metastore/api/GrantRevokeRoleRequest.java   |   1059 -
 .../metastore/api/GrantRevokeRoleResponse.java  |    390 -
 .../hive/metastore/api/GrantRevokeType.java     |     45 -
 .../hive/metastore/api/HeartbeatRequest.java    |    489 -
 .../metastore/api/HeartbeatTxnRangeRequest.java |    482 -
 .../api/HeartbeatTxnRangeResponse.java          |    588 -
 .../hive/metastore/api/HiveObjectPrivilege.java |    729 -
 .../hive/metastore/api/HiveObjectRef.java       |    872 -
 .../hive/metastore/api/HiveObjectType.java      |     54 -
 .../apache/hadoop/hive/metastore/api/Index.java |   1374 -
 .../api/IndexAlreadyExistsException.java        |    395 -
 .../metastore/api/InsertEventRequestData.java   |    700 -
 .../metastore/api/InvalidInputException.java    |    395 -
 .../metastore/api/InvalidObjectException.java   |    395 -
 .../api/InvalidOperationException.java          |    395 -
 .../api/InvalidPartitionException.java          |    395 -
 .../hive/metastore/api/LockComponent.java       |   1158 -
 .../hadoop/hive/metastore/api/LockLevel.java    |     48 -
 .../hadoop/hive/metastore/api/LockRequest.java  |    861 -
 .../hadoop/hive/metastore/api/LockResponse.java |    500 -
 .../hadoop/hive/metastore/api/LockState.java    |     51 -
 .../hadoop/hive/metastore/api/LockType.java     |     48 -
 .../hive/metastore/api/LongColumnStatsData.java |    790 -
 .../hive/metastore/api/MetaException.java       |    395 -
 .../hive/metastore/api/MetadataPpdResult.java   |    517 -
 .../hive/metastore/api/NoSuchLockException.java |    395 -
 .../metastore/api/NoSuchObjectException.java    |    395 -
 .../hive/metastore/api/NoSuchTxnException.java  |    395 -
 .../api/NotNullConstraintsRequest.java          |    490 -
 .../api/NotNullConstraintsResponse.java         |    443 -
 .../hive/metastore/api/NotificationEvent.java   |   1006 -
 .../metastore/api/NotificationEventRequest.java |    490 -
 .../api/NotificationEventResponse.java          |    443 -
 .../hive/metastore/api/OpenTxnRequest.java      |    702 -
 .../hive/metastore/api/OpenTxnsResponse.java    |    438 -
 .../apache/hadoop/hive/metastore/api/Order.java |    497 -
 .../hadoop/hive/metastore/api/Partition.java    |   1229 -
 .../hive/metastore/api/PartitionEventType.java  |     42 -
 .../api/PartitionListComposingSpec.java         |    449 -
 .../hive/metastore/api/PartitionSpec.java       |    826 -
 .../api/PartitionSpecWithSharedSD.java          |    558 -
 .../hive/metastore/api/PartitionWithoutSD.java  |   1016 -
 .../metastore/api/PartitionsByExprRequest.java  |    815 -
 .../metastore/api/PartitionsByExprResult.java   |    542 -
 .../metastore/api/PartitionsStatsRequest.java   |    790 -
 .../metastore/api/PartitionsStatsResult.java    |    490 -
 .../hive/metastore/api/PrimaryKeysRequest.java  |    490 -
 .../hive/metastore/api/PrimaryKeysResponse.java |    443 -
 .../metastore/api/PrincipalPrivilegeSet.java    |    906 -
 .../hive/metastore/api/PrincipalType.java       |     48 -
 .../hadoop/hive/metastore/api/PrivilegeBag.java |    449 -
 .../hive/metastore/api/PrivilegeGrantInfo.java  |    815 -
 .../metastore/api/PutFileMetadataRequest.java   |    710 -
 .../metastore/api/PutFileMetadataResult.java    |    283 -
 .../hive/metastore/api/RequestPartsSpec.java    |    438 -
 .../hadoop/hive/metastore/api/ResourceType.java |     48 -
 .../hadoop/hive/metastore/api/ResourceUri.java  |    511 -
 .../apache/hadoop/hive/metastore/api/Role.java  |    601 -
 .../hive/metastore/api/RolePrincipalGrant.java  |   1035 -
 .../hive/metastore/api/SQLForeignKey.java       |   1715 -
 .../metastore/api/SQLNotNullConstraint.java     |   1005 -
 .../hive/metastore/api/SQLPrimaryKey.java       |   1103 -
 .../hive/metastore/api/SQLUniqueConstraint.java |   1103 -
 .../hadoop/hive/metastore/api/Schema.java       |    605 -
 .../hadoop/hive/metastore/api/SerDeInfo.java    |    655 -
 .../api/SetPartitionsStatsRequest.java          |    550 -
 .../hive/metastore/api/ShowCompactRequest.java  |    283 -
 .../hive/metastore/api/ShowCompactResponse.java |    443 -
 .../api/ShowCompactResponseElement.java         |   1641 -
 .../hive/metastore/api/ShowLocksRequest.java    |    710 -
 .../hive/metastore/api/ShowLocksResponse.java   |    449 -
 .../metastore/api/ShowLocksResponseElement.java |   1929 -
 .../hadoop/hive/metastore/api/SkewedInfo.java   |    834 -
 .../hive/metastore/api/StorageDescriptor.java   |   1748 -
 .../metastore/api/StringColumnStatsData.java    |    782 -
 .../apache/hadoop/hive/metastore/api/Table.java |   1948 -
 .../hadoop/hive/metastore/api/TableMeta.java    |    701 -
 .../hive/metastore/api/TableStatsRequest.java   |    640 -
 .../hive/metastore/api/TableStatsResult.java    |    443 -
 .../hive/metastore/api/ThriftHiveMetastore.java | 186370 ---------------
 .../hive/metastore/api/TxnAbortedException.java |    395 -
 .../hadoop/hive/metastore/api/TxnInfo.java      |   1220 -
 .../hive/metastore/api/TxnOpenException.java    |    395 -
 .../hadoop/hive/metastore/api/TxnState.java     |     48 -
 .../apache/hadoop/hive/metastore/api/Type.java  |    768 -
 .../metastore/api/UniqueConstraintsRequest.java |    490 -
 .../api/UniqueConstraintsResponse.java          |    443 -
 .../hive/metastore/api/UnknownDBException.java  |    395 -
 .../api/UnknownPartitionException.java          |    395 -
 .../metastore/api/UnknownTableException.java    |    395 -
 .../hive/metastore/api/UnlockRequest.java       |    387 -
 .../hadoop/hive/metastore/api/Version.java      |    499 -
 .../metastore/api/hive_metastoreConstants.java  |     87 -
 .../gen-php/metastore/ThriftHiveMetastore.php   |  46667 ----
 .../src/gen/thrift/gen-php/metastore/Types.php  |  20941 --
 .../hive_metastore/ThriftHiveMetastore-remote   |   1291 -
 .../hive_metastore/ThriftHiveMetastore.py       |  37904 ----
 .../thrift/gen-py/hive_metastore/__init__.py    |      1 -
 .../thrift/gen-py/hive_metastore/constants.py   |     35 -
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |  14886 --
 .../thrift/gen-rb/hive_metastore_constants.rb   |     57 -
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |   3444 -
 .../gen/thrift/gen-rb/thrift_hive_metastore.rb  |  10557 -
 .../hadoop/hive/metastore/DatabaseProduct.java  |      2 +-
 .../apache/hadoop/hive/metastore/Deadline.java  |      4 +-
 .../hive/metastore/FileMetadataManager.java     |      7 +-
 .../hive/metastore/HMSMetricsListener.java      |     47 +-
 .../hadoop/hive/metastore/HiveAlterHandler.java |     22 +-
 .../hadoop/hive/metastore/HiveMetaHook.java     |      4 +
 .../hadoop/hive/metastore/HiveMetaStore.java    |    606 +-
 .../hive/metastore/HiveMetaStoreClient.java     |     78 +-
 .../hadoop/hive/metastore/IMetaStoreClient.java |     89 +-
 .../hive/metastore/IMetaStoreSchemaInfo.java    |      2 +-
 .../hive/metastore/MetaStoreDirectSql.java      |    464 +-
 .../hive/metastore/MetaStoreEventListener.java  |     45 +-
 .../MetaStoreEventListenerConstants.java        |     41 +
 .../hive/metastore/MetaStoreFilterHook.java     |      5 +-
 .../metastore/MetaStoreListenerNotifier.java    |     52 +-
 .../metastore/MetaStorePreEventListener.java    |      5 +-
 .../hive/metastore/MetaStoreSchemaInfo.java     |      4 +-
 .../hadoop/hive/metastore/MetaStoreThread.java  |      2 +-
 .../hadoop/hive/metastore/MetaStoreUtils.java   |    314 +-
 .../metastore/NumDistinctValueEstimator.java    |    367 -
 .../hadoop/hive/metastore/ObjectStore.java      |    178 +-
 .../hive/metastore/PartFilterExprUtil.java      |      2 +-
 .../apache/hadoop/hive/metastore/RawStore.java  |     23 +-
 .../hive/metastore/ReplChangeManager.java       |    153 +-
 .../hive/metastore/RetryingHMSHandler.java      |      4 +-
 .../hive/metastore/StatObjectConverter.java     |     87 +-
 .../hive/metastore/TUGIBasedProcessor.java      |      2 +-
 .../TransactionalMetaStoreEventListener.java    |     39 +
 .../TransactionalValidationListener.java        |     44 +-
 .../apache/hadoop/hive/metastore/Warehouse.java |     21 +-
 .../hive/metastore/cache/CachedStore.java       |    169 +-
 .../hive/metastore/cache/SharedCache.java       |     10 +-
 .../aggr/BinaryColumnStatsAggregator.java       |     64 +
 .../aggr/BooleanColumnStatsAggregator.java      |     65 +
 .../columnstats/aggr/ColumnStatsAggregator.java |     33 +
 .../aggr/ColumnStatsAggregatorFactory.java      |    113 +
 .../aggr/DateColumnStatsAggregator.java         |    362 +
 .../aggr/DecimalColumnStatsAggregator.java      |    375 +
 .../aggr/DoubleColumnStatsAggregator.java       |    349 +
 .../aggr/IExtrapolatePartStatus.java            |     47 +
 .../aggr/LongColumnStatsAggregator.java         |    348 +
 .../aggr/StringColumnStatsAggregator.java       |    305 +
 .../cache/DateColumnStatsDataInspector.java     |    124 +
 .../cache/DecimalColumnStatsDataInspector.java  |    124 +
 .../cache/DoubleColumnStatsDataInspector.java   |    124 +
 .../cache/LongColumnStatsDataInspector.java     |    124 +
 .../cache/StringColumnStatsDataInspector.java   |    125 +
 .../merge/BinaryColumnStatsMerger.java          |     35 +
 .../merge/BooleanColumnStatsMerger.java         |     35 +
 .../columnstats/merge/ColumnStatsMerger.java    |     31 +
 .../merge/ColumnStatsMergerFactory.java         |    120 +
 .../merge/DateColumnStatsMerger.java            |     59 +
 .../merge/DecimalColumnStatsMerger.java         |     61 +
 .../merge/DoubleColumnStatsMerger.java          |     54 +
 .../merge/LongColumnStatsMerger.java            |     54 +
 .../merge/StringColumnStatsMerger.java          |     54 +
 .../datasource/BoneCPDataSourceProvider.java    |     95 +
 .../datasource/DataSourceProvider.java          |     79 +
 .../datasource/DataSourceProviderFactory.java   |     47 +
 .../datasource/HikariCPDataSourceProvider.java  |     97 +
 .../hive/metastore/datasource/package-info.java |     23 +
 .../metastore/events/AddForeignKeyEvent.java    |     37 +
 .../hive/metastore/events/AddIndexEvent.java    |      4 +
 .../events/AddNotNullConstraintEvent.java       |     37 +
 .../metastore/events/AddPartitionEvent.java     |      4 +
 .../metastore/events/AddPrimaryKeyEvent.java    |     38 +
 .../events/AddUniqueConstraintEvent.java        |     37 +
 .../hive/metastore/events/AlterIndexEvent.java  |      4 +
 .../metastore/events/AlterPartitionEvent.java   |      4 +
 .../hive/metastore/events/AlterTableEvent.java  |      4 +
 .../metastore/events/ConfigChangeEvent.java     |      4 +
 .../metastore/events/CreateDatabaseEvent.java   |      4 +
 .../metastore/events/CreateFunctionEvent.java   |      4 +
 .../hive/metastore/events/CreateTableEvent.java |      4 +
 .../metastore/events/DropConstraintEvent.java   |     47 +
 .../metastore/events/DropDatabaseEvent.java     |      4 +
 .../metastore/events/DropFunctionEvent.java     |      4 +
 .../hive/metastore/events/DropIndexEvent.java   |      4 +
 .../metastore/events/DropPartitionEvent.java    |      4 +
 .../hive/metastore/events/DropTableEvent.java   |      4 +
 .../hive/metastore/events/InsertEvent.java      |      6 +-
 .../hive/metastore/events/ListenerEvent.java    |      5 +-
 .../events/LoadPartitionDoneEvent.java          |      4 +
 .../hive/metastore/events/PreAddIndexEvent.java |      4 +
 .../metastore/events/PreAddPartitionEvent.java  |      4 +
 .../metastore/events/PreAlterDatabaseEvent.java |     47 +
 .../metastore/events/PreAlterIndexEvent.java    |      4 +
 .../events/PreAlterPartitionEvent.java          |      4 +
 .../metastore/events/PreAlterTableEvent.java    |      4 +
 .../events/PreAuthorizationCallEvent.java       |      4 +
 .../events/PreCreateDatabaseEvent.java          |      4 +
 .../metastore/events/PreCreateTableEvent.java   |      4 +
 .../metastore/events/PreDropDatabaseEvent.java  |      4 +
 .../metastore/events/PreDropIndexEvent.java     |      4 +
 .../metastore/events/PreDropPartitionEvent.java |      4 +
 .../metastore/events/PreDropTableEvent.java     |      4 +
 .../hive/metastore/events/PreEventContext.java  |      8 +-
 .../events/PreLoadPartitionDoneEvent.java       |      4 +
 .../metastore/events/PreReadDatabaseEvent.java  |      4 +
 .../metastore/events/PreReadTableEvent.java     |      4 +
 .../hbase/AggrStatsInvalidatorFilter.java       |    121 -
 .../hadoop/hive/metastore/hbase/Counter.java    |     53 -
 .../hive/metastore/hbase/HBaseConnection.java   |     96 -
 .../metastore/hbase/HBaseFilterPlanUtil.java    |    612 -
 .../hive/metastore/hbase/HBaseImport.java       |    619 -
 .../hive/metastore/hbase/HBaseReadWrite.java    |   2943 -
 .../hive/metastore/hbase/HBaseSchemaTool.java   |    192 -
 .../hadoop/hive/metastore/hbase/HBaseStore.java |   2964 -
 .../hadoop/hive/metastore/hbase/HBaseUtils.java |   1781 -
 .../hive/metastore/hbase/MetadataStore.java     |      2 +-
 .../hive/metastore/hbase/ObjectCache.java       |     81 -
 .../hive/metastore/hbase/PartitionCache.java    |    168 -
 .../metastore/hbase/PartitionKeyComparator.java |    292 -
 .../hbase/SharedStorageDescriptor.java          |    251 -
 .../hadoop/hive/metastore/hbase/StatsCache.java |    323 -
 .../metastore/hbase/TephraHBaseConnection.java  |    127 -
 .../metastore/hbase/VanillaHBaseConnection.java |    137 -
 .../stats/BinaryColumnStatsAggregator.java      |     64 -
 .../stats/BooleanColumnStatsAggregator.java     |     65 -
 .../hbase/stats/ColumnStatsAggregator.java      |     34 -
 .../stats/ColumnStatsAggregatorFactory.java     |    104 -
 .../stats/DecimalColumnStatsAggregator.java     |    351 -
 .../stats/DoubleColumnStatsAggregator.java      |    325 -
 .../hbase/stats/IExtrapolatePartStatus.java     |     47 -
 .../hbase/stats/LongColumnStatsAggregator.java  |    323 -
 .../stats/StringColumnStatsAggregator.java      |    105 -
 .../stats/merge/BinaryColumnStatsMerger.java    |     35 -
 .../stats/merge/BooleanColumnStatsMerger.java   |     35 -
 .../hbase/stats/merge/ColumnStatsMerger.java    |     34 -
 .../stats/merge/ColumnStatsMergerFactory.java   |    150 -
 .../stats/merge/DateColumnStatsMerger.java      |     55 -
 .../stats/merge/DecimalColumnStatsMerger.java   |     55 -
 .../stats/merge/DoubleColumnStatsMerger.java    |     48 -
 .../stats/merge/LongColumnStatsMerger.java      |     48 -
 .../stats/merge/StringColumnStatsMerger.java    |     49 -
 .../metastore/hooks/JDOConnectionURLHook.java   |      4 +
 .../messaging/AddForeignKeyMessage.java         |     36 +
 .../messaging/AddNotNullConstraintMessage.java  |     36 +
 .../messaging/AddPartitionMessage.java          |      2 +
 .../messaging/AddPrimaryKeyMessage.java         |     35 +
 .../messaging/AddUniqueConstraintMessage.java   |     36 +
 .../messaging/AlterPartitionMessage.java        |      2 +
 .../metastore/messaging/AlterTableMessage.java  |      2 +
 .../metastore/messaging/CreateTableMessage.java |      2 +
 .../messaging/DropConstraintMessage.java        |     29 +
 .../messaging/DropPartitionMessage.java         |      2 +
 .../metastore/messaging/DropTableMessage.java   |      2 +
 .../hive/metastore/messaging/EventMessage.java  |      7 +-
 .../hive/metastore/messaging/EventUtils.java    |     13 +
 .../hive/metastore/messaging/InsertMessage.java |      2 +
 .../messaging/MessageDeserializer.java          |     36 +-
 .../metastore/messaging/MessageFactory.java     |     53 +-
 .../event/filters/DatabaseAndTableFilter.java   |      2 +-
 .../json/JSONAddForeignKeyMessage.java          |    101 +
 .../json/JSONAddNotNullConstraintMessage.java   |     96 +
 .../messaging/json/JSONAddPartitionMessage.java |      8 +-
 .../json/JSONAddPrimaryKeyMessage.java          |    101 +
 .../json/JSONAddUniqueConstraintMessage.java    |     98 +
 .../json/JSONAlterPartitionMessage.java         |      8 +-
 .../messaging/json/JSONAlterTableMessage.java   |      8 +-
 .../messaging/json/JSONCreateTableMessage.java  |     18 +-
 .../json/JSONDropConstraintMessage.java         |     90 +
 .../json/JSONDropPartitionMessage.java          |     15 +-
 .../messaging/json/JSONDropTableMessage.java    |     13 +-
 .../messaging/json/JSONInsertMessage.java       |      8 +-
 .../messaging/json/JSONMessageDeserializer.java |     50 +
 .../messaging/json/JSONMessageFactory.java      |     61 +-
 .../hive/metastore/parser/ExpressionTree.java   |      4 +-
 .../hive/metastore/tools/HiveSchemaHelper.java  |      2 +-
 .../hadoop/hive/metastore/txn/TxnDbUtil.java    |      5 +-
 .../hadoop/hive/metastore/txn/TxnHandler.java   |    254 +-
 .../hadoop/hive/metastore/txn/TxnStore.java     |      2 +-
 .../hadoop/hive/metastore/txn/TxnUtils.java     |      2 +-
 .../src/main/resources/thrift-replacements.txt  |     60 -
 .../hive/metastore/model/MColumnDescriptor.java |     51 -
 .../hive/metastore/model/MConstraint.java       |    188 -
 .../hive/metastore/model/MDBPrivilege.java      |    132 -
 .../hadoop/hive/metastore/model/MDatabase.java  |    128 -
 .../hive/metastore/model/MDelegationToken.java  |     45 -
 .../hive/metastore/model/MFieldSchema.java      |     80 -
 .../hadoop/hive/metastore/model/MFunction.java  |    119 -
 .../hive/metastore/model/MGlobalPrivilege.java  |    120 -
 .../hadoop/hive/metastore/model/MIndex.java     |    200 -
 .../hadoop/hive/metastore/model/MMasterKey.java |     55 -
 .../metastore/model/MMetastoreDBProperties.java |     56 -
 .../hive/metastore/model/MNotificationLog.java  |     98 -
 .../metastore/model/MNotificationNextId.java    |     42 -
 .../hadoop/hive/metastore/model/MOrder.java     |     62 -
 .../hadoop/hive/metastore/model/MPartition.java |    155 -
 .../model/MPartitionColumnPrivilege.java        |    161 -
 .../model/MPartitionColumnStatistics.java       |    258 -
 .../hive/metastore/model/MPartitionEvent.java   |     90 -
 .../metastore/model/MPartitionPrivilege.java    |    139 -
 .../hive/metastore/model/MPrincipalDesc.java    |     59 -
 .../hive/metastore/model/MResourceUri.java      |     49 -
 .../hadoop/hive/metastore/model/MRole.java      |     80 -
 .../hadoop/hive/metastore/model/MRoleMap.java   |    120 -
 .../hadoop/hive/metastore/model/MSerDeInfo.java |     81 -
 .../metastore/model/MStorageDescriptor.java     |    277 -
 .../hive/metastore/model/MStringList.java       |     62 -
 .../hadoop/hive/metastore/model/MTable.java     |    256 -
 .../metastore/model/MTableColumnPrivilege.java  |    160 -
 .../metastore/model/MTableColumnStatistics.java |    249 -
 .../hive/metastore/model/MTablePrivilege.java   |    139 -
 .../hadoop/hive/metastore/model/MType.java      |    105 -
 .../hive/metastore/model/MVersionTable.java     |     57 -
 metastore/src/model/package.jdo                 |   1081 -
 .../metastore/hbase/hbase_metastore_proto.proto |    368 -
 .../DummyRawStoreControlledCommit.java          |     29 +-
 .../DummyRawStoreForJdoConnection.java          |     26 +-
 .../InjectableBehaviourObjectStore.java         |     26 +-
 .../hadoop/hive/metastore/TestObjectStore.java  |     25 +-
 .../hadoop/hive/metastore/TestOldSchema.java    |    220 +
 .../hive/metastore/cache/TestCachedStore.java   |    230 +-
 .../TestDataSourceProviderFactory.java          |    167 +
 .../hadoop/hive/metastore/hbase/MockUtils.java  |    219 -
 .../hbase/TestHBaseAggregateStatsCache.java     |    316 -
 ...stHBaseAggregateStatsCacheWithBitVector.java |    184 -
 .../TestHBaseAggregateStatsExtrapolation.java   |    716 -
 .../TestHBaseAggregateStatsNDVUniformDist.java  |    580 -
 .../hbase/TestHBaseFilterPlanUtil.java          |    483 -
 .../hive/metastore/hbase/TestHBaseStore.java    |   1978 -
 .../hbase/TestHBaseStoreBitVector.java          |    622 -
 .../metastore/hbase/TestHBaseStoreCached.java   |    365 -
 .../hbase/TestSharedStorageDescriptor.java      |    152 -
 pom.xml                                         |     55 +-
 ql/if/queryplan.thrift                          |      3 +
 ql/pom.xml                                      |     10 +-
 ql/src/gen/thrift/gen-cpp/queryplan_types.cpp   |     12 +-
 ql/src/gen/thrift/gen-cpp/queryplan_types.h     |      5 +-
 .../hadoop/hive/ql/plan/api/StageType.java      |     11 +-
 ql/src/gen/thrift/gen-php/Types.php             |      6 +
 ql/src/gen/thrift/gen-py/queryplan/ttypes.py    |      9 +
 ql/src/gen/thrift/gen-rb/queryplan_types.rb     |      7 +-
 .../UDAFTemplates/VectorUDAFAvg.txt             |    255 +-
 .../UDAFTemplates/VectorUDAFAvgDecimal.txt      |    566 +
 .../UDAFTemplates/VectorUDAFAvgDecimalMerge.txt |    597 +
 .../UDAFTemplates/VectorUDAFAvgMerge.txt        |    547 +
 .../UDAFTemplates/VectorUDAFAvgTimestamp.txt    |    517 +
 .../UDAFTemplates/VectorUDAFMinMax.txt          |    184 +-
 .../UDAFTemplates/VectorUDAFMinMaxDecimal.txt   |     30 +-
 .../VectorUDAFMinMaxIntervalDayTime.txt         |     26 +-
 .../UDAFTemplates/VectorUDAFMinMaxString.txt    |     29 +-
 .../UDAFTemplates/VectorUDAFMinMaxTimestamp.txt |     26 +-
 .../UDAFTemplates/VectorUDAFSum.txt             |    139 +-
 .../UDAFTemplates/VectorUDAFVar.txt             |    194 +-
 .../UDAFTemplates/VectorUDAFVarDecimal.txt      |    219 +-
 .../UDAFTemplates/VectorUDAFVarMerge.txt        |    573 +
 .../UDAFTemplates/VectorUDAFVarTimestamp.txt    |    477 +
 .../hadoop/hive/llap/ChannelOutputStream.java   |     37 +-
 .../hadoop/hive/llap/LlapCacheAwareFs.java      |    422 +
 .../hive/llap/LlapOutputFormatService.java      |      4 +-
 .../hadoop/hive/llap/LlapRecordWriter.java      |      6 +-
 .../hadoop/hive/ql/CompilationOpContext.java    |     13 +-
 .../java/org/apache/hadoop/hive/ql/Context.java |      2 +-
 .../java/org/apache/hadoop/hive/ql/Driver.java  |     12 +
 .../org/apache/hadoop/hive/ql/ErrorMsg.java     |     40 +-
 .../hadoop/hive/ql/HiveDriverRunHook.java       |      4 +
 .../hive/ql/HiveDriverRunHookContext.java       |      4 +
 .../apache/hadoop/hive/ql/QueryProperties.java  |      1 +
 .../hadoop/hive/ql/exec/CollectOperator.java    |      5 +
 .../hadoop/hive/ql/exec/ColumnStatsTask.java    |     46 +-
 .../hive/ql/exec/ColumnStatsUpdateTask.java     |     24 +-
 .../hadoop/hive/ql/exec/CommonJoinOperator.java |     17 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |    152 +-
 .../hadoop/hive/ql/exec/DemuxOperator.java      |      5 +
 .../apache/hadoop/hive/ql/exec/Description.java |      5 +
 .../apache/hadoop/hive/ql/exec/ExplainTask.java |     27 +-
 .../hadoop/hive/ql/exec/FetchOperator.java      |      6 -
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |     77 +-
 .../hadoop/hive/ql/exec/ForwardOperator.java    |      5 +
 .../hadoop/hive/ql/exec/FunctionRegistry.java   |      4 +-
 .../hadoop/hive/ql/exec/FunctionTask.java       |     28 +-
 .../hadoop/hive/ql/exec/GroupByOperator.java    |      2 +-
 .../hive/ql/exec/HashTableDummyOperator.java    |      5 +-
 .../hive/ql/exec/HashTableSinkOperator.java     |     10 +-
 .../hadoop/hive/ql/exec/KeyWrapperFactory.java  |      6 +
 .../ql/exec/LateralViewForwardOperator.java     |      5 +
 .../hadoop/hive/ql/exec/ListSinkOperator.java   |      5 +
 .../hadoop/hive/ql/exec/MapJoinOperator.java    |     18 +
 .../apache/hadoop/hive/ql/exec/MapOperator.java |      3 +
 .../hadoop/hive/ql/exec/MapredContext.java      |      2 +
 .../apache/hadoop/hive/ql/exec/MoveTask.java    |     48 +-
 .../apache/hadoop/hive/ql/exec/MuxOperator.java |      5 +
 .../apache/hadoop/hive/ql/exec/NodeUtils.java   |     68 +-
 .../hadoop/hive/ql/exec/ObjectCacheFactory.java |      9 +-
 .../apache/hadoop/hive/ql/exec/Operator.java    |     75 +
 .../hadoop/hive/ql/exec/OperatorFactory.java    |      3 +
 .../hadoop/hive/ql/exec/OperatorUtils.java      |     39 +-
 .../hive/ql/exec/OrcFileMergeOperator.java      |     38 +-
 .../hadoop/hive/ql/exec/ReplCopyTask.java       |    157 +-
 .../hive/ql/exec/SerializationUtilities.java    |      7 +-
 .../ql/exec/SparkHashTableSinkOperator.java     |     17 +-
 .../hadoop/hive/ql/exec/StatsNoJobTask.java     |      5 -
 .../hadoop/hive/ql/exec/TableScanOperator.java  |     11 +-
 .../apache/hadoop/hive/ql/exec/TaskFactory.java |     10 +-
 .../org/apache/hadoop/hive/ql/exec/UDF.java     |      3 +
 .../hive/ql/exec/UDFArgumentException.java      |      4 +
 .../ql/exec/UDFArgumentLengthException.java     |      5 +
 .../hive/ql/exec/UDFArgumentTypeException.java  |      6 +-
 .../hadoop/hive/ql/exec/UDFMethodResolver.java  |      1 +
 .../hadoop/hive/ql/exec/UnionOperator.java      |      5 +
 .../apache/hadoop/hive/ql/exec/Utilities.java   |    509 +-
 .../hive/ql/exec/errors/ErrorAndSolution.java   |      5 +-
 .../hadoop/hive/ql/exec/mr/MapRedTask.java      |      2 +-
 .../ql/exec/persistence/FlatRowContainer.java   |      2 +-
 .../hive/ql/exec/persistence/RowContainer.java  |     10 +-
 .../hadoop/hive/ql/exec/repl/ReplDumpTask.java  |    325 +
 .../hadoop/hive/ql/exec/repl/ReplDumpWork.java  |     75 +
 .../hive/ql/exec/repl/ReplStateLogTask.java     |     51 +
 .../hive/ql/exec/repl/ReplStateLogWork.java     |    100 +
 .../ql/exec/repl/bootstrap/ReplLoadTask.java    |    289 +
 .../ql/exec/repl/bootstrap/ReplLoadWork.java    |     70 +
 .../repl/bootstrap/events/BootstrapEvent.java   |     28 +
 .../repl/bootstrap/events/DatabaseEvent.java    |     34 +
 .../repl/bootstrap/events/FunctionEvent.java    |     33 +
 .../repl/bootstrap/events/PartitionEvent.java   |     26 +
 .../exec/repl/bootstrap/events/TableEvent.java  |     42 +
 .../filesystem/BootstrapEventsIterator.java     |    181 +
 .../filesystem/DatabaseEventsIterator.java      |    145 +
 .../events/filesystem/FSDatabaseEvent.java      |     88 +
 .../events/filesystem/FSFunctionEvent.java      |     39 +
 .../events/filesystem/FSPartitionEvent.java     |     84 +
 .../events/filesystem/FSTableEvent.java         |    123 +
 .../exec/repl/bootstrap/load/LoadDatabase.java  |    129 +
 .../exec/repl/bootstrap/load/LoadFunction.java  |     87 +
 .../repl/bootstrap/load/ReplicationState.java   |     58 +
 .../exec/repl/bootstrap/load/TaskTracker.java   |    120 +
 .../bootstrap/load/table/LoadPartitions.java    |    316 +
 .../repl/bootstrap/load/table/LoadTable.java    |    245 +
 .../repl/bootstrap/load/table/TableContext.java |     49 +
 .../exec/repl/bootstrap/load/util/Context.java  |     37 +
 .../repl/bootstrap/load/util/PathUtils.java     |    105 +
 .../hive/ql/exec/spark/HashTableLoader.java     |      2 +-
 .../ql/exec/spark/LocalHiveSparkClient.java     |      2 +-
 .../ql/exec/spark/RemoteHiveSparkClient.java    |     19 +-
 .../hive/ql/exec/spark/RepartitionShuffler.java |     42 -
 .../exec/spark/SparkDynamicPartitionPruner.java |     13 +-
 .../hive/ql/exec/spark/SparkPlanGenerator.java  |      6 +-
 .../ql/exec/spark/SparkReduceRecordHandler.java |    331 +-
 .../hive/ql/exec/spark/SparkUtilities.java      |     20 +
 .../ql/exec/spark/session/SparkSessionImpl.java |     11 +-
 .../hadoop/hive/ql/exec/tez/DagUtils.java       |     36 +-
 .../hive/ql/exec/tez/MapRecordSource.java       |     12 +
 .../hadoop/hive/ql/exec/tez/ObjectCache.java    |      5 +
 .../hive/ql/exec/tez/ReduceRecordSource.java    |     80 +-
 .../ql/exec/tez/RestrictedConfigChecker.java    |     79 +
 .../ql/exec/tez/SessionExpirationTracker.java   |    235 +
 .../hadoop/hive/ql/exec/tez/TezSessionPool.java |    165 +
 .../hive/ql/exec/tez/TezSessionPoolManager.java |    551 +-
 .../hive/ql/exec/tez/TezSessionPoolSession.java |    158 +
 .../apache/hadoop/hive/ql/exec/tez/TezTask.java |     84 +-
 .../ql/exec/tez/tools/KeyValueInputMerger.java  |     50 +-
 .../vector/VectorAppMasterEventOperator.java    |      3 +-
 .../hive/ql/exec/vector/VectorAssignRow.java    |    410 +-
 .../hive/ql/exec/vector/VectorBatchDebug.java   |    105 +
 .../ql/exec/vector/VectorDeserializeRow.java    |    492 +-
 .../hive/ql/exec/vector/VectorExtractRow.java   |     40 +-
 .../ql/exec/vector/VectorFilterOperator.java    |      2 +-
 .../ql/exec/vector/VectorGroupByOperator.java   |     69 +-
 .../ql/exec/vector/VectorLimitOperator.java     |      2 +-
 .../exec/vector/VectorMapJoinBaseOperator.java  |      3 +-
 .../hive/ql/exec/vector/VectorMapOperator.java  |     48 +-
 .../exec/vector/VectorSMBMapJoinOperator.java   |      2 +-
 .../ql/exec/vector/VectorSelectOperator.java    |     12 +-
 .../ql/exec/vector/VectorizationContext.java    |    247 +-
 .../ql/exec/vector/VectorizedBatchUtil.java     |     61 +-
 .../ql/exec/vector/VectorizedRowBatchCtx.java   |     55 +-
 .../vector/expressions/CastDateToTimestamp.java |    131 +
 .../vector/expressions/IfExprColumnNull.java    |    109 +
 .../vector/expressions/IfExprNullColumn.java    |    109 +
 .../VectorExpressionWriterFactory.java          |    188 +-
 .../VectorInBloomFilterColDynamicValue.java     |     18 +-
 .../aggregates/VectorAggregateExpression.java   |     17 +-
 .../aggregates/VectorUDAFAvgDecimal.java        |    521 -
 .../aggregates/VectorUDAFAvgTimestamp.java      |    488 -
 .../aggregates/VectorUDAFBloomFilter.java       |     60 +-
 .../aggregates/VectorUDAFBloomFilterMerge.java  |     43 +-
 .../expressions/aggregates/VectorUDAFCount.java |     28 +-
 .../aggregates/VectorUDAFCountMerge.java        |     28 +-
 .../aggregates/VectorUDAFCountStar.java         |     20 +-
 .../aggregates/VectorUDAFStdPopTimestamp.java   |    533 -
 .../aggregates/VectorUDAFStdSampTimestamp.java  |    533 -
 .../aggregates/VectorUDAFSumDecimal.java        |     39 +-
 .../aggregates/VectorUDAFSumTimestamp.java      |    434 +
 .../aggregates/VectorUDAFVarPopTimestamp.java   |    533 -
 .../aggregates/VectorUDAFVarSampTimestamp.java  |    533 -
 .../mapjoin/VectorMapJoinCommonOperator.java    |     35 +-
 .../VectorMapJoinGenerateResultOperator.java    |      6 +-
 .../fast/VectorMapJoinFastTableContainer.java   |      8 +-
 .../VectorMapJoinOptimizedCreateHashTable.java  |      6 +-
 .../exec/vector/ptf/VectorPTFEvaluatorBase.java |    122 +
 .../vector/ptf/VectorPTFEvaluatorCount.java     |    108 +
 .../vector/ptf/VectorPTFEvaluatorCountStar.java |     79 +
 .../ptf/VectorPTFEvaluatorDecimalAvg.java       |    162 +
 .../VectorPTFEvaluatorDecimalFirstValue.java    |    117 +
 .../ptf/VectorPTFEvaluatorDecimalLastValue.java |    113 +
 .../ptf/VectorPTFEvaluatorDecimalMax.java       |    146 +
 .../ptf/VectorPTFEvaluatorDecimalMin.java       |    146 +
 .../ptf/VectorPTFEvaluatorDecimalSum.java       |    140 +
 .../vector/ptf/VectorPTFEvaluatorDenseRank.java |     77 +
 .../vector/ptf/VectorPTFEvaluatorDoubleAvg.java |    153 +
 .../ptf/VectorPTFEvaluatorDoubleFirstValue.java |    113 +
 .../ptf/VectorPTFEvaluatorDoubleLastValue.java  |    109 +
 .../vector/ptf/VectorPTFEvaluatorDoubleMax.java |    136 +
 .../vector/ptf/VectorPTFEvaluatorDoubleMin.java |    136 +
 .../vector/ptf/VectorPTFEvaluatorDoubleSum.java |    134 +
 .../vector/ptf/VectorPTFEvaluatorLongAvg.java   |    153 +
 .../ptf/VectorPTFEvaluatorLongFirstValue.java   |    113 +
 .../ptf/VectorPTFEvaluatorLongLastValue.java    |    109 +
 .../vector/ptf/VectorPTFEvaluatorLongMax.java   |    136 +
 .../vector/ptf/VectorPTFEvaluatorLongMin.java   |    136 +
 .../vector/ptf/VectorPTFEvaluatorLongSum.java   |    134 +
 .../exec/vector/ptf/VectorPTFEvaluatorRank.java |     81 +
 .../vector/ptf/VectorPTFEvaluatorRowNumber.java |     77 +
 .../exec/vector/ptf/VectorPTFGroupBatches.java  |    216 +
 .../ql/exec/vector/ptf/VectorPTFOperator.java   |    570 +
 .../ql/exec/vector/udf/VectorUDFAdaptor.java    |    171 +-
 .../hive/ql/hooks/ExecuteWithHookContext.java   |      5 +
 .../org/apache/hadoop/hive/ql/hooks/Hook.java   |      6 +
 .../hadoop/hive/ql/hooks/HookContext.java       |      4 +
 .../hadoop/hive/ql/hooks/PostExecute.java       |      4 +
 .../apache/hadoop/hive/ql/hooks/PreExecute.java |      4 +
 .../hadoop/hive/ql/hooks/QueryLifeTimeHook.java |      5 +
 .../hive/ql/hooks/QueryLifeTimeHookContext.java |      4 +
 .../hooks/QueryLifeTimeHookWithParseHooks.java  |      5 +
 .../hadoop/hive/ql/io/AcidInputFormat.java      |     11 +-
 .../hadoop/hive/ql/io/AcidOutputFormat.java     |     33 +-
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |    152 +-
 .../apache/hadoop/hive/ql/io/BucketCodec.java   |    128 +
 .../hive/ql/io/CombineHiveInputFormat.java      |      2 +-
 .../org/apache/hadoop/hive/ql/io/HdfsUtils.java |     40 +-
 .../hadoop/hive/ql/io/HiveFileFormatUtils.java  |      2 +-
 .../hadoop/hive/ql/io/HiveInputFormat.java      |    106 +-
 .../io/LlapCacheOnlyInputFormatInterface.java   |     28 +
 .../hadoop/hive/ql/io/RecordIdentifier.java     |     21 +-
 .../hadoop/hive/ql/io/orc/ExternalCache.java    |      2 +-
 .../hive/ql/io/orc/OrcFileKeyWrapper.java       |     36 +-
 .../io/orc/OrcFileStripeMergeRecordReader.java  |      5 +-
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |    190 +-
 .../hadoop/hive/ql/io/orc/OrcOutputFormat.java  |      2 +-
 .../hive/ql/io/orc/OrcRawRecordMerger.java      |    773 +-
 .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java |    160 +-
 .../apache/hadoop/hive/ql/io/orc/OrcSplit.java  |     10 +-
 .../io/orc/VectorizedOrcAcidRowBatchReader.java |    207 +-
 .../hive/ql/io/orc/encoded/EncodedReader.java   |      3 +-
 .../ql/io/orc/encoded/EncodedReaderImpl.java    |    538 +-
 .../hive/ql/io/orc/encoded/IncompleteCb.java    |      7 +
 .../hadoop/hive/ql/io/orc/encoded/IoTrace.java  |    478 +
 .../hadoop/hive/ql/io/orc/encoded/Reader.java   |      2 +-
 .../hive/ql/io/orc/encoded/ReaderImpl.java      |      6 +-
 .../ql/io/parquet/MapredParquetInputFormat.java |     47 +-
 .../io/parquet/MapredParquetOutputFormat.java   |     30 +-
 .../ql/io/parquet/ParquetRecordReaderBase.java  |     54 +-
 .../hive/ql/io/parquet/ProjectionPusher.java    |      4 +-
 .../parquet/VectorizedParquetInputFormat.java   |     22 +-
 .../ql/io/parquet/convert/ETypeConverter.java   |     24 +-
 .../hive/ql/io/parquet/convert/Repeated.java    |      1 -
 .../parquet/read/DataWritableReadSupport.java   |     21 +-
 .../read/ParquetFilterPredicateConverter.java   |     13 +-
 .../read/ParquetRecordReaderWrapper.java        |     26 +-
 .../ql/io/parquet/serde/ParquetTableUtils.java  |     31 +-
 .../ql/io/parquet/timestamp/NanoTimeUtils.java  |    247 +-
 .../vector/ParquetFooterInputFromCache.java     |    196 +
 .../vector/VectorizedParquetRecordReader.java   |    202 +-
 .../vector/VectorizedPrimitiveColumnReader.java |     21 +-
 .../parquet/write/DataWritableWriteSupport.java |     14 +-
 .../ql/io/parquet/write/DataWritableWriter.java |     10 +-
 .../write/ParquetRecordWriterWrapper.java       |     24 +
 .../apache/hadoop/hive/ql/lib/RuleRegExp.java   |      3 +
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java    |     18 +-
 .../hive/ql/lockmgr/EmbeddedLockManager.java    |     22 +-
 .../zookeeper/ZooKeeperHiveLockManager.java     |     13 +-
 .../hadoop/hive/ql/log/LogDivertAppender.java   |     15 +-
 .../hive/ql/log/LogDivertAppenderForTest.java   |      8 +-
 .../apache/hadoop/hive/ql/metadata/Hive.java    |    174 +-
 .../hadoop/hive/ql/metadata/HiveException.java  |     38 +-
 .../metadata/HiveMaterializedViewsRegistry.java |      9 +-
 .../hive/ql/metadata/HiveStorageHandler.java    |      4 +
 .../metadata/HiveStoragePredicateHandler.java   |      6 +
 .../hadoop/hive/ql/metadata/InputEstimator.java |      4 +
 .../ql/metadata/SessionHiveMetaStoreClient.java |     13 +
 .../hadoop/hive/ql/metadata/VirtualColumn.java  |     14 +
 .../formatting/MetaDataFormatUtils.java         |     35 +-
 .../BucketingSortingReduceSinkOptimizer.java    |      7 +-
 .../optimizer/ConstantPropagateProcFactory.java |      6 +-
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   |     89 +-
 .../ql/optimizer/CountDistinctRewriteProc.java  |     64 +-
 .../DynamicPartitionPruningOptimization.java    |    151 +-
 .../hive/ql/optimizer/GenMapRedUtils.java       |      5 +
 .../hive/ql/optimizer/MapJoinProcessor.java     |      5 +-
 .../ql/optimizer/OperatorComparatorFactory.java |    552 -
 .../hadoop/hive/ql/optimizer/Optimizer.java     |      4 +-
 .../ql/optimizer/ReduceSinkMapJoinProc.java     |      6 +-
 .../hive/ql/optimizer/SharedScanOptimizer.java  |    626 -
 .../hive/ql/optimizer/SharedWorkOptimizer.java  |   1177 +
 .../optimizer/SortedDynPartitionOptimizer.java  |      1 +
 .../ql/optimizer/SparkRemoveDynamicPruning.java |     80 +
 .../SparkRemoveDynamicPruningBySize.java        |     65 -
 .../hive/ql/optimizer/StatsOptimizer.java       |     17 +-
 .../ql/optimizer/calcite/HiveCalciteUtil.java   |      9 +-
 .../calcite/HiveConfPlannerContext.java         |     34 +
 .../optimizer/calcite/HivePlannerContext.java   |      7 +-
 .../hive/ql/optimizer/calcite/HiveType.java     |     48 +
 .../optimizer/calcite/HiveTypeSystemImpl.java   |     17 +-
 .../ql/optimizer/calcite/RelOptHiveTable.java   |     63 +-
 .../calcite/cost/HiveAlgorithmsUtil.java        |     10 +-
 .../calcite/cost/HiveDefaultCostModel.java      |      2 +-
 .../calcite/cost/HiveOnTezCostModel.java        |     35 +-
 .../calcite/reloperators/HiveAggregate.java     |     13 +-
 .../calcite/reloperators/HiveIntersect.java     |     10 +
 .../calcite/reloperators/HiveJoin.java          |      5 +-
 .../calcite/reloperators/HiveTableScan.java     |     50 +-
 .../calcite/reloperators/HiveUnion.java         |      9 +
 .../rules/HiveAggregateJoinTransposeRule.java   |      2 +-
 .../calcite/rules/HiveAggregateReduceRule.java  |    126 +
 .../HiveDruidProjectFilterTransposeRule.java    |     48 -
 .../rules/HiveExpandDistinctAggregatesRule.java |      3 +-
 .../rules/HiveFilterSetOpTransposeRule.java     |     98 +-
 .../calcite/rules/HiveJoinAddNotNullRule.java   |     42 +-
 .../HiveJoinPushTransitivePredicatesRule.java   |      2 +-
 .../rules/HiveReduceExpressionsRule.java        |      2 +-
 .../HiveReduceExpressionsWithStatsRule.java     |      2 +-
 .../calcite/rules/HiveRelDecorrelator.java      |    306 +-
 .../calcite/rules/HiveRelFieldTrimmer.java      |      2 +-
 .../calcite/rules/HiveRemoveSqCountCheck.java   |    142 +
 .../calcite/rules/HiveSemiJoinRule.java         |     96 +-
 .../calcite/rules/HiveSortJoinReduceRule.java   |      3 +-
 .../rules/HiveSortLimitPullUpConstantsRule.java |      2 +-
 .../calcite/rules/HiveSortRemoveRule.java       |      2 +-
 .../calcite/rules/HiveSortUnionReduceRule.java  |      2 +-
 .../calcite/rules/HiveSubQueryRemoveRule.java   |    620 +-
 .../calcite/rules/HiveUnionMergeRule.java       |     84 +
 .../rules/HiveUnionPullUpConstantsRule.java     |      2 +-
 .../HiveMaterializedViewFilterScanRule.java     |     92 -
 .../rules/views/HiveMaterializedViewRule.java   |     38 +
 .../MaterializedViewSubstitutionVisitor.java    |    292 -
 .../rules/views/SubstitutionVisitor.java        |   2458 -
 .../calcite/stats/HiveRelMdSelectivity.java     |     33 +-
 .../calcite/translator/ASTBuilder.java          |     31 +-
 .../calcite/translator/ExprNodeConverter.java   |     32 +-
 .../calcite/translator/RexNodeConverter.java    |     65 +-
 .../translator/SqlFunctionConverter.java        |      7 +-
 .../calcite/translator/TypeConverter.java       |     17 +-
 .../correlation/CorrelationUtilities.java       |     51 +-
 .../correlation/ReduceSinkDeDuplication.java    |    476 +-
 .../ReduceSinkDeDuplicationUtils.java           |    584 +
 .../ReduceSinkJoinDeDuplication.java            |    255 +
 .../ql/optimizer/lineage/ExprProcFactory.java   |     48 +-
 .../hive/ql/optimizer/lineage/Generator.java    |     27 +
 .../ql/optimizer/lineage/OpProcFactory.java     |     36 +-
 .../hive/ql/optimizer/physical/LlapDecider.java |      2 +-
 .../hive/ql/optimizer/physical/Vectorizer.java  |    944 +-
 .../hive/ql/optimizer/ppr/PartitionPruner.java  |     23 +-
 .../spark/CombineEquivalentWorkResolver.java    |     79 +-
 .../spark/SetSparkReducerParallelism.java       |    141 +-
 .../optimizer/spark/SparkMapJoinOptimizer.java  |     50 +
 .../spark/SparkPartitionPruningSinkDesc.java    |     17 +-
 .../stats/annotation/StatsRulesProcFactory.java |    263 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |     70 +-
 .../hadoop/hive/ql/parse/ColumnStatsList.java   |     64 +
 .../ql/parse/ColumnStatsSemanticAnalyzer.java   |     36 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java      |     49 +-
 .../apache/hadoop/hive/ql/parse/EximUtil.java   |     39 +-
 .../hive/ql/parse/ExportSemanticAnalyzer.java   |    243 +-
 .../hive/ql/parse/FunctionSemanticAnalyzer.java |      4 +-
 .../hadoop/hive/ql/parse/GenTezUtils.java       |    232 +-
 .../org/apache/hadoop/hive/ql/parse/HiveLexer.g |      2 +-
 .../apache/hadoop/hive/ql/parse/HiveParser.g    |     19 +-
 .../hive/ql/parse/HiveSemanticAnalyzerHook.java |      4 +
 .../parse/HiveSemanticAnalyzerHookContext.java  |      9 +
 .../HiveSemanticAnalyzerHookContextImpl.java    |     12 +
 .../hive/ql/parse/ImportSemanticAnalyzer.java   |    177 +-
 .../hive/ql/parse/LoadSemanticAnalyzer.java     |      5 +
 .../hadoop/hive/ql/parse/ParseContext.java      |     30 +-
 .../hive/ql/parse/PrunedPartitionList.java      |     21 +-
 .../apache/hadoop/hive/ql/parse/QBSubQuery.java |     23 +-
 .../ql/parse/ReplicationSemanticAnalyzer.java   |    782 +-
 .../hadoop/hive/ql/parse/ReplicationSpec.java   |    146 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |    158 +-
 .../hive/ql/parse/SemanticAnalyzerFactory.java  |      8 +-
 .../hadoop/hive/ql/parse/SemanticException.java |      5 +-
 .../hadoop/hive/ql/parse/SemiJoinHint.java      |     13 +-
 .../hadoop/hive/ql/parse/TezCompiler.java       |     85 +-
 .../hive/ql/parse/TypeCheckProcFactory.java     |     21 +-
 .../ql/parse/UpdateDeleteSemanticAnalyzer.java  |     10 +-
 .../hadoop/hive/ql/parse/repl/CopyUtils.java    |    136 +
 .../hadoop/hive/ql/parse/repl/DumpType.java     |     35 +
 .../hadoop/hive/ql/parse/repl/ReplLogger.java   |     36 +
 .../hadoop/hive/ql/parse/repl/ReplState.java    |     57 +
 .../dump/BootStrapReplicationSpecFunction.java  |      5 +-
 .../hive/ql/parse/repl/dump/HiveWrapper.java    |      2 +-
 .../ql/parse/repl/dump/PartitionExport.java     |    119 +
 .../hive/ql/parse/repl/dump/TableExport.java    |    274 +
 .../hadoop/hive/ql/parse/repl/dump/Utils.java   |     32 +
 .../repl/dump/events/AddForeignKeyHandler.java  |     41 +
 .../events/AddNotNullConstraintHandler.java     |     41 +
 .../repl/dump/events/AddPartitionHandler.java   |      3 +-
 .../repl/dump/events/AddPrimaryKeyHandler.java  |     41 +
 .../dump/events/AddUniqueConstraintHandler.java |     41 +
 .../repl/dump/events/CreateTableHandler.java    |      8 +-
 .../repl/dump/events/DropConstraintHandler.java |     41 +
 .../repl/dump/events/EventHandlerFactory.java   |      5 +
 .../ql/parse/repl/dump/io/FileOperations.java   |    112 +
 .../parse/repl/dump/io/FunctionSerializer.java  |      2 +-
 .../parse/repl/dump/io/PartitionSerializer.java |     11 +-
 .../ql/parse/repl/dump/io/TableSerializer.java  |     22 +-
 .../repl/dump/log/BootstrapDumpLogger.java      |     71 +
 .../repl/dump/log/IncrementalDumpLogger.java    |     55 +
 .../repl/dump/log/state/BootstrapDumpBegin.java |     47 +
 .../repl/dump/log/state/BootstrapDumpEnd.java   |     59 +
 .../dump/log/state/BootstrapDumpFunction.java   |     44 +
 .../repl/dump/log/state/BootstrapDumpTable.java |     52 +
 .../dump/log/state/IncrementalDumpBegin.java    |     43 +
 .../repl/dump/log/state/IncrementalDumpEnd.java |     54 +
 .../dump/log/state/IncrementalDumpEvent.java    |     51 +
 .../hive/ql/parse/repl/load/DumpMetaData.java   |     11 +-
 .../parse/repl/load/EventDumpDirComparator.java |     38 +
 .../parse/repl/load/UpdatedMetaDataTracker.java |     76 +
 .../repl/load/log/BootstrapLoadLogger.java      |     66 +
 .../repl/load/log/IncrementalLoadLogger.java    |     55 +
 .../repl/load/log/state/BootstrapLoadBegin.java |     51 +
 .../repl/load/log/state/BootstrapLoadEnd.java   |     59 +
 .../load/log/state/BootstrapLoadFunction.java   |     44 +
 .../repl/load/log/state/BootstrapLoadTable.java |     52 +
 .../load/log/state/IncrementalLoadBegin.java    |     47 +
 .../repl/load/log/state/IncrementalLoadEnd.java |     54 +
 .../load/log/state/IncrementalLoadEvent.java    |     51 +
 .../load/message/AbstractMessageHandler.java    |     25 +-
 .../repl/load/message/AddForeignKeyHandler.java |     70 +
 .../message/AddNotNullConstraintHandler.java    |     71 +
 .../repl/load/message/AddPrimaryKeyHandler.java |     68 +
 .../message/AddUniqueConstraintHandler.java     |     69 +
 .../load/message/CreateFunctionHandler.java     |     20 +-
 .../load/message/DropConstraintHandler.java     |     50 +
 .../repl/load/message/DropFunctionHandler.java  |      5 +-
 .../repl/load/message/DropPartitionHandler.java |      5 +-
 .../repl/load/message/DropTableHandler.java     |      5 +-
 .../parse/repl/load/message/InsertHandler.java  |      3 +-
 .../parse/repl/load/message/MessageHandler.java |     21 +-
 .../load/message/RenamePartitionHandler.java    |     15 +-
 .../repl/load/message/RenameTableHandler.java   |     24 +-
 .../parse/repl/load/message/TableHandler.java   |     23 +-
 .../load/message/TruncatePartitionHandler.java  |     11 +-
 .../repl/load/message/TruncateTableHandler.java |      8 +-
 .../ql/parse/spark/GenSparkProcContext.java     |      5 +
 .../hive/ql/parse/spark/GenSparkUtils.java      |    124 +-
 .../hive/ql/parse/spark/GenSparkWork.java       |      2 +-
 .../hive/ql/parse/spark/SparkCompiler.java      |     32 +-
 .../SparkPartitionPruningSinkOperator.java      |     51 +-
 .../parse/spark/SparkProcessAnalyzeTable.java   |     17 +-
 .../hive/ql/parse/spark/SplitOpTreeForDPP.java  |    131 +-
 .../hive/ql/plan/AbstractOperatorDesc.java      |      9 +
 .../hadoop/hive/ql/plan/AlterDatabaseDesc.java  |     37 +-
 .../hadoop/hive/ql/plan/AlterTableDesc.java     |     48 +-
 .../hadoop/hive/ql/plan/AppMasterEventDesc.java |     12 +
 .../hadoop/hive/ql/plan/ColStatistics.java      |      3 -
 .../hive/ql/plan/CommonMergeJoinDesc.java       |      9 +
 .../ql/plan/ConditionalResolverMergeFiles.java  |     11 +-
 .../hadoop/hive/ql/plan/CreateFunctionDesc.java |     15 +-
 .../hadoop/hive/ql/plan/DescTableDesc.java      |      4 +-
 .../hadoop/hive/ql/plan/DropFunctionDesc.java   |     16 +-
 .../hive/ql/plan/DynamicPruningEventDesc.java   |     24 +
 .../hadoop/hive/ql/plan/DynamicValue.java       |      4 +
 .../hadoop/hive/ql/plan/ExprNodeDescUtils.java  |     21 +
 .../hadoop/hive/ql/plan/FileSinkDesc.java       |     29 +
 .../apache/hadoop/hive/ql/plan/FilterDesc.java  |     12 +
 .../apache/hadoop/hive/ql/plan/GroupByDesc.java |     82 +-
 .../hadoop/hive/ql/plan/HashTableSinkDesc.java  |     10 +
 .../hadoop/hive/ql/plan/HiveOperation.java      |      4 +
 .../hadoop/hive/ql/plan/ImportTableDesc.java    |     71 +-
 .../apache/hadoop/hive/ql/plan/JoinDesc.java    |     15 +
 .../hive/ql/plan/LateralViewJoinDesc.java       |     11 +
 .../apache/hadoop/hive/ql/plan/LimitDesc.java   |      9 +
 .../apache/hadoop/hive/ql/plan/MapJoinDesc.java |     19 +-
 .../org/apache/hadoop/hive/ql/plan/MapWork.java |     49 +-
 .../hadoop/hive/ql/plan/OperatorDesc.java       |      1 +
 .../org/apache/hadoop/hive/ql/plan/PTFDesc.java |    112 +
 .../hadoop/hive/ql/plan/ReduceSinkDesc.java     |     25 +
 .../hive/ql/plan/RenamePartitionDesc.java       |     22 +-
 .../hadoop/hive/ql/plan/ReplCopyWork.java       |     82 +-
 .../apache/hadoop/hive/ql/plan/ScriptDesc.java  |     11 +
 .../apache/hadoop/hive/ql/plan/SelectDesc.java  |     12 +
 .../apache/hadoop/hive/ql/plan/Statistics.java  |     23 +-
 .../hadoop/hive/ql/plan/TableScanDesc.java      |     26 +
 .../hadoop/hive/ql/plan/TezEdgeProperty.java    |      7 +-
 .../hadoop/hive/ql/plan/TruncateTableDesc.java  |     11 +-
 .../apache/hadoop/hive/ql/plan/UDTFDesc.java    |     12 +
 .../hadoop/hive/ql/plan/VectorGroupByDesc.java  |     18 +
 .../hadoop/hive/ql/plan/VectorMapJoinDesc.java  |     24 +-
 .../hadoop/hive/ql/plan/VectorPTFDesc.java      |    360 +
 .../hadoop/hive/ql/plan/VectorPTFInfo.java      |    161 +
 .../hadoop/hive/ql/plan/ptf/WindowFrameDef.java |      6 +-
 .../hive/ql/ppd/SyntheticJoinPredicate.java     |      2 +-
 .../hadoop/hive/ql/processors/DfsProcessor.java |     60 +-
 .../hadoop/hive/ql/processors/SetProcessor.java |     11 +
 .../AuthorizationPreEventListener.java          |     19 +
 .../authorization/plugin/HiveOperationType.java |      4 +
 .../plugin/sqlstd/Operation2Privilege.java      |     31 +
 .../sqlstd/SQLStdHiveAccessController.java      |      3 +-
 .../SQLStdHiveAuthorizationValidator.java       |      9 +
 .../hadoop/hive/ql/session/LineageState.java    |     14 +
 .../hadoop/hive/ql/session/OperationLog.java    |      6 +-
 .../hive/ql/stats/ClientStatsPublisher.java     |      6 +
 .../hadoop/hive/ql/stats/StatsAggregator.java   |      6 +-
 .../hive/ql/stats/StatsCollectionContext.java   |      5 +
 .../hadoop/hive/ql/stats/StatsPublisher.java    |      6 +-
 .../apache/hadoop/hive/ql/stats/StatsUtils.java |    289 +-
 .../hive/ql/txn/compactor/CompactorMR.java      |     14 +-
 .../apache/hadoop/hive/ql/udf/UDFToBoolean.java |      2 +-
 .../apache/hadoop/hive/ql/udf/UDFToInteger.java |      5 +-
 .../apache/hadoop/hive/ql/udf/UDFToString.java  |      4 +-
 .../generic/AbstractGenericUDAFResolver.java    |      1 +
 .../hadoop/hive/ql/udf/generic/Collector.java   |      4 +
 .../DecimalNumDistinctValueEstimator.java       |     42 -
 .../DoubleNumDistinctValueEstimator.java        |     39 -
 .../hive/ql/udf/generic/GenericUDAFAverage.java |     10 +-
 .../ql/udf/generic/GenericUDAFBloomFilter.java  |     32 +-
 .../ql/udf/generic/GenericUDAFComputeStats.java |    381 +-
 .../ql/udf/generic/GenericUDAFEvaluator.java    |      8 +
 .../GenericUDAFMkCollectionEvaluator.java       |      2 +-
 .../udf/generic/GenericUDAFParameterInfo.java   |      4 +
 .../ql/udf/generic/GenericUDAFResolver2.java    |      1 +
 .../ql/udf/generic/GenericUDAFStdSample.java    |     13 +-
 .../hive/ql/udf/generic/GenericUDAFSum.java     |     25 +-
 .../udf/generic/GenericUDAFVarianceSample.java  |     13 +-
 .../hadoop/hive/ql/udf/generic/GenericUDF.java  |     12 +-
 .../hive/ql/udf/generic/GenericUDFDate.java     |      4 +-
 .../ql/udf/generic/GenericUDFInBloomFilter.java |     16 +-
 .../hive/ql/udf/generic/GenericUDFLeadLag.java  |      4 +-
 .../ql/udf/generic/GenericUDFTimestamp.java     |      3 +-
 .../generic/GenericUDFToTimestampLocalTZ.java   |    113 +
 .../ql/udf/generic/GenericUDFToTimestampTZ.java |     89 -
 .../hadoop/hive/ql/udf/generic/GenericUDTF.java |      5 +-
 .../ql/udf/generic/GenericUDTFGetSplits.java    |     20 +-
 .../generic/LongNumDistinctValueEstimator.java  |     43 -
 .../udf/generic/NumDistinctValueEstimator.java  |    383 -
 .../StringNumDistinctValueEstimator.java        |     39 -
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |    250 +-
 .../apache/hadoop/hive/ql/TestTxnCommands2.java |    553 +-
 .../ql/TestTxnCommands2WithSplitUpdate.java     |    586 -
 ...ommands2WithSplitUpdateAndVectorization.java |      4 +-
 .../hadoop/hive/ql/TestTxnCommandsBase.java     |    162 +
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java |    297 +
 .../hive/ql/exec/TestKeyWrapperFactory.java     |    113 +
 .../hadoop/hive/ql/exec/TestUtilities.java      |    224 +
 .../ql/exec/persistence/TestRowContainer.java   |     75 +
 .../repl/bootstrap/load/TaskTrackerTest.java    |     46 +
 .../hive/ql/exec/spark/TestSparkTask.java       |     49 +
 .../hive/ql/exec/tez/SampleTezSessionState.java |      3 +-
 .../hadoop/hive/ql/exec/tez/TestDagUtils.java   |     69 +
 .../hive/ql/exec/tez/TestTezSessionPool.java    |     46 +-
 .../hadoop/hive/ql/exec/tez/TestTezTask.java    |      6 +-
 .../hive/ql/exec/util/DescriptionTest.java      |     31 +
 .../CollectorTestOperator.java                  |     47 +
 .../CountCollectorTestOperator.java             |     48 +
 .../CountVectorCollectorTestOperator.java       |     47 +
 .../RowCollectorTestOperator.java               |     52 +
 .../RowCollectorTestOperatorBase.java           |     32 +
 .../RowVectorCollectorTestOperator.java         |     63 +
 .../ql/exec/util/rowobjects/RowTestObjects.java |    100 +
 .../exec/util/rowobjects/RowTestObjectsMap.java |     77 +
 .../util/rowobjects/RowTestObjectsMultiSet.java |     91 +
 .../exec/vector/TestVectorGroupByOperator.java  |      2 +-
 .../exec/vector/TestVectorSelectOperator.java   |      3 +-
 .../expressions/TestVectorMathFunctions.java    |     22 +
 .../vector/expressions/TestVectorTypeCasts.java |     16 +
 .../exec/vector/mapjoin/MapJoinTestConfig.java  |    658 +
 .../ql/exec/vector/mapjoin/MapJoinTestData.java |    272 +
 .../vector/mapjoin/MapJoinTestDescription.java  |    239 +
 .../vector/mapjoin/TestMapJoinOperator.java     |    549 +
 .../util/batchgen/TestVectorBatchGenerate.java  |     57 +
 .../batchgen/VectorBatchGenerateStream.java     |     63 +
 .../util/batchgen/VectorBatchGenerateUtil.java  |     92 +
 .../util/batchgen/VectorBatchGenerator.java     |    306 +
 .../batchgen/VectorColumnGroupGenerator.java    |    494 +
 .../apache/hadoop/hive/ql/io/TestAcidUtils.java |     64 +-
 .../hive/ql/io/orc/TestInputOutputFormat.java   |    772 +-
 .../hive/ql/io/orc/TestOrcRawRecordMerger.java  |    745 +-
 .../hive/ql/io/orc/TestOrcRecordUpdater.java    |     51 +-
 .../TestVectorizedOrcAcidRowBatchReader.java    |      8 +-
 .../io/orc/encoded/TestEncodedReaderImpl.java   |     14 +-
 .../io/parquet/AbstractTestParquetDirect.java   |      7 +-
 .../ql/io/parquet/TestDataWritableWriter.java   |     65 +-
 .../parquet/TestMapredParquetOutputFormat.java  |     10 -
 .../io/parquet/TestParquetRowGroupFilter.java   |      3 -
 .../io/parquet/TestVectorizedColumnReader.java  |      2 +-
 .../parquet/VectorizedColumnReaderTestBase.java |     16 +-
 .../io/parquet/convert/TestETypeConverter.java  |    124 -
 .../read/TestParquetFilterPredicate.java        |     56 +-
 .../serde/TestParquetTimestampUtils.java        |    263 +
 .../io/parquet/timestamp/TestNanoTimeUtils.java |    256 -
 .../TestParquetTimestampConverter.java          |    264 -
 .../hive/ql/lockmgr/TestDbTxnManager2.java      |      6 -
 ...TestGenMapRedUtilsCreateConditionalTask.java |      2 +
 .../calcite/TestCBORuleFiredOnlyOnce.java       |      2 +-
 .../ql/optimizer/physical/TestVectorizer.java   |     10 +-
 .../parse/TestReplicationSemanticAnalyzer.java  |     71 +-
 .../hive/ql/parse/repl/CopyUtilsTest.java       |     47 +
 .../repl/load/TestEventDumpDirComparator.java   |     44 +
 .../PrimaryToReplicaResourceFunctionTest.java   |     18 +
 .../processors/TestCommandProcessorFactory.java |      2 +
 .../hive/ql/txn/compactor/CompactorTest.java    |      8 +
 .../hive/ql/txn/compactor/TestWorker.java       |    125 +-
 .../queries/clientnegative/acid_overwrite.q     |      8 -
 .../alter_table_non_partitioned_table_cascade.q |      4 +
 .../queries/clientnegative/create_not_acid.q    |      2 +-
 .../clientnegative/load_data_into_acid.q        |     22 +
 .../parquet_alter_part_table_drop_columns.q     |     22 +
 .../parquet_int96_alter_invalid_timezone.q      |      5 -
 .../parquet_int96_create_invalid_timezone.q     |      3 -
 .../clientnegative/repl_dump_requires_admin.q   |     38 +
 .../clientnegative/repl_load_requires_admin.q   |     41 +
 .../subquery_scalar_implicit_gby.q              |      1 -
 .../clientpositive/acid_bucket_pruning.q        |      8 +-
 .../clientpositive/acid_insert_overwrite.q      |     73 +
 .../queries/clientpositive/acid_no_buckets.q    |    210 +
 .../alter_partition_onto_nocurrent_db.q         |     15 +
 ...lter_table_update_status_disable_bitvector.q |    139 +
 .../clientpositive/authorization_view_8.q       |     23 +
 ql/src/test/queries/clientpositive/bitvector.q  |      3 +
 .../clientpositive/bucket_num_reducers_acid.q   |     30 +
 .../clientpositive/bucket_num_reducers_acid2.q  |     33 +
 ql/src/test/queries/clientpositive/char_udf1.q  |      4 +-
 .../queries/clientpositive/cmdwithcomments.q    |      2 +
 .../queries/clientpositive/compute_stats_date.q |      2 +-
 .../clientpositive/compute_stats_decimal.q      |      2 +-
 .../clientpositive/compute_stats_double.q       |      2 +-
 .../queries/clientpositive/compute_stats_long.q |      2 +-
 .../clientpositive/compute_stats_string.q       |      2 +-
 ql/src/test/queries/clientpositive/concat_op.q  |      8 +-
 .../queries/clientpositive/constprog_semijoin.q |      2 +
 .../clientpositive/correlationoptimizer14.q     |      1 +
 .../queries/clientpositive/count_dist_rewrite.q |      2 +
 ql/src/test/queries/clientpositive/dfscmd.q     |      7 +
 .../clientpositive/dynamic_semijoin_reduction.q |     22 +-
 .../dynamic_semijoin_reduction_3.q              |      2 +-
 .../dynamic_semijoin_reduction_sw.q             |     60 +
 .../clientpositive/explaindenpendencydiffengs.q |     10 +
 .../test/queries/clientpositive/filter_union.q  |     56 +
 ql/src/test/queries/clientpositive/fm-sketch.q  |     58 +
 .../clientpositive/groupby_distinct_samekey.q   |      1 +
 ql/src/test/queries/clientpositive/hll.q        |     57 +
 ql/src/test/queries/clientpositive/innerjoin1.q |     19 +
 ql/src/test/queries/clientpositive/input20.q    |      8 +-
 ql/src/test/queries/clientpositive/input33.q    |      8 +-
 .../test/queries/clientpositive/input3_limit.q  |      6 +-
 .../queries/clientpositive/insert_orig_table.q  |     20 +-
 .../clientpositive/insert_overwrite_directory.q |      4 +-
 .../clientpositive/insert_values_orig_table.q   |     20 +-
 .../insert_values_orig_table_use_metadata.q     |     23 +-
 .../test/queries/clientpositive/jdbc_handler.q  |      7 +-
 ql/src/test/queries/clientpositive/join47.q     |    204 +
 .../queries/clientpositive/join_max_hashtable.q |     13 +
 ql/src/test/queries/clientpositive/llap_smb.q   |     53 +
 .../test/queries/clientpositive/localtimezone.q |     95 +
 ql/src/test/queries/clientpositive/mapjoin47.q  |    206 +
 .../clientpositive/materialized_view_create.q   |      6 +
 .../materialized_view_create_rewrite.q          |      2 +
 ql/src/test/queries/clientpositive/mm_all.q     |     45 -
 .../test/queries/clientpositive/mm_loaddata.q   |     53 +
 .../queries/clientpositive/orc_empty_table.q    |     11 +
 .../clientpositive/orc_merge_incompat_schema.q  |     47 +
 .../orc_merge_incompat_writer_version.q         |     30 +
 .../clientpositive/parquet_int96_timestamp.q    |     85 -
 .../clientpositive/parquet_no_row_serde.q       |     25 +
 .../clientpositive/parquet_ppd_decimal.q        |      1 +
 .../clientpositive/parquet_predicate_pushdown.q |      1 +
 .../parquet_predicate_pushdown_2.q              |      7 +
 .../parquet_timestamp_conversion.q              |     13 -
 .../test/queries/clientpositive/parquet_types.q |      1 +
 .../parquet_types_vectorization.q               |      1 +
 ql/src/test/queries/clientpositive/partialdhj.q |     54 +
 .../test/queries/clientpositive/perf/query11.q  |     77 +
 .../test/queries/clientpositive/perf/query28.q  |      2 +
 ql/src/test/queries/clientpositive/ppd2.q       |      2 +
 .../queries/clientpositive/ppd_join_filter.q    |      2 +
 .../reduceSinkDeDuplication_pRS_key_empty.q     |     10 +-
 .../reduce_deduplicate_extended.q               |      1 +
 .../test/queries/clientpositive/semijoin_hint.q |     32 +-
 .../clientpositive/set_processor_namespaces.q   |      8 +
 ql/src/test/queries/clientpositive/skewjoin.q   |      1 +
 .../queries/clientpositive/smb_mapjoin_20.q     |     13 +
 .../queries/clientpositive/smb_mapjoin_47.q     |    211 +
 .../clientpositive/spark_constprog_dpp.q        |     17 +
 .../spark_dynamic_partition_pruning.q           |     56 +-
 .../spark_dynamic_partition_pruning_2.q         |     16 +
 .../spark_dynamic_partition_pruning_3.q         |    228 +
 ...ark_dynamic_partition_pruning_mapjoin_only.q |     50 +
 .../spark_explain_groupbyshuffle.q              |      8 +
 .../spark_multi_insert_parallel_orderby.q       |     67 +
 .../queries/clientpositive/spark_union_merge.q  |     41 +
 .../spark_use_ts_stats_for_mapjoin.q            |      1 +
 ...spark_vectorized_dynamic_partition_pruning.q |      2 +-
 .../test/queries/clientpositive/subquery_in.q   |     50 +-
 .../queries/clientpositive/subquery_scalar.q    |     49 +
 ql/src/test/queries/clientpositive/sysdb.q      |     12 +-
 .../test/queries/clientpositive/timestamptz.q   |     16 +-
 .../test/queries/clientpositive/timestamptz_1.q |      4 +-
 .../test/queries/clientpositive/timestamptz_2.q |      3 +-
 .../clientpositive/truncate_column_buckets.q    |      4 +-
 .../queries/clientpositive/udf_stddev_samp.q    |      9 +
 .../test/queries/clientpositive/udf_var_samp.q  |      9 +
 .../test/queries/clientpositive/varchar_udf1.q  |      4 +-
 .../queries/clientpositive/vector_aggregate_9.q |     16 +-
 .../clientpositive/vector_between_columns.q     |      1 +
 .../queries/clientpositive/vector_complex_all.q |     41 +
 .../clientpositive/vector_groupby_cube1.q       |     17 +-
 .../vector_groupby_grouping_id1.q               |     21 +
 .../vector_groupby_grouping_id2.q               |     52 +
 .../vector_groupby_grouping_window.q            |      1 +
 .../clientpositive/vector_groupby_reduce.q      |     73 +-
 .../vector_outer_reference_windowed.q           |     91 +
 .../clientpositive/vector_ptf_part_simple.q     |    240 +-
 .../clientpositive/vector_tablesample_rows.q    |      8 +-
 .../test/queries/clientpositive/vector_udf1.q   |     74 +-
 .../queries/clientpositive/vector_windowing.q   |    791 +
 .../vector_windowing_expressions.q              |     94 +
 .../clientpositive/vector_windowing_gby.q       |     21 +
 .../clientpositive/vector_windowing_gby2.q      |     48 +
 .../vector_windowing_multipartitioning.q        |     73 +
 .../clientpositive/vector_windowing_navfn.q     |      2 +
 .../vector_windowing_order_null.q               |     58 +
 .../vector_windowing_range_multiorder.q         |     68 +
 .../clientpositive/vector_windowing_rank.q      |    117 +
 .../clientpositive/vector_windowing_streaming.q |     85 +
 .../vector_windowing_windowspec.q               |     70 +
 .../vector_windowing_windowspec4.q              |     37 +
 .../queries/clientpositive/vectorization_0.q    |     15 +-
 .../queries/clientpositive/vectorization_1.q    |     22 +
 .../queries/clientpositive/vectorization_10.q   |     24 +
 .../queries/clientpositive/vectorization_11.q   |     15 +
 .../queries/clientpositive/vectorization_12.q   |     32 +
 .../queries/clientpositive/vectorization_13.q   |      2 +-
 .../queries/clientpositive/vectorization_14.q   |      2 +-
 .../queries/clientpositive/vectorization_15.q   |      5 +-
 .../queries/clientpositive/vectorization_16.q   |      2 +-
 .../queries/clientpositive/vectorization_17.q   |      2 +-
 .../queries/clientpositive/vectorization_2.q    |     23 +
 .../queries/clientpositive/vectorization_3.q    |     25 +
 .../queries/clientpositive/vectorization_4.q    |     23 +
 .../queries/clientpositive/vectorization_5.q    |     20 +
 .../queries/clientpositive/vectorization_6.q    |     21 +
 .../queries/clientpositive/vectorization_7.q    |      2 +-
 .../queries/clientpositive/vectorization_8.q    |      2 +-
 .../queries/clientpositive/vectorization_9.q    |      2 +-
 .../queries/clientpositive/vectorized_casts.q   |      4 +-
 .../clientpositive/vectorized_distinct_gby.q    |      7 +-
 .../vectorized_multi_output_select.q            |     28 +
 .../queries/clientpositive/vectorized_parquet.q |      4 +
 .../clientpositive/vectorized_parquet_types.q   |      1 +
 .../queries/clientpositive/vectorized_ptf.q     |      1 +
 .../clientpositive/vectorized_timestamp.q       |     27 +-
 .../results/clientnegative/acid_overwrite.q.out |     19 -
 ...er_table_non_partitioned_table_cascade.q.out |     21 +
 .../authorization_uri_export.q.out              |      1 +
 .../clientnegative/create_not_acid.q.out        |      4 +-
 .../clientnegative/delete_non_acid_table.q.out  |      2 +-
 .../results/clientnegative/dyn_part_max.q.out   |      1 -
 .../clientnegative/load_data_into_acid.q.out    |     33 +
 .../parquet_alter_part_table_drop_columns.q.out |     53 +
 .../parquet_int96_alter_invalid_timezone.q.out  |     13 -
 .../parquet_int96_create_invalid_timezone.q.out |      5 -
 .../repl_dump_requires_admin.q.out              |     69 +
 .../repl_load_requires_admin.q.out              |     80 +
 .../set_hiveconf_internal_variable0.q.out       |      2 +-
 .../set_hiveconf_internal_variable1.q.out       |      2 +-
 .../subquery_scalar_implicit_gby.q.out          |      1 -
 .../clientnegative/update_non_acid_table.q.out  |      2 +-
 .../clientpositive/acid_insert_overwrite.q.out  |    395 +
 .../clientpositive/acid_table_stats.q.out       |     14 +-
 .../clientpositive/alterColumnStats.q.out       |     16 +-
 .../clientpositive/alterColumnStatsPart.q.out   |      6 +-
 .../alter_partition_onto_nocurrent_db.q.out     |     78 +
 .../alter_partition_update_status.q.out         |     30 +-
 .../alter_table_column_stats.q.out              |    672 +-
 .../alter_table_update_status.q.out             |    384 +-
 ..._table_update_status_disable_bitvector.q.out |    708 +
 .../clientpositive/analyze_tbl_part.q.out       |     18 +-
 .../annotate_stats_deep_filters.q.out           |      4 +-
 .../clientpositive/annotate_stats_groupby.q.out |     10 +-
 .../clientpositive/annotate_stats_join.q.out    |     44 +-
 .../annotate_stats_join_pkfk.q.out              |     28 +-
 .../clientpositive/annotate_stats_select.q.out  |      4 +-
 .../clientpositive/autoColumnStats_4.q.out      |     16 +-
 .../clientpositive/autoColumnStats_5.q.out      |     50 +-
 .../clientpositive/autoColumnStats_6.q.out      |      6 +-
 .../clientpositive/autoColumnStats_7.q.out      |     12 +-
 .../clientpositive/autoColumnStats_8.q.out      |     22 +-
 .../clientpositive/autoColumnStats_9.q.out      |     32 +-
 .../results/clientpositive/auto_join0.q.out     |    109 +-
 .../results/clientpositive/auto_join12.q.out    |     22 +-
 .../results/clientpositive/auto_join15.q.out    |     47 +-
 .../results/clientpositive/auto_join16.q.out    |     16 +-
 .../results/clientpositive/auto_join20.q.out    |     94 +-
 .../results/clientpositive/auto_join26.q.out    |      8 +-
 .../results/clientpositive/auto_join27.q.out    |      8 +-
 .../results/clientpositive/auto_join31.q.out    |    286 +-
 .../results/clientpositive/auto_join4.q.out     |      6 +-
 .../results/clientpositive/auto_join5.q.out     |      6 +-
 .../results/clientpositive/auto_join8.q.out     |      6 +-
 .../auto_join_without_localtask.q.out           |     74 +-
 .../results/clientpositive/avro_decimal.q.out   |      8 +-
 .../clientpositive/avro_decimal_native.q.out    |      8 +-
 .../materialized_view_create_rewrite.q.out      |      8 +
 .../clientpositive/beeline/smb_mapjoin_12.q.out |     75 +-
 .../test/results/clientpositive/bitvector.q.out |     10 +
 .../bucket_num_reducers_acid.q.out              |     33 +
 .../bucket_num_reducers_acid2.q.out             |     44 +
 .../bucketsortoptimize_insert_8.q.out           |    342 +-
 .../cbo_rp_annotate_stats_groupby.q.out         |     10 +-
 .../clientpositive/cbo_rp_auto_join0.q.out      |     98 +-
 .../clientpositive/cbo_rp_auto_join1.q.out      |     18 +-
 .../results/clientpositive/cbo_rp_join0.q.out   |     12 +-
 .../clientpositive/cbo_rp_outer_join_ppr.q.out  |     44 +-
 .../test/results/clientpositive/char_udf1.q.out |     10 +-
 .../clientpositive/cmdwithcomments.q.out        |     17 +-
 .../clientpositive/colstats_all_nulls.q.out     |     16 +-
 ...names_with_leading_and_trailing_spaces.q.out |     16 +-
 .../column_pruner_multiple_children.q.out       |     22 +-
 .../clientpositive/columnstats_partlvl.q.out    |     92 +-
 .../clientpositive/columnstats_partlvl_dp.q.out |     84 +-
 .../clientpositive/columnstats_quoting.q.out    |     24 +-
 .../clientpositive/columnstats_tbllvl.q.out     |    144 +-
 .../test/results/clientpositive/combine2.q.out  |      6 +-
 .../results/clientpositive/compustat_avro.q.out |     16 +-
 .../clientpositive/compute_stats_date.q.out     |     34 +-
 .../clientpositive/compute_stats_decimal.q.out  |    Bin 1633 -> 1512 bytes
 .../clientpositive/compute_stats_double.q.out   |    Bin 1539 -> 1443 bytes
 .../compute_stats_empty_table.q.out             |      6 +-
 .../clientpositive/compute_stats_long.q.out     |    Bin 1498 -> 1376 bytes
 .../clientpositive/compute_stats_string.q.out   |    Bin 1517 -> 1448 bytes
 .../test/results/clientpositive/concat_op.q.out |     24 +-
 .../confirm_initial_tbl_stats.q.out             |     86 +-
 .../test/results/clientpositive/constGby.q.out  |      2 +-
 .../clientpositive/constant_prop_2.q.out        |      6 +-
 .../clientpositive/constantfolding.q.out        |      4 +-
 .../clientpositive/constprog_partitioner.q.out  |     10 +-
 .../clientpositive/correlated_join_keys.q.out   |     10 +-
 .../clientpositive/correlationoptimizer10.q.out |      4 +-
 .../clientpositive/correlationoptimizer11.q.out |      8 +-
 .../clientpositive/correlationoptimizer12.q.out |      4 +-
 .../clientpositive/correlationoptimizer13.q.out |     32 +-
 .../clientpositive/correlationoptimizer15.q.out |      4 +-
 .../clientpositive/correlationoptimizer7.q.out  |     24 +-
 .../clientpositive/correlationoptimizer8.q.out  |    252 +-
 .../clientpositive/correlationoptimizer9.q.out  |    156 +-
 .../clientpositive/count_dist_rewrite.q.out     |     12 +-
 .../results/clientpositive/create_view.q.out    |      4 +-
 .../clientpositive/cross_join_merge.q.out       |     80 +-
 .../results/clientpositive/ctas_colname.q.out   |      4 +-
 .../results/clientpositive/decimal_stats.q.out  |      8 +-
 .../results/clientpositive/decimal_udf.q.out    |     24 +-
 .../results/clientpositive/deleteAnalyze.q.out  |      8 +-
 .../clientpositive/describe_syntax.q.out        |     24 +-
 .../results/clientpositive/describe_table.q.out |     32 +-
 ql/src/test/results/clientpositive/dfscmd.q.out |      1 +
 .../display_colstats_tbllvl.q.out               |    128 +-
 .../clientpositive/distinct_windowing.q.out     |      8 +-
 .../distinct_windowing_no_cbo.q.out             |     12 +-
 .../results/clientpositive/druid_basic2.q.out   |    103 +-
 .../clientpositive/druid_intervals.q.out        |    130 +-
 .../clientpositive/druid_timeseries.q.out       |      6 +-
 .../results/clientpositive/druid_topn.q.out     |    268 +-
 .../clientpositive/dynamic_rdd_cache.q.out      |   1431 -
 .../dynpart_sort_optimization_acid.q.out        |   1475 -
 .../encrypted/encryption_move_tbl.q.out         |     36 +-
 .../results/clientpositive/except_all.q.out     |     48 +-
 .../exec_parallel_column_stats.q.out            |     12 +-
 .../clientpositive/explain_logical.q.out        |     32 +-
 .../explaindenpendencydiffengs.q.out            |     52 +
 .../extrapolate_part_stats_full.q.out           |      6 +-
 .../extrapolate_part_stats_partial.q.out        |     20 +-
 .../results/clientpositive/filter_aggr.q.out    |    228 +-
 .../results/clientpositive/filter_union.q.out   |    593 +
 .../test/results/clientpositive/fm-sketch.q.out |    323 +
 .../test/results/clientpositive/fold_case.q.out |     14 +-
 .../clientpositive/fouter_join_ppr.q.out        |     84 +-
 .../results/clientpositive/groupby4_map.q.out   |      4 +-
 .../clientpositive/groupby4_map_skew.q.out      |      4 +-
 .../results/clientpositive/groupby_cube1.q.out  |     20 +-
 .../groupby_distinct_samekey.q.out              |     45 +-
 .../groupby_grouping_window.q.out               |      2 +-
 .../clientpositive/groupby_position.q.out       |     56 +-
 .../clientpositive/groupby_rollup1.q.out        |     12 +-
 .../clientpositive/groupby_sort_11.q.out        |     18 +-
 .../clientpositive/groupby_sort_1_23.q.out      |    254 +-
 .../results/clientpositive/groupby_sort_2.q.out |      6 +-
 .../results/clientpositive/groupby_sort_3.q.out |     12 +-
 .../results/clientpositive/groupby_sort_4.q.out |     12 +-
 .../results/clientpositive/groupby_sort_5.q.out |     18 +-
 .../results/clientpositive/groupby_sort_6.q.out |     20 +-
 .../results/clientpositive/groupby_sort_7.q.out |      6 +-
 .../results/clientpositive/groupby_sort_9.q.out |      6 +-
 .../clientpositive/groupby_sort_skew_1_23.q.out |    254 +-
 .../clientpositive/groupby_sort_test_1.q.out    |      6 +-
 ql/src/test/results/clientpositive/hll.q.out    |    323 +
 .../identity_project_remove_skip.q.out          |    189 +-
 .../clientpositive/index_auto_mult_tables.q.out |     52 +-
 .../index_auto_mult_tables_compact.q.out        |     52 +-
 .../clientpositive/infer_bucket_sort.q.out      |      4 +-
 .../infer_bucket_sort_grouping_operators.q.out  |     18 +-
 .../results/clientpositive/innerjoin1.q.out     |    214 +
 .../test/results/clientpositive/input20.q.out   |     21 +-
 .../test/results/clientpositive/input30.q.out   |      6 +-
 .../test/results/clientpositive/input31.q.out   |      4 +-
 .../test/results/clientpositive/input32.q.out   |      4 +-
 .../test/results/clientpositive/input33.q.out   |     21 +-
 .../test/results/clientpositive/input39.q.out   |      2 +-
 .../results/clientpositive/input3_limit.q.out   |     44 +-
 .../test/results/clientpositive/input41.q.out   |      2 +-
 .../clientpositive/insert_orig_table.q.out      |     68 +-
 .../insert_overwrite_directory.q.out            |      8 +-
 .../insert_values_orig_table.q.out              |     68 +-
 .../insert_values_orig_table_use_metadata.q.out |    116 +-
 ql/src/test/results/clientpositive/join12.q.out |     30 +-
 ql/src/test/results/clientpositive/join16.q.out |     22 +-
 ql/src/test/results/clientpositive/join29.q.out |     48 +-
 ql/src/test/results/clientpositive/join30.q.out |      8 +-
 ql/src/test/results/clientpositive/join31.q.out |      6 +-
 ql/src/test/results/clientpositive/join32.q.out |     80 +-
 ql/src/test/results/clientpositive/join33.q.out |     80 +-
 ql/src/test/results/clientpositive/join35.q.out |     64 +-
 ql/src/test/results/clientpositive/join38.q.out |     28 +-
 ql/src/test/results/clientpositive/join4.q.out  |      8 +-
 ql/src/test/results/clientpositive/join40.q.out |      6 +-
 ql/src/test/results/clientpositive/join47.q.out |   1715 +
 ql/src/test/results/clientpositive/join5.q.out  |      8 +-
 ql/src/test/results/clientpositive/join8.q.out  |      8 +-
 .../clientpositive/join_alt_syntax.q.out        |    272 +-
 .../clientpositive/join_cond_pushdown_2.q.out   |    112 +-
 .../clientpositive/join_cond_pushdown_4.q.out   |    112 +-
 .../clientpositive/join_filters_overlap.q.out   |      4 +-
 .../results/clientpositive/join_parse.q.out     |    102 +-
 .../limit_pushdown_negative.q.out               |     96 +-
 .../list_bucket_query_oneskew_2.q.out           |     10 +-
 .../llap/acid_bucket_pruning.q.out              |     30 +-
 .../clientpositive/llap/acid_no_buckets.q.out   |   1976 +
 .../llap/authorization_view_8.q.out             |     69 +
 .../clientpositive/llap/autoColumnStats_2.q.out |     64 +-
 .../clientpositive/llap/auto_join0.q.out        |     51 +-
 .../clientpositive/llap/auto_join1.q.out        |      6 +-
 .../clientpositive/llap/auto_join21.q.out       |     10 +-
 .../clientpositive/llap/auto_join29.q.out       |     90 +-
 .../clientpositive/llap/auto_join30.q.out       |    630 +-
 .../llap/auto_smb_mapjoin_14.q.out              |     44 +-
 .../llap/auto_sortmerge_join_9.q.out            |    234 +-
 .../clientpositive/llap/bucket_groupby.q.out    |     72 +-
 .../llap/bucket_map_join_tez1.q.out             |      7 -
 .../llap/bucket_map_join_tez2.q.out             |     34 +-
 .../results/clientpositive/llap/cluster.q.out   |     40 +-
 ...names_with_leading_and_trailing_spaces.q.out |     16 +-
 .../llap/column_table_stats.q.out               |     68 +-
 .../llap/column_table_stats_orc.q.out           |     50 +-
 .../llap/columnstats_part_coltype.q.out         |    126 +-
 .../llap/correlationoptimizer1.q.out            |    376 +-
 .../llap/correlationoptimizer2.q.out            |    186 +-
 .../llap/correlationoptimizer3.q.out            |    296 +-
 .../llap/correlationoptimizer4.q.out            |    220 +-
 .../llap/correlationoptimizer6.q.out            |    532 +-
 .../results/clientpositive/llap/count.q.out     |    124 +-
 .../llap/count_dist_rewrite.q.out               |     66 +-
 .../clientpositive/llap/cross_join.q.out        |      8 +-
 .../clientpositive/llap/deleteAnalyze.q.out     |      8 +-
 .../llap/dynamic_partition_pruning.q.out        |    317 +-
 .../llap/dynamic_semijoin_reduction.q.out       |    863 +-
 .../llap/dynamic_semijoin_reduction_3.q.out     |      4 +-
 .../llap/dynamic_semijoin_reduction_sw.q.out    |    518 +
 .../llap/dynamic_semijoin_user_level.q.out      |    172 +-
 .../llap/dynpart_sort_optimization2.q.out       |     20 +-
 .../clientpositive/llap/except_distinct.q.out   |     52 +-
 .../clientpositive/llap/explainuser_1.q.out     |   1505 +-
 .../clientpositive/llap/explainuser_2.q.out     |    931 +-
 .../clientpositive/llap/explainuser_4.q.out     |     14 +-
 .../extrapolate_part_stats_partial_ndv.q.out    |    132 +-
 .../clientpositive/llap/filter_union.q.out      |    539 +
 .../results/clientpositive/llap/groupby1.q.out  |      6 +-
 .../results/clientpositive/llap/groupby2.q.out  |      6 +-
 .../llap/groupby_resolution.q.out               |     78 +-
 .../results/clientpositive/llap/having.q.out    |     66 +-
 .../llap/hybridgrace_hashjoin_1.q.out           |     12 +-
 .../llap/hybridgrace_hashjoin_2.q.out           |     24 +-
 .../llap/identity_project_remove_skip.q.out     |     71 +-
 .../clientpositive/llap/insert_orig_table.q.out |     68 +-
 .../clientpositive/llap/intersect_all.q.out     |     40 +-
 .../llap/intersect_distinct.q.out               |     40 +-
 .../clientpositive/llap/intersect_merge.q.out   |    204 +-
 .../clientpositive/llap/jdbc_handler.q.out      |     32 +-
 .../results/clientpositive/llap/join1.q.out     |      6 +-
 .../clientpositive/llap/join32_lessSize.q.out   |    175 +-
 .../llap/join_max_hashtable.q.out               |    222 +-
 .../llap/limit_join_transpose.q.out             |    214 +-
 .../clientpositive/llap/limit_pushdown.q.out    |     68 +-
 .../clientpositive/llap/limit_pushdown3.q.out   |     64 +-
 .../results/clientpositive/llap/lineage2.q.out  |      4 +-
 .../results/clientpositive/llap/lineage3.q.out  |      4 +-
 .../results/clientpositive/llap/llap_smb.q.out  |    359 +
 .../clientpositive/llap/llap_stats.q.out        |     16 +-
 .../llap/llap_vector_nohybridgrace.q.out        |     12 +-
 .../clientpositive/llap/llapdecider.q.out       |     88 +-
 .../results/clientpositive/llap/merge1.q.out    |     16 +-
 .../results/clientpositive/llap/merge2.q.out    |     12 +-
 .../results/clientpositive/llap/mergejoin.q.out |     16 +-
 .../llap/metadata_only_queries.q.out            |    112 +-
 .../test/results/clientpositive/llap/mrr.q.out  |    153 +-
 .../clientpositive/llap/multiMapJoin2.q.out     |    255 +-
 .../clientpositive/llap/offset_limit.q.out      |     10 +-
 .../llap/offset_limit_ppd_optimizer.q.out       |     58 +-
 .../clientpositive/llap/orc_create.q.out        |      2 +-
 .../llap/orc_merge_incompat_schema.q.out        |    161 +
 .../orc_merge_incompat_writer_version.q.out     |    108 +
 .../clientpositive/llap/orc_ppd_basic.q.out     |    234 +-
 .../results/clientpositive/llap/parallel.q.out  |      2 +-
 .../clientpositive/llap/parallel_colstats.q.out |     26 +-
 .../llap/parquet_predicate_pushdown.q.out       |     18 +-
 .../llap/parquet_types_vectorization.q.out      |    849 +
 .../clientpositive/llap/partialdhj.q.out        |    460 +
 .../llap/partition_shared_scan.q.out            |      7 -
 .../test/results/clientpositive/llap/ptf.q.out  |    154 +-
 .../clientpositive/llap/ptf_streaming.q.out     |     96 +-
 .../llap/reduce_deduplicate_extended.q.out      |     74 +-
 .../schema_evol_orc_vec_part_all_complex.q.out  |     72 +-
 .../schema_evol_text_vec_part_all_complex.q.out |     72 +-
 ...hema_evol_text_vecrow_part_all_complex.q.out |     72 +-
 .../clientpositive/llap/semijoin_hint.q.out     |    175 +-
 .../results/clientpositive/llap/skewjoin.q.out  |     12 +-
 .../clientpositive/llap/skewjoinopt15.q.out     |      4 +-
 .../clientpositive/llap/smb_mapjoin_14.q.out    |     54 +-
 .../clientpositive/llap/smb_mapjoin_15.q.out    |      4 +-
 .../clientpositive/llap/stats_only_null.q.out   |      6 +-
 .../clientpositive/llap/subquery_exists.q.out   |     22 +-
 .../clientpositive/llap/subquery_in.q.out       |   1015 +-
 .../clientpositive/llap/subquery_multi.q.out    |    531 +-
 .../clientpositive/llap/subquery_notin.q.out    |    740 +-
 .../clientpositive/llap/subquery_null_agg.q.out |     55 +-
 .../clientpositive/llap/subquery_scalar.q.out   |   2227 +-
 .../clientpositive/llap/subquery_select.q.out   |    244 +-
 .../clientpositive/llap/subquery_views.q.out    |    111 +-
 .../results/clientpositive/llap/sysdb.q.out     |    252 +-
 .../llap/table_access_keys_stats.q.out          |     36 +-
 .../results/clientpositive/llap/tez_dml.q.out   |     12 +-
 .../llap/tez_dynpart_hashjoin_1.q.out           |     24 +-
 .../llap/tez_dynpart_hashjoin_2.q.out           |     24 +-
 .../llap/tez_dynpart_hashjoin_3.q.out           |      4 +-
 .../clientpositive/llap/tez_join_hash.q.out     |     20 +-
 .../clientpositive/llap/tez_join_tests.q.out    |     37 +-
 .../clientpositive/llap/tez_joins_explain.q.out |     37 +-
 .../clientpositive/llap/tez_smb_empty.q.out     |     16 +-
 .../clientpositive/llap/tez_smb_main.q.out      |      4 +-
 .../results/clientpositive/llap/tez_union.q.out |     64 +-
 .../clientpositive/llap/tez_union2.q.out        |     16 +-
 .../llap/tez_union_group_by.q.out               |      2 +-
 .../llap/tez_union_multiinsert.q.out            |     68 +-
 .../llap/tez_vector_dynpart_hashjoin_1.q.out    |     24 +-
 .../llap/tez_vector_dynpart_hashjoin_2.q.out    |     24 +-
 .../results/clientpositive/llap/union2.q.out    |      4 +-
 .../results/clientpositive/llap/union4.q.out    |      6 +-
 .../results/clientpositive/llap/union5.q.out    |      4 +-
 .../results/clientpositive/llap/union6.q.out    |      4 +-
 .../results/clientpositive/llap/union7.q.out    |     20 +-
 .../results/clientpositive/llap/union9.q.out    |      6 +-
 .../clientpositive/llap/unionDistinct_1.q.out   |    283 +-
 .../clientpositive/llap/union_remove_26.q.out   |     54 +-
 .../clientpositive/llap/union_top_level.q.out   |     30 +-
 .../clientpositive/llap/varchar_udf1.q.out      |     10 +-
 .../llap/vector_adaptor_usage_mode.q.out        |     43 +-
 .../llap/vector_aggregate_9.q.out               |    311 +-
 .../llap/vector_auto_smb_mapjoin_14.q.out       |    104 +-
 .../clientpositive/llap/vector_between_in.q.out |     44 +-
 .../llap/vector_binary_join_groupby.q.out       |      8 +
 .../llap/vector_cast_constant.q.out             |     34 +-
 .../clientpositive/llap/vector_char_2.q.out     |      8 +
 .../clientpositive/llap/vector_coalesce_2.q.out |      8 +
 .../llap/vector_complex_all.q.out               |   1209 +-
 .../llap/vector_complex_join.q.out              |     77 +-
 .../clientpositive/llap/vector_count.q.out      |     70 +-
 .../llap/vector_count_distinct.q.out            |      8 +
 .../clientpositive/llap/vector_date_1.q.out     |      2 +-
 .../llap/vector_decimal_aggregate.q.out         |     45 +-
 .../llap/vector_decimal_precision.q.out         |     32 +-
 .../llap/vector_decimal_round.q.out             |     44 +-
 .../llap/vector_decimal_udf.q.out               |      6 +-
 .../clientpositive/llap/vector_distinct_2.q.out |      4 +
 .../llap/vector_empty_where.q.out               |     32 +
 .../clientpositive/llap/vector_groupby4.q.out   |      4 +
 .../clientpositive/llap/vector_groupby6.q.out   |      4 +
 .../clientpositive/llap/vector_groupby_3.q.out  |      4 +
 .../llap/vector_groupby_cube1.q.out             |    196 +-
 .../llap/vector_groupby_grouping_id1.q.out      |    800 +
 .../llap/vector_groupby_grouping_id2.q.out      |   2117 +
 .../llap/vector_groupby_grouping_id3.q.out      |      8 +
 .../llap/vector_groupby_grouping_sets3.q.out    |      8 +-
 .../llap/vector_groupby_grouping_sets4.q.out    |     27 -
 .../vector_groupby_grouping_sets_grouping.q.out |      2 +-
 .../llap/vector_groupby_grouping_window.q.out   |      4 +-
 .../llap/vector_groupby_mapjoin.q.out           |     22 +-
 .../llap/vector_groupby_reduce.q.out            |   2568 +-
 .../llap/vector_groupby_rollup1.q.out           |     12 +-
 .../llap/vector_grouping_sets.q.out             |      8 +
 .../llap/vector_include_no_sel.q.out            |      4 +
 .../clientpositive/llap/vector_inner_join.q.out |      2 +
 .../llap/vector_interval_mapjoin.q.out          |      8 +-
 .../clientpositive/llap/vector_join30.q.out     |   1170 +-
 .../llap/vector_left_outer_join.q.out           |      4 +-
 .../llap/vector_leftsemi_mapjoin.q.out          |    168 +
 .../llap/vector_mapjoin_reduce.q.out            |     52 +-
 .../llap/vector_mr_diff_schema_alias.q.out      |     34 +-
 .../llap/vector_null_projection.q.out           |      8 +-
 .../llap/vector_number_compare_projection.q.out |    186 +-
 .../clientpositive/llap/vector_nvl.q.out        |      2 +-
 .../clientpositive/llap/vector_orderby_5.q.out  |      4 +
 .../llap/vector_outer_join1.q.out               |     16 +-
 .../llap/vector_outer_join2.q.out               |      8 +-
 .../llap/vector_outer_reference_windowed.q.out  |   1435 +
 .../llap/vector_partition_diff_num_cols.q.out   |     20 +
 .../llap/vector_partitioned_date_time.q.out     |     40 +-
 .../llap/vector_ptf_part_simple.q.out           |   3898 +-
 .../llap/vector_reduce_groupby_decimal.q.out    |      4 +
 .../llap/vector_string_concat.q.out             |      4 +
 .../llap/vector_tablesample_rows.q.out          |     93 +-
 .../clientpositive/llap/vector_udf1.q.out       |   1137 +-
 .../llap/vector_when_case_null.q.out            |     12 +-
 .../clientpositive/llap/vector_windowing.q.out  |   9768 +
 .../llap/vector_windowing_expressions.q.out     |   2022 +
 .../llap/vector_windowing_gby.q.out             |    314 +
 .../llap/vector_windowing_gby2.q.out            |   1158 +
 .../vector_windowing_multipartitioning.q.out    |  11587 +
 .../llap/vector_windowing_navfn.q.out           |    218 +-
 .../llap/vector_windowing_order_null.q.out      |   1232 +
 .../vector_windowing_range_multiorder.q.out     |  12584 ++
 .../llap/vector_windowing_rank.q.out            |   1860 +
 .../llap/vector_windowing_streaming.q.out       |   1036 +
 .../llap/vector_windowing_windowspec.q.out      |   2374 +
 .../llap/vector_windowing_windowspec4.q.out     |    223 +
 .../clientpositive/llap/vectorization_0.q.out   |    281 +-
 .../clientpositive/llap/vectorization_1.q.out   |    175 +
 .../clientpositive/llap/vectorization_10.q.out  |    118 +
 .../clientpositive/llap/vectorization_11.q.out  |    100 +
 .../clientpositive/llap/vectorization_12.q.out  |    240 +
 .../clientpositive/llap/vectorization_13.q.out  |    100 +-
 .../clientpositive/llap/vectorization_14.q.out  |     86 +-
 .../clientpositive/llap/vectorization_15.q.out  |    155 +-
 .../clientpositive/llap/vectorization_16.q.out  |     68 +-
 .../clientpositive/llap/vectorization_17.q.out  |     41 +-
 .../clientpositive/llap/vectorization_2.q.out   |    179 +
 .../clientpositive/llap/vectorization_3.q.out   |    184 +
 .../clientpositive/llap/vectorization_4.q.out   |    178 +
 .../clientpositive/llap/vectorization_5.q.out   |    173 +
 .../clientpositive/llap/vectorization_6.q.out   |    112 +
 .../clientpositive/llap/vectorization_7.q.out   |     18 +-
 .../clientpositive/llap/vectorization_8.q.out   |     34 +-
 .../clientpositive/llap/vectorization_9.q.out   |     68 +-
 .../llap/vectorization_limit.q.out              |     83 +-
 .../llap/vectorization_pushdown.q.out           |     10 +-
 .../llap/vectorization_short_regress.q.out      |    376 +-
 .../llap/vectorized_bucketmapjoin1.q.out        |     21 +-
 .../clientpositive/llap/vectorized_case.q.out   |     20 +-
 .../clientpositive/llap/vectorized_casts.q.out  |     89 +-
 .../llap/vectorized_date_funcs.q.out            |      4 +
 .../llap/vectorized_distinct_gby.q.out          |    150 +-
 .../vectorized_dynamic_partition_pruning.q.out  |    340 +-
 .../vectorized_dynamic_semijoin_reduction.q.out |     60 +-
 ...vectorized_dynamic_semijoin_reduction2.q.out |     30 +-
 .../llap/vectorized_mapjoin.q.out               |     36 +-
 .../llap/vectorized_mapjoin2.q.out              |      8 +-
 .../llap/vectorized_multi_output_select.q.out   |    201 +
 .../llap/vectorized_nested_mapjoin.q.out        |      6 +-
 .../llap/vectorized_parquet.q.out               |     28 +-
 .../llap/vectorized_parquet_types.q.out         |     36 +-
 .../clientpositive/llap/vectorized_ptf.q.out    |    298 +-
 .../llap/vectorized_shufflejoin.q.out           |     26 +-
 .../llap/vectorized_timestamp.q.out             |    630 +-
 .../llap/vectorized_timestamp_funcs.q.out       |     70 +-
 .../results/clientpositive/llap/windowing.q.out |      2 +-
 .../clientpositive/llap/windowing_gby.q.out     |      4 +-
 .../llap/windowing_windowspec.q.out             |    955 +
 .../results/clientpositive/localtimezone.q.out  |    558 +
 .../clientpositive/louter_join_ppr.q.out        |     84 +-
 .../test/results/clientpositive/mapjoin47.q.out |   1790 +
 .../test/results/clientpositive/masking_4.q.out |     16 +-
 .../materialized_view_create.q.out              |     40 +
 .../materialized_view_create_rewrite.q.out      |      8 +
 ql/src/test/results/clientpositive/merge1.q.out |      6 +-
 ql/src/test/results/clientpositive/merge2.q.out |      6 +-
 .../clientpositive/metadata_only_queries.q.out  |    112 +-
 ql/src/test/results/clientpositive/mm_all.q.out |    296 -
 .../results/clientpositive/mm_loaddata.q.out    |    296 +
 .../clientpositive/multi_insert_gby2.q.out      |     40 +-
 .../clientpositive/multi_insert_gby3.q.out      |    189 +-
 .../clientpositive/nested_column_pruning.q.out  |     42 +-
 .../results/clientpositive/notable_alias1.q.out |     24 +-
 .../results/clientpositive/notable_alias2.q.out |     24 +-
 .../test/results/clientpositive/nullgroup.q.out |      8 +-
 .../results/clientpositive/nullgroup2.q.out     |     76 +-
 .../results/clientpositive/nullgroup3.q.out     |      8 +-
 .../results/clientpositive/nullgroup4.q.out     |     26 +-
 .../nullgroup4_multi_distinct.q.out             |     12 +-
 .../results/clientpositive/orc_create.q.out     |      2 +-
 .../clientpositive/orc_empty_table.q.out        |     68 +
 .../orc_merge_incompat_schema.q.out             |    161 +
 .../orc_merge_incompat_writer_version.q.out     |    108 +
 .../results/clientpositive/outer_join_ppr.q.out |     40 +-
 .../outer_reference_windowed.q.out              |     10 +-
 .../clientpositive/parallel_colstats.q.out      |     24 +-
 .../clientpositive/parquet_analyze.q.out        |      4 +-
 .../parquet_int96_timestamp.q.out               |    191 +-
 .../parquet_mixed_partition_formats.q.out       |      6 +-
 .../clientpositive/parquet_no_row_serde.q.out   |    190 +
 .../parquet_predicate_pushdown_2.q.out          |     38 +
 .../parquet_timestamp_conversion.q.out          |     68 -
 .../clientpositive/partial_column_stats.q.out   |     20 +-
 .../clientpositive/partition_boolexpr.q.out     |      4 +-
 .../partition_coltype_literals.q.out            |     44 +-
 ql/src/test/results/clientpositive/pcs.q.out    |      2 +-
 .../results/clientpositive/perf/query1.q.out    |     40 +-
 .../results/clientpositive/perf/query10.q.out   |     52 +-
 .../results/clientpositive/perf/query11.q.out   |    351 +
 .../results/clientpositive/perf/query14.q.out   |   1876 +-
 .../results/clientpositive/perf/query16.q.out   |    265 +-
 .../results/clientpositive/perf/query17.q.out   |    236 +-
 .../results/clientpositive/perf/query2.q.out    |     22 +-
 .../results/clientpositive/perf/query23.q.out   |    180 +-
 .../results/clientpositive/perf/query24.q.out   |    162 +-
 .../results/clientpositive/perf/query25.q.out   |    232 +-
 .../results/clientpositive/perf/query28.q.out   |    184 +-
 .../results/clientpositive/perf/query29.q.out   |    232 +-
 .../results/clientpositive/perf/query30.q.out   |    102 +-
 .../results/clientpositive/perf/query31.q.out   |     90 +-
 .../results/clientpositive/perf/query32.q.out   |     18 +-
 .../results/clientpositive/perf/query33.q.out   |    178 +-
 .../results/clientpositive/perf/query35.q.out   |     32 +-
 .../results/clientpositive/perf/query38.q.out   |    270 +-
 .../results/clientpositive/perf/query39.q.out   |     30 +-
 .../results/clientpositive/perf/query4.q.out    |   1118 +-
 .../results/clientpositive/perf/query44.q.out   |     29 +-
 .../results/clientpositive/perf/query45.q.out   |      6 +-
 .../results/clientpositive/perf/query46.q.out   |      6 +-
 .../results/clientpositive/perf/query47.q.out   |    222 +-
 .../results/clientpositive/perf/query49.q.out   |     22 +-
 .../results/clientpositive/perf/query5.q.out    |    122 +-
 .../results/clientpositive/perf/query50.q.out   |     72 +-
 .../results/clientpositive/perf/query51.q.out   |      6 +-
 .../results/clientpositive/perf/query54.q.out   |    458 +-
 .../results/clientpositive/perf/query56.q.out   |    178 +-
 .../results/clientpositive/perf/query57.q.out   |    222 +-
 .../results/clientpositive/perf/query58.q.out   |     96 +-
 .../results/clientpositive/perf/query59.q.out   |     20 +-
 .../results/clientpositive/perf/query6.q.out    |    306 +-
 .../results/clientpositive/perf/query60.q.out   |    178 +-
 .../results/clientpositive/perf/query61.q.out   |     30 +-
 .../results/clientpositive/perf/query64.q.out   |    734 +-
 .../results/clientpositive/perf/query65.q.out   |      6 +-
 .../results/clientpositive/perf/query66.q.out   |     40 +-
 .../results/clientpositive/perf/query68.q.out   |      6 +-
 .../results/clientpositive/perf/query69.q.out   |     32 +-
 .../results/clientpositive/perf/query70.q.out   |     22 +-
 .../results/clientpositive/perf/query71.q.out   |     80 +-
 .../results/clientpositive/perf/query72.q.out   |    270 +-
 .../results/clientpositive/perf/query74.q.out   |    544 +-
 .../results/clientpositive/perf/query75.q.out   |    110 +-
 .../results/clientpositive/perf/query76.q.out   |     84 +-
 .../results/clientpositive/perf/query77.q.out   |    188 +-
 .../results/clientpositive/perf/query78.q.out   |    104 +-
 .../results/clientpositive/perf/query8.q.out    |     10 +-
 .../results/clientpositive/perf/query80.q.out   |    172 +-
 .../results/clientpositive/perf/query81.q.out   |    104 +-
 .../results/clientpositive/perf/query83.q.out   |    311 +-
 .../results/clientpositive/perf/query85.q.out   |    148 +-
 .../results/clientpositive/perf/query87.q.out   |    248 +-
 .../results/clientpositive/perf/query88.q.out   |    138 +-
 .../results/clientpositive/perf/query9.q.out    |     90 +-
 .../results/clientpositive/perf/query90.q.out   |     30 +-
 .../results/clientpositive/perf/query92.q.out   |     18 +-
 .../results/clientpositive/perf/query94.q.out   |    265 +-
 .../results/clientpositive/perf/query95.q.out   |     80 +-
 .../results/clientpositive/perf/query97.q.out   |     16 +-
 .../test/results/clientpositive/plan_json.q.out |      2 +-
 ql/src/test/results/clientpositive/ppd2.q.out   |     52 +-
 .../test/results/clientpositive/ppd_gby.q.out   |     32 +-
 .../test/results/clientpositive/ppd_gby2.q.out  |     44 +-
 .../results/clientpositive/ppd_gby_join.q.out   |     64 +-
 .../test/results/clientpositive/ppd_join.q.out  |     48 +-
 .../test/results/clientpositive/ppd_join4.q.out |     23 +-
 .../clientpositive/ppd_outer_join1.q.out        |     40 +-
 .../clientpositive/ppd_outer_join2.q.out        |     40 +-
 .../clientpositive/ppd_outer_join3.q.out        |     40 +-
 .../clientpositive/ppd_outer_join4.q.out        |     60 +-
 .../test/results/clientpositive/ppd_union.q.out |     82 +-
 ql/src/test/results/clientpositive/ppd_vc.q.out |     18 +-
 .../results/clientpositive/ppd_windowing1.q.out |     66 +-
 .../results/clientpositive/ptfgroupbyjoin.q.out |      2 +-
 .../results/clientpositive/quotedid_basic.q.out |      4 +-
 .../clientpositive/rcfile_null_value.q.out      |      8 +-
 .../reduceSinkDeDuplication_pRS_key_empty.q.out |    Bin 10132 -> 10596 bytes
 .../reduce_deduplicate_extended2.q.out          |     89 +-
 .../clientpositive/remove_exprs_stats.q.out     |     12 +-
 .../rename_external_partition_location.q.out    |     24 +-
 .../rename_table_update_column_stats.q.out      |     96 +-
 .../clientpositive/router_join_ppr.q.out        |     84 +-
 .../test/results/clientpositive/row__id.q.out   |     18 +-
 .../test/results/clientpositive/semijoin2.q.out |      4 +-
 .../test/results/clientpositive/semijoin4.q.out |      4 +-
 .../test/results/clientpositive/semijoin5.q.out |      8 +-
 .../set_processor_namespaces.q.out              |      2 +
 .../results/clientpositive/setop_subq.q.out     |     36 +-
 .../show_create_table_delimited.q.out           |      2 +-
 .../clientpositive/skewjoin_mapjoin1.q.out      |     16 +-
 .../clientpositive/skewjoin_mapjoin10.q.out     |     16 +-
 .../clientpositive/skewjoin_mapjoin5.q.out      |     24 +-
 .../results/clientpositive/skewjoinopt1.q.out   |      8 +-
 .../results/clientpositive/skewjoinopt2.q.out   |      8 +-
 .../results/clientpositive/skewjoinopt9.q.out   |     24 +-
 .../results/clientpositive/smb_mapjoin_12.q.out |     75 +-
 .../results/clientpositive/smb_mapjoin_20.q.out |   1060 +-
 .../results/clientpositive/smb_mapjoin_47.q.out |   1542 +
 .../spark/annotate_stats_join.q.out             |     44 +-
 .../clientpositive/spark/auto_join0.q.out       |     43 +-
 .../clientpositive/spark/auto_join12.q.out      |     22 +-
 .../clientpositive/spark/auto_join15.q.out      |     35 +-
 .../clientpositive/spark/auto_join16.q.out      |     16 +-
 .../clientpositive/spark/auto_join20.q.out      |     62 +-
 .../clientpositive/spark/auto_join26.q.out      |      4 +-
 .../clientpositive/spark/auto_join27.q.out      |      2 +-
 .../clientpositive/spark/auto_join30.q.out      |    793 +-
 .../clientpositive/spark/auto_join31.q.out      |    116 +-
 .../clientpositive/spark/auto_join4.q.out       |      6 +-
 .../clientpositive/spark/auto_join5.q.out       |      6 +-
 .../clientpositive/spark/auto_join8.q.out       |      6 +-
 .../spark/auto_join_reordering_values.q.out     |     10 +-
 .../spark/auto_join_without_localtask.q.out     |     28 +-
 .../spark/auto_smb_mapjoin_14.q.out             |     24 +-
 .../spark/auto_sortmerge_join_1.q.out           |      6 +-
 .../spark/auto_sortmerge_join_12.q.out          |     10 +-
 .../spark/auto_sortmerge_join_3.q.out           |      6 +-
 .../spark/auto_sortmerge_join_4.q.out           |      6 +-
 .../spark/auto_sortmerge_join_5.q.out           |      4 +-
 .../spark/auto_sortmerge_join_7.q.out           |      8 +-
 .../spark/auto_sortmerge_join_8.q.out           |      8 +-
 .../spark/auto_sortmerge_join_9.q.out           |     66 +-
 .../spark/avro_decimal_native.q.out             |      8 +-
 .../spark/bucket_map_join_spark1.q.out          |      8 +-
 .../spark/bucket_map_join_spark2.q.out          |      8 +-
 .../spark/bucket_map_join_spark3.q.out          |      8 +-
 .../spark/bucket_map_join_spark4.q.out          |     12 +-
 .../spark/bucket_map_join_tez2.q.out            |     28 +-
 .../clientpositive/spark/bucketmapjoin1.q.out   |      8 +-
 .../clientpositive/spark/bucketmapjoin2.q.out   |     14 +-
 .../clientpositive/spark/bucketmapjoin3.q.out   |      8 +-
 .../clientpositive/spark/bucketmapjoin4.q.out   |      8 +-
 .../spark/constprog_partitioner.q.out           |     10 +-
 .../spark/constprog_semijoin.q.out              |     20 +-
 .../results/clientpositive/spark/count.q.out    |    124 +-
 .../spark/dynamic_rdd_cache.q.out               |     75 +-
 .../spark/explaindenpendencydiffengs.q.out      |     52 +
 .../spark/filter_join_breaktask.q.out           |      6 +-
 .../results/clientpositive/spark/groupby3.q.out |      2 +-
 .../clientpositive/spark/groupby4_map.q.out     |      4 +-
 .../spark/groupby4_map_skew.q.out               |      4 +-
 .../clientpositive/spark/groupby_cube1.q.out    |     20 +-
 .../clientpositive/spark/groupby_map_ppr.q.out  |      4 +-
 .../spark/groupby_map_ppr_multi_distinct.q.out  |      4 +-
 .../clientpositive/spark/groupby_position.q.out |     62 +-
 .../clientpositive/spark/groupby_ppr.q.out      |      4 +-
 .../spark/groupby_ppr_multi_distinct.q.out      |      4 +-
 .../spark/groupby_resolution.q.out              |      2 +-
 .../clientpositive/spark/groupby_rollup1.q.out  |     12 +-
 .../spark/groupby_sort_1_23.q.out               |    240 +-
 .../spark/groupby_sort_skew_1_23.q.out          |    240 +-
 .../spark/identity_project_remove_skip.q.out    |     70 +-
 .../clientpositive/spark/index_bitmap3.q.out    |      4 +-
 .../spark/index_bitmap_auto.q.out               |      4 +-
 .../spark/infer_bucket_sort_map_operators.q.out |      8 +-
 .../infer_bucket_sort_reducers_power_two.q.out  |      2 +-
 .../results/clientpositive/spark/join12.q.out   |     30 +-
 .../results/clientpositive/spark/join16.q.out   |     22 +-
 .../results/clientpositive/spark/join17.q.out   |      4 +-
 .../results/clientpositive/spark/join26.q.out   |      6 +-
 .../results/clientpositive/spark/join29.q.out   |     48 +-
 .../results/clientpositive/spark/join30.q.out   |      4 +-
 .../results/clientpositive/spark/join31.q.out   |      2 +-
 .../results/clientpositive/spark/join32.q.out   |     84 +-
 .../clientpositive/spark/join32_lessSize.q.out  |    104 +-
 .../results/clientpositive/spark/join33.q.out   |     84 +-
 .../results/clientpositive/spark/join34.q.out   |      6 +-
 .../results/clientpositive/spark/join35.q.out   |     66 +-
 .../results/clientpositive/spark/join38.q.out   |     24 +-
 .../results/clientpositive/spark/join4.q.out    |      8 +-
 .../results/clientpositive/spark/join5.q.out    |      8 +-
 .../results/clientpositive/spark/join8.q.out    |      8 +-
 .../results/clientpositive/spark/join9.q.out    |      4 +-
 .../clientpositive/spark/join_alt_syntax.q.out  |    220 +-
 .../spark/join_cond_pushdown_2.q.out            |     92 +-
 .../spark/join_cond_pushdown_4.q.out            |     92 +-
 .../spark/join_filters_overlap.q.out            |     38 +-
 .../clientpositive/spark/limit_pushdown.q.out   |     72 +-
 .../clientpositive/spark/louter_join_ppr.q.out  |    108 +-
 .../clientpositive/spark/mapjoin_mapjoin.q.out  |     12 +-
 .../results/clientpositive/spark/merge1.q.out   |      6 +-
 .../results/clientpositive/spark/merge2.q.out   |      6 +-
 .../spark/metadata_only_queries.q.out           |    112 +-
 .../spark/multi_insert_gby2.q.out               |     20 +-
 .../spark/multi_insert_gby3.q.out               |    126 +-
 .../clientpositive/spark/nullgroup.q.out        |      8 +-
 .../clientpositive/spark/nullgroup2.q.out       |     76 +-
 .../clientpositive/spark/nullgroup4.q.out       |     26 +-
 .../spark/nullgroup4_multi_distinct.q.out       |     12 +-
 .../spark/optimize_nullscan.q.out               |     58 +-
 .../clientpositive/spark/outer_join_ppr.q.out   |     52 +-
 .../spark/parquet_int96_timestamp.q.out         |    718 -
 .../test/results/clientpositive/spark/pcr.q.out |     20 +-
 .../clientpositive/spark/ppd_gby_join.q.out     |     64 +-
 .../results/clientpositive/spark/ppd_join.q.out |     48 +-
 .../clientpositive/spark/ppd_join_filter.q.out  |     16 +-
 .../clientpositive/spark/ppd_outer_join1.q.out  |     40 +-
 .../clientpositive/spark/ppd_outer_join2.q.out  |     40 +-
 .../clientpositive/spark/ppd_outer_join3.q.out  |     40 +-
 .../clientpositive/spark/ppd_outer_join4.q.out  |     60 +-
 .../test/results/clientpositive/spark/ptf.q.out |    114 +-
 .../clientpositive/spark/ptf_streaming.q.out    |     62 +-
 .../clientpositive/spark/router_join_ppr.q.out  |    108 +-
 .../clientpositive/spark/skewjoinopt1.q.out     |      8 +-
 .../clientpositive/spark/skewjoinopt15.q.out    |      8 +-
 .../clientpositive/spark/skewjoinopt2.q.out     |      8 +-
 .../clientpositive/spark/skewjoinopt9.q.out     |     24 +-
 .../clientpositive/spark/smb_mapjoin_14.q.out   |     54 +-
 .../clientpositive/spark/smb_mapjoin_15.q.out   |     20 +-
 .../clientpositive/spark/smb_mapjoin_20.q.out   |   1063 +-
 .../spark/spark_constprog_dpp.q.out             |    138 +
 .../spark/spark_dynamic_partition_pruning.q.out |   3343 +-
 .../spark_dynamic_partition_pruning_2.q.out     |    213 +-
 .../spark_dynamic_partition_pruning_3.q.out     |   1791 +
 ...dynamic_partition_pruning_mapjoin_only.q.out |    492 +
 .../spark/spark_explain_groupbyshuffle.q.out    |    116 +
 .../spark/spark_explainuser_1.q.out             |   1106 +-
 .../spark_multi_insert_parallel_orderby.q.out   |   2013 +
 .../spark/spark_union_merge.q.out               |    564 +
 .../spark/spark_use_op_stats.q.out              |     46 +-
 .../spark/spark_use_ts_stats_for_mapjoin.q.out  |     10 +-
 .../results/clientpositive/spark/stats1.q.out   |      4 +-
 .../clientpositive/spark/stats_only_null.q.out  |      6 +-
 .../clientpositive/spark/subquery_exists.q.out  |     10 +-
 .../clientpositive/spark/subquery_in.q.out      |    660 +-
 .../spark/table_access_keys_stats.q.out         |     36 +-
 .../clientpositive/spark/tez_join_tests.q.out   |     26 +-
 .../spark/tez_joins_explain.q.out               |     26 +-
 .../clientpositive/spark/transform_ppr1.q.out   |      8 +-
 .../clientpositive/spark/transform_ppr2.q.out   |      4 +-
 .../spark/truncate_column_buckets.q.out         |      8 +-
 .../results/clientpositive/spark/union10.q.out  |      8 +-
 .../results/clientpositive/spark/union11.q.out  |      6 +-
 .../results/clientpositive/spark/union12.q.out  |      8 +-
 .../results/clientpositive/spark/union14.q.out  |      4 +-
 .../results/clientpositive/spark/union15.q.out  |      6 +-
 .../results/clientpositive/spark/union16.q.out  |      2 +-
 .../results/clientpositive/spark/union17.q.out  |      8 +-
 .../results/clientpositive/spark/union18.q.out  |      8 +-
 .../results/clientpositive/spark/union19.q.out  |      8 +-
 .../results/clientpositive/spark/union2.q.out   |      4 +-
 .../results/clientpositive/spark/union20.q.out  |      4 +-
 .../results/clientpositive/spark/union22.q.out  |      6 +-
 .../results/clientpositive/spark/union24.q.out  |     56 +-
 .../results/clientpositive/spark/union25.q.out  |      4 +-
 .../results/clientpositive/spark/union30.q.out  |     46 +-
 .../results/clientpositive/spark/union31.q.out  |     18 +-
 .../results/clientpositive/spark/union4.q.out   |      6 +-
 .../results/clientpositive/spark/union5.q.out   |      4 +-
 .../results/clientpositive/spark/union6.q.out   |      4 +-
 .../results/clientpositive/spark/union7.q.out   |      4 +-
 .../results/clientpositive/spark/union9.q.out   |      6 +-
 .../clientpositive/spark/union_ppr.q.out        |      8 +-
 .../clientpositive/spark/union_remove_1.q.out   |      6 +-
 .../clientpositive/spark/union_remove_10.q.out  |      8 +-
 .../clientpositive/spark/union_remove_13.q.out  |      8 +-
 .../clientpositive/spark/union_remove_15.q.out  |      8 +-
 .../clientpositive/spark/union_remove_16.q.out  |      8 +-
 .../clientpositive/spark/union_remove_18.q.out  |      6 +-
 .../clientpositive/spark/union_remove_19.q.out  |     54 +-
 .../clientpositive/spark/union_remove_2.q.out   |      6 +-
 .../clientpositive/spark/union_remove_20.q.out  |      6 +-
 .../clientpositive/spark/union_remove_22.q.out  |      6 +-
 .../clientpositive/spark/union_remove_23.q.out  |     10 +-
 .../clientpositive/spark/union_remove_24.q.out  |      6 +-
 .../clientpositive/spark/union_remove_25.q.out  |      6 +-
 .../clientpositive/spark/union_remove_4.q.out   |      6 +-
 .../clientpositive/spark/union_remove_5.q.out   |      6 +-
 .../clientpositive/spark/union_remove_6.q.out   |      6 +-
 .../spark/union_remove_6_subq.q.out             |     16 +-
 .../clientpositive/spark/union_remove_7.q.out   |      8 +-
 .../clientpositive/spark/union_remove_8.q.out   |      8 +-
 .../clientpositive/spark/union_remove_9.q.out   |      8 +-
 .../clientpositive/spark/union_view.q.out       |     24 +-
 .../spark/vector_between_in.q.out               |     52 +-
 .../spark/vector_cast_constant.q.out            |     33 +-
 .../spark/vector_count_distinct.q.out           |     10 +-
 .../spark/vector_decimal_aggregate.q.out        |     46 +-
 .../spark/vector_distinct_2.q.out               |      6 +-
 .../clientpositive/spark/vector_elt.q.out       |     12 +-
 .../clientpositive/spark/vector_groupby_3.q.out |      6 +-
 .../spark/vector_inner_join.q.out               |      2 +
 .../spark/vector_left_outer_join.q.out          |     16 +-
 .../spark/vector_mapjoin_reduce.q.out           |     26 +-
 .../clientpositive/spark/vector_orderby_5.q.out |      6 +-
 .../spark/vector_outer_join1.q.out              |     44 +-
 .../spark/vector_outer_join2.q.out              |     20 +-
 .../spark/vector_string_concat.q.out            |      6 +-
 .../spark/vector_tablesample_rows.q.out         |    424 +
 .../clientpositive/spark/vectorization_0.q.out  |    365 +-
 .../clientpositive/spark/vectorization_1.q.out  |    173 +
 .../clientpositive/spark/vectorization_10.q.out |    117 +
 .../clientpositive/spark/vectorization_11.q.out |     99 +
 .../clientpositive/spark/vectorization_12.q.out |    238 +
 .../clientpositive/spark/vectorization_13.q.out |    142 +-
 .../clientpositive/spark/vectorization_14.q.out |    105 +-
 .../clientpositive/spark/vectorization_15.q.out |    174 +-
 .../clientpositive/spark/vectorization_16.q.out |     83 +-
 .../clientpositive/spark/vectorization_17.q.out |     53 +-
 .../clientpositive/spark/vectorization_2.q.out  |    177 +
 .../clientpositive/spark/vectorization_3.q.out  |    182 +
 .../clientpositive/spark/vectorization_4.q.out  |    176 +
 .../clientpositive/spark/vectorization_5.q.out  |    171 +
 .../clientpositive/spark/vectorization_6.q.out  |    111 +
 .../clientpositive/spark/vectorization_7.q.out  |     18 +-
 .../clientpositive/spark/vectorization_8.q.out  |     18 +-
 .../clientpositive/spark/vectorization_9.q.out  |     83 +-
 .../spark/vectorization_div0.q.out              |     36 +-
 .../spark/vectorization_pushdown.q.out          |     15 +-
 .../spark/vectorization_short_regress.q.out     |    490 +-
 .../clientpositive/spark/vectorized_case.q.out  |     44 +-
 .../spark/vectorized_mapjoin.q.out              |     47 +-
 .../spark/vectorized_math_funcs.q.out           |      8 +-
 .../spark/vectorized_nested_mapjoin.q.out       |     24 +-
 .../clientpositive/spark/vectorized_ptf.q.out   |    331 +-
 .../spark/vectorized_shufflejoin.q.out          |     43 +-
 .../spark/vectorized_string_funcs.q.out         |      8 +-
 .../spark/vectorized_timestamp_funcs.q.out      |     68 +-
 .../clientpositive/spark/windowing.q.out        |      2 +-
 ql/src/test/results/clientpositive/stats1.q.out |      4 +-
 .../clientpositive/stats_only_null.q.out        |      6 +-
 ql/src/test/results/clientpositive/subq2.q.out  |     24 +-
 .../clientpositive/subquery_exists.q.out        |     10 +-
 .../clientpositive/subquery_exists_having.q.out |     28 +-
 .../clientpositive/subquery_in_having.q.out     |   1439 +-
 .../clientpositive/subquery_notexists.q.out     |     32 +-
 .../subquery_notexists_having.q.out             |     32 +-
 .../clientpositive/subquery_notin_having.q.out  |     26 +-
 .../subquery_unqualcolumnrefs.q.out             |     86 +-
 .../symlink_text_input_format.q.out             |      4 +-
 .../temp_table_display_colstats_tbllvl.q.out    |    116 +-
 .../clientpositive/tez/explainanalyze_1.q.out   |     10 +-
 .../clientpositive/tez/explainanalyze_2.q.out   |    370 +-
 .../clientpositive/tez/explainanalyze_3.q.out   |      4 +-
 .../clientpositive/tez/explainanalyze_4.q.out   |     12 +-
 .../clientpositive/tez/explainanalyze_5.q.out   |     20 +-
 .../clientpositive/tez/explainuser_3.q.out      |      6 +-
 .../tez/hybridgrace_hashjoin_1.q.out            |     12 +-
 .../tez/hybridgrace_hashjoin_2.q.out            |     24 +-
 .../tez/vectorization_limit.q.out               |     82 +-
 .../results/clientpositive/timestamptz.q.out    |     48 +-
 .../results/clientpositive/timestamptz_1.q.out  |     26 +-
 .../results/clientpositive/timestamptz_2.q.out  |     34 +-
 .../truncate_column_buckets.q.out               |      8 +-
 .../results/clientpositive/tunable_ndv.q.out    |     60 +-
 .../clientpositive/udf_stddev_samp.q.out        |     66 +-
 .../results/clientpositive/udf_var_samp.q.out   |     66 +-
 .../results/clientpositive/udtf_explode.q.out   |      4 +-
 .../test/results/clientpositive/union10.q.out   |      8 +-
 .../test/results/clientpositive/union11.q.out   |      6 +-
 .../test/results/clientpositive/union12.q.out   |      8 +-
 .../test/results/clientpositive/union14.q.out   |      4 +-
 .../test/results/clientpositive/union15.q.out   |      6 +-
 .../test/results/clientpositive/union16.q.out   |     50 +-
 .../test/results/clientpositive/union17.q.out   |      8 +-
 .../test/results/clientpositive/union18.q.out   |      8 +-
 .../test/results/clientpositive/union19.q.out   |      8 +-
 ql/src/test/results/clientpositive/union2.q.out |      4 +-
 .../test/results/clientpositive/union20.q.out   |      4 +-
 .../test/results/clientpositive/union24.q.out   |     56 +-
 .../test/results/clientpositive/union25.q.out   |      4 +-
 .../test/results/clientpositive/union30.q.out   |     52 +-
 .../test/results/clientpositive/union31.q.out   |     18 +-
 .../test/results/clientpositive/union34.q.out   |      4 +-
 ql/src/test/results/clientpositive/union4.q.out |      6 +-
 ql/src/test/results/clientpositive/union5.q.out |      4 +-
 ql/src/test/results/clientpositive/union6.q.out |      4 +-
 ql/src/test/results/clientpositive/union7.q.out |      4 +-
 ql/src/test/results/clientpositive/union9.q.out |      6 +-
 .../clientpositive/union_pos_alias.q.out        |     10 +-
 .../results/clientpositive/union_remove_1.q.out |     12 +-
 .../clientpositive/union_remove_10.q.out        |      6 +-
 .../clientpositive/union_remove_13.q.out        |      6 +-
 .../clientpositive/union_remove_15.q.out        |     12 +-
 .../clientpositive/union_remove_16.q.out        |     12 +-
 .../clientpositive/union_remove_18.q.out        |     12 +-
 .../clientpositive/union_remove_19.q.out        |    108 +-
 .../results/clientpositive/union_remove_2.q.out |      6 +-
 .../clientpositive/union_remove_20.q.out        |     12 +-
 .../clientpositive/union_remove_22.q.out        |     12 +-
 .../clientpositive/union_remove_23.q.out        |     10 +-
 .../clientpositive/union_remove_24.q.out        |     12 +-
 .../clientpositive/union_remove_25.q.out        |     12 +-
 .../results/clientpositive/union_remove_4.q.out |     12 +-
 .../results/clientpositive/union_remove_5.q.out |      6 +-
 .../results/clientpositive/union_remove_6.q.out |     12 +-
 .../clientpositive/union_remove_6_subq.q.out    |     30 +-
 .../results/clientpositive/union_remove_7.q.out |     12 +-
 .../results/clientpositive/union_remove_8.q.out |      6 +-
 .../results/clientpositive/union_remove_9.q.out |      6 +-
 .../results/clientpositive/union_view.q.out     |     24 +-
 .../clientpositive/unionall_unbalancedppd.q.out |     35 +-
 .../clientpositive/vector_aggregate_9.q.out     |    239 +-
 .../vector_binary_join_groupby.q.out            |      8 +
 .../clientpositive/vector_cast_constant.q.out   |     16 +-
 .../results/clientpositive/vector_char_2.q.out  |      8 +
 .../clientpositive/vector_coalesce_2.q.out      |      8 +
 .../clientpositive/vector_complex_all.q.out     |      6 +-
 .../clientpositive/vector_complex_join.q.out    |     28 +-
 .../results/clientpositive/vector_count.q.out   |     78 +-
 .../results/clientpositive/vector_date_1.q.out  |      2 +-
 .../vector_decimal_aggregate.q.out              |     20 +-
 .../vector_decimal_precision.q.out              |     16 +-
 .../clientpositive/vector_decimal_round.q.out   |     44 +-
 .../clientpositive/vector_distinct_2.q.out      |      4 +
 .../clientpositive/vector_empty_where.q.out     |     32 +
 .../clientpositive/vector_groupby4.q.out        |      4 +
 .../clientpositive/vector_groupby6.q.out        |      4 +
 .../clientpositive/vector_groupby_3.q.out       |      4 +
 .../clientpositive/vector_groupby_mapjoin.q.out |      8 +
 .../clientpositive/vector_groupby_reduce.q.out  |   2522 +-
 .../clientpositive/vector_grouping_sets.q.out   |      8 +
 .../clientpositive/vector_include_no_sel.q.out  |      4 +
 .../vector_interval_mapjoin.q.out               |      6 +-
 .../clientpositive/vector_mapjoin_reduce.q.out  |     20 +-
 .../vector_mr_diff_schema_alias.q.out           |     26 +-
 .../clientpositive/vector_null_projection.q.out |      8 +-
 .../results/clientpositive/vector_nvl.q.out     |      2 +-
 .../clientpositive/vector_orderby_5.q.out       |      4 +
 .../clientpositive/vector_outer_join1.q.out     |      4 +
 .../clientpositive/vector_outer_join2.q.out     |      4 +
 .../clientpositive/vector_outer_join3.q.out     |      6 +-
 .../clientpositive/vector_outer_join4.q.out     |      2 +-
 .../vector_outer_reference_windowed.q.out       |   1221 +
 .../clientpositive/vector_ptf_part_simple.q.out |   2895 +
 .../vector_reduce_groupby_decimal.q.out         |      4 +
 .../clientpositive/vector_string_concat.q.out   |      4 +
 .../vector_tablesample_rows.q.out               |    100 +-
 .../clientpositive/vector_when_case_null.q.out  |     12 +-
 .../clientpositive/vector_windowing.q.out       |   9149 +
 .../vector_windowing_expressions.q.out          |   1822 +
 .../clientpositive/vector_windowing_gby.q.out   |    252 +
 .../clientpositive/vector_windowing_gby2.q.out  |   1015 +
 .../vector_windowing_multipartitioning.q.out    |  11288 +
 .../vector_windowing_order_null.q.out           |    989 +
 .../vector_windowing_range_multiorder.q.out     |  12239 +
 .../clientpositive/vector_windowing_rank.q.out  |   1543 +
 .../vector_windowing_streaming.q.out            |    836 +
 .../vector_windowing_windowspec.q.out           |   2060 +
 .../vector_windowing_windowspec4.q.out          |    211 +
 .../clientpositive/vectorization_1.q.out        |    146 +
 .../clientpositive/vectorization_10.q.out       |    114 +
 .../clientpositive/vectorization_11.q.out       |     96 +
 .../clientpositive/vectorization_12.q.out       |   1284 +-
 .../clientpositive/vectorization_13.q.out       |     43 +-
 .../clientpositive/vectorization_14.q.out       |   1301 +-
 .../clientpositive/vectorization_15.q.out       |    155 +-
 .../clientpositive/vectorization_16.q.out       |    364 +-
 .../clientpositive/vectorization_17.q.out       |     27 +-
 .../clientpositive/vectorization_2.q.out        |    150 +
 .../clientpositive/vectorization_3.q.out        |    155 +
 .../clientpositive/vectorization_4.q.out        |    149 +
 .../clientpositive/vectorization_5.q.out        |    144 +
 .../clientpositive/vectorization_6.q.out        |    108 +
 .../clientpositive/vectorization_7.q.out        |     10 +-
 .../clientpositive/vectorization_8.q.out        |     10 +-
 .../clientpositive/vectorization_9.q.out        |    364 +-
 .../clientpositive/vectorization_limit.q.out    |     28 +-
 .../clientpositive/vectorization_pushdown.q.out |      2 +-
 .../clientpositive/vectorized_case.q.out        |     20 +-
 .../clientpositive/vectorized_casts.q.out       |     85 +-
 .../clientpositive/vectorized_date_funcs.q.out  |      4 +
 .../vectorized_distinct_gby.q.out               |     78 +-
 .../clientpositive/vectorized_mapjoin.q.out     |     16 +-
 .../clientpositive/vectorized_mapjoin2.q.out    |     12 +-
 .../vectorized_parquet_types.q.out              |     16 +-
 .../clientpositive/vectorized_shufflejoin.q.out |     19 +-
 .../clientpositive/vectorized_timestamp.q.out   |    314 +-
 .../vectorized_timestamp_funcs.q.out            |     49 +-
 .../test/results/clientpositive/view_cbo.q.out  |     12 +-
 .../results/clientpositive/windowing_gby2.q.out |     12 +-
 .../clientpositive/windowing_navfn.q.out        |      2 +-
 .../clientpositive/windowing_streaming.q.out    |      6 +-
 serde/if/serde.thrift                           |      6 +-
 .../src/gen/thrift/gen-cpp/serde_constants.cpp  |      6 +-
 serde/src/gen/thrift/gen-cpp/serde_constants.h  |      2 +-
 .../hadoop/hive/serde/serdeConstants.java       |      6 +-
 .../org/apache/hadoop/hive/serde/Types.php      |     10 +-
 .../org_apache_hadoop_hive_serde/constants.py   |      6 +-
 serde/src/gen/thrift/gen-rb/serde_constants.rb  |      6 +-
 .../apache/hadoop/hive/serde2/SerDeUtils.java   |      6 +-
 .../binarysortable/BinarySortableSerDe.java     |     25 +-
 .../serde2/io/TimestampLocalTZWritable.java     |    449 +
 .../hive/serde2/io/TimestampTZWritable.java     |    427 -
 .../hadoop/hive/serde2/lazy/LazyFactory.java    |      6 +-
 .../hive/serde2/lazy/LazySerDeParameters.java   |     19 +-
 .../hive/serde2/lazy/LazyTimestampLocalTZ.java  |    102 +
 .../hive/serde2/lazy/LazyTimestampTZ.java       |     91 -
 .../hadoop/hive/serde2/lazy/LazyUtils.java      |      6 +-
 .../lazy/fast/LazySimpleDeserializeRead.java    |     14 +-
 .../LazyPrimitiveObjectInspectorFactory.java    |      8 +-
 .../LazyTimestampLocalTZObjectInspector.java    |     52 +
 .../LazyTimestampTZObjectInspector.java         |     43 -
 .../serde2/lazybinary/LazyBinaryFactory.java    |      6 +-
 .../hive/serde2/lazybinary/LazyBinarySerDe.java |      8 +-
 .../lazybinary/LazyBinaryTimestampLocalTZ.java  |     43 +
 .../lazybinary/LazyBinaryTimestampTZ.java       |     36 -
 .../hive/serde2/lazybinary/LazyBinaryUtils.java |      6 +-
 .../ObjectInspectorConverters.java              |     14 +-
 .../objectinspector/ObjectInspectorUtils.java   |     22 +-
 .../PrimitiveObjectInspector.java               |      6 +-
 .../JavaTimestampLocalTZObjectInspector.java    |    103 +
 .../JavaTimestampTZObjectInspector.java         |     76 -
 .../PrimitiveObjectInspectorConverter.java      |     23 +-
 .../PrimitiveObjectInspectorFactory.java        |     32 +-
 .../PrimitiveObjectInspectorUtils.java          |    123 +-
 ...SettableTimestampLocalTZObjectInspector.java |     34 +
 .../SettableTimestampTZObjectInspector.java     |     34 -
 .../TimestampLocalTZObjectInspector.java        |     29 +
 .../primitive/TimestampTZObjectInspector.java   |     29 -
 ...ConstantTimestampLocalTZObjectInspector.java |     40 +
 ...tableConstantTimestampTZObjectInspector.java |     36 -
 ...WritableTimestampLocalTZObjectInspector.java |    107 +
 .../WritableTimestampTZObjectInspector.java     |     79 -
 .../serde2/thrift/ThriftJDBCBinarySerDe.java    |      2 +-
 .../apache/hadoop/hive/serde2/thrift/Type.java  |     10 +-
 .../typeinfo/TimestampLocalTZTypeInfo.java      |    104 +
 .../hive/serde2/typeinfo/TypeInfoFactory.java   |     21 +-
 .../hive/serde2/typeinfo/TypeInfoUtils.java     |     28 +-
 .../hive/serde2/io/TestTimestampTZWritable.java |     20 +-
 .../hive/serde2/typeinfo/TestTypeInfoUtils.java |     34 +-
 service-rpc/if/TCLIService.thrift               |      8 +-
 .../thrift/gen-cpp/TCLIService_constants.cpp    |      2 +-
 .../gen/thrift/gen-cpp/TCLIService_types.cpp    |      4 +-
 .../src/gen/thrift/gen-cpp/TCLIService_types.h  |      2 +-
 .../rpc/thrift/TCLIServiceConstants.java        |      4 +-
 .../apache/hive/service/rpc/thrift/TTypeId.java |      4 +-
 service-rpc/src/gen/thrift/gen-php/Types.php    |      6 +-
 .../gen/thrift/gen-py/TCLIService/constants.py  |      2 +-
 .../src/gen/thrift/gen-py/TCLIService/ttypes.py |      6 +-
 .../thrift/gen-rb/t_c_l_i_service_constants.rb  |      2 +-
 .../gen/thrift/gen-rb/t_c_l_i_service_types.rb  |      6 +-
 service/pom.xml                                 |     11 +
 .../org/apache/hive/tmpl/QueryProfileTmpl.jamon |     14 +-
 .../apache/hive/service/cli/ColumnValue.java    |      2 +-
 .../apache/hive/service/cli/TypeDescriptor.java |      2 +-
 .../operation/ExecuteStatementOperation.java    |      8 +-
 .../hive/service/cli/operation/Operation.java   |     16 +-
 .../service/cli/operation/SQLOperation.java     |      5 +-
 .../service/cli/session/HiveSessionHook.java    |      4 +
 .../service/cli/thrift/ThriftCLIService.java    |      6 +-
 .../cli/thrift/ThriftHttpCLIService.java        |     19 +-
 .../service/servlet/QueryProfileServlet.java    |      4 +-
 .../apache/hadoop/hive/shims/Hadoop23Shims.java |     60 +-
 .../hadoop/hive/shims/TestHadoop23Shims.java    |     14 +-
 .../apache/hadoop/hive/shims/HadoopShims.java   |     10 +-
 .../hive/thrift/HadoopThriftAuthBridge.java     |     16 +
 .../apache/hive/spark/client/SparkClient.java   |      9 +-
 .../hive/spark/client/SparkClientFactory.java   |     22 +-
 .../hive/spark/client/SparkClientImpl.java      |     68 +-
 .../hive/spark/client/rpc/RpcConfiguration.java |      6 +-
 .../hive/spark/client/rpc/RpcDispatcher.java    |      7 +-
 .../apache/hive/spark/client/rpc/RpcServer.java |      1 -
 .../apache/hive/spark/client/rpc/TestRpc.java   |     55 +-
 standalone-metastore/pom.xml                    |    232 +
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp  |  74036 ++++++
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.h    |  23462 ++
 .../ThriftHiveMetastore_server.skeleton.cpp     |    844 +
 .../thrift/gen-cpp/hive_metastore_constants.cpp |     65 +
 .../thrift/gen-cpp/hive_metastore_constants.h   |     48 +
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp |  21819 ++
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |   9034 +
 .../hive/metastore/api/AbortTxnRequest.java     |    387 +
 .../hive/metastore/api/AbortTxnsRequest.java    |    438 +
 .../metastore/api/AddDynamicPartitions.java     |    864 +
 .../metastore/api/AddForeignKeyRequest.java     |    443 +
 .../api/AddNotNullConstraintRequest.java        |    443 +
 .../metastore/api/AddPartitionsRequest.java     |    849 +
 .../hive/metastore/api/AddPartitionsResult.java |    447 +
 .../metastore/api/AddPrimaryKeyRequest.java     |    443 +
 .../api/AddUniqueConstraintRequest.java         |    443 +
 .../hadoop/hive/metastore/api/AggrStats.java    |    542 +
 .../metastore/api/AlreadyExistsException.java   |    395 +
 .../metastore/api/BinaryColumnStatsData.java    |    696 +
 .../metastore/api/BooleanColumnStatsData.java   |    696 +
 .../metastore/api/CacheFileMetadataRequest.java |    703 +
 .../metastore/api/CacheFileMetadataResult.java  |    387 +
 .../hive/metastore/api/CheckLockRequest.java    |    589 +
 .../metastore/api/ClearFileMetadataRequest.java |    438 +
 .../metastore/api/ClearFileMetadataResult.java  |    283 +
 .../hive/metastore/api/ClientCapabilities.java  |    441 +
 .../hive/metastore/api/ClientCapability.java    |     45 +
 .../hive/metastore/api/CmRecycleRequest.java    |    488 +
 .../hive/metastore/api/CmRecycleResponse.java   |    283 +
 .../hive/metastore/api/ColumnStatistics.java    |    549 +
 .../metastore/api/ColumnStatisticsData.java     |    675 +
 .../metastore/api/ColumnStatisticsDesc.java     |    798 +
 .../hive/metastore/api/ColumnStatisticsObj.java |    593 +
 .../hive/metastore/api/CommitTxnRequest.java    |    387 +
 .../hive/metastore/api/CompactionRequest.java   |    977 +
 .../hive/metastore/api/CompactionResponse.java  |    583 +
 .../hive/metastore/api/CompactionType.java      |     45 +
 .../api/ConfigValSecurityException.java         |    395 +
 .../api/CurrentNotificationEventId.java         |    387 +
 .../hive/metastore/api/DataOperationType.java   |     57 +
 .../hadoop/hive/metastore/api/Database.java     |   1095 +
 .../apache/hadoop/hive/metastore/api/Date.java  |    387 +
 .../hive/metastore/api/DateColumnStatsData.java |    823 +
 .../hadoop/hive/metastore/api/Decimal.java      |    497 +
 .../metastore/api/DecimalColumnStatsData.java   |    823 +
 .../metastore/api/DoubleColumnStatsData.java    |    799 +
 .../metastore/api/DropConstraintRequest.java    |    591 +
 .../hive/metastore/api/DropPartitionsExpr.java  |    505 +
 .../metastore/api/DropPartitionsRequest.java    |   1112 +
 .../metastore/api/DropPartitionsResult.java     |    447 +
 .../hive/metastore/api/EnvironmentContext.java  |    447 +
 .../hive/metastore/api/EventRequestType.java    |     48 +
 .../hadoop/hive/metastore/api/FieldSchema.java  |    603 +
 .../metastore/api/FileMetadataExprType.java     |     42 +
 .../hive/metastore/api/FireEventRequest.java    |    861 +
 .../metastore/api/FireEventRequestData.java     |    309 +
 .../hive/metastore/api/FireEventResponse.java   |    283 +
 .../hive/metastore/api/ForeignKeysRequest.java  |    707 +
 .../hive/metastore/api/ForeignKeysResponse.java |    443 +
 .../hadoop/hive/metastore/api/Function.java     |   1199 +
 .../hadoop/hive/metastore/api/FunctionType.java |     42 +
 .../metastore/api/GetAllFunctionsResponse.java  |    447 +
 .../api/GetFileMetadataByExprRequest.java       |    773 +
 .../api/GetFileMetadataByExprResult.java        |    553 +
 .../metastore/api/GetFileMetadataRequest.java   |    438 +
 .../metastore/api/GetFileMetadataResult.java    |    540 +
 .../metastore/api/GetOpenTxnsInfoResponse.java  |    542 +
 .../hive/metastore/api/GetOpenTxnsResponse.java |    750 +
 .../api/GetPrincipalsInRoleRequest.java         |    389 +
 .../api/GetPrincipalsInRoleResponse.java        |    443 +
 .../api/GetRoleGrantsForPrincipalRequest.java   |    502 +
 .../api/GetRoleGrantsForPrincipalResponse.java  |    443 +
 .../hive/metastore/api/GetTableRequest.java     |    605 +
 .../hive/metastore/api/GetTableResult.java      |    394 +
 .../hive/metastore/api/GetTablesRequest.java    |    659 +
 .../hive/metastore/api/GetTablesResult.java     |    443 +
 .../api/GrantRevokePrivilegeRequest.java        |    620 +
 .../api/GrantRevokePrivilegeResponse.java       |    390 +
 .../metastore/api/GrantRevokeRoleRequest.java   |   1059 +
 .../metastore/api/GrantRevokeRoleResponse.java  |    390 +
 .../hive/metastore/api/GrantRevokeType.java     |     45 +
 .../hive/metastore/api/HeartbeatRequest.java    |    489 +
 .../metastore/api/HeartbeatTxnRangeRequest.java |    482 +
 .../api/HeartbeatTxnRangeResponse.java          |    588 +
 .../hive/metastore/api/HiveObjectPrivilege.java |    729 +
 .../hive/metastore/api/HiveObjectRef.java       |    872 +
 .../hive/metastore/api/HiveObjectType.java      |     54 +
 .../apache/hadoop/hive/metastore/api/Index.java |   1374 +
 .../api/IndexAlreadyExistsException.java        |    395 +
 .../metastore/api/InsertEventRequestData.java   |    700 +
 .../metastore/api/InvalidInputException.java    |    395 +
 .../metastore/api/InvalidObjectException.java   |    395 +
 .../api/InvalidOperationException.java          |    395 +
 .../api/InvalidPartitionException.java          |    395 +
 .../hive/metastore/api/LockComponent.java       |   1158 +
 .../hadoop/hive/metastore/api/LockLevel.java    |     48 +
 .../hadoop/hive/metastore/api/LockRequest.java  |    861 +
 .../hadoop/hive/metastore/api/LockResponse.java |    500 +
 .../hadoop/hive/metastore/api/LockState.java    |     51 +
 .../hadoop/hive/metastore/api/LockType.java     |     48 +
 .../hive/metastore/api/LongColumnStatsData.java |    799 +
 .../hive/metastore/api/MetaException.java       |    395 +
 .../hive/metastore/api/MetadataPpdResult.java   |    517 +
 .../hive/metastore/api/NoSuchLockException.java |    395 +
 .../metastore/api/NoSuchObjectException.java    |    395 +
 .../hive/metastore/api/NoSuchTxnException.java  |    395 +
 .../api/NotNullConstraintsRequest.java          |    490 +
 .../api/NotNullConstraintsResponse.java         |    443 +
 .../hive/metastore/api/NotificationEvent.java   |   1006 +
 .../metastore/api/NotificationEventRequest.java |    490 +
 .../api/NotificationEventResponse.java          |    443 +
 .../api/NotificationEventsCountRequest.java     |    488 +
 .../api/NotificationEventsCountResponse.java    |    387 +
 .../hive/metastore/api/OpenTxnRequest.java      |    702 +
 .../hive/metastore/api/OpenTxnsResponse.java    |    438 +
 .../apache/hadoop/hive/metastore/api/Order.java |    497 +
 .../hadoop/hive/metastore/api/Partition.java    |   1229 +
 .../hive/metastore/api/PartitionEventType.java  |     42 +
 .../api/PartitionListComposingSpec.java         |    449 +
 .../hive/metastore/api/PartitionSpec.java       |    826 +
 .../api/PartitionSpecWithSharedSD.java          |    558 +
 .../hive/metastore/api/PartitionWithoutSD.java  |   1016 +
 .../metastore/api/PartitionsByExprRequest.java  |    815 +
 .../metastore/api/PartitionsByExprResult.java   |    542 +
 .../metastore/api/PartitionsStatsRequest.java   |    790 +
 .../metastore/api/PartitionsStatsResult.java    |    490 +
 .../hive/metastore/api/PrimaryKeysRequest.java  |    490 +
 .../hive/metastore/api/PrimaryKeysResponse.java |    443 +
 .../metastore/api/PrincipalPrivilegeSet.java    |    906 +
 .../hive/metastore/api/PrincipalType.java       |     48 +
 .../hadoop/hive/metastore/api/PrivilegeBag.java |    449 +
 .../hive/metastore/api/PrivilegeGrantInfo.java  |    815 +
 .../metastore/api/PutFileMetadataRequest.java   |    710 +
 .../metastore/api/PutFileMetadataResult.java    |    283 +
 .../hive/metastore/api/RequestPartsSpec.java    |    438 +
 .../hadoop/hive/metastore/api/ResourceType.java |     48 +
 .../hadoop/hive/metastore/api/ResourceUri.java  |    511 +
 .../apache/hadoop/hive/metastore/api/Role.java  |    601 +
 .../hive/metastore/api/RolePrincipalGrant.java  |   1035 +
 .../hive/metastore/api/SQLForeignKey.java       |   1715 +
 .../metastore/api/SQLNotNullConstraint.java     |   1005 +
 .../hive/metastore/api/SQLPrimaryKey.java       |   1103 +
 .../hive/metastore/api/SQLUniqueConstraint.java |   1103 +
 .../hadoop/hive/metastore/api/Schema.java       |    605 +
 .../hadoop/hive/metastore/api/SerDeInfo.java    |    655 +
 .../api/SetPartitionsStatsRequest.java          |    550 +
 .../hive/metastore/api/ShowCompactRequest.java  |    283 +
 .../hive/metastore/api/ShowCompactResponse.java |    443 +
 .../api/ShowCompactResponseElement.java         |   1641 +
 .../hive/metastore/api/ShowLocksRequest.java    |    710 +
 .../hive/metastore/api/ShowLocksResponse.java   |    449 +
 .../metastore/api/ShowLocksResponseElement.java |   1929 +
 .../hadoop/hive/metastore/api/SkewedInfo.java   |    834 +
 .../hive/metastore/api/StorageDescriptor.java   |   1748 +
 .../metastore/api/StringColumnStatsData.java    |    791 +
 .../apache/hadoop/hive/metastore/api/Table.java |   1948 +
 .../hadoop/hive/metastore/api/TableMeta.java    |    701 +
 .../hive/metastore/api/TableStatsRequest.java   |    640 +
 .../hive/metastore/api/TableStatsResult.java    |    443 +
 .../hive/metastore/api/ThriftHiveMetastore.java | 188205 ++++++++++++++++
 .../hive/metastore/api/TxnAbortedException.java |    395 +
 .../hadoop/hive/metastore/api/TxnInfo.java      |   1220 +
 .../hive/metastore/api/TxnOpenException.java    |    395 +
 .../hadoop/hive/metastore/api/TxnState.java     |     48 +
 .../apache/hadoop/hive/metastore/api/Type.java  |    768 +
 .../metastore/api/UniqueConstraintsRequest.java |    490 +
 .../api/UniqueConstraintsResponse.java          |    443 +
 .../hive/metastore/api/UnknownDBException.java  |    395 +
 .../api/UnknownPartitionException.java          |    395 +
 .../metastore/api/UnknownTableException.java    |    395 +
 .../hive/metastore/api/UnlockRequest.java       |    387 +
 .../hadoop/hive/metastore/api/Version.java      |    499 +
 .../metastore/api/hive_metastoreConstants.java  |     87 +
 .../gen-php/metastore/ThriftHiveMetastore.php   |  47128 ++++
 .../src/gen/thrift/gen-php/metastore/Types.php  |  21262 ++
 .../hive_metastore/ThriftHiveMetastore-remote   |   1305 +
 .../hive_metastore/ThriftHiveMetastore.py       |  38297 ++++
 .../thrift/gen-py/hive_metastore/__init__.py    |      1 +
 .../thrift/gen-py/hive_metastore/constants.py   |     35 +
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |  15163 ++
 .../thrift/gen-rb/hive_metastore_constants.rb   |     57 +
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |   3516 +
 .../gen/thrift/gen-rb/thrift_hive_metastore.rb  |  10672 +
 .../hadoop/hive/metastore/ThreadPool.java       |     63 +
 .../hive/metastore/conf/MetastoreConf.java      |   1514 +
 .../hive/metastore/metrics/JsonReporter.java    |    166 +
 .../hive/metastore/metrics/JvmPauseMonitor.java |    222 +
 .../hadoop/hive/metastore/metrics/Metrics.java  |    243 +
 .../metastore/metrics/MetricsConstants.java     |     46 +
 .../hive/metastore/metrics/PerfLogger.java      |    194 +
 .../hive/metastore/model/MColumnDescriptor.java |     51 +
 .../hive/metastore/model/MConstraint.java       |    188 +
 .../hive/metastore/model/MDBPrivilege.java      |    132 +
 .../hadoop/hive/metastore/model/MDatabase.java  |    128 +
 .../hive/metastore/model/MDelegationToken.java  |     45 +
 .../hive/metastore/model/MFieldSchema.java      |     80 +
 .../hadoop/hive/metastore/model/MFunction.java  |    119 +
 .../hive/metastore/model/MGlobalPrivilege.java  |    120 +
 .../hadoop/hive/metastore/model/MIndex.java     |    200 +
 .../hadoop/hive/metastore/model/MMasterKey.java |     55 +
 .../metastore/model/MMetastoreDBProperties.java |     56 +
 .../hive/metastore/model/MNotificationLog.java  |     98 +
 .../metastore/model/MNotificationNextId.java    |     42 +
 .../hadoop/hive/metastore/model/MOrder.java     |     62 +
 .../hadoop/hive/metastore/model/MPartition.java |    155 +
 .../model/MPartitionColumnPrivilege.java        |    161 +
 .../model/MPartitionColumnStatistics.java       |    272 +
 .../hive/metastore/model/MPartitionEvent.java   |     90 +
 .../metastore/model/MPartitionPrivilege.java    |    139 +
 .../hive/metastore/model/MPrincipalDesc.java    |     59 +
 .../hive/metastore/model/MResourceUri.java      |     49 +
 .../hadoop/hive/metastore/model/MRole.java      |     80 +
 .../hadoop/hive/metastore/model/MRoleMap.java   |    120 +
 .../hadoop/hive/metastore/model/MSerDeInfo.java |     81 +
 .../metastore/model/MStorageDescriptor.java     |    277 +
 .../hive/metastore/model/MStringList.java       |     62 +
 .../hadoop/hive/metastore/model/MTable.java     |    256 +
 .../metastore/model/MTableColumnPrivilege.java  |    160 +
 .../metastore/model/MTableColumnStatistics.java |    263 +
 .../hive/metastore/model/MTablePrivilege.java   |    139 +
 .../hadoop/hive/metastore/model/MType.java      |    105 +
 .../hive/metastore/model/MVersionTable.java     |     57 +
 .../security/DelegationTokenIdentifier.java     |     52 +
 .../security/DelegationTokenSecretManager.java  |    125 +
 .../security/DelegationTokenStore.java          |    118 +
 .../security/HadoopThriftAuthBridge.java        |    684 +
 .../security/HadoopThriftAuthBridge23.java      |    114 +
 .../metastore/security/MemoryTokenStore.java    |    136 +
 .../MetastoreDelegationTokenManager.java        |    172 +
 .../metastore/security/TFilterTransport.java    |     99 +
 .../security/TUGIAssumingTransport.java         |     73 +
 .../security/TUGIContainingTransport.java       |     96 +
 .../TokenStoreDelegationTokenSecretManager.java |    333 +
 .../hadoop/hive/metastore/utils/HdfsUtils.java  |    125 +
 .../hive/metastore/utils/SecurityUtils.java     |     40 +
 .../hive/metastore/utils/StringUtils.java       |    112 +
 .../MetastoreDelegationTokenSupport.java        |     68 +
 .../src/main/resources/package.jdo              |   1087 +
 .../src/main/resources/thrift-replacements.txt  |     69 +
 .../src/main/thrift/hive_metastore.thrift       |   1613 +
 .../hive/metastore/conf/TestMetastoreConf.java  |    414 +
 .../hive/metastore/metrics/TestMetrics.java     |    200 +
 .../hive/metastore/utils/TestHdfsUtils.java     |    193 +
 .../apache/hadoop/hive/common/io/Allocator.java |     17 +
 .../apache/hadoop/hive/common/io/DataCache.java |      6 +
 .../hadoop/hive/common/io/DiskRangeList.java    |     80 +-
 .../hive/common/io/FileMetadataCache.java       |     51 +
 .../io/encoded/MemoryBufferOrBuffers.java       |     24 +
 .../hive/common/type/FastHiveDecimal.java       |     18 +
 .../hive/common/type/FastHiveDecimalImpl.java   |     71 +-
 .../hive/ql/io/sarg/SearchArgumentImpl.java     |      5 +-
 .../hive/serde2/io/HiveDecimalWritable.java     |     58 +
 .../apache/hive/common/util/BloomFilter.java    |     37 +-
 .../apache/hive/common/util/BloomKFilter.java   |    472 +
 .../hive/common/io/TestDiskRangeList.java       |    106 +
 .../hive/common/type/HiveDecimalTestBase.java   |     10 +
 .../hive/common/type/TestHiveDecimal.java       |     73 +
 .../hive/common/util/TestBloomFilter.java       |    136 +-
 .../hive/common/util/TestBloomKFilter.java      |    699 +
 .../ptest2/conf/deployed/master-mr2.properties  |     57 +-
 .../ptest2/conf/server-env.properties.example   |     23 +
 .../ptest/api/server/ExecutionController.java   |      5 +-
 .../hive/ptest/api/server/TestExecutor.java     |     16 +-
 .../org/apache/hive/ptest/execution/PTest.java  |     18 +-
 .../hive/ptest/execution/conf/Context.java      |     84 +
 .../conf/ExecutionContextConfiguration.java     |     20 +-
 .../ptest/execution/conf/TestConfiguration.java |     20 +-
 .../hive/ptest/execution/conf/TestParser.java   |      3 +-
 .../hive/ptest/execution/TestJIRAService.java   |      6 +-
 .../hive/ptest/execution/conf/TestContext.java  |     68 +
 .../execution/conf/TestTestConfiguration.java   |     10 +-
 .../apache/hadoop/hive/tools/GenVectorCode.java |    345 +-
 2788 files changed, 803465 insertions(+), 684284 deletions(-)
----------------------------------------------------------------------

[38/50] [abbrv] hive git commit: HIVE-15212 : merge branch into master - fix some tests (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-15212 : merge branch into master - fix some tests (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a02a61b3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a02a61b3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a02a61b3

Branch: refs/heads/master
Commit: a02a61b3e242732cb57f21dc18358e205113da8a
Parents: a0c546f
Author: sergey <se...@apache.org>
Authored: Thu Sep 28 20:33:30 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Sep 28 20:33:30 2017 -0700

----------------------------------------------------------------------
 ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java | 1 -
 ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java     | 5 +++++
 2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a02a61b3/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
index 1a9b39f..b7d65ec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
@@ -61,7 +61,6 @@ public class ReplCopyTask extends Task<ReplCopyWork> implements Serializable {
     FileSystem dstFs = null;
     Path toPath = null;
     try {
-      // TODO: merge with real CopyTask logic?
       Path fromPath = work.getFromPaths()[0];
       toPath = work.getToPaths()[0];
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a02a61b3/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index d4282ed..5fd4ecc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileChecksum;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hive.common.BlobStorageUtils;
@@ -3895,6 +3896,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
 
   private void cleanUpOneDirectoryForReplace(Path path, FileSystem fs,
       PathFilter pathFilter, HiveConf conf, boolean purge) throws IOException, HiveException {
+    if (conf.getBoolVar(HiveConf.ConfVars.REPLCMENABLED)) {
+      recycleDirToCmPath(path, purge);
+    }
     FileStatus[] statuses = fs.listStatus(path, pathFilter);
     if (statuses == null || statuses.length == 0) return;
     if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
@@ -3904,6 +3908,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
       }
       Utilities.FILE_OP_LOGGER.trace(s);
     }
+
     if (!trashFiles(fs, statuses, conf, purge)) {
       throw new HiveException("Old path " + path + " has not been cleaned up.");
     }

[26/50] [abbrv] hive git commit: HIVE-15212 : merge branch into master - some initial cleanup and fixes (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-15212 : merge branch into master - some initial cleanup and fixes (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/543e0b01
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/543e0b01
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/543e0b01

Branch: refs/heads/master
Commit: 543e0b01a27fa5a51ac3adcfb6c878c403c6be7b
Parents: 9a9f254
Author: sergey <se...@apache.org>
Authored: Fri Sep 15 12:51:35 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Sep 15 12:51:35 2017 -0700

----------------------------------------------------------------------
 .../hadoop/hive/metastore/MetaStoreUtils.java   |  3 +-
 .../hadoop/hive/metastore/TestObjectStore.java  |  3 +-
 .../hive/ql/exec/HashTableSinkOperator.java     |  3 +-
 .../apache/hadoop/hive/ql/exec/MoveTask.java    |  2 +-
 .../hadoop/hive/ql/exec/ReplCopyTask.java       |  2 +-
 .../apache/hadoop/hive/ql/exec/Utilities.java   | 32 ++++++++++++--------
 .../apache/hadoop/hive/ql/metadata/Hive.java    |  2 +-
 .../BucketingSortingReduceSinkOptimizer.java    |  1 -
 .../hive/ql/optimizer/GenMapRedUtils.java       |  7 ++---
 .../hive/ql/optimizer/StatsOptimizer.java       |  6 ----
 .../hive/ql/parse/ImportSemanticAnalyzer.java   | 20 ++++++------
 .../hive/ql/plan/BucketMapJoinContext.java      |  1 -
 .../apache/hadoop/hive/ql/plan/MoveWork.java    |  1 -
 .../apache/hadoop/hive/ql/TestTxnCommands2.java | 17 ++++++-----
 14 files changed, 49 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 49005b9..faec3a8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -1977,7 +1977,6 @@ public class MetaStoreUtils {
    * @param params table properties
    * @return true if table is an INSERT_ONLY table, false otherwise
    */
-  // TODO# also check that transactional is true
   public static boolean isInsertOnlyTable(Map<String, String> params) {
     return isInsertOnlyTable(params, false);
   }
@@ -1998,7 +1997,7 @@ public class MetaStoreUtils {
 
    /** The method for altering table props; may set the table to MM, non-MM, or not affect MM. */
   public static Boolean isToInsertOnlyTable(Map<String, String> props) {
-    // TODO# Setting these separately is a very hairy issue in certain combinations, since we
+    // TODO: Setting these separately is a very hairy issue in certain combinations, since we
     //       cannot decide what type of table this becomes without taking both into account, and
     //       in many cases the conversion might be illegal.
     //       The only thing we allow is tx = true w/o tx-props, for backward compat.

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 176fc4c..e6661f9 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.metastore;
 
 
 import com.codahale.metrics.Counter;
+import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
@@ -57,8 +58,6 @@ import org.mockito.Mockito;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableList;
 import javax.jdo.Query;
 import java.util.Arrays;
 import java.util.HashMap;

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
index f4f20f0..0b8045e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
@@ -301,9 +301,10 @@ public class HashTableSinkOperator extends TerminalOperator<HashTableSinkDesc> i
       String bigBucketFileName = getExecContext().getCurrentBigBucketFile();
       String fileName = getExecContext().getLocalWork().getBucketFileName(bigBucketFileName);
       // get the tmp URI path; it will be a hdfs path if not local mode
-      // TODO# this doesn't work... the path for writer and reader mismatch
+      // TODO: this doesn't work... the path for writer and reader mismatch
       //      Dump the side-table for tag ... -local-10004/HashTable-Stage-1/MapJoin-a-00-(ds%3D2008-04-08)mm_2.hashtable
       //      Load back 1 hashtable file      -local-10004/HashTable-Stage-1/MapJoin-a-00-srcsortbucket3outof4.txt.hashtable
+      //      Hive3 probably won't support MR so do we really care? 
       String dumpFilePrefix = conf.getDumpFilePrefix();
       Path path = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName);
       console.printInfo(Utilities.now() + "\tDump the side-table for tag: " + tag +

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 34d0598..739ce18 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -268,7 +268,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
 
   @Override
   public int execute(DriverContext driverContext) {
-    if (work.isNoop()) return 0; // TODO# temporary flag for HIVE-14990
+    if (work.isNoop()) return 0;
     Utilities.LOG14535.info("Executing MoveWork " + System.identityHashCode(work)
         + " with " + work.getLoadFileWork() + "; " + work.getLoadTableWork() + "; "
         + work.getLoadMultiFilesWork());

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
index 86e21cb..1a9b39f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
@@ -61,7 +61,7 @@ public class ReplCopyTask extends Task<ReplCopyWork> implements Serializable {
     FileSystem dstFs = null;
     Path toPath = null;
     try {
-      // TODO# merge with real CopyTask logic
+      // TODO: merge with real CopyTask logic?
       Path fromPath = work.getFromPaths()[0];
       toPath = work.getToPaths()[0];
 

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index c6d4616..3f6fb36 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1124,9 +1124,6 @@ public final class Utilities {
     if (orig.getName().indexOf(tmpPrefix) == 0) {
       return orig;
     }
-    if (orig.getName().contains("=1")) {
-      LOG.error("TODO# creating tmp path from " + orig, new Exception());
-    }
     return new Path(orig.getParent(), tmpPrefix + orig.getName());
   }
 
@@ -1466,7 +1463,8 @@ public final class Utilities {
         Set<Path> filesKept = new HashSet<Path>();
         perfLogger.PerfLogBegin("FileSinkOperator", "RemoveTempOrDuplicateFiles");
         // remove any tmp file or double-committed output files
-        List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(fs, statuses, dpCtx, conf, hconf, filesKept);
+        List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(
+            fs, statuses, dpCtx, conf, hconf, filesKept);
         perfLogger.PerfLogEnd("FileSinkOperator", "RemoveTempOrDuplicateFiles");
         // create empty buckets if necessary
         if (emptyBuckets.size() > 0) {
@@ -1578,11 +1576,12 @@ public final class Utilities {
    * @return a list of path names corresponding to should-be-created empty buckets.
    */
   public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
-      DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, Set<Path> filesKept) throws IOException {
+      DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, Set<Path> filesKept)
+          throws IOException {
     int dpLevels = dpCtx == null ? 0 : dpCtx.getNumDPCols(),
-        numBuckets = (conf != null && conf.getTable() != null)
-          ? conf.getTable().getNumBuckets() : 0;
-    return removeTempOrDuplicateFiles(fs, fileStats, dpLevels, numBuckets, hconf, null, 0, false);
+        numBuckets = (conf != null && conf.getTable() != null) ? conf.getTable().getNumBuckets() : 0;
+    return removeTempOrDuplicateFiles(
+        fs, fileStats, dpLevels, numBuckets, hconf, null, 0, false, filesKept);
   }
   
   private static boolean removeEmptyDpDirectory(FileSystem fs, Path path) throws IOException {
@@ -1598,7 +1597,8 @@ public final class Utilities {
   }
 
   public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
-      int dpLevels, int numBuckets, Configuration hconf, Long txnId, int stmtId, boolean isMmTable) throws IOException {
+      int dpLevels, int numBuckets, Configuration hconf, Long txnId, int stmtId, boolean isMmTable,
+      Set<Path> filesKept) throws IOException {
     if (fileStats == null) {
       return null;
     }
@@ -1625,6 +1625,9 @@ public final class Utilities {
           Utilities.LOG14535.info("removeTempOrDuplicateFiles processing files in MM directory " + mmDir);
         }
         taskIDToFile = removeTempOrDuplicateFilesNonMm(items, fs);
+        if (filesKept != null && taskIDToFile != null) {
+          addFilesToPathSet(taskIDToFile.values(), filesKept);
+        }
 
         // TODO: not clear why two if conditions are different. Preserve the existing logic for now.
         addBucketFileToResults(taskIDToFile, numBuckets, hconf, result);
@@ -1636,6 +1639,9 @@ public final class Utilities {
       }
       if (!isMmTable) {
         taskIDToFile = removeTempOrDuplicateFilesNonMm(items, fs);
+        if (filesKept != null && taskIDToFile != null) {
+          addFilesToPathSet(taskIDToFile.values(), filesKept);
+        }
       } else {
         if (items.length > 1) {
           throw new IOException("Unexpected directories for non-DP MM: " + Arrays.toString(items));
@@ -1647,6 +1653,9 @@ public final class Utilities {
         Utilities.LOG14535.info(
             "removeTempOrDuplicateFiles processing files in MM directory "  + mmDir);
         taskIDToFile = removeTempOrDuplicateFilesNonMm(fs.listStatus(mmDir), fs);
+        if (filesKept != null && taskIDToFile != null) {
+          addFilesToPathSet(taskIDToFile.values(), filesKept);
+        }
       }
       // TODO: not clear why two if conditions are different. Preserve the existing logic for now.
       addBucketFileToResults2(taskIDToFile, numBuckets, hconf, result);
@@ -4194,7 +4203,6 @@ public final class Utilities {
     }
 
     Utilities.LOG14535.info("Looking for manifests in: " + manifestDir + " (" + txnId + ")");
-    // TODO# may be wrong if there are no splits (empty insert/CTAS)
     List<Path> manifests = new ArrayList<>();
     if (fs.exists(manifestDir)) {
       FileStatus[] manifestFiles = fs.listStatus(manifestDir);
@@ -4275,8 +4283,8 @@ public final class Utilities {
     for (int i = 0; i < mmDirectories.size(); ++i) {
       finalResults[i] = new PathOnlyFileStatus(mmDirectories.get(i));
     }
-    List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(
-        fs, finalResults, dpLevels, mbc == null ? 0 : mbc.numBuckets, hconf, txnId, stmtId, isMmTable);
+    List<Path> emptyBuckets = Utilities.removeTempOrDuplicateFiles(fs, finalResults, dpLevels,
+        mbc == null ? 0 : mbc.numBuckets, hconf, txnId, stmtId, isMmTable, null);
     // create empty buckets if necessary
     if (emptyBuckets.size() > 0) {
       assert mbc != null;

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index ff5ffe9..e51e648 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1911,7 +1911,7 @@ public class Hive {
 private void walkDirTree(FileStatus fSta, FileSystem fSys,
     Map<List<String>, String> skewedColValueLocationMaps, Path newPartPath, SkewedInfo skewedInfo)
     throws IOException {
-  // TODO# HERE broken
+  // TODO: may be broken? no LB bugs for now but if any are found.
 
   /* Base Case. It's leaf. */
   if (!fSta.isDir()) {

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
index 09c3079..85dcdd7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
@@ -237,7 +237,6 @@ public class BucketingSortingReduceSinkOptimizer extends Transform {
       Map<String, Integer> bucketFileNameMapping = new HashMap<String, Integer>();
       for (int pos = 0; pos < srcs.length; pos++) {
         if (ShimLoader.getHadoopShims().isDirectory(srcs[pos])) {
-          // TODO# HERE
           throw new RuntimeException("Was expecting '" + srcs[pos].getPath() + "' to be bucket file.");
         }
         bucketFileNameMapping.put(srcs[pos].getPath().getName(), pos);

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index 02e6512..6dfa607 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -1371,7 +1371,7 @@ public final class GenMapRedUtils {
       dummyMv = new MoveWork(null, null, null,
          new LoadFileDesc(inputDirName, finalName, true, null, null, false), false);
     } else {
-      // TODO# create the noop MoveWork to avoid q file changes for now. Should be removed.
+      // TODO# create the noop MoveWork to avoid q file changes for now. Should be removed w/the flag just before merge
       dummyMv = new MoveWork(null, null, null,
           new LoadFileDesc(inputDirName, finalName, true, null, null, false), false);
       dummyMv.setNoop(true);
@@ -1380,13 +1380,11 @@ public final class GenMapRedUtils {
     // MM directory, the original MoveTask still commits based on the parent. Note that this path
     // can only be triggered for a merge that's part of insert for now; MM tables do not support
     // concatenate. Keeping the old logic for non-MM tables with temp directories and stuff.
-    // TODO# is this correct?
     Path fsopPath = srcMmWriteId != null ? fsInputDesc.getFinalDirName() : finalName;
     Utilities.LOG14535.info("Looking for MoveTask to make it dependant on the conditional tasks");
 
     Task<MoveWork> mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(
         mvTasks, fsopPath, fsInputDesc.isMmTable());
-    // TODO# questionable master merge here
     ConditionalTask cndTsk = GenMapRedUtils.createCondTask(conf, currTask, dummyMv, work,
         fsInputDesc.getMergeInputDirName(), finalName, mvTask, dependencyTask);
 
@@ -1847,7 +1845,7 @@ public final class GenMapRedUtils {
           srcDir = srcDir.getParent();
         }
       } else if (mvWork.getLoadTableWork() != null) {
-        srcDir = mvWork.getLoadTableWork().getSourcePath(); // TODO# THIS
+        srcDir = mvWork.getLoadTableWork().getSourcePath();
       }
       Utilities.LOG14535.info("Observing MoveWork " + System.identityHashCode(mvWork)
           + " with " + srcDir + "(from " + (isLfd ? "LFD" : "LTD") + ") while looking for "
@@ -1970,7 +1968,6 @@ public final class GenMapRedUtils {
     Task<MoveWork> mvTask = null;
 
     if (!chDir) {
-      // TODO# is it correct to always use MM dir in MM case here? Where does MoveTask point?
       Utilities.LOG14535.info("Looking for MoveTask from createMoveTask");
       mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(
           mvTasks, fsOp.getConf().getFinalDirName(), fsOp.getConf().isMmTable());

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
index 0fe1ea3..7b2a1a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
@@ -342,12 +342,6 @@ public class StatsOptimizer extends Transform {
           return null;  // todo we can collapse this part of tree into single TS
         }
 
-        /* TODO# seems to be removed in master?
-        Table tbl = tsOp.getConf().getTableMetadata();
-        if (AcidUtils.isFullAcidTable(tbl)) {
-          Logger.info("Table " + tbl.getTableName() + " is ACID table. Skip StatsOptimizer.");
-          return null;
-        }*/
         List<Object> oneRow = new ArrayList<Object>();
 
         Hive hive = Hive.get(pctx.getConf());

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index a41cc8b..814910d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -314,9 +314,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
 
     Long txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
     int stmtId = 0;
-    //todo due to the master merge, tblDesc is no longer CreateTableDesc, but ImportTableDesc
+    // TODO: due to the master merge, tblDesc is no longer CreateTableDesc, but ImportTableDesc
     /*
-    if (txnId != null) {
+    if (isAcid(txnId)) {
       tblDesc.setInitialMmWriteId(txnId);
     }
     */
@@ -374,10 +374,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
 
     Task<?> copyTask = null;
     if (replicationSpec.isInReplicationScope()) {
-      if (isSourceMm || txnId != null) {
+      if (isSourceMm || isAcid(txnId)) {
         // TODO: ReplCopyTask is completely screwed. Need to support when it's not as screwed.
-        throw new RuntimeException(
-            "Not supported right now because Replication is completely screwed");
+        throw new RuntimeException("Replicating MM and ACID tables is not supported");
       }
       copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, destPath, x.getConf());
     } else {
@@ -397,6 +396,10 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
     return loadTableTask;
   }
 
+  private static boolean isAcid(Long txnId) {
+    return (txnId != null) && (txnId != 0);
+  }
+
   private static Task<?> createTableTask(ImportTableDesc tableDesc, EximUtil.SemanticAnalyzerWrapperContext x){
     return tableDesc.getCreateTableTask(x.getInputs(), x.getOutputs(), x.getConf());
   }
@@ -458,10 +461,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
 
       Task<?> copyTask = null;
       if (replicationSpec.isInReplicationScope()) {
-        if (isSourceMm || txnId != null) {
+        if (isSourceMm || isAcid(txnId)) {
           // TODO: ReplCopyTask is completely screwed. Need to support when it's not as screwed.
-          throw new RuntimeException(
-              "Not supported right now because Replication is completely screwed");
+          throw new RuntimeException("Replicating MM and ACID tables is not supported");
         }
         copyTask = ReplCopyTask.getLoadCopyTask(
             replicationSpec, new Path(srcLocation), destPath, x.getConf());
@@ -480,7 +482,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       loadTableWork.setInheritTableSpecs(false);
       // Do not commit the write ID from each task; need to commit once.
       // TODO: we should just change the import to use a single MoveTask, like dynparts.
-      loadTableWork.setIntermediateInMmWrite(txnId != null);
+      loadTableWork.setIntermediateInMmWrite(isAcid(txnId));
       Task<?> loadPartTask = TaskFactory.get(new MoveWork(
           x.getInputs(), x.getOutputs(), loadTableWork, null, false), x.getConf());
       copyTask.addDependentTask(loadPartTask);

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java
index 96f722c..4587c32 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BucketMapJoinContext.java
@@ -39,7 +39,6 @@ public class BucketMapJoinContext implements Serializable {
 
   private static final long serialVersionUID = 1L;
 
-  // TODO# this is completely broken, esp. w/load into bucketed tables (should perhaps be forbidden for MM tables)
   // table alias (small) --> input file name (big) --> target file names (small)
   private Map<String, Map<String, List<String>>> aliasBucketFileNameMapping;
   private String mapJoinBigTableAlias;

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
index 50adc42..8594edf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
@@ -153,7 +153,6 @@ public class MoveWork implements Serializable {
     this.srcLocal = srcLocal;
   }
 
-  // TODO# temporary test flag
   public void setNoop(boolean b) {
     this.isNoop = true;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/543e0b01/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 9171e60..a8a1fd6 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -368,18 +368,19 @@ public class TestTxnCommands2 {
      */
     String[][] expected = {
       {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t0\t13",  "bucket_00000"},
-      {"{\"transactionid\":18,\"bucketid\":536870912,\"rowid\":0}\t0\t15", "bucket_00000"},
-      {"{\"transactionid\":20,\"bucketid\":536870912,\"rowid\":0}\t0\t17", "bucket_00000"},
-      {"{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\t0\t120", "bucket_00000"},
+      {"{\"transactionid\":20,\"bucketid\":536870912,\"rowid\":0}\t0\t15", "bucket_00000"},
+      {"{\"transactionid\":22,\"bucketid\":536870912,\"rowid\":0}\t0\t17", "bucket_00000"},
+      {"{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t0\t120", "bucket_00000"},
       {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":1}\t1\t2",   "bucket_00001"},
       {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":3}\t1\t4",   "bucket_00001"},
       {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":2}\t1\t5",   "bucket_00001"},
       {"{\"transactionid\":0,\"bucketid\":536936448,\"rowid\":4}\t1\t6",   "bucket_00001"},
-      {"{\"transactionid\":18,\"bucketid\":536936448,\"rowid\":0}\t1\t16", "bucket_00001"}
+      {"{\"transactionid\":20,\"bucketid\":536936448,\"rowid\":0}\t1\t16", "bucket_00001"}
     };
     Assert.assertEquals("Unexpected row count before compaction", expected.length, rs.size());
     for(int i = 0; i < expected.length; i++) {
-      Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+      Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i) + "; expected " + expected[i][0],
+          rs.get(i).startsWith(expected[i][0]));
     }
     //run Compaction
     runStatementOnDriver("alter table "+ TestTxnCommands2.Table.NONACIDORCTBL +" compact 'major'");
@@ -763,7 +764,7 @@ public class TestTxnCommands2 {
           Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         } else if (numDelta == 2) {
-          Assert.assertEquals("delta_0000023_0000023_0000", status[i].getPath().getName());
+          Assert.assertEquals("delta_0000025_0000025_0000", status[i].getPath().getName());
           Assert.assertEquals(1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         }
@@ -772,7 +773,7 @@ public class TestTxnCommands2 {
         FileStatus[] buckets = fs.listStatus(status[i].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
         Arrays.sort(buckets);
         if (numDeleteDelta == 1) {
-          Assert.assertEquals("delete_delta_0000022_0000022_0000", status[i].getPath().getName());
+          Assert.assertEquals("delete_delta_0000024_0000024_0000", status[i].getPath().getName());
           Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         }
@@ -819,7 +820,7 @@ public class TestTxnCommands2 {
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         } else if (numBase == 2) {
           // The new base dir now has two bucket files, since the delta dir has two bucket files
-          Assert.assertEquals("base_0000023", status[i].getPath().getName());
+          Assert.assertEquals("base_0000025", status[i].getPath().getName());
           Assert.assertEquals(1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         }

[36/50] [abbrv] hive git commit: HIVE-15212 : merge branch into master - clean up some pre-merge TODOs (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-15212 : merge branch into master - clean up some pre-merge TODOs (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/55d9ceee
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/55d9ceee
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/55d9ceee

Branch: refs/heads/master
Commit: 55d9ceeea314763ff1db32be8f8c7452963ec831
Parents: 379d9ba
Author: sergey <se...@apache.org>
Authored: Wed Sep 27 14:20:46 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed Sep 27 14:20:46 2017 -0700

----------------------------------------------------------------------
 .../hive/ql/exec/AbstractFileMergeOperator.java |  14 ++-
 .../apache/hadoop/hive/ql/exec/CopyTask.java    |   4 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |  16 ++-
 .../hadoop/hive/ql/exec/FetchOperator.java      |   8 +-
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |  56 ++++++----
 .../hadoop/hive/ql/exec/ImportCommitTask.java   |   5 +-
 .../hadoop/hive/ql/exec/JoinOperator.java       |   5 +-
 .../apache/hadoop/hive/ql/exec/MoveTask.java    |  32 +++---
 .../hive/ql/exec/OrcFileMergeOperator.java      |   2 +-
 .../hadoop/hive/ql/exec/StatsNoJobTask.java     |  11 +-
 .../apache/hadoop/hive/ql/exec/StatsTask.java   |   5 +-
 .../apache/hadoop/hive/ql/exec/Utilities.java   | 102 ++++++++++++-------
 .../hive/ql/io/CombineHiveInputFormat.java      |   8 +-
 .../hadoop/hive/ql/io/HiveInputFormat.java      |  26 ++---
 .../apache/hadoop/hive/ql/metadata/Hive.java    |  57 +++++++----
 .../formatting/TextMetaDataFormatter.java       |   1 -
 .../ql/optimizer/AbstractBucketJoinProc.java    |   2 +-
 .../hive/ql/optimizer/GenMapRedUtils.java       |  32 ++++--
 .../optimizer/unionproc/UnionProcFactory.java   |   5 +-
 .../hadoop/hive/ql/parse/GenTezUtils.java       |   6 +-
 .../hive/ql/parse/ImportSemanticAnalyzer.java   |  15 +--
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  15 ++-
 .../hadoop/hive/ql/parse/TaskCompiler.java      |   9 +-
 .../ql/plan/ConditionalResolverMergeFiles.java  |  15 ++-
 .../hadoop/hive/ql/plan/LoadFileDesc.java       |   4 +-
 .../hadoop/hive/ql/plan/LoadTableDesc.java      |  17 ++--
 .../apache/hadoop/hive/ql/plan/MoveWork.java    |   4 +-
 .../hive/ql/stats/fs/FSStatsAggregator.java     |   7 +-
 .../hive/ql/stats/fs/FSStatsPublisher.java      |   7 +-
 29 files changed, 300 insertions(+), 190 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
index 6b98ecd..b163a1e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
@@ -123,8 +123,10 @@ public abstract class AbstractFileMergeOperator<T extends FileMergeDesc>
       finalPath = new Path(tp, taskId);
       outPath = new Path(ttp, Utilities.toTempPath(taskId));
     }
-    Utilities.LOG14535.info("Paths for merge " + taskId + ": tmp " + tmpPath + ", task "
-        + taskTmpPath + ", final " + finalPath + ", out " + outPath);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Paths for merge " + taskId + ": tmp " + tmpPath + ", task "
+          + taskTmpPath + ", final " + finalPath + ", out " + outPath);
+    }
   }
 
   /**
@@ -173,7 +175,9 @@ public abstract class AbstractFileMergeOperator<T extends FileMergeDesc>
 
     Path newTmpPath = new Path(tmpPath, newPath);
     if (!fs.exists(newTmpPath)) {
-      Utilities.LOG14535.info("Creating " + newTmpPath);
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("Creating " + newTmpPath);
+      }
       fs.mkdirs(newTmpPath);
     }
 
@@ -199,7 +203,9 @@ public abstract class AbstractFileMergeOperator<T extends FileMergeDesc>
   }
 
   protected void fixTmpPath(Path path) throws IOException {
-    Utilities.LOG14535.info("Calling fixTmpPath with " + path);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Calling fixTmpPath with " + path);
+    }
     // Fix temp path for alter table ... concatenate
     if (isListBucketingAlterTableConcatenate) {
       if (this.tmpPathFixedConcatenate) {

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
index 1f223f5..7299ed1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
@@ -113,7 +113,9 @@ public class CopyTask extends Task<CopyWork> implements Serializable {
     if (mmDirs == null || mmDirs.length == 0) return null;
     List<FileStatus> allFiles = new ArrayList<FileStatus>();
     for (FileStatus mmDir : mmDirs) {
-      Utilities.LOG14535.info("Found source MM directory " + mmDir.getPath());
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("Found source MM directory " + mmDir.getPath());
+      }
       matchFilesOneDir(fs, mmDir.getPath(), allFiles);
     }
     return allFiles.toArray(new FileStatus[allFiles.size()]);

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 8414a78..9502635 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4162,7 +4162,9 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
       Path src = allMmDirs.get(i);
       Path tgt = src.getParent();
       String prefix = src.getName().substring(prefixLen + 1) + "_";
-      Utilities.LOG14535.info("Will move " + src + " to " + tgt + " (prefix " + prefix + ")");
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("Will move " + src + " to " + tgt + " (prefix " + prefix + ")");
+      }
       targetPaths.add(tgt);
       targetPrefix.add(prefix);
     }
@@ -4214,7 +4216,9 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
   }
 
   private static void ensureDelete(FileSystem fs, Path path, String what) throws IOException {
-    Utilities.LOG14535.info("Deleting " + what + " " + path);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Deleting " + what + " " + path);
+    }
     try {
       if (!fs.delete(path, true)) throw new IOException("delete returned false");
     } catch (Exception ex) {
@@ -4255,14 +4259,18 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
         Path src = part.getDataLocation(), tgt = new Path(src, mmDir);
         srcs.add(src);
         tgts.add(tgt);
-        Utilities.LOG14535.info("Will move " + src + " to " + tgt);
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("Will move " + src + " to " + tgt);
+        }
       }
     } else {
       checkMmLb(tbl);
       Path src = tbl.getDataLocation(), tgt = new Path(src, mmDir);
       srcs.add(src);
       tgts.add(tgt);
-      Utilities.LOG14535.info("Will move " + src + " to " + tgt);
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("Will move " + src + " to " + tgt);
+      }
     }
     // Don't set inputs and outputs - the locks have already been taken so it's pointless.
     MoveWork mw = new MoveWork(null, null, null, null, false, SessionState.get().getLineageState());

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
index d2d9946..d09b12b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
@@ -269,7 +269,6 @@ public class FetchOperator implements Serializable {
     while (iterPath.hasNext()) {
       currPath = iterPath.next();
       currDesc = iterPartDesc.next();
-      Utilities.LOG14535.debug("Considering " + currPath);
       if (isNonNativeTable) {
         return true;
       }
@@ -286,7 +285,6 @@ public class FetchOperator implements Serializable {
         }
       }
     }
-    Utilities.LOG14535.debug("Done with all the paths");
     return false;
   }
 
@@ -381,7 +379,9 @@ public class FetchOperator implements Serializable {
       Utilities.copyTableJobPropertiesToConf(currDesc.getTableDesc(), job);
       InputFormat inputFormat = getInputFormatFromCache(formatter, job);
       String inputs = processCurrPathForMmWriteIds(inputFormat);
-      Utilities.LOG14535.info("Setting fetch inputs to " + inputs);
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("Setting fetch inputs to " + inputs);
+      }
       if (inputs == null) return null;
       job.set("mapred.input.dir", inputs);
 
@@ -414,7 +414,7 @@ public class FetchOperator implements Serializable {
       validTxnList = null;  // non-MM case
     }
     if (validTxnList != null) {
-      Utilities.LOG14535.info("Observing " + currDesc.getTableName() + ": " + validTxnList);
+      Utilities.FILE_OP_LOGGER.info("Processing " + currDesc.getTableName() + " for MM paths");
     }
 
     Path[] dirs = HiveInputFormat.processPathsForMmRead(Lists.newArrayList(currPath), job, validTxnList);

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 3544884..280d3cf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -185,9 +185,11 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
         txnId = conf.getTransactionId();
         stmtId = conf.getStatementId();
       }
-      Utilities.LOG14535.info("new FSPaths for " + numFiles + " files, dynParts = " + bDynParts
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("new FSPaths for " + numFiles + " files, dynParts = " + bDynParts
           + ": tmpPath " + tmpPath + ", task path " + taskOutputTempPath
           + " (spec path " + specPath + ")"/*, new Exception()*/);
+      }
 
       outPaths = new Path[numFiles];
       finalPaths = new Path[numFiles];
@@ -254,7 +256,9 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
         throws IOException, HiveException {
       if ((bDynParts || isSkewedStoredAsSubDirectories)
           && !fs.exists(finalPaths[idx].getParent())) {
-        Utilities.LOG14535.info("commit making path for dyn/skew: " + finalPaths[idx].getParent());
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("commit making path for dyn/skew: " + finalPaths[idx].getParent());
+        }
         FileUtils.mkdir(fs, finalPaths[idx].getParent(), hconf);
       }
       // If we're updating or deleting there may be no file to close.  This can happen
@@ -266,7 +270,10 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
       boolean needToRename = outPaths[idx] != null && ((acidOp != Operation.UPDATE
           && acidOp != Operation.DELETE) || fs.exists(outPaths[idx]));
       if (needToRename && outPaths[idx] != null) {
-        Utilities.LOG14535.info("committing " + outPaths[idx] + " to " + finalPaths[idx] + " (" + isMmTable + ")");
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("committing " + outPaths[idx] + " to "
+              + finalPaths[idx] + " (" + isMmTable + ")");
+        }
         if (isMmTable) {
           assert outPaths[idx].equals(finalPaths[idx]);
           commitPaths.add(outPaths[idx]);
@@ -381,14 +388,10 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
     }
 
     public void addToStat(String statType, long amount) {
-      if ("rowCount".equals(statType)) {
-        Utilities.LOG14535.info("Adding " + statType + " = " + amount + " to " + System.identityHashCode(this));
-      }
       stat.addToStat(statType, amount);
     }
 
     public Collection<String> getStoredStats() {
-      Utilities.LOG14535.info("Getting stats from " + System.identityHashCode(this));
       return stat.getStoredStats();
     }
   } // class FSPaths
@@ -459,8 +462,10 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
         unionPath = null;
       }
     }
-    Utilities.LOG14535.info("Setting up FSOP " + System.identityHashCode(this) + " ("
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Setting up FSOP " + System.identityHashCode(this) + " ("
         + conf.isLinkedFileSink() + ") with " + taskId + " and " + specPath + " + " + unionPath);
+    }
   }
 
   /** Kryo ctor. */
@@ -544,9 +549,11 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
 
       if (!bDynParts) {
         fsp = new FSPaths(specPath, conf.isMmTable());
-        Utilities.LOG14535.info("creating new paths " + System.identityHashCode(fsp)
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("creating new paths " + System.identityHashCode(fsp)
             + " from ctor; childSpec " + unionPath + ": tmpPath " + fsp.getTmpPath()
             + ", task path " + fsp.getTaskOutputTempPath());
+        }
 
         // Create all the files - this is required because empty files need to be created for
         // empty buckets
@@ -719,10 +726,11 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
       throws HiveException {
     try {
       fsp.initializeBucketPaths(filesIdx, taskId, isNativeTable, isSkewedStoredAsSubDirectories);
-      Utilities.LOG14535.info("createBucketForFileIdx " + filesIdx + ": final path " + fsp.finalPaths[filesIdx]
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("createBucketForFileIdx " + filesIdx + ": final path " + fsp.finalPaths[filesIdx]
           + "; out path " + fsp.outPaths[filesIdx] +" (spec path " + specPath + ", tmp path "
-          + fsp.getTmpPath() + ", task " + taskId + ")"/*, new Exception()*/);
-
+          + fsp.getTmpPath() + ", task " + taskId + ")");
+      }
       if (LOG.isInfoEnabled()) {
         LOG.info("New Final Path: FS " + fsp.finalPaths[filesIdx]);
       }
@@ -1002,7 +1010,6 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
   protected FSPaths lookupListBucketingPaths(String lbDirName) throws HiveException {
     FSPaths fsp2 = valToPaths.get(lbDirName);
     if (fsp2 == null) {
-      Utilities.LOG14535.info("lookupListBucketingPaths for " + lbDirName);
       fsp2 = createNewPaths(lbDirName);
     }
     return fsp2;
@@ -1018,9 +1025,11 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
   private FSPaths createNewPaths(String dirName) throws HiveException {
     FSPaths fsp2 = new FSPaths(specPath, conf.isMmTable());
     fsp2.configureDynPartPath(dirName, !conf.isMmTable() && isUnionDp ? unionPath : null);
-    Utilities.LOG14535.info("creating new paths " + System.identityHashCode(fsp2) + " for "
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("creating new paths " + System.identityHashCode(fsp2) + " for "
         + dirName + ", childSpec " + unionPath + ": tmpPath " + fsp2.getTmpPath()
         + ", task path " + fsp2.getTaskOutputTempPath());
+    }
     if(!conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED)) {
       createBucketFiles(fsp2);
       valToPaths.put(dirName, fsp2);
@@ -1286,7 +1295,9 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
           specPath = conf.getParentDir();
           unionSuffix = conf.getDirName().getName();
         }
-        Utilities.LOG14535.info("jobCloseOp using specPath " + specPath);
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("jobCloseOp using specPath " + specPath);
+        }
         if (!conf.isMmTable()) {
           Utilities.mvFileToFinalPath(specPath, hconf, success, LOG, dpCtx, conf, reporter);
         } else {
@@ -1352,7 +1363,6 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
   }
 
   private void publishStats() throws HiveException {
-    Utilities.LOG14535.error("FSOP publishStats called.");
     boolean isStatsReliable = conf.isStatsReliable();
 
     // Initializing a stats publisher
@@ -1383,8 +1393,10 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
     for (Map.Entry<String, FSPaths> entry : valToPaths.entrySet()) {
       String fspKey = entry.getKey();     // DP/LB
       FSPaths fspValue = entry.getValue();
-      Utilities.LOG14535.info("Observing entry for stats " + fspKey
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("Observing entry for stats " + fspKey
           + " => FSP with tmpPath " + fspValue.getTmpPath());
+      }
       // for bucketed tables, hive.optimize.sort.dynamic.partition optimization
       // adds the taskId to the fspKey.
       if (conf.getDpSortState().equals(DPSortState.PARTITION_BUCKET_SORTED)) {
@@ -1397,7 +1409,6 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
         // not be retrieved from staging table and hence not aggregated. To avoid this issue
         // we will remove the taskId from the key which is redundant anyway.
         fspKey = fspKey.split(taskID)[0];
-        Utilities.LOG14535.info("Adjusting fspKey for stats to " + fspKey);
       }
 
       // split[0] = DP, split[1] = LB
@@ -1409,14 +1420,17 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
       String prefix = conf.getTableInfo().getTableName().toLowerCase();
       prefix = Utilities.join(prefix, spSpec, dpSpec);
       prefix = prefix.endsWith(Path.SEPARATOR) ? prefix : prefix + Path.SEPARATOR;
-      Utilities.LOG14535.info("Prefix for stats " + prefix + " (from " + spSpec + ", " + dpSpec + ")");
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace(
+            "Prefix for stats " + prefix + " (from " + spSpec + ", " + dpSpec + ")");
+      }
 
       Map<String, String> statsToPublish = new HashMap<String, String>();
       for (String statType : fspValue.getStoredStats()) {
         statsToPublish.put(statType, Long.toString(fspValue.stat.getStat(statType)));
       }
       if (!statsPublisher.publishStat(prefix, statsToPublish)) {
-        Utilities.LOG14535.error("Failed to publish stats");
+        LOG.error("Failed to publish stats");
         // The original exception is lost.
         // Not changing the interface to maintain backward compatibility
         if (isStatsReliable) {
@@ -1426,7 +1440,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
     }
     sContext.setIndexForTezUnion(this.getIndexForTezUnion());
     if (!statsPublisher.closeConnection(sContext)) {
-      Utilities.LOG14535.error("Failed to close stats");
+      LOG.error("Failed to close stats");
       // The original exception is lost.
       // Not changing the interface to maintain backward compatibility
       if (isStatsReliable) {

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/ImportCommitTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ImportCommitTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ImportCommitTask.java
index 27db9a4..b54d457 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ImportCommitTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ImportCommitTask.java
@@ -33,11 +33,12 @@ public class ImportCommitTask extends Task<ImportCommitWork> {
 
   @Override
   public int execute(DriverContext driverContext) {
-    Utilities.LOG14535.info("Executing ImportCommit for " + work.getTxnId());
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Executing ImportCommit for " + work.getTxnId());
+    }
 
     try {
       if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
-        Utilities.LOG14535.info("Exiting due to explain");
         return 0;
       }
       return 0;

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
index 3f5cdf5..65b2f87 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
@@ -231,13 +231,12 @@ public class JoinOperator extends CommonJoinOperator<JoinDesc> implements Serial
         // Step1: rename tmp output folder to intermediate path. After this
         // point, updates from speculative tasks still writing to tmpPath
         // will not appear in finalPath.
-        log.info("Moving tmp dir: " + tmpPath + " to: " + intermediatePath);
-        Utilities.LOG14535.info("Moving tmp dir: " + tmpPath + " to: " + intermediatePath + "(spec " + specPath + ")");
+        Utilities.FILE_OP_LOGGER.info("Moving tmp dir: " + tmpPath + " to: " + intermediatePath + "(spec " + specPath + ")");
         Utilities.rename(fs, tmpPath, intermediatePath);
         // Step2: remove any tmp file or double-committed output files
         Utilities.removeTempOrDuplicateFiles(fs, intermediatePath);
         // Step3: move to the file destination
-        log.info("Moving tmp dir: " + intermediatePath + " to: " + specPath);
+        Utilities.FILE_OP_LOGGER.info("Moving tmp dir: " + intermediatePath + " to: " + specPath);
         Utilities.renameOrMoveFiles(fs, intermediatePath, specPath);
       }
     } else {

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index ed527b8..e644845 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -267,9 +267,11 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
   @Override
   public int execute(DriverContext driverContext) {
     if (work.isNoop()) return 0;
-    Utilities.LOG14535.info("Executing MoveWork " + System.identityHashCode(work)
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Executing MoveWork " + System.identityHashCode(work)
         + " with " + work.getLoadFileWork() + "; " + work.getLoadTableWork() + "; "
         + work.getLoadMultiFilesWork());
+    }
 
     try {
       if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
@@ -284,9 +286,9 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
         Path targetPath = lfd.getTargetDir();
         Path sourcePath = lfd.getSourcePath();
         if (targetPath.equals(sourcePath)) {
-          Utilities.LOG14535.info("MoveTask not moving LFD " + sourcePath);
+          Utilities.FILE_OP_LOGGER.debug("MoveTask not moving " + sourcePath);
         } else {
-          Utilities.LOG14535.info("MoveTask moving LFD " + sourcePath + " to " + targetPath);
+          Utilities.FILE_OP_LOGGER.debug("MoveTask moving " + sourcePath + " to " + targetPath);
           if(lfd.getWriteType() == AcidUtils.Operation.INSERT) {
             //'targetPath' is table root of un-partitioned table/partition
             //'sourcePath' result of 'select ...' part of CTAS statement
@@ -316,7 +318,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
             if (!destFs.exists(destPath.getParent())) {
               destFs.mkdirs(destPath.getParent());
             }
-            Utilities.LOG14535.info("MoveTask moving LMFD " + srcPath + " to " + destPath);
+            Utilities.FILE_OP_LOGGER.debug("MoveTask moving (multi-file) " + srcPath + " to " + destPath);
             moveFile(srcPath, destPath, isDfsDir);
           } else {
             if (!destFs.exists(destPath)) {
@@ -328,11 +330,11 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
               for (FileStatus child : children) {
                 Path childSrc = child.getPath();
                 Path childDest = new Path(destPath, filePrefix + childSrc.getName());
-                Utilities.LOG14535.info("MoveTask moving LMFD " + childSrc + " to " + childDest);
+                Utilities.FILE_OP_LOGGER.debug("MoveTask moving (multi-file) " + childSrc + " to " + childDest);
                 moveFile(childSrc, childDest, isDfsDir);
               }
             } else {
-              Utilities.LOG14535.info("MoveTask skipping empty directory LMFD " + srcPath);
+              Utilities.FILE_OP_LOGGER.debug("MoveTask skipping empty directory (multi-file) " + srcPath);
             }
             if (!srcFs.delete(srcPath, false)) {
               throw new IOException("Couldn't delete " + srcPath + " after moving all the files");
@@ -356,7 +358,9 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
           mesg.append(')');
         }
         String mesg_detail = " from " + tbd.getSourcePath();
-        Utilities.LOG14535.info("" + mesg.toString() + " " + mesg_detail);
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace(mesg.toString() + " " + mesg_detail);
+        }
         console.printInfo(mesg.toString(), mesg_detail);
         Table table = db.getTable(tbd.getTable().getTableName());
 
@@ -369,10 +373,9 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
         DataContainer dc = null;
         if (tbd.getPartitionSpec().size() == 0) {
           dc = new DataContainer(table.getTTable());
-          Utilities.LOG14535.info("loadTable called from " + tbd.getSourcePath() + " into " + tbd.getTable().getTableName());
-          if (tbd.isMmTable() && !tbd.isCommitMmWrite()) {
-            throw new HiveException(
-                "Only single-partition LoadTableDesc can skip commiting write ID");
+          if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+            Utilities.FILE_OP_LOGGER.trace("loadTable called from " + tbd.getSourcePath()
+              + " into " + tbd.getTable().getTableName());
           }
           db.loadTable(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getReplace(),
               work.isSrcLocal(), isSkewedStoredAsDirs(tbd), isFullAcidOp, hasFollowingStatsTask(),
@@ -448,8 +451,10 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
       TaskInformation ti) throws HiveException, IOException, InvalidOperationException {
     List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(),  tbd.getPartitionSpec());
     db.validatePartitionNameCharacters(partVals);
-    Utilities.LOG14535.info("loadPartition called from " + tbd.getSourcePath()
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath()
         + " into " + tbd.getTable().getTableName());
+    }
     db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(),
         tbd.getPartitionSpec(), tbd.getReplace(),
         tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(),
@@ -488,9 +493,6 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
     // iterate over it and call loadPartition() here.
     // The reason we don't do inside HIVE-1361 is the latter is large and we
     // want to isolate any potential issue it may introduce.
-    if (tbd.isMmTable() && !tbd.isCommitMmWrite()) {
-      throw new HiveException("Only single-partition LoadTableDesc can skip commiting write ID");
-    }
     Map<Map<String, String>, Partition> dp =
       db.loadDynamicPartitions(
         tbd.getSourcePath(),

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
index a1ad243..50160a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
@@ -96,7 +96,7 @@ public class OrcFileMergeOperator extends
 
       filePath = k.getInputPath().toUri().getPath();
 
-      Utilities.LOG14535.info("OrcFileMergeOperator processing " + filePath);
+      Utilities.FILE_OP_LOGGER.info("OrcFileMergeOperator processing " + filePath);
 
 
       fixTmpPath(k.getInputPath().getParent());

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
index 65363ed..ad5a96b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
@@ -239,7 +239,7 @@ public class StatsNoJobTask extends Task<StatsNoJobWork> implements Serializable
         Map<String, String> parameters = tTable.getParameters();
         try {
           Path dir = new Path(tTable.getSd().getLocation());
-          Utilities.LOG14535.info("Aggregating stats for " + dir);
+          LOG.debug("Aggregating stats for " + dir);
           long numRows = 0;
           long rawDataSize = 0;
           long fileSize = 0;
@@ -249,7 +249,7 @@ public class StatsNoJobTask extends Task<StatsNoJobWork> implements Serializable
 
           boolean statsAvailable = false;
           for(FileStatus file: fileList) {
-            Utilities.LOG14535.info("Computing stats for " + file);
+            LOG.debug("Computing stats for " + file);
             if (!file.isDir()) {
               InputFormat<?, ?> inputFormat = ReflectionUtil.newInstance(
                   table.getInputFormatClass(), jc);
@@ -286,11 +286,10 @@ public class StatsNoJobTask extends Task<StatsNoJobWork> implements Serializable
             db.alterTable(tableFullName, new Table(tTable), environmentContext);
 
             String msg = "Table " + tableFullName + " stats: [" + toString(parameters) + ']';
-            Utilities.LOG14535.debug(msg);
+            if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+              Utilities.FILE_OP_LOGGER.trace(msg);
+            }
             console.printInfo(msg);
-          } else {
-            String msg = "Table " + tableFullName + " does not provide stats.";
-            Utilities.LOG14535.debug(msg);
           }
         } catch (Exception e) {
           console.printInfo("[Warning] could not update stats for " + tableFullName + ".",

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
index 91a9c88..1725629 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
@@ -217,7 +217,10 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
         if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) {
           console.printInfo("Table " + tableFullName + " stats: [" + toString(parameters) + ']');
         }
-        Utilities.LOG14535.info("Table " + tableFullName + " stats: [" + toString(parameters) + ']');
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace(
+              "Table " + tableFullName + " stats: [" + toString(parameters) + ']');
+        }
       } else {
         // Partitioned table:
         // Need to get the old stats of the partition

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index b1c7844..cb3cb03 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -220,9 +220,12 @@ import java.util.concurrent.Executors;
  */
 @SuppressWarnings({ "nls", "deprecation" })
 public final class Utilities {
-
-  // TODO# remove when merging; convert some statements to local loggers, remove others
-  public static final Logger LOG14535 = LoggerFactory.getLogger("Log14535");
+  /**
+   * A logger mostly used to trace-log the details of Hive table file operations. Filtering the
+   * logs for FileOperations (with trace logs present) allows one to debug what Hive has done with
+   * various files and directories while committing writes, as well as reading.
+   */
+  public static final Logger FILE_OP_LOGGER = LoggerFactory.getLogger("FileOperations");
 
   /**
    * The object in the reducer are composed of these top level fields.
@@ -1475,7 +1478,9 @@ public final class Utilities {
           perfLogger.PerfLogEnd("FileSinkOperator", "CreateEmptyBuckets");
         }
         // move to the file destination
-        Utilities.LOG14535.info("Moving tmp dir: " + tmpPath + " to: " + specPath);
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("Moving tmp dir: " + tmpPath + " to: " + specPath);
+        }
         perfLogger.PerfLogBegin("FileSinkOperator", "RenameOrMoveFiles");
         if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVE_EXEC_MOVE_FILES_FROM_SOURCE_DIR)) {
           // HIVE-17113 - avoid copying files that may have been written to the temp dir by runaway tasks,
@@ -1487,10 +1492,14 @@ public final class Utilities {
         perfLogger.PerfLogEnd("FileSinkOperator", "RenameOrMoveFiles");
       }
     } else {
-      Utilities.LOG14535.info("deleting tmpPath " + tmpPath);
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("deleting tmpPath " + tmpPath);
+      }
       fs.delete(tmpPath, true);
     }
-    Utilities.LOG14535.info("deleting taskTmpPath " + taskTmpPath);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("deleting taskTmpPath " + taskTmpPath);
+    }
     fs.delete(taskTmpPath, true);
   }
 
@@ -1533,7 +1542,9 @@ public final class Utilities {
     }
 
     for (Path path : paths) {
-      Utilities.LOG14535.info("creating empty bucket for " + path);
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("creating empty bucket for " + path);
+      }
       RecordWriter writer = HiveFileFormatUtils.getRecordWriter(
           jc, hiveOutputFormat, outputClass, isCompressed,
           tableInfo.getProperties(), path, reporter);
@@ -1610,7 +1621,6 @@ public final class Utilities {
         assert parts[i].isDirectory() : "dynamic partition " + parts[i].getPath()
             + " is not a directory";
         Path path = parts[i].getPath();
-        Utilities.LOG14535.info("removeTempOrDuplicateFiles looking at DP " + path);
         if (removeEmptyDpDirectory(fs, path)) {
           parts[i] = null;
           continue;
@@ -1622,7 +1632,10 @@ public final class Utilities {
           if (!mmDir.getName().equals(AcidUtils.deltaSubdir(txnId, txnId, stmtId))) {
             throw new IOException("Unexpected non-MM directory name " + mmDir);
           }
-          Utilities.LOG14535.info("removeTempOrDuplicateFiles processing files in MM directory " + mmDir);
+          if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+            Utilities.FILE_OP_LOGGER.trace(
+                "removeTempOrDuplicateFiles processing files in MM directory " + mmDir);
+          }
         }
         taskIDToFile = removeTempOrDuplicateFilesNonMm(items, fs);
         if (filesKept != null && taskIDToFile != null) {
@@ -1650,8 +1663,10 @@ public final class Utilities {
         if (!mmDir.getName().equals(AcidUtils.deltaSubdir(txnId, txnId, stmtId))) {
           throw new IOException("Unexpected non-MM directory " + mmDir);
         }
-        Utilities.LOG14535.info(
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace(
             "removeTempOrDuplicateFiles processing files in MM directory "  + mmDir);
+        }
         taskIDToFile = removeTempOrDuplicateFilesNonMm(fs.listStatus(mmDir), fs);
         if (filesKept != null && taskIDToFile != null) {
           addFilesToPathSet(taskIDToFile.values(), filesKept);
@@ -1687,7 +1702,6 @@ public final class Utilities {
       HashMap<String, FileStatus> taskIDToFile, int numBuckets, List<Path> result) {
     String taskID1 = taskIDToFile.keySet().iterator().next();
     Path bucketPath = taskIDToFile.values().iterator().next().getPath();
-    Utilities.LOG14535.info("Bucket path " + bucketPath);
     for (int j = 0; j < numBuckets; ++j) {
       addBucketFileIfMissing(result, taskIDToFile, taskID1, bucketPath, j);
     }
@@ -1700,7 +1714,9 @@ public final class Utilities {
       // create empty bucket, file name should be derived from taskID2
       URI bucketUri = bucketPath.toUri();
       String path2 = replaceTaskIdFromFilename(bucketUri.getPath().toString(), j);
-      Utilities.LOG14535.info("Creating an empty bucket file " + path2);
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("Creating an empty bucket file " + path2);
+      }
       result.add(new Path(bucketUri.getScheme(), bucketUri.getAuthority(), path2));
     }
   }
@@ -1714,7 +1730,9 @@ public final class Utilities {
 
     for (FileStatus one : files) {
       if (isTempPath(one)) {
-        Utilities.LOG14535.info("removeTempOrDuplicateFiles deleting " + one.getPath()/*, new Exception()*/);
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("removeTempOrDuplicateFiles deleting " + one.getPath());
+        }
         if (!fs.delete(one.getPath(), true)) {
           throw new IOException("Unable to delete tmp file: " + one.getPath());
         }
@@ -1729,7 +1747,10 @@ public final class Utilities {
   private static void ponderRemovingTempOrDuplicateFile(FileSystem fs,
       FileStatus file, HashMap<String, FileStatus> taskIdToFile) throws IOException {
     String taskId = getPrefixedTaskIdFromFilename(file.getPath().getName());
-    Utilities.LOG14535.info("removeTempOrDuplicateFiles pondering " + file.getPath() + ", taskId " + taskId);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("removeTempOrDuplicateFiles looking at "
+          + file.getPath() + ", taskId " + taskId);
+    }
 
     FileStatus otherFile = taskIdToFile.get(taskId);
     taskIdToFile.put(taskId, (otherFile == null) ? file :
@@ -2601,10 +2622,13 @@ public final class Utilities {
         // generate a full partition specification
         LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<String, String>(partSpec);
         if (!Warehouse.makeSpecFromName(fullPartSpec, partPath, new HashSet<String>(partSpec.keySet()))) {
-          Utilities.LOG14535.warn("Ignoring invalid DP directory " + partPath);
+          Utilities.FILE_OP_LOGGER.warn("Ignoring invalid DP directory " + partPath);
           continue;
         }
-        Utilities.LOG14535.info("Adding partition spec from " + partPath + ": " + fullPartSpec);
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("Adding partition spec from "
+              + partPath + ": " + fullPartSpec);
+        }
         fullPartSpecs.add(fullPartSpec);
       }
       return fullPartSpecs;
@@ -4037,14 +4061,16 @@ public final class Utilities {
             + " (when shortened to " + dir + ")");
       }
       String subDir = dir.substring(relRoot.length());
-      Utilities.LOG14535.info("Looking at " + subDir + " from " + lfs.getPath());
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("Looking at " + subDir + " from " + lfs.getPath());
+      }
       // If sorted, we'll skip a bunch of files.
       if (lastRelDir != null && subDir.startsWith(lastRelDir)) continue;
       int startIx = skipLevels > 0 ? -1 : 0;
       for (int i = 0; i < skipLevels; ++i) {
         startIx = subDir.indexOf(Path.SEPARATOR_CHAR, startIx + 1);
         if (startIx == -1) {
-          Utilities.LOG14535.info("Expected level of nesting (" + skipLevels + ") is not "
+          Utilities.FILE_OP_LOGGER.info("Expected level of nesting (" + skipLevels + ") is not "
               + " present in " + subDir + " (from " + lfs.getPath() + ")");
           break;
         }
@@ -4052,13 +4078,12 @@ public final class Utilities {
       if (startIx == -1) continue;
       int endIx = subDir.indexOf(Path.SEPARATOR_CHAR, startIx + 1);
       if (endIx == -1) {
-        Utilities.LOG14535.info("Expected level of nesting (" + (skipLevels + 1) + ") is not "
+        Utilities.FILE_OP_LOGGER.info("Expected level of nesting (" + (skipLevels + 1) + ") is not "
             + " present in " + subDir + " (from " + lfs.getPath() + ")");
         continue;
       }
       lastRelDir = subDir = subDir.substring(0, endIx);
       Path candidate = new Path(relRoot, subDir);
-      Utilities.LOG14535.info("Considering MM directory candidate " + candidate);
       if (!filter.accept(candidate)) continue;
       results.add(fs.makeQualified(candidate));
     }
@@ -4073,7 +4098,6 @@ public final class Utilities {
     }
     sb.append(Path.SEPARATOR).append(AcidUtils.deltaSubdir(txnId, txnId, stmtId));
     Path pathPattern = new Path(path, sb.toString());
-    Utilities.LOG14535.info("Looking for files via: " + pathPattern);
     return statusToPath(fs.globStatus(pathPattern, filter));
   }
 
@@ -4084,11 +4108,11 @@ public final class Utilities {
         fs, specPath, dpLevels, lbLevels, filter, txnId, stmtId, conf);
     if (files != null) {
       for (Path path : files) {
-        Utilities.LOG14535.info("Deleting " + path + " on failure");
+        Utilities.FILE_OP_LOGGER.info("Deleting " + path + " on failure");
         tryDelete(fs, path);
       }
     }
-    Utilities.LOG14535.info("Deleting " + manifestDir + " on failure");
+    Utilities.FILE_OP_LOGGER.info("Deleting " + manifestDir + " on failure");
     fs.delete(manifestDir, true);
   }
 
@@ -4099,7 +4123,7 @@ public final class Utilities {
     // We assume one FSOP per task (per specPath), so we create it in specPath.
     Path manifestPath = getManifestDir(specPath, txnId, stmtId, unionSuffix);
     manifestPath = new Path(manifestPath, taskId + MANIFEST_EXTENSION);
-    Utilities.LOG14535.info("Writing manifest to " + manifestPath + " with " + commitPaths);
+    Utilities.FILE_OP_LOGGER.info("Writing manifest to " + manifestPath + " with " + commitPaths);
     try {
       // Don't overwrite the manifest... should fail if we have collisions.
       try (FSDataOutputStream out = fs.create(manifestPath, false)) {
@@ -4144,7 +4168,7 @@ public final class Utilities {
       return;
     }
 
-    Utilities.LOG14535.info("Looking for manifests in: " + manifestDir + " (" + txnId + ")");
+    Utilities.FILE_OP_LOGGER.debug("Looking for manifests in: " + manifestDir + " (" + txnId + ")");
     List<Path> manifests = new ArrayList<>();
     if (fs.exists(manifestDir)) {
       FileStatus[] manifestFiles = fs.listStatus(manifestDir);
@@ -4152,21 +4176,21 @@ public final class Utilities {
         for (FileStatus status : manifestFiles) {
           Path path = status.getPath();
           if (path.getName().endsWith(MANIFEST_EXTENSION)) {
-            Utilities.LOG14535.info("Reading manifest " + path);
+            Utilities.FILE_OP_LOGGER.info("Reading manifest " + path);
             manifests.add(path);
           }
         }
       }
     } else {
-      Utilities.LOG14535.info("No manifests found - query produced no output");
+      Utilities.FILE_OP_LOGGER.info("No manifests found - query produced no output");
       manifestDir = null;
     }
 
-    Utilities.LOG14535.info("Looking for files in: " + specPath);
+    Utilities.FILE_OP_LOGGER.debug("Looking for files in: " + specPath);
     JavaUtils.IdPathFilter filter = new JavaUtils.IdPathFilter(txnId, stmtId, true);
     if (isMmCtas && !fs.exists(specPath)) {
       // TODO: do we also need to do this when creating an empty partition from select?
-      Utilities.LOG14535.info("Creating table directory for CTAS with no output at " + specPath);
+      Utilities.FILE_OP_LOGGER.info("Creating table directory for CTAS with no output at " + specPath);
       FileUtils.mkdir(fs, specPath, hconf);
     }
     Path[] files = getMmDirectoryCandidates(
@@ -4174,7 +4198,9 @@ public final class Utilities {
     ArrayList<Path> mmDirectories = new ArrayList<>();
     if (files != null) {
       for (Path path : files) {
-        Utilities.LOG14535.info("Looking at path: " + path);
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("Looking at path: " + path);
+        }
         mmDirectories.add(path);
       }
     }
@@ -4193,14 +4219,14 @@ public final class Utilities {
     }
 
     if (manifestDir != null) {
-      Utilities.LOG14535.info("Deleting manifest directory " + manifestDir);
+      Utilities.FILE_OP_LOGGER.info("Deleting manifest directory " + manifestDir);
       tryDelete(fs, manifestDir);
       if (unionSuffix != null) {
         // Also delete the parent directory if we are the last union FSOP to execute.
         manifestDir = manifestDir.getParent();
         FileStatus[] remainingFiles = fs.listStatus(manifestDir);
         if (remainingFiles == null || remainingFiles.length == 0) {
-          Utilities.LOG14535.info("Deleting manifest directory " + manifestDir);
+          Utilities.FILE_OP_LOGGER.info("Deleting manifest directory " + manifestDir);
           tryDelete(fs, manifestDir);
         }
       }
@@ -4257,15 +4283,17 @@ public final class Utilities {
         // Found the right union directory; treat it as "our" MM directory.
         cleanMmDirectory(childPath, fs, null, committed);
       } else {
-        Utilities.LOG14535.info("FSOP for " + unionSuffix
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("FSOP for " + unionSuffix
             + " is ignoring the other side of the union " + childPath.getName());
+        }
       }
     }
   }
 
   private static void deleteUncommitedFile(Path childPath, FileSystem fs)
       throws IOException, HiveException {
-    Utilities.LOG14535.info("Deleting " + childPath + " that was not committed");
+    Utilities.FILE_OP_LOGGER.info("Deleting " + childPath + " that was not committed");
     // We should actually succeed here - if we fail, don't commit the query.
     if (!fs.delete(childPath, true)) {
       throw new HiveException("Failed to delete an uncommitted path " + childPath);
@@ -4278,7 +4306,9 @@ public final class Utilities {
    */
   public static List<Path> getValidMmDirectoriesFromTableOrPart(Path path, Configuration conf,
       ValidTxnList validTxnList, int lbLevels) throws IOException {
-    Utilities.LOG14535.info("Looking for valid MM paths under " + path);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Looking for valid MM paths under " + path);
+    }
     // NULL means this directory is entirely valid.
     List<Path> result = null;
     FileSystem fs = path.getFileSystem(conf);
@@ -4289,7 +4319,7 @@ public final class Utilities {
       Path childPath = file.getPath();
       Long txnId = JavaUtils.extractTxnId(childPath);
       if (!file.isDirectory() || txnId == null || !validTxnList.isTxnValid(txnId)) {
-        Utilities.LOG14535.info("Skipping path " + childPath);
+        Utilities.FILE_OP_LOGGER.debug("Skipping path " + childPath);
         if (result == null) {
           result = new ArrayList<>(children.length - 1);
           for (int j = 0; j < i; ++j) {

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
index 6f2aaf0..93de69f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
@@ -121,14 +121,12 @@ public class CombineHiveInputFormat<K extends WritableComparable, V extends Writ
           isMmNonMerge = !isMerge && MetaStoreUtils.isInsertOnlyTable(tbl.getProperties());
         } else {
           // This would be the case for obscure tasks like truncate column (unsupported for MM).
-          Utilities.LOG14535.warn("Assuming not insert-only; no table in partition spec " + part);
+          Utilities.FILE_OP_LOGGER.warn("Assuming not insert-only; no table in partition spec " + part);
         }
 
         if (isAvoidSplitCombine || isMmNonMerge) {
-          //if (LOG.isDebugEnabled()) {
-            Utilities.LOG14535.info("The path [" + paths[i + start] +
-                "] is being parked for HiveInputFormat.getSplits");
-          //}
+          Utilities.FILE_OP_LOGGER.info("The path [" + paths[i + start] +
+              "] is being parked for HiveInputFormat.getSplits");
           nonCombinablePathIndices.add(i + start);
         }
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index c3b9190..fba5b7e 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -520,7 +520,9 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
   private static void processForWriteIds(Path dir, JobConf conf,
       ValidTxnList validTxnList, List<Path> finalPaths) throws IOException {
     FileSystem fs = dir.getFileSystem(conf);
-    Utilities.LOG14535.warn("Checking " + dir + " (root) for inputs");
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Checking " + dir + " (root) for inputs");
+    }
     // Ignore nullscan-optimized paths.
     if (fs instanceof NullScanFileSystem) {
       finalPaths.add(dir);
@@ -537,20 +539,20 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
       boolean hadAcidState = false;   // whether getAcidState has been called for currDir
       for (FileStatus file : files) {
         Path path = file.getPath();
-        Utilities.LOG14535.warn("Checking " + path + " for inputs");
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("Checking " + path + " for inputs");
+        }
         if (!file.isDirectory()) {
-          Utilities.LOG14535.warn("Ignoring a file not in MM directory " + path);
+          Utilities.FILE_OP_LOGGER.warn("Ignoring a file not in MM directory " + path);
         } else if (JavaUtils.extractTxnId(path) == null) {
           subdirs.add(path);
-        } else {
-          if (!hadAcidState) {
-            AcidUtils.Directory dirInfo = AcidUtils.getAcidState(currDir, conf, validTxnList, Ref.from(false), true, null);
-            hadAcidState = true;
-            // todo for IOW, we also need to count in base dir, if any
-            for (AcidUtils.ParsedDelta delta : dirInfo.getCurrentDirectories()) {
-              Utilities.LOG14535.info("Adding input " + delta.getPath());
-              finalPaths.add(delta.getPath());
-            }
+        } else if (!hadAcidState) {
+          AcidUtils.Directory dirInfo = AcidUtils.getAcidState(currDir, conf, validTxnList, Ref.from(false), true, null);
+          hadAcidState = true;
+          // TODO: for IOW, we also need to count in base dir, if any
+          for (AcidUtils.ParsedDelta delta : dirInfo.getCurrentDirectories()) {
+            Utilities.FILE_OP_LOGGER.debug("Adding input " + delta.getPath());
+            finalPaths.add(delta.getPath());
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 39808b2..d4282ed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1724,12 +1724,17 @@ public class Hive {
       // TODO: this assumes both paths are qualified; which they are, currently.
       if (isMmTableWrite && loadPath.equals(newPartPath)) {
         // MM insert query, move itself is a no-op.
-        Utilities.LOG14535.info("not moving " + loadPath + " to " + newPartPath + " (MM)");
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("not moving " + loadPath + " to " + newPartPath + " (MM)");
+        }
         assert !isAcid;
         if (areEventsForDmlNeeded(tbl, oldPart)) {
           newFiles = listFilesCreatedByQuery(loadPath, txnId, stmtId);
         }
-        Utilities.LOG14535.info("maybe deleting stuff from " + oldPartPath + " (new " + newPartPath + ") for replace");
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("maybe deleting stuff from " + oldPartPath
+              + " (new " + newPartPath + ") for replace");
+        }
         if (replace && oldPartPath != null) {
           boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
           deleteOldPathForReplace(newPartPath, oldPartPath, getConf(), isAutoPurge,
@@ -1745,7 +1750,9 @@ public class Hive {
           destPath = new Path(destPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
           filter = replace ? new JavaUtils.IdPathFilter(txnId, stmtId, false, true) : filter;
         }
-        Utilities.LOG14535.info("moving " + loadPath + " to " + destPath);
+        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+          Utilities.FILE_OP_LOGGER.trace("moving " + loadPath + " to " + destPath);
+        }
         if (replace || (oldPart == null && !isAcid)) {
           boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
           replaceFiles(tbl.getPath(), loadPath, destPath, oldPartPath, getConf(),
@@ -1916,7 +1923,9 @@ private void walkDirTree(FileStatus fSta, FileSystem fSys,
 
   /* Base Case. It's leaf. */
   if (!fSta.isDir()) {
-    Utilities.LOG14535.info("Processing LB leaf " + fSta.getPath());
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Processing LB leaf " + fSta.getPath());
+    }
     /* construct one location map if not exists. */
     constructOneLBLocationMap(fSta, skewedColValueLocationMaps, newPartPath, skewedInfo);
     return;
@@ -1925,7 +1934,9 @@ private void walkDirTree(FileStatus fSta, FileSystem fSys,
   /* dfs. */
   FileStatus[] children = fSys.listStatus(fSta.getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER);
   if (children != null) {
-    Utilities.LOG14535.info("Processing LB dir " + fSta.getPath());
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Processing LB dir " + fSta.getPath());
+    }
     for (FileStatus child : children) {
       walkDirTree(child, fSys, skewedColValueLocationMaps, newPartPath, skewedInfo);
     }
@@ -1974,8 +1985,10 @@ private void constructOneLBLocationMap(FileStatus fSta,
   for (int i = 0; i < (dirNames.length - dirsToTake); ++i) {
     lbdPath = lbdPath.getParent();
   }
-  Utilities.LOG14535.info("Saving LB location " + lbdPath + " based on "
+  if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+    Utilities.FILE_OP_LOGGER.trace("Saving LB location " + lbdPath + " based on "
       + colCount + " keys and " + fSta.getPath());
+  }
   if ((skewedValue.size() > 0) && (skewedValue.size() == colCount)
       && !skewedColValueLocationMaps.containsKey(skewedValue)) {
     skewedColValueLocationMaps.put(skewedValue, lbdPath.toString());
@@ -1993,7 +2006,6 @@ private void constructOneLBLocationMap(FileStatus fSta,
    */
   private Map<List<String>, String> constructListBucketingLocationMap(Path newPartPath,
       SkewedInfo skewedInfo) throws IOException, FileNotFoundException {
-    Utilities.LOG14535.info("Constructing list bucketing map for " + newPartPath);
     Map<List<String>, String> skewedColValueLocationMaps = new HashMap<List<String>, String>();
     FileSystem fSys = newPartPath.getFileSystem(conf);
     walkDirTree(fSys.getFileStatus(newPartPath),
@@ -2021,7 +2033,6 @@ private void constructOneLBLocationMap(FileStatus fSta,
             throw new HiveException("partition " + s.getPath() + " is not a directory!");
           }
           Path dpPath = s.getPath();
-          Utilities.LOG14535.info("Found DP " + dpPath);
           validPartitions.add(dpPath);
         }
       } else {
@@ -2034,7 +2045,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
           for (int i = 0; i < numLB; ++i) {
             dpPath = dpPath.getParent(); // Now skip the LB directories, if any...
           }
-          Utilities.LOG14535.info("Found DP " + dpPath);
+          if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+            Utilities.FILE_OP_LOGGER.trace("Found DP " + dpPath);
+          }
           validPartitions.add(dpPath);
         }
       }
@@ -2111,7 +2124,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
         final LinkedHashMap<String, String> fullPartSpec = Maps.newLinkedHashMap(partSpec);
         if (!Warehouse.makeSpecFromName(
             fullPartSpec, partPath, new HashSet<String>(partSpec.keySet()))) {
-          Utilities.LOG14535.warn("Ignoring invalid DP directory " + partPath);
+          Utilities.FILE_OP_LOGGER.warn("Ignoring invalid DP directory " + partPath);
           continue;
         }
         futures.add(pool.submit(new Callable<Void>() {
@@ -2123,7 +2136,6 @@ private void constructOneLBLocationMap(FileStatus fSta,
               LOG.info("New loading path = " + partPath + " with partSpec " + fullPartSpec);
 
               // load the partition
-              Utilities.LOG14535.info("loadPartition called for DPP from " + partPath + " to " + tbl.getTableName());
               Partition newPartition = loadPartition(partPath, tbl, fullPartSpec,
                   replace, true, numLB > 0,
                   false, isAcid, hasFollowingStatsTask, txnId, stmtId);
@@ -2227,7 +2239,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
     }
     // TODO: this assumes both paths are qualified; which they are, currently.
     if (isMmTable && loadPath.equals(tbl.getPath())) {
-      Utilities.LOG14535.info("not moving " + loadPath + " to " + tbl.getPath());
+      Utilities.FILE_OP_LOGGER.debug("not moving " + loadPath + " to " + tbl.getPath());
       if (replace) {
         Path tableDest = tbl.getPath();
         boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
@@ -2245,7 +2257,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
         destPath = new Path(destPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
         filter = replace ? new JavaUtils.IdPathFilter(txnId, stmtId, false, true) : filter;
       }
-      Utilities.LOG14535.info("moving " + loadPath + " to " + tblPath + " (replace = " + replace + ")");
+      Utilities.FILE_OP_LOGGER.debug("moving " + loadPath + " to " + tblPath + " (replace = " + replace + ")");
       if (replace) {
         boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
         replaceFiles(tblPath, loadPath, destPath, tblPath,
@@ -3431,7 +3443,9 @@ private void constructOneLBLocationMap(FileStatus fSta,
                 Executors.newFixedThreadPool(conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25),
                 new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Move-Thread-%d").build()) : null;
             if (destIsSubDirOfSrc && !destFs.exists(destf)) {
-              Utilities.LOG14535.info("Creating " + destf);
+              if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+                Utilities.FILE_OP_LOGGER.trace("Creating " + destf);
+              }
               destFs.mkdirs(destf);
             }
             /* Move files one by one because source is a subdirectory of destination */
@@ -3831,7 +3845,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
 
   private void deleteOldPathForReplace(Path destPath, Path oldPath, HiveConf conf, boolean purge,
       PathFilter pathFilter, boolean isMmTable, int lbLevels) throws HiveException {
-    Utilities.LOG14535.info("Deleting old paths for replace in " + destPath + " and old path " + oldPath);
+    Utilities.FILE_OP_LOGGER.debug("Deleting old paths for replace in " + destPath
+        + " and old path " + oldPath);
     boolean isOldPathUnderDestf = false;
     try {
       FileSystem oldFs = oldPath.getFileSystem(conf);
@@ -3860,7 +3875,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
             if (!lbDir.isDirectory()) {
               throw new HiveException("Unexpected path during overwrite: " + lbPath);
             }
-            Utilities.LOG14535.info("Cleaning up LB directory " + lbPath);
+            Utilities.FILE_OP_LOGGER.info("Cleaning up LB directory " + lbPath);
             cleanUpOneDirectoryForReplace(lbPath, oldFs, pathFilter, conf, purge);
           }
         }
@@ -3882,11 +3897,13 @@ private void constructOneLBLocationMap(FileStatus fSta,
       PathFilter pathFilter, HiveConf conf, boolean purge) throws IOException, HiveException {
     FileStatus[] statuses = fs.listStatus(path, pathFilter);
     if (statuses == null || statuses.length == 0) return;
-    String s = "Deleting files under " + path + " for replace: ";
-    for (FileStatus file : statuses) {
-      s += file.getPath().getName() + ", ";
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      String s = "Deleting files under " + path + " for replace: ";
+      for (FileStatus file : statuses) {
+        s += file.getPath().getName() + ", ";
+      }
+      Utilities.FILE_OP_LOGGER.trace(s);
     }
-    Utilities.LOG14535.info(s);
     if (!trashFiles(fs, statuses, conf, purge)) {
       throw new HiveException("Old path " + path + " has not been cleaned up.");
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
index 41bbdfa..3c25896 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
@@ -407,7 +407,6 @@ class TextMetaDataFormatter implements MetaDataFormatter {
   }
 
   private void processDir(FileStatus status, FileSystem fs, FileData fd) throws IOException {
-    Utilities.LOG14535.info("Processing dir for status: " + status.getPath());
     long accessTime = status.getAccessTime();
     long updateTime = status.getModificationTime();
     if (accessTime > fd.lastAccessTime) {

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
index 02bdd91..66ea4e2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
@@ -265,7 +265,7 @@ abstract public class AbstractBucketJoinProc implements NodeProcessor {
 
       Table tbl = tso.getConf().getTableMetadata();
       if (MetaStoreUtils.isInsertOnlyTable(tbl.getParameters())) {
-        Utilities.LOG14535.debug("No bucketed join on MM table " + tbl.getTableName());
+        Utilities.FILE_OP_LOGGER.debug("No bucketed join on MM table " + tbl.getTableName());
         return false;
       }
       if (tbl.isPartitioned()) {

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index 6a56407..9ffda7e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -1266,9 +1266,11 @@ public final class GenMapRedUtils {
     // 1. create the operator tree
     //
     FileSinkDesc fsInputDesc = fsInput.getConf();
-    Utilities.LOG14535.info("Creating merge work from " + System.identityHashCode(fsInput)
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Creating merge work from " + System.identityHashCode(fsInput)
         + " with write ID " + (fsInputDesc.isMmTable() ? fsInputDesc.getTransactionId() : null)
         + " into " + finalName);
+    }
 
     boolean isBlockMerge = (conf.getBoolVar(ConfVars.HIVEMERGERCFILEBLOCKLEVEL) &&
         fsInputDesc.getTableInfo().getInputFileFormatClass().equals(RCFileInputFormat.class)) ||
@@ -1367,12 +1369,11 @@ public final class GenMapRedUtils {
     MoveWork dummyMv = null;
     if (srcMmWriteId == null) {
       // Only create the movework for non-MM table. No action needed for a MM table.
-      Utilities.LOG14535.info("creating dummy movetask for merge (with lfd)");
       dummyMv = new MoveWork(null, null, null,
           new LoadFileDesc(inputDirName, finalName, true, null, null, false), false,
           SessionState.get().getLineageState());
     } else {
-      // TODO# create the noop MoveWork to avoid q file changes for now. Should be removed w/the flag just before merge
+      // TODO# noop MoveWork to avoid q file changes in HIVE-14990. Remove (w/the flag) after merge.
       dummyMv = new MoveWork(null, null, null,
           new LoadFileDesc(inputDirName, finalName, true, null, null, false), false,
           SessionState.get().getLineageState());
@@ -1383,7 +1384,6 @@ public final class GenMapRedUtils {
     // can only be triggered for a merge that's part of insert for now; MM tables do not support
     // concatenate. Keeping the old logic for non-MM tables with temp directories and stuff.
     Path fsopPath = srcMmWriteId != null ? fsInputDesc.getFinalDirName() : finalName;
-    Utilities.LOG14535.info("Looking for MoveTask to make it dependant on the conditional tasks");
 
     Task<MoveWork> mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(
         mvTasks, fsopPath, fsInputDesc.isMmTable());
@@ -1565,7 +1565,6 @@ public final class GenMapRedUtils {
     TableDesc tblDesc = fsDesc.getTableInfo();
     aliases.add(inputDirStr); // dummy alias: just use the input path
 
-    Utilities.LOG14535.info("createMRWorkForMergingFiles for " + inputDir);
     // constructing the default MapredWork
     MapredWork cMrPlan = GenMapRedUtils.getMapRedWorkFromConf(conf);
     MapWork cplan = cMrPlan.getMapWork();
@@ -1616,7 +1615,9 @@ public final class GenMapRedUtils {
           + " format other than RCFile or ORCFile");
     }
 
-    Utilities.LOG14535.info("creating mergefilework from " + inputDirs + " to " + finalName);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("creating mergefilework from " + inputDirs + " to " + finalName);
+    }
     // create the merge file work
     MergeFileWork work = new MergeFileWork(inputDirs, finalName,
         hasDynamicPartitions, tblDesc.getInputFileFormatClass().getName(), tblDesc);
@@ -1749,7 +1750,9 @@ public final class GenMapRedUtils {
       Task<? extends Serializable> currTask, MoveWork mvWork, Serializable mergeWork,
       Path condInputPath, Path condOutputPath, Task<MoveWork> moveTaskToLink,
       DependencyCollectionTask dependencyTask) {
-    Utilities.LOG14535.info("Creating conditional merge task for " + condInputPath);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Creating conditional merge task for " + condInputPath);
+    }
     // Create a dummy task if no move is needed.
     Serializable moveWork = mvWork != null ? mvWork : new DependencyCollectionWork();
 
@@ -1849,9 +1852,11 @@ public final class GenMapRedUtils {
       } else if (mvWork.getLoadTableWork() != null) {
         srcDir = mvWork.getLoadTableWork().getSourcePath();
       }
-      Utilities.LOG14535.info("Observing MoveWork " + System.identityHashCode(mvWork)
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("Observing MoveWork " + System.identityHashCode(mvWork)
           + " with " + srcDir + "(from " + (isLfd ? "LFD" : "LTD") + ") while looking for "
           + fsopFinalDir + "(mm = " + isMmFsop + ")");
+      }
 
       if ((srcDir != null) && srcDir.equals(fsopFinalDir)) {
         return mvTsk;
@@ -1958,11 +1963,17 @@ public final class GenMapRedUtils {
         if (fileSinkDesc.isLinkedFileSink()) {
           for (FileSinkDesc fsConf : fileSinkDesc.getLinkedFileSinkDesc()) {
             fsConf.setDirName(new Path(tmpDir, fsConf.getDirName().getName()));
-            Utilities.LOG14535.info("createMoveTask setting tmpDir for LinkedFileSink chDir " + fsConf.getDirName() + "; dest was " + fileSinkDesc.getDestPath());
+            if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+              Utilities.FILE_OP_LOGGER.trace("createMoveTask setting tmpDir for LinkedFileSink chDir "
+                + fsConf.getDirName() + "; dest was " + fileSinkDesc.getDestPath());
+            }
           }
         } else {
           fileSinkDesc.setDirName(tmpDir);
-          Utilities.LOG14535.info("createMoveTask setting tmpDir  chDir " + tmpDir + "; dest was " + fileSinkDesc.getDestPath());
+          if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+            Utilities.FILE_OP_LOGGER.trace("createMoveTask setting tmpDir chDir "
+              + tmpDir + "; dest was " + fileSinkDesc.getDestPath());
+          }
         }
       }
     }
@@ -1970,7 +1981,6 @@ public final class GenMapRedUtils {
     Task<MoveWork> mvTask = null;
 
     if (!chDir) {
-      Utilities.LOG14535.info("Looking for MoveTask from createMoveTask");
       mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(
           mvTasks, fsOp.getConf().getFinalDirName(), fsOp.getConf().isMmTable());
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
index bc26c5e..3f38bd1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
@@ -223,7 +223,10 @@ public final class UnionProcFactory {
           FileSinkDesc fileSinkDesc = (FileSinkDesc) fileSinkOp.getConf().clone();
           fileSinkDesc.setDirName(new Path(parentDirName, AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + parent.getIdentifier()));
           fileSinkDesc.setLinkedFileSink(true);
-          Utilities.LOG14535.info("Created LinkedFileSink for union " + fileSinkDesc.getDirName() + "; parent " + parentDirName);
+          if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+            Utilities.FILE_OP_LOGGER.trace("Created LinkedFileSink for union " + fileSinkDesc.getDirName()
+                + "; parent " + parentDirName);
+          }
           parent.setChildOperators(null);
           Operator<? extends OperatorDesc> tmpFileSinkOp =
             OperatorFactory.getAndMakeChild(fileSinkDesc, parent.getSchema(), parent);

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index cf1d14c..01cb2b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -316,7 +316,8 @@ public class GenTezUtils {
         linked.add(desc);
 
         desc.setDirName(new Path(path, AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + linked.size()));
-        Utilities.LOG14535.info("removing union - new desc with " + desc.getDirName() + "; parent " + path);
+        Utilities.FILE_OP_LOGGER.debug("removing union - new desc with "
+            + desc.getDirName() + "; parent " + path);
         desc.setLinkedFileSink(true);
         desc.setLinkedFileSinkDesc(linked);
       }
@@ -384,7 +385,8 @@ public class GenTezUtils {
       // If underlying data is RCFile or OrcFile, RCFileBlockMerge task or
       // OrcFileStripeMerge task would be created.
       LOG.info("using CombineHiveInputformat for the merge job");
-      Utilities.LOG14535.info("will generate MR work for merging files from " + fileSink.getConf().getDirName() + " to " + finalName);
+      Utilities.FILE_OP_LOGGER.debug("will generate MR work for merging files from "
+          + fileSink.getConf().getDirName() + " to " + finalName);
       GenMapRedUtils.createMRWorkForMergingFiles(fileSink, finalName,
           context.dependencyTask, context.moveTask,
           hconf, context.currentTask);

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index b9f28eb..41b5156 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -367,9 +367,11 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
     Path dataPath = new Path(fromURI.toString(), EximUtil.DATA_PATH_NAME);
     Path destPath = !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtPath)
         : new Path(tgtPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
-    Utilities.LOG14535.info("adding import work for table with source location: "
-        + dataPath + "; table: " + tgtPath + "; copy destination " + destPath + "; mm "
-        + txnId + " (src " + isSourceMm + ") for " + (table == null ? "a new table" : table.getTableName()));
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("adding import work for table with source location: " +
+        dataPath + "; table: " + tgtPath + "; copy destination " + destPath + "; mm " + txnId +
+        " (src " + isSourceMm + ") for " + (table == null ? "a new table" : table.getTableName()));
+    }
 
     Task<?> copyTask = null;
     if (replicationSpec.isInReplicationScope()) {
@@ -458,9 +460,11 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       Path destPath = !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtLocation)
           : new Path(tgtLocation, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
       Path moveTaskSrc =  !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? destPath : tgtLocation;
-      Utilities.LOG14535.info("adding import work for partition with source location: "
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("adding import work for partition with source location: "
           + srcLocation + "; target: " + tgtLocation + "; copy dest " + destPath + "; mm "
           + txnId + " (src " + isSourceMm + ") for " + partSpecToString(partSpec.getPartSpec()));
+      }
 
 
       Task<?> copyTask = null;
@@ -484,9 +488,6 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       loadTableWork.setTxnId(txnId);
       loadTableWork.setStmtId(stmtId);
       loadTableWork.setInheritTableSpecs(false);
-      // Do not commit the write ID from each task; need to commit once.
-      // TODO: we should just change the import to use a single MoveTask, like dynparts.
-      loadTableWork.setIntermediateInMmWrite(isAcid(txnId));
       Task<?> loadPartTask = TaskFactory.get(new MoveWork(
           x.getInputs(), x.getOutputs(), loadTableWork, null, false, SessionState.get().getLineageState()), x.getConf());
       copyTask.addDependentTask(loadPartTask);

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index df50aab..29d6520 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6834,7 +6834,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
       } else {
         queryTmpdir = ctx.getTempDirForPath(dest_path, true);
       }
-      Utilities.LOG14535.info("create filesink w/DEST_TABLE specifying " + queryTmpdir + " from " + dest_path);
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("create filesink w/DEST_TABLE specifying " + queryTmpdir
+            + " from " + dest_path);
+      }
       if (dpCtx != null) {
         // set the root of the temporary path where dynamic partition columns will populate
         dpCtx.setRootPath(queryTmpdir);
@@ -6914,7 +6917,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
 
       isMmTable = MetaStoreUtils.isInsertOnlyTable(dest_tab.getParameters());
       queryTmpdir = isMmTable ? dest_path : ctx.getTempDirForPath(dest_path, true);
-      Utilities.LOG14535.info("create filesink w/DEST_PARTITION specifying " + queryTmpdir + " from " + dest_path);
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("create filesink w/DEST_PARTITION specifying "
+            + queryTmpdir + " from " + dest_path);
+      }
       table_desc = Utilities.getTableDesc(dest_tab);
 
       // Add sorting/bucketing if needed
@@ -6996,7 +7002,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         try {
           Path qPath = FileUtils.makeQualified(dest_path, conf);
           queryTmpdir = isMmTable ? qPath : ctx.getTempDirForPath(qPath, true);
-          Utilities.LOG14535.info("Setting query directory " + queryTmpdir + " from " + dest_path + " (" + isMmTable + ")");
+          if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+            Utilities.FILE_OP_LOGGER.trace("Setting query directory " + queryTmpdir
+                  + " from " + dest_path + " (" + isMmTable + ")");
+          }
         } catch (Exception e) { 
           throw new SemanticException("Error creating temporary folder on: "
               + dest_path, e);

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 752a934..a63f709 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -342,9 +342,14 @@ public abstract class TaskCompiler {
       dataSinkForCtas.setDirName(location);
       location = new Path(location, AcidUtils.deltaSubdir(txnIdForCtas, txnIdForCtas, stmtId));
       lfd.setSourcePath(location);
-      Utilities.LOG14535.info("Setting MM CTAS to  " + location);
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("Setting MM CTAS to " + location);
+      }
+    }
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Location for LFD is being set to "
+        + location + "; moving from " + lfd.getSourcePath());
     }
-    Utilities.LOG14535.info("Location for LFD is being set to " + location + "; moving from " + lfd.getSourcePath());
     lfd.setTargetDir(location);
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
index a0e024c..129347b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
@@ -171,7 +171,7 @@ public class ConditionalResolverMergeFiles implements ConditionalResolver,
           if(lbLevel == 0) {
             // static partition without list bucketing
             long totalSz = getMergeSize(inpFs, dirPath, avgConditionSize);
-            Utilities.LOG14535.info("merge resolve simple case - totalSz " + totalSz + " from " + dirPath);
+            Utilities.FILE_OP_LOGGER.debug("merge resolve simple case - totalSz " + totalSz + " from " + dirPath);
 
             if (totalSz >= 0) { // add the merge job
               setupMapRedWork(conf, work, trgtSize, totalSz);
@@ -186,7 +186,7 @@ public class ConditionalResolverMergeFiles implements ConditionalResolver,
           }
         }
       } else {
-        Utilities.LOG14535.info("Resolver returning movetask for " + dirPath);
+        Utilities.FILE_OP_LOGGER.info("Resolver returning movetask for " + dirPath);
         resTsks.add(mvTask);
       }
     } catch (IOException e) {
@@ -229,7 +229,6 @@ public class ConditionalResolverMergeFiles implements ConditionalResolver,
       Task<? extends Serializable> mrTask, Task<? extends Serializable> mrAndMvTask, Path dirPath,
       FileSystem inpFs, ConditionalResolverMergeFilesCtx ctx, MapWork work, int dpLbLevel)
       throws IOException {
-    Utilities.LOG14535.info("generateActualTasks for " + dirPath);
     DynamicPartitionCtx dpCtx = ctx.getDPCtx();
     // get list of dynamic partitions
     FileStatus[] status = HiveStatsUtils.getFileStatusRecurse(dirPath, dpLbLevel, inpFs);
@@ -240,7 +239,7 @@ public class ConditionalResolverMergeFiles implements ConditionalResolver,
     Path path = ptpi.keySet().iterator().next();
     PartitionDesc partDesc = ptpi.get(path);
     TableDesc tblDesc = partDesc.getTableDesc();
-    Utilities.LOG14535.info("merge resolver removing " + path);
+    Utilities.FILE_OP_LOGGER.debug("merge resolver removing " + path);
     work.removePathToPartitionInfo(path); // the root path is not useful anymore
 
     // cleanup pathToAliases
@@ -263,14 +262,14 @@ public class ConditionalResolverMergeFiles implements ConditionalResolver,
         PartitionDesc pDesc = (dpCtx != null) ? generateDPFullPartSpec(dpCtx, status, tblDesc, i)
             : partDesc;
         if (pDesc == null) {
-          Utilities.LOG14535.warn("merger ignoring invalid DP path " + status[i].getPath());
+          Utilities.FILE_OP_LOGGER.warn("merger ignoring invalid DP path " + status[i].getPath());
           continue;
         }
-        Utilities.LOG14535.info("merge resolver will merge " + status[i].getPath());
+        Utilities.FILE_OP_LOGGER.debug("merge resolver will merge " + status[i].getPath());
         work.resolveDynamicPartitionStoredAsSubDirsMerge(conf, status[i].getPath(), tblDesc,
             aliases, pDesc);
       } else {
-        Utilities.LOG14535.info("merge resolver will move " + status[i].getPath());
+        Utilities.FILE_OP_LOGGER.debug("merge resolver will move " + status[i].getPath());
 
         toMove.add(status[i].getPath());
       }
@@ -375,7 +374,7 @@ public class ConditionalResolverMergeFiles implements ConditionalResolver,
       long totalSz = 0;
       int numFiles = 0;
       for (FileStatus fStat : fStats) {
-        Utilities.LOG14535.info("Resolver looking at " + fStat.getPath());
+        Utilities.FILE_OP_LOGGER.debug("Resolver looking at " + fStat.getPath());
         if (fStat.isDir()) {
           AverageSize avgSzDir = getAverageSize(inpFs, fStat.getPath());
           if (avgSzDir.getTotalSize() < 0) {

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
index d90acce..30d9912 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
@@ -73,7 +73,9 @@ public class LoadFileDesc extends LoadDesc implements Serializable {
       final boolean isDfsDir, final String columns,
       final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) {
     super(sourcePath, writeType);
-    Utilities.LOG14535.info("creating LFD from " + sourcePath + " to " + targetDir);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("creating LFD from " + sourcePath + " to " + targetDir);
+    }
     this.targetDir = targetDir;
     this.isDfsDir = isDfsDir;
     this.columns = columns;

[30/50] [abbrv] hive git commit: HIVE-15212 : merge branch into master - some test/out fixes 3 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-15212 : merge branch into master - some test/out fixes 3 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6f0be79b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6f0be79b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6f0be79b

Branch: refs/heads/master
Commit: 6f0be79ba93bac5469ceb003c224ed356507476c
Parents: 1fb04f3
Author: sergey <se...@apache.org>
Authored: Thu Sep 21 17:43:27 2017 +1000
Committer: sergey <se...@apache.org>
Committed: Thu Sep 21 17:43:27 2017 +1000

----------------------------------------------------------------------
 ql/src/test/queries/clientpositive/mm_all.q     | 31 +++----
 ql/src/test/results/clientpositive/mm_all.q.out | 94 ++++++--------------
 2 files changed, 42 insertions(+), 83 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/6f0be79b/ql/src/test/queries/clientpositive/mm_all.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_all.q b/ql/src/test/queries/clientpositive/mm_all.q
index e23260f..b93e13e 100644
--- a/ql/src/test/queries/clientpositive/mm_all.q
+++ b/ql/src/test/queries/clientpositive/mm_all.q
@@ -209,11 +209,9 @@ drop table multi0_2_mm;
 create table multi0_1_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only");
 create table multi0_2_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only");
 
---from intermediate
---insert overwrite table multi0_1_mm select key, p
---insert overwrite table multi0_2_mm select p, key;
-insert into table multi0_1_mm select key, p from intermediate;
-insert into table multi0_2_mm select p, key from intermediate;
+from intermediate
+insert overwrite table multi0_1_mm select key, p
+insert overwrite table multi0_2_mm select p, key;
 
 select * from multi0_1_mm order by key, key2;
 select * from multi0_2_mm order by key, key2;
@@ -222,11 +220,10 @@ set hive.merge.mapredfiles=true;
 set hive.merge.sparkfiles=true;
 set hive.merge.tezfiles=true;
 
---from intermediate
---insert into table multi0_1_mm select p, key
---insert overwrite table multi0_2_mm select key, p;
-insert into table multi0_1_mm select p, key from intermediate;
-insert into table multi0_2_mm select key, p from intermediate;
+from intermediate
+insert into table multi0_1_mm select p, key
+insert overwrite table multi0_2_mm select key, p;
+
 select * from multi0_1_mm order by key, key2;
 select * from multi0_2_mm order by key, key2;
 
@@ -243,23 +240,27 @@ create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties(
 from intermediate
 insert into table multi1_mm partition(p=1) select p, key
 insert into table multi1_mm partition(p=2) select key, p;
+
 select * from multi1_mm order by key, key2, p;
---from intermediate
---insert into table multi1_mm partition(p=2) select p, key
---insert overwrite table multi1_mm partition(p=1) select key, p;
-insert into table multi1_mm partition(p=2) select p, key from intermediate;
-insert into table multi1_mm partition(p=1) select key, p from intermediate;
+
+from intermediate
+insert into table multi1_mm partition(p=2) select p, key
+insert overwrite table multi1_mm partition(p=1) select key, p;
+
 select * from multi1_mm order by key, key2, p;
 
 from intermediate
 insert into table multi1_mm partition(p) select p, key, p
 insert into table multi1_mm partition(p=1) select key, p;
+
 select key, key2, p from multi1_mm order by key, key2, p;
 
 from intermediate
 insert into table multi1_mm partition(p) select p, key, 1
 insert into table multi1_mm partition(p=1) select key, p;
+
 select key, key2, p from multi1_mm order by key, key2, p;
+
 drop table multi1_mm;
 
 

http://git-wip-us.apache.org/repos/asf/hive/blob/6f0be79b/ql/src/test/results/clientpositive/mm_all.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/mm_all.q.out b/ql/src/test/results/clientpositive/mm_all.q.out
index ea60414..4162ce9 100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@ -1342,36 +1342,28 @@ POSTHOOK: query: create table multi0_2_mm (key int, key2 int)  tblproperties("tr
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@multi0_2_mm
-PREHOOK: query: insert into table multi0_1_mm select key, p from intermediate
+PREHOOK: query: from intermediate
+insert overwrite table multi0_1_mm select key, p
+insert overwrite table multi0_2_mm select p, key
 PREHOOK: type: QUERY
 PREHOOK: Input: default@intermediate
 PREHOOK: Input: default@intermediate@p=455
 PREHOOK: Input: default@intermediate@p=456
 PREHOOK: Input: default@intermediate@p=457
 PREHOOK: Output: default@multi0_1_mm
-POSTHOOK: query: insert into table multi0_1_mm select key, p from intermediate
+PREHOOK: Output: default@multi0_2_mm
+POSTHOOK: query: from intermediate
+insert overwrite table multi0_1_mm select key, p
+insert overwrite table multi0_2_mm select p, key
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@intermediate
 POSTHOOK: Input: default@intermediate@p=455
 POSTHOOK: Input: default@intermediate@p=456
 POSTHOOK: Input: default@intermediate@p=457
 POSTHOOK: Output: default@multi0_1_mm
+POSTHOOK: Output: default@multi0_2_mm
 POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
-PREHOOK: query: insert into table multi0_2_mm select p, key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@multi0_2_mm
-POSTHOOK: query: insert into table multi0_2_mm select p, key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@multi0_2_mm
 POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 PREHOOK: query: select * from multi0_1_mm order by key, key2
@@ -1396,42 +1388,28 @@ POSTHOOK: query: select * from multi0_2_mm order by key, key2
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@multi0_2_mm
 #### A masked pattern was here ####
-455	97
-455	98
-456	0
-456	10
-457	100
-457	103
-PREHOOK: query: insert into table multi0_1_mm select p, key from intermediate
+PREHOOK: query: from intermediate
+insert into table multi0_1_mm select p, key
+insert overwrite table multi0_2_mm select key, p
 PREHOOK: type: QUERY
 PREHOOK: Input: default@intermediate
 PREHOOK: Input: default@intermediate@p=455
 PREHOOK: Input: default@intermediate@p=456
 PREHOOK: Input: default@intermediate@p=457
 PREHOOK: Output: default@multi0_1_mm
-POSTHOOK: query: insert into table multi0_1_mm select p, key from intermediate
+PREHOOK: Output: default@multi0_2_mm
+POSTHOOK: query: from intermediate
+insert into table multi0_1_mm select p, key
+insert overwrite table multi0_2_mm select key, p
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@intermediate
 POSTHOOK: Input: default@intermediate@p=455
 POSTHOOK: Input: default@intermediate@p=456
 POSTHOOK: Input: default@intermediate@p=457
 POSTHOOK: Output: default@multi0_1_mm
+POSTHOOK: Output: default@multi0_2_mm
 POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table multi0_2_mm select key, p from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@multi0_2_mm
-POSTHOOK: query: insert into table multi0_2_mm select key, p from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@multi0_2_mm
 POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 PREHOOK: query: select * from multi0_1_mm order by key, key2
@@ -1462,18 +1440,6 @@ POSTHOOK: query: select * from multi0_2_mm order by key, key2
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@multi0_2_mm
 #### A masked pattern was here ####
-0	456
-10	456
-97	455
-98	455
-100	457
-103	457
-455	97
-455	98
-456	0
-456	10
-457	100
-457	103
 PREHOOK: query: drop table multi0_1_mm
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@multi0_1_mm
@@ -1550,38 +1516,30 @@ POSTHOOK: Input: default@multi1_mm@p=2
 456	10	1
 457	100	1
 457	103	1
-PREHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@multi1_mm@p=2
-POSTHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@multi1_mm@p=2
-POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
-POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
+PREHOOK: query: from intermediate
+insert into table multi1_mm partition(p=2) select p, key
+insert overwrite table multi1_mm partition(p=1) select key, p
 PREHOOK: type: QUERY
 PREHOOK: Input: default@intermediate
 PREHOOK: Input: default@intermediate@p=455
 PREHOOK: Input: default@intermediate@p=456
 PREHOOK: Input: default@intermediate@p=457
 PREHOOK: Output: default@multi1_mm@p=1
-POSTHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
+PREHOOK: Output: default@multi1_mm@p=2
+POSTHOOK: query: from intermediate
+insert into table multi1_mm partition(p=2) select p, key
+insert overwrite table multi1_mm partition(p=1) select key, p
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@intermediate
 POSTHOOK: Input: default@intermediate@p=455
 POSTHOOK: Input: default@intermediate@p=456
 POSTHOOK: Input: default@intermediate@p=457
 POSTHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: Output: default@multi1_mm@p=2
 POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 PREHOOK: query: select * from multi1_mm order by key, key2, p
 PREHOOK: type: QUERY
 PREHOOK: Input: default@multi1_mm

[18/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --cc common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index ff8a7aa,e4b09a2..21c4cc9
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@@ -1864,14 -1855,9 +1857,16 @@@ public class HiveConf extends Configura
          " of the lock manager is dumped to log file.  This is for debugging.  See also " +
          "hive.lock.numretries and hive.lock.sleep.between.retries."),
  
-     HIVE_TXN_OPERATIONAL_PROPERTIES("hive.txn.operational.properties", 0,
+     HIVE_TXN_OPERATIONAL_PROPERTIES("hive.txn.operational.properties", 1,
 -        "This is intended to be used as an internal property for future versions of ACID. (See\n" +
 -        "HIVE-14035 for details.)"),
 +        "Sets the operational properties that control the appropriate behavior for various\n"
 +        + "versions of the Hive ACID subsystem. Mostly it is intended to be used as an internal property\n"
 +        + "for future versions of ACID. (See HIVE-14035 for details.)\n"
 +        + "0: Turn on the legacy mode for ACID\n"
 +        + "1: Enable split-update feature found in the newer version of Hive ACID subsystem\n"
 +        + "2: Hash-based merge, which combines delta files using GRACE hash join based approach (not implemented)\n"
-         + "3: Make the table 'quarter-acid' as it only supports insert. But it doesn't require ORC or bucketing."),
++        + "3: Make the table 'quarter-acid' as it only supports insert. But it doesn't require ORC or bucketing.\n"
++        + "This is intended to be used as an internal property for future versions of ACID. (See\n" +
++          "HIVE-14035 for details.)"),
  
      HIVE_MAX_OPEN_TXNS("hive.max.open.txns", 100000, "Maximum number of open transactions. If \n" +
          "current open transactions reach this limit, future open transaction requests will be \n" +

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 913e333,5812a1b..ab5f7b7
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@@ -192,13 -194,14 +197,13 @@@ public class HiveMetaStore extends Thri
      };
    };
  
 -  /**
 -   * default port on which to start the Hive server
 -   */
    public static final String ADMIN = "admin";
    public static final String PUBLIC = "public";
 +  /** MM write states. */
 +  public static final char MM_WRITE_OPEN = 'o', MM_WRITE_COMMITTED = 'c', MM_WRITE_ABORTED = 'a';
  
    private static HadoopThriftAuthBridge.Server saslServer;
-   private static HiveDelegationTokenManager delegationTokenManager;
+   private static MetastoreDelegationTokenManager delegationTokenManager;
    private static boolean useSasl;
  
    public static final String NO_FILTER_STRING = "";
@@@ -437,41 -495,14 +497,15 @@@
          }
        }
  
-       //Start Metrics for Embedded mode
+       //Start Metrics
        if (hiveConf.getBoolVar(ConfVars.METASTORE_METRICS)) {
-         try {
-           MetricsFactory.init(hiveConf);
-         } catch (Exception e) {
-           // log exception, but ignore inability to start
-           LOG.error("error in Metrics init: " + e.getClass().getName() + " "
-               + e.getMessage(), e);
-         }
-       }
- 
-       Metrics metrics = MetricsFactory.getInstance();
-       if (metrics != null && hiveConf.getBoolVar(ConfVars.METASTORE_INIT_METADATA_COUNT_ENABLED)) {
          LOG.info("Begin calculating metadata count metrics.");
+         Metrics.initialize(hiveConf);
+         databaseCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_DATABASES);
+         tableCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_TABLES);
+         partCount = Metrics.getOrCreateGauge(MetricsConstants.TOTAL_PARTITIONS);
          updateMetrics();
-         LOG.info("Finished metadata count metrics: " + initDatabaseCount + " databases, " + initTableCount +
-           " tables, " + initPartCount + " partitions.");
-         metrics.addGauge(MetricsConstant.INIT_TOTAL_DATABASES, new MetricsVariable<Object>() {
-           @Override
-           public Object getValue() {
-             return initDatabaseCount;
-           }
-         });
-         metrics.addGauge(MetricsConstant.INIT_TOTAL_TABLES, new MetricsVariable<Object>() {
-           @Override
-           public Object getValue() {
-             return initTableCount;
-           }
-         });
-         metrics.addGauge(MetricsConstant.INIT_TOTAL_PARTITIONS, new MetricsVariable<Object>() {
-           @Override
-           public Object getValue() {
-             return initPartCount;
-           }
-         });
++
        }
  
        preListeners = MetaStoreUtils.getMetaStoreListeners(MetaStorePreEventListener.class,

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 8e89477,70451c4..37da2f8
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@@ -59,12 -60,11 +60,12 @@@ import org.apache.hadoop.hive.conf.Hive
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
  import org.apache.hadoop.hive.conf.HiveConfUtil;
  import org.apache.hadoop.hive.metastore.api.*;
 +import org.apache.hadoop.hive.metastore.TableType;
+ import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
  import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+ import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
  import org.apache.hadoop.hive.metastore.txn.TxnUtils;
- import org.apache.hadoop.hive.shims.ShimLoader;
  import org.apache.hadoop.hive.shims.Utils;
- import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
  import org.apache.hadoop.security.UserGroupInformation;
  import org.apache.hadoop.util.StringUtils;
  import org.apache.thrift.TApplicationException;

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index e7bbd62,bbe13fd..49005b9
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@@ -1981,59 -1967,156 +1970,212 @@@ public class MetaStoreUtils 
      return cols;
    }
  
 +  // TODO The following two utility methods can be moved to AcidUtils once no class in metastore is relying on them,
 +  // right now ObjectStore.getAllMmTablesForCleanup is calling these method
 +  /**
 +   * Checks if a table is an ACID table that only supports INSERT, but not UPDATE/DELETE
 +   * @param params table properties
 +   * @return true if table is an INSERT_ONLY table, false otherwise
 +   */
 +  // TODO# also check that transactional is true
 +  public static boolean isInsertOnlyTable(Map<String, String> params) {
 +    return isInsertOnlyTable(params, false);
 +  }
 +
 +  public static boolean isInsertOnlyTable(Map<String, String> params, boolean isCtas) {
 +    String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
 +    return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
 +  }
 +
 +   public static boolean isInsertOnlyTable(Properties params) {
 +    // TODO#  redirect for now - fix before merge
 +    HashMap<String, String> testMap = new HashMap<String, String>();
 +    for (String n  : params.stringPropertyNames()) {
 +      testMap.put(n, params.getProperty(n));
 +    }
 +    return isInsertOnlyTable(testMap);
 +  }
 +
 +   /** The method for altering table props; may set the table to MM, non-MM, or not affect MM. */
 +  public static Boolean isToInsertOnlyTable(Map<String, String> props) {
 +    // TODO# Setting these separately is a very hairy issue in certain combinations, since we
 +    //       cannot decide what type of table this becomes without taking both into account, and
 +    //       in many cases the conversion might be illegal.
 +    //       The only thing we allow is tx = true w/o tx-props, for backward compat.
 +    String transactional = props.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
 +    String transactionalProp = props.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
 +    if (transactional == null && transactionalProp == null) return null; // Not affected.
 +    boolean isSetToTxn = "true".equalsIgnoreCase(transactional);
 +    if (transactionalProp == null) {
 +      if (isSetToTxn) return false; // Assume the full ACID table.
 +      throw new RuntimeException("Cannot change '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL
 +          + "' without '" + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "'");
 +    }
 +    if (!"insert_only".equalsIgnoreCase(transactionalProp)) return false; // Not MM.
 +    if (!isSetToTxn) {
 +      throw new RuntimeException("Cannot set '"
 +          + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "' to 'insert_only' without "
 +          + "setting '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL + "' to 'true'");
 +    }
 +    return true;
 +  }
 +
 +  public static boolean isRemovedInsertOnlyTable(Set<String> removedSet) {
 +    boolean hasTxn = removedSet.contains(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL),
 +        hasProps = removedSet.contains(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
 +    return hasTxn || hasProps;
 +  }
++
+   // given a list of partStats, this function will give you an aggr stats
+   public static List<ColumnStatisticsObj> aggrPartitionStats(List<ColumnStatistics> partStats,
+       String dbName, String tableName, List<String> partNames, List<String> colNames,
+       boolean useDensityFunctionForNDVEstimation, double ndvTuner)
+       throws MetaException {
+     // 1. group by the stats by colNames
+     // map the colName to List<ColumnStatistics>
+     Map<String, List<ColumnStatistics>> map = new HashMap<>();
+     for (ColumnStatistics css : partStats) {
+       List<ColumnStatisticsObj> objs = css.getStatsObj();
+       for (ColumnStatisticsObj obj : objs) {
+         List<ColumnStatisticsObj> singleObj = new ArrayList<>();
+         singleObj.add(obj);
+         ColumnStatistics singleCS = new ColumnStatistics(css.getStatsDesc(), singleObj);
+         if (!map.containsKey(obj.getColName())) {
+           map.put(obj.getColName(), new ArrayList<ColumnStatistics>());
+         }
+         map.get(obj.getColName()).add(singleCS);
+       }
+     }
+     return aggrPartitionStats(map,dbName,tableName,partNames,colNames,useDensityFunctionForNDVEstimation, ndvTuner);
+   }
+ 
+   public static List<ColumnStatisticsObj> aggrPartitionStats(
+       Map<String, List<ColumnStatistics>> map, String dbName, String tableName,
+       final List<String> partNames, List<String> colNames,
+       final boolean useDensityFunctionForNDVEstimation,final double ndvTuner) throws MetaException {
+     List<ColumnStatisticsObj> colStats = new ArrayList<>();
+     // 2. Aggregate stats for each column in a separate thread
+     if (map.size()< 1) {
+       //stats are absent in RDBMS
+       LOG.debug("No stats data found for: dbName=" +dbName +" tblName=" + tableName +
+           " partNames= " + partNames + " colNames=" + colNames );
+       return colStats;
+     }
+     final ExecutorService pool = Executors.newFixedThreadPool(Math.min(map.size(), 16),
+         new ThreadFactoryBuilder().setDaemon(true).setNameFormat("aggr-col-stats-%d").build());
+     final List<Future<ColumnStatisticsObj>> futures = Lists.newLinkedList();
+ 
+     long start = System.currentTimeMillis();
+     for (final Entry<String, List<ColumnStatistics>> entry : map.entrySet()) {
+       futures.add(pool.submit(new Callable<ColumnStatisticsObj>() {
+         @Override
+         public ColumnStatisticsObj call() throws Exception {
+           List<ColumnStatistics> css = entry.getValue();
+           ColumnStatsAggregator aggregator = ColumnStatsAggregatorFactory.getColumnStatsAggregator(css
+               .iterator().next().getStatsObj().iterator().next().getStatsData().getSetField(),
+               useDensityFunctionForNDVEstimation, ndvTuner);
+           ColumnStatisticsObj statsObj = aggregator.aggregate(entry.getKey(), partNames, css);
+           return statsObj;
+         }}));
+     }
+     pool.shutdown();
+     for (Future<ColumnStatisticsObj> future : futures) {
+       try {
+         colStats.add(future.get());
+       } catch (InterruptedException | ExecutionException e) {
+         pool.shutdownNow();
+         LOG.debug(e.toString());
+         throw new MetaException(e.toString());
+       }
+     }
+     LOG.debug("Time for aggr col stats in seconds: {} Threads used: {}",
+       ((System.currentTimeMillis() - (double)start))/1000, Math.min(map.size(), 16));
+     return colStats;
+   }
+ 
+ 
+   /**
+    * Produce a hash for the storage descriptor
+    * @param sd storage descriptor to hash
+    * @param md message descriptor to use to generate the hash
+    * @return the hash as a byte array
+    */
+   public static byte[] hashStorageDescriptor(StorageDescriptor sd, MessageDigest md)  {
+     // Note all maps and lists have to be absolutely sorted.  Otherwise we'll produce different
+     // results for hashes based on the OS or JVM being used.
+     md.reset();
+     for (FieldSchema fs : sd.getCols()) {
+       md.update(fs.getName().getBytes(ENCODING));
+       md.update(fs.getType().getBytes(ENCODING));
+       if (fs.getComment() != null) md.update(fs.getComment().getBytes(ENCODING));
+     }
+     if (sd.getInputFormat() != null) {
+       md.update(sd.getInputFormat().getBytes(ENCODING));
+     }
+     if (sd.getOutputFormat() != null) {
+       md.update(sd.getOutputFormat().getBytes(ENCODING));
+     }
+     md.update(sd.isCompressed() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
+     md.update(Integer.toString(sd.getNumBuckets()).getBytes(ENCODING));
+     if (sd.getSerdeInfo() != null) {
+       SerDeInfo serde = sd.getSerdeInfo();
+       if (serde.getName() != null) {
+         md.update(serde.getName().getBytes(ENCODING));
+       }
+       if (serde.getSerializationLib() != null) {
+         md.update(serde.getSerializationLib().getBytes(ENCODING));
+       }
+       if (serde.getParameters() != null) {
+         SortedMap<String, String> params = new TreeMap<>(serde.getParameters());
+         for (Entry<String, String> param : params.entrySet()) {
+           md.update(param.getKey().getBytes(ENCODING));
+           md.update(param.getValue().getBytes(ENCODING));
+         }
+       }
+     }
+     if (sd.getBucketCols() != null) {
+       List<String> bucketCols = new ArrayList<>(sd.getBucketCols());
+       for (String bucket : bucketCols) md.update(bucket.getBytes(ENCODING));
+     }
+     if (sd.getSortCols() != null) {
+       SortedSet<Order> orders = new TreeSet<>(sd.getSortCols());
+       for (Order order : orders) {
+         md.update(order.getCol().getBytes(ENCODING));
+         md.update(Integer.toString(order.getOrder()).getBytes(ENCODING));
+       }
+     }
+     if (sd.getSkewedInfo() != null) {
+       SkewedInfo skewed = sd.getSkewedInfo();
+       if (skewed.getSkewedColNames() != null) {
+         SortedSet<String> colnames = new TreeSet<>(skewed.getSkewedColNames());
+         for (String colname : colnames) md.update(colname.getBytes(ENCODING));
+       }
+       if (skewed.getSkewedColValues() != null) {
+         SortedSet<String> sortedOuterList = new TreeSet<>();
+         for (List<String> innerList : skewed.getSkewedColValues()) {
+           SortedSet<String> sortedInnerList = new TreeSet<>(innerList);
+           sortedOuterList.add(StringUtils.join(sortedInnerList, "."));
+         }
+         for (String colval : sortedOuterList) md.update(colval.getBytes(ENCODING));
+       }
+       if (skewed.getSkewedColValueLocationMaps() != null) {
+         SortedMap<String, String> sortedMap = new TreeMap<>();
+         for (Entry<List<String>, String> smap : skewed.getSkewedColValueLocationMaps().entrySet()) {
+           SortedSet<String> sortedKey = new TreeSet<>(smap.getKey());
+           sortedMap.put(StringUtils.join(sortedKey, "."), smap.getValue());
+         }
+         for (Entry<String, String> e : sortedMap.entrySet()) {
+           md.update(e.getKey().getBytes(ENCODING));
+           md.update(e.getValue().getBytes(ENCODING));
+         }
+       }
+       md.update(sd.isStoredAsSubDirectories() ? "true".getBytes(ENCODING) : "false".getBytes(ENCODING));
+     }
+ 
+     return md.digest();
+   }
+ 
+   public static double decimalToDouble(Decimal decimal) {
+     return new BigDecimal(new BigInteger(decimal.getUnscaled()), decimal.getScale()).doubleValue();
+   }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 88afe03,0db1bc0..903b2c7
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@@ -59,9 -58,11 +59,12 @@@ import javax.jdo.PersistenceManagerFact
  import javax.jdo.Query;
  import javax.jdo.Transaction;
  import javax.jdo.datastore.DataStoreCache;
 +import javax.jdo.datastore.JDOConnection;
  import javax.jdo.identity.IntIdentity;
+ import javax.sql.DataSource;
  
+ import com.codahale.metrics.Counter;
+ import com.codahale.metrics.MetricRegistry;
  import org.apache.commons.lang.ArrayUtils;
  import org.apache.commons.lang.exception.ExceptionUtils;
  import org.apache.hadoop.conf.Configurable;
@@@ -415,10 -431,13 +434,14 @@@ public class ObjectStore implements Raw
      pm = getPersistenceManager();
      isInitialized = pm != null;
      if (isInitialized) {
 +      dbType = determineDatabaseProduct();
        expressionProxy = createExpressionProxy(hiveConf);
        if (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL)) {
-         directSql = new MetaStoreDirectSql(pm, hiveConf, dbType);
+         String schema = prop.getProperty("javax.jdo.mapping.Schema");
+         if (schema != null && schema.isEmpty()) {
+           schema = null;
+         }
+         directSql = new MetaStoreDirectSql(pm, hiveConf, schema);
        }
      }
      LOG.debug("RawStore: " + this + ", with PersistenceManager: " + pm +
@@@ -2733,8 -2759,7 +2783,8 @@@
        boolean isConfigEnabled = HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL)
            && (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL_DDL) || !isInTxn);
        if (isConfigEnabled && directSql == null) {
 -        directSql = new MetaStoreDirectSql(pm, getConf());
 +        dbType = determineDatabaseProduct();
-         directSql = new MetaStoreDirectSql(pm, getConf(), dbType);
++        directSql = new MetaStoreDirectSql(pm, getConf(), "");
        }
  
        if (!allowJdo && isConfigEnabled && !directSql.isCompatibleDatastore()) {

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index ed05381,3a3d184..dabede4
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@@ -39,8 -39,6 +39,7 @@@ public final class TransactionalValidat
  
    // These constants are also imported by org.apache.hadoop.hive.ql.io.AcidUtils.
    public static final String DEFAULT_TRANSACTIONAL_PROPERTY = "default";
-   public static final String LEGACY_TRANSACTIONAL_PROPERTY = "legacy";
 +  public static final String INSERTONLY_TRANSACTIONAL_PROPERTY = "insert_only";
  
    TransactionalValidationListener(Configuration conf) {
      super(conf);
@@@ -104,13 -114,10 +115,13 @@@
        //normalize prop name
        parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, transactionalValue);
      }
-     if ("true".equalsIgnoreCase(transactionalValue)) {
+     if ("true".equalsIgnoreCase(transactionalValue) && !"true".equalsIgnoreCase(oldTransactionalValue)) {
+       //only need to check conformance if alter table enabled aicd
        if (!conformToAcid(newTable)) {
 -        throw new MetaException("The table must be stored using an ACID compliant format (such as ORC)");
 +        // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing
 +        if (transactionalPropertiesValue == null || !"insert_only".equalsIgnoreCase(transactionalPropertiesValue)) {
-           throw new MetaException("The table must be bucketed and stored using an ACID compliant" +
-               " format (such as ORC)");
++          throw new MetaException("The table must be stored using an ACID compliant format (such as ORC)");
 +        }
        }
  
        if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
@@@ -204,13 -194,9 +204,12 @@@
        return;
      }
  
 -    if ("true".equalsIgnoreCase(transactionalValue)) {
 +    if ("true".equalsIgnoreCase(transactional)) {
        if (!conformToAcid(newTable)) {
 -        throw new MetaException("The table must be stored using an ACID compliant format (such as ORC)");
 +        // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing
 +        if (transactionalProperties == null || !"insert_only".equalsIgnoreCase(transactionalProperties)) {
-           throw new MetaException("The table must be bucketed and stored using an ACID compliant" +
-               " format (such as ORC)");
++          throw new MetaException("The table must be stored using an ACID compliant format (such as ORC)");
 +        }
        }
  
        if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
@@@ -289,8 -273,6 +286,7 @@@
      boolean isValid = false;
      switch (transactionalProperties) {
        case DEFAULT_TRANSACTIONAL_PROPERTY:
-       case LEGACY_TRANSACTIONAL_PROPERTY:
 +      case INSERTONLY_TRANSACTIONAL_PROPERTY:
          isValid = true;
          break;
        default:

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 906591d,93d1ba6..a97ba1a
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@@ -18,7 -18,11 +18,16 @@@
  package org.apache.hadoop.hive.metastore.cache;
  
  import java.nio.ByteBuffer;
- import java.util.*;
+ import java.util.ArrayList;
++import java.util.ArrayList;
++import java.util.HashMap;
+ import java.util.HashMap;
+ import java.util.LinkedList;
++import java.util.LinkedList;
+ import java.util.List;
++import java.util.List;
++import java.util.Map;
+ import java.util.Map;
  import java.util.concurrent.Executors;
  import java.util.concurrent.ScheduledExecutorService;
  import java.util.concurrent.ThreadFactory;

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --cc metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 6d851a0,fb16cfc..0f7827b
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@@ -110,7 -114,14 +112,12 @@@ public class DummyRawStoreForJdoConnect
    }
  
    @Override
+   public boolean isActiveTransaction() {
+     return false;
+   }
+ 
+   @Override
    public void rollbackTransaction() {
 -
 -
    }
  
    @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 9542990,08228af..7573fb2
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@@ -379,13 -362,14 +377,14 @@@ public class TestObjectStore 
    public void testDirectSqlErrorMetrics() throws Exception {
      HiveConf conf = new HiveConf();
      conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED, true);
-     conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name()
-         + "," + MetricsReporting.JMX.name());
+     Metrics.initialize(conf);
  
-     MetricsFactory.init(conf);
-     CodahaleMetrics metrics = (CodahaleMetrics) MetricsFactory.getInstance();
+     // recall setup so that we get an object store with the metrics initalized
+     setUp();
+     Counter directSqlErrors =
+         Metrics.getRegistry().getCounters().get(MetricsConstants.DIRECTSQL_ERRORS);
  
 -    objectStore.new GetDbHelper("foo", null, true, true) {
 +    objectStore.new GetDbHelper("foo", true, true) {
        @Override
        protected Database getSqlResult(ObjectStore.GetHelper<Database> ctx) throws MetaException {
          return null;
@@@ -398,11 -382,9 +397,9 @@@
        }
      }.run(false);
  
-     String json = metrics.dumpJson();
-     MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER,
-         MetricsConstant.DIRECTSQL_ERRORS, "");
+     Assert.assertEquals(0, directSqlErrors.getCount());
  
 -    objectStore.new GetDbHelper("foo", null, true, true) {
 +    objectStore.new GetDbHelper("foo", true, true) {
        @Override
        protected Database getSqlResult(ObjectStore.GetHelper<Database> ctx) throws MetaException {
          throw new RuntimeException();

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 82a9fad,acc2390..b6eff94
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@@ -3846,11 -3867,42 +3884,22 @@@ public class DDLTask extends Task<DDLWo
            throw new HiveException(ErrorMsg.REPLACE_CANNOT_DROP_COLUMNS, alterTbl.getOldName());
          }
        }
+ 
+       boolean partitioned = tbl.isPartitioned();
+       boolean droppingColumns = alterTbl.getNewCols().size() < sd.getCols().size();
+       if (ParquetHiveSerDe.isParquetTable(tbl) &&
+           isSchemaEvolutionEnabled(tbl) &&
+           !alterTbl.getIsCascade() &&
+           droppingColumns && partitioned) {
+         LOG.warn("Cannot drop columns from a partitioned parquet table without the CASCADE option");
+         throw new HiveException(ErrorMsg.REPLACE_CANNOT_DROP_COLUMNS,
+             alterTbl.getOldName());
+       }
        sd.setCols(alterTbl.getNewCols());
      } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDPROPS) {
 -      if (StatsSetupConst.USER.equals(environmentContext.getProperties()
 -              .get(StatsSetupConst.STATS_GENERATED))) {
 -        environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
 -      }
 -      if (part != null) {
 -        part.getTPartition().getParameters().putAll(alterTbl.getProps());
 -      } else {
 -        tbl.getTTable().getParameters().putAll(alterTbl.getProps());
 -      }
 +      return alterTableAddProps(alterTbl, tbl, part, environmentContext);
      } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.DROPPROPS) {
 -      Iterator<String> keyItr = alterTbl.getProps().keySet().iterator();
 -      if (StatsSetupConst.USER.equals(environmentContext.getProperties()
 -          .get(StatsSetupConst.STATS_GENERATED))) {
 -        // drop a stats parameter, which triggers recompute stats update automatically
 -        environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
 -      }
 -      while (keyItr.hasNext()) {
 -        if (part != null) {
 -          part.getTPartition().getParameters().remove(keyItr.next());
 -        } else {
 -          tbl.getTTable().getParameters().remove(keyItr.next());
 -        }
 -      }
 +      return alterTableDropProps(alterTbl, tbl, part, environmentContext);
      } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDSERDEPROPS) {
        StorageDescriptor sd = retrieveStorageDescriptor(tbl, part);
        sd.getSerdeInfo().getParameters().putAll(alterTbl.getProps());
@@@ -4002,219 -4054,7 +4051,215 @@@
        throw new HiveException(ErrorMsg.UNSUPPORTED_ALTER_TBL_OP, alterTbl.getOp().toString());
      }
  
 -    return 0;
 +    return null;
 +  }
 +
 +  private List<Task<?>> alterTableDropProps(AlterTableDesc alterTbl, Table tbl,
 +      Partition part, EnvironmentContext environmentContext) throws HiveException {
 +    if (StatsSetupConst.USER.equals(environmentContext.getProperties()
 +        .get(StatsSetupConst.STATS_GENERATED))) {
 +      // drop a stats parameter, which triggers recompute stats update automatically
 +      environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
 +    }
 +
 +    List<Task<?>> result = null;
 +    if (part == null) {
 +      Set<String> removedSet = alterTbl.getProps().keySet();
 +      boolean isFromMmTable = MetaStoreUtils.isInsertOnlyTable(tbl.getParameters()),
 +          isRemoved = MetaStoreUtils.isRemovedInsertOnlyTable(removedSet);
 +      if (isFromMmTable && isRemoved) {
 +        result = generateRemoveMmTasks(tbl);
 +      }
 +    }
 +    Iterator<String> keyItr = alterTbl.getProps().keySet().iterator();
 +    while (keyItr.hasNext()) {
 +      if (part != null) {
 +        part.getTPartition().getParameters().remove(keyItr.next());
 +      } else {
 +        tbl.getTTable().getParameters().remove(keyItr.next());
 +      }
 +    }
 +    return result;
 +  }
 +
 +  private List<Task<?>> generateRemoveMmTasks(Table tbl) throws HiveException {
 +    // To avoid confusion from nested MM directories when table is converted back and forth, we
 +    // want to rename mm_ dirs to remove the prefix; however, given the unpredictable nested
 +    // directory handling in Hive/MR, we will instead move all the files into the root directory.
 +    // We will also delete any directories that are not committed. 
 +    // Note that this relies on locks. Note also that we only do the renames AFTER the metastore
 +    // operation commits. Deleting uncommitted things is safe, but moving stuff before we convert
 +    // could cause data loss.
 +    List<Path> allMmDirs = new ArrayList<>();
 +    if (tbl.isStoredAsSubDirectories()) {
 +      // TODO: support this? we only bail because it's a PITA and hardly anyone seems to care.
 +      throw new HiveException("Converting list bucketed tables stored as subdirectories "
 +          + " to and from MM is not supported");
 +    }
 +    List<String> bucketCols = tbl.getBucketCols();
 +    if (bucketCols != null && !bucketCols.isEmpty()
 +        && HiveConf.getBoolVar(conf, ConfVars.HIVE_STRICT_CHECKS_BUCKETING)) {
 +      throw new HiveException("Converting bucketed tables from MM is not supported by default; "
 +          + "copying files from multiple MM directories may potentially break the buckets. You "
 +          + "can set " + ConfVars.HIVE_STRICT_CHECKS_BUCKETING.varname
 +          + " to false for this query if you want to force the conversion.");
 +    }
 +    Hive db = getHive();
 +    String value = conf.get(ValidTxnList.VALID_TXNS_KEY);
 +    ValidTxnList validTxnList = value == null ? new ValidReadTxnList() : new ValidReadTxnList(value);
 +    if (tbl.getPartitionKeys().size() > 0) {
 +      PartitionIterable parts = new PartitionIterable(db, tbl, null,
 +          HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
 +      Iterator<Partition> partIter = parts.iterator();
 +      while (partIter.hasNext()) {
 +        Partition part = partIter.next();
 +        checkMmLb(part);
 +        handleRemoveMm(part.getDataLocation(), validTxnList, allMmDirs);
 +      }
 +    } else {
 +      checkMmLb(tbl);
 +      handleRemoveMm(tbl.getDataLocation(), validTxnList, allMmDirs);
 +    }
 +    List<Path> targetPaths = new ArrayList<>(allMmDirs.size());
 +    List<String> targetPrefix = new ArrayList<>(allMmDirs.size());
 +    int prefixLen = JavaUtils.DELTA_PREFIX.length();
 +    for (int i = 0; i < allMmDirs.size(); ++i) {
 +      Path src = allMmDirs.get(i);
 +      Path tgt = src.getParent();
 +      String prefix = src.getName().substring(prefixLen + 1) + "_";
 +      Utilities.LOG14535.info("Will move " + src + " to " + tgt + " (prefix " + prefix + ")");
 +      targetPaths.add(tgt);
 +      targetPrefix.add(prefix);
 +    }
 +    // Don't set inputs and outputs - the locks have already been taken so it's pointless.
 +    MoveWork mw = new MoveWork(null, null, null, null, false);
 +    mw.setMultiFilesDesc(new LoadMultiFilesDesc(
 +        allMmDirs, targetPaths, targetPrefix, true, null, null));
 +    return Lists.<Task<?>>newArrayList(TaskFactory.get(mw, conf));
 +  }
 +
 +  private void checkMmLb(Table tbl) throws HiveException {
 +    if (!tbl.isStoredAsSubDirectories()) return;
 +    // TODO: support this?
 +    throw new HiveException("Converting list bucketed tables stored as subdirectories "
 +        + " to and from MM is not supported");
 +  }
 +
 +  private void checkMmLb(Partition part) throws HiveException {
 +    if (!part.isStoredAsSubDirectories()) return;
 +    // TODO: support this?
 +    throw new HiveException("Converting list bucketed tables stored as subdirectories "
 +        + " to and from MM is not supported. Please create a table in the desired format.");
 +  }
 +
 +  private void handleRemoveMm(
 +      Path path, ValidTxnList validTxnList, List<Path> result) throws HiveException {
 +    // Note: doesn't take LB into account; that is not presently supported here (throws above).
 +    try {
 +      FileSystem fs = path.getFileSystem(conf);
 +      for (FileStatus file : fs.listStatus(path)) {
 +        Path childPath = file.getPath();
 +        if (!file.isDirectory()) {
 +          ensureDelete(fs, childPath, "a non-directory file");
 +          continue;
 +        }
 +        Long writeId = JavaUtils.extractTxnId(childPath);
 +        if (writeId == null) {
 +          ensureDelete(fs, childPath, "an unknown directory");
 +        } else if (!validTxnList.isTxnValid(writeId)) {
 +          // Assume no concurrent active writes - we rely on locks here. We could check and fail.
 +          ensureDelete(fs, childPath, "an uncommitted directory");
 +        } else {
 +          result.add(childPath);
 +        }
 +      }
 +    } catch (IOException ex) {
 +      throw new HiveException(ex);
 +    }
 +  }
 +
 +  private static void ensureDelete(FileSystem fs, Path path, String what) throws IOException {
 +    Utilities.LOG14535.info("Deleting " + what + " " + path);
 +    try {
 +      if (!fs.delete(path, true)) throw new IOException("delete returned false");
 +    } catch (Exception ex) {
 +      String error = "Couldn't delete " + path + "; cannot remove MM setting from the table";
 +      LOG.error(error, ex);
 +      throw (ex instanceof IOException) ? (IOException)ex : new IOException(ex);
 +    }
 +  }
 +
 +  private List<Task<?>> generateAddMmTasks(Table tbl) throws HiveException {
 +    // We will move all the files in the table/partition directories into the first MM
 +    // directory, then commit the first write ID.
 +    List<Path> srcs = new ArrayList<>(), tgts = new ArrayList<>();
 +    long mmWriteId = 0;
 +    try {
 +      HiveTxnManager txnManager = SessionState.get().getTxnMgr();
 +      if (txnManager.isTxnOpen()) {
 +        mmWriteId = txnManager.getCurrentTxnId();
 +      } else {
 +        mmWriteId = txnManager.openTxn(new Context(conf), conf.getUser());
 +        txnManager.commitTxn();
 +      }
 +    } catch (Exception e) {
 +      String errorMessage = "FAILED: Error in acquiring locks: " + e.getMessage();
 +      console.printError(errorMessage, "\n"
 +          + org.apache.hadoop.util.StringUtils.stringifyException(e));
 +    }
 +    int stmtId = 0;
 +    String mmDir = AcidUtils.deltaSubdir(mmWriteId, mmWriteId, stmtId);
 +    Hive db = getHive();
 +    if (tbl.getPartitionKeys().size() > 0) {
 +      PartitionIterable parts = new PartitionIterable(db, tbl, null,
 +          HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
 +      Iterator<Partition> partIter = parts.iterator();
 +      while (partIter.hasNext()) {
 +        Partition part = partIter.next();
 +        checkMmLb(part);
 +        Path src = part.getDataLocation(), tgt = new Path(src, mmDir);
 +        srcs.add(src);
 +        tgts.add(tgt);
 +        Utilities.LOG14535.info("Will move " + src + " to " + tgt);
 +      }
 +    } else {
 +      checkMmLb(tbl);
 +      Path src = tbl.getDataLocation(), tgt = new Path(src, mmDir);
 +      srcs.add(src);
 +      tgts.add(tgt);
 +      Utilities.LOG14535.info("Will move " + src + " to " + tgt);
 +    }
 +    // Don't set inputs and outputs - the locks have already been taken so it's pointless.
 +    MoveWork mw = new MoveWork(null, null, null, null, false);
 +    mw.setMultiFilesDesc(new LoadMultiFilesDesc(srcs, tgts, true, null, null));
 +    ImportCommitWork icw = new ImportCommitWork(tbl.getDbName(), tbl.getTableName(), mmWriteId, stmtId);
 +    Task<?> mv = TaskFactory.get(mw, conf), ic = TaskFactory.get(icw, conf);
 +    mv.addDependentTask(ic);
 +    return Lists.<Task<?>>newArrayList(mv);
 +  }
 +
 +  private List<Task<?>> alterTableAddProps(AlterTableDesc alterTbl, Table tbl,
 +      Partition part, EnvironmentContext environmentContext) throws HiveException {
 +    if (StatsSetupConst.USER.equals(environmentContext.getProperties()
 +        .get(StatsSetupConst.STATS_GENERATED))) {
 +      environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
 +    }
-     if(alterTbl.getProps().containsKey(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY)) {
-       NanoTimeUtils.validateTimeZone(
-           alterTbl.getProps().get(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY));
-     }
 +    List<Task<?>> result = null;
 +    if (part != null) {
 +      part.getTPartition().getParameters().putAll(alterTbl.getProps());
 +    } else {
 +      boolean isFromMmTable = MetaStoreUtils.isInsertOnlyTable(tbl.getParameters());
 +      Boolean isToMmTable = MetaStoreUtils.isToInsertOnlyTable(alterTbl.getProps());
 +      if (isToMmTable != null) {
 +        if (!isFromMmTable && isToMmTable) {
 +          result = generateAddMmTasks(tbl);
 +        } else if (isFromMmTable && !isToMmTable) {
 +          result = generateRemoveMmTasks(tbl);
 +        }
 +      }
 +      tbl.getTTable().getParameters().putAll(alterTbl.getProps());
 +    }
 +    return result;
    }
  
    private int dropConstraint(Hive db, AlterTableDesc alterTbl)
@@@ -4583,22 -4426,8 +4631,8 @@@
        }
      }
  
-     // If HIVE_PARQUET_INT96_DEFAULT_UTC_WRITE_ZONE is set to True, then set new Parquet tables timezone
-     // to UTC by default (only if the table property is not set)
-     if (ParquetHiveSerDe.isParquetTable(tbl)) {
-       SessionState ss = SessionState.get();
-       String parquetTimezone = tbl.getProperty(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY);
-       if (parquetTimezone == null || parquetTimezone.isEmpty()) {
-         if (ss.getConf().getBoolVar(ConfVars.HIVE_PARQUET_INT96_DEFAULT_UTC_WRITE_ZONE)) {
-           tbl.setProperty(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY, ParquetTableUtils.PARQUET_INT96_NO_ADJUSTMENT_ZONE);
-         }
-       } else {
-         NanoTimeUtils.validateTimeZone(parquetTimezone);
-       }
-     }
- 
      // create the table
 -    if (crtTbl.getReplaceMode()){
 +    if (crtTbl.getReplaceMode()) {
        // replace-mode creates are really alters using CreateTableDesc.
        try {
          db.alterTable(tbl.getDbName()+"."+tbl.getTableName(),tbl,null);

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
index 0582f94,13750cd..d2d9946
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
@@@ -382,14 -368,7 +379,11 @@@ public class FetchOperator implements S
  
        Class<? extends InputFormat> formatter = currDesc.getInputFileFormatClass();
        Utilities.copyTableJobPropertiesToConf(currDesc.getTableDesc(), job);
-       if (ParquetHiveSerDe.class.getName().equals(currDesc.getTableDesc().getSerdeClassName())) {
-         ParquetTableUtils.setParquetTimeZoneIfAbsent(job, currDesc.getTableDesc().getProperties());
-       }
        InputFormat inputFormat = getInputFormatFromCache(formatter, job);
 +      String inputs = processCurrPathForMmWriteIds(inputFormat);
 +      Utilities.LOG14535.info("Setting fetch inputs to " + inputs);
 +      if (inputs == null) return null;
 +      job.set("mapred.input.dir", inputs);
  
        InputSplit[] splits = inputFormat.getSplits(job, 1);
        FetchInputFormatSplit[] inputSplits = new FetchInputFormatSplit[splits.length];

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 7e1b8fa,bc265eb..3544884
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@@ -41,11 -28,11 +41,13 @@@ import org.apache.hadoop.hive.common.St
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.conf.HiveConfUtil;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
  import org.apache.hadoop.hive.ql.CompilationOpContext;
  import org.apache.hadoop.hive.ql.ErrorMsg;
 +import org.apache.hadoop.hive.ql.exec.Utilities.MissingBucketsContext;
  import org.apache.hadoop.hive.ql.io.AcidUtils;
 +import org.apache.hadoop.hive.ql.io.AcidUtils.Operation;
+ import org.apache.hadoop.hive.ql.io.BucketCodec;
  import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
  import org.apache.hadoop.hive.ql.io.HiveKey;
  import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
@@@ -165,28 -149,11 +167,28 @@@ public class FileSinkOperator extends T
      Path[] finalPaths;
      RecordWriter[] outWriters;
      RecordUpdater[] updaters;
-     private Stat stat;
+     Stat stat;
 +    int acidLastBucket = -1;
 +    int acidFileOffset = -1;
 +    private boolean isMmTable;
 +    private Long txnId;
 +    private int stmtId;
 +
 +    public FSPaths(Path specPath, boolean isMmTable) {
 +      this.isMmTable = isMmTable;
 +      if (!isMmTable) {
 +        tmpPath = Utilities.toTempPath(specPath);
 +        taskOutputTempPath = Utilities.toTaskTempPath(specPath);
 +      } else {
 +        tmpPath = specPath;
 +        taskOutputTempPath = null; // Should not be used.
 +        txnId = conf.getTransactionId();
 +        stmtId = conf.getStatementId();
 +      }
 +      Utilities.LOG14535.info("new FSPaths for " + numFiles + " files, dynParts = " + bDynParts
 +          + ": tmpPath " + tmpPath + ", task path " + taskOutputTempPath
 +          + " (spec path " + specPath + ")"/*, new Exception()*/);
  
 -    public FSPaths(Path specPath) {
 -      tmpPath = Utilities.toTempPath(specPath);
 -      taskOutputTempPath = Utilities.toTaskTempPath(specPath);
        outPaths = new Path[numFiles];
        finalPaths = new Path[numFiles];
        outWriters = new RecordWriter[numFiles];
@@@ -881,10 -758,10 +885,10 @@@
        // for a given operator branch prediction should work quite nicely on it.
        // RecordUpdateer expects to get the actual row, not a serialized version of it.  Thus we
        // pass the row rather than recordValue.
 -      if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
 +      if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID || conf.isMmTable()) {
-         rowOutWriters[writerOffset].write(recordValue);
+         rowOutWriters[findWriterOffset(row)].write(recordValue);
        } else if (conf.getWriteType() == AcidUtils.Operation.INSERT) {
-         fpaths.updaters[writerOffset].insert(conf.getTransactionId(), row);
+         fpaths.updaters[findWriterOffset(row)].insert(conf.getTransactionId(), row);
        } else {
          // TODO I suspect we could skip much of the stuff above this in the function in the case
          // of update and delete.  But I don't understand all of the side effects of the above

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index e86ca3a,cde2805..34d0598
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@@ -403,232 -579,6 +436,231 @@@ public class MoveTask extends Task<Move
        return (1);
      }
    }
 +
 +  private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd,
 +      TaskInformation ti) throws HiveException, IOException, InvalidOperationException {
 +    List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(),  tbd.getPartitionSpec());
 +    db.validatePartitionNameCharacters(partVals);
 +    Utilities.LOG14535.info("loadPartition called from " + tbd.getSourcePath()
 +        + " into " + tbd.getTable().getTableName());
 +    db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(),
 +        tbd.getPartitionSpec(), tbd.getReplace(),
 +        tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(),
 +        work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID &&
 +            !tbd.isMmTable(),
 +        hasFollowingStatsTask(), tbd.getTxnId(), tbd.getStmtId());
 +    Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);
 +
 +    // See the comment inside updatePartitionBucketSortColumns.
 +    if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) {
 +      updatePartitionBucketSortColumns(db, table, partn, ti.bucketCols,
 +          ti.numBuckets, ti.sortCols);
 +    }
 +
 +    DataContainer dc = new DataContainer(table.getTTable(), partn.getTPartition());
 +    // add this partition to post-execution hook
 +    if (work.getOutputs() != null) {
 +      DDLTask.addIfAbsentByName(new WriteEntity(partn,
 +        getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs());
 +    }
 +    return dc;
 +  }
 +
 +  private DataContainer handleDynParts(Hive db, Table table, LoadTableDesc tbd,
 +      TaskInformation ti, DynamicPartitionCtx dpCtx) throws HiveException,
 +      IOException, InvalidOperationException {
 +    DataContainer dc;
 +    List<LinkedHashMap<String, String>> dps = Utilities.getFullDPSpecs(conf, dpCtx);
 +
 +    console.printInfo(System.getProperty("line.separator"));
 +    long startTime = System.currentTimeMillis();
 +    // load the list of DP partitions and return the list of partition specs
 +    // TODO: In a follow-up to HIVE-1361, we should refactor loadDynamicPartitions
 +    // to use Utilities.getFullDPSpecs() to get the list of full partSpecs.
 +    // After that check the number of DPs created to not exceed the limit and
 +    // iterate over it and call loadPartition() here.
 +    // The reason we don't do inside HIVE-1361 is the latter is large and we
 +    // want to isolate any potential issue it may introduce.
 +    if (tbd.isMmTable() && !tbd.isCommitMmWrite()) {
 +      throw new HiveException("Only single-partition LoadTableDesc can skip commiting write ID");
 +    }
 +    Map<Map<String, String>, Partition> dp =
 +      db.loadDynamicPartitions(
 +        tbd.getSourcePath(),
 +        tbd.getTable().getTableName(),
 +        tbd.getPartitionSpec(),
 +        tbd.getReplace(),
 +        dpCtx.getNumDPCols(),
 +        (tbd.getLbCtx() == null) ? 0 : tbd.getLbCtx().calculateListBucketingLevel(),
 +        work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID &&
 +            !tbd.isMmTable(),
 +        SessionState.get().getTxnMgr().getCurrentTxnId(), tbd.getStmtId(), hasFollowingStatsTask(),
 +        work.getLoadTableWork().getWriteType());
 +
 +    // publish DP columns to its subscribers
 +    if (dps != null && dps.size() > 0) {
 +      pushFeed(FeedType.DYNAMIC_PARTITIONS, dp.values());
 +    }
 +
 +    String loadTime = "\t Time taken to load dynamic partitions: "  +
 +        (System.currentTimeMillis() - startTime)/1000.0 + " seconds";
 +    console.printInfo(loadTime);
 +    LOG.info(loadTime);
 +
 +    if (dp.size() == 0 && conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) {
 +      throw new HiveException("This query creates no partitions." +
 +          " To turn off this error, set hive.error.on.empty.partition=false.");
 +    }
 +
 +    startTime = System.currentTimeMillis();
 +    // for each partition spec, get the partition
 +    // and put it to WriteEntity for post-exec hook
 +    for(Map.Entry<Map<String, String>, Partition> entry : dp.entrySet()) {
 +      Partition partn = entry.getValue();
 +
 +      // See the comment inside updatePartitionBucketSortColumns.
 +      if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) {
 +        updatePartitionBucketSortColumns(
 +            db, table, partn, ti.bucketCols, ti.numBuckets, ti.sortCols);
 +      }
 +
 +      WriteEntity enty = new WriteEntity(partn,
 +        getWriteType(tbd, work.getLoadTableWork().getWriteType()));
 +      if (work.getOutputs() != null) {
 +        DDLTask.addIfAbsentByName(enty, work.getOutputs());
 +      }
 +      // Need to update the queryPlan's output as well so that post-exec hook get executed.
 +      // This is only needed for dynamic partitioning since for SP the the WriteEntity is
 +      // constructed at compile time and the queryPlan already contains that.
 +      // For DP, WriteEntity creation is deferred at this stage so we need to update
 +      // queryPlan here.
 +      if (queryPlan.getOutputs() == null) {
 +        queryPlan.setOutputs(new LinkedHashSet<WriteEntity>());
 +      }
 +      queryPlan.getOutputs().add(enty);
 +
 +      // update columnar lineage for each partition
 +      dc = new DataContainer(table.getTTable(), partn.getTPartition());
 +
 +      // Don't set lineage on delete as we don't have all the columns
 +      if (SessionState.get() != null &&
 +          work.getLoadTableWork().getWriteType() != AcidUtils.Operation.DELETE &&
 +          work.getLoadTableWork().getWriteType() != AcidUtils.Operation.UPDATE) {
 +        SessionState.get().getLineageState().setLineage(tbd.getSourcePath(), dc,
 +            table.getCols());
 +      }
 +      LOG.info("\tLoading partition " + entry.getKey());
 +    }
 +    console.printInfo("\t Time taken for adding to write entity : " +
 +        (System.currentTimeMillis() - startTime)/1000.0 + " seconds");
 +    dc = null; // reset data container to prevent it being added again.
 +    return dc;
 +  }
 +
 +  private void inferTaskInformation(TaskInformation ti) {
 +    // Find the first ancestor of this MoveTask which is some form of map reduce task
 +    // (Either standard, local, or a merge)
 +    while (ti.task.getParentTasks() != null && ti.task.getParentTasks().size() == 1) {
 +      ti.task = (Task)ti.task.getParentTasks().get(0);
 +      // If it was a merge task or a local map reduce task, nothing can be inferred
 +      if (ti.task instanceof MergeFileTask || ti.task instanceof MapredLocalTask) {
 +        break;
 +      }
 +
 +      // If it's a standard map reduce task, check what, if anything, it inferred about
 +      // the directory this move task is moving
 +      if (ti.task instanceof MapRedTask) {
 +        MapredWork work = (MapredWork)ti.task.getWork();
 +        MapWork mapWork = work.getMapWork();
 +        ti.bucketCols = mapWork.getBucketedColsByDirectory().get(ti.path);
 +        ti.sortCols = mapWork.getSortedColsByDirectory().get(ti.path);
 +        if (work.getReduceWork() != null) {
 +          ti.numBuckets = work.getReduceWork().getNumReduceTasks();
 +        }
 +
 +        if (ti.bucketCols != null || ti.sortCols != null) {
 +          // This must be a final map reduce task (the task containing the file sink
 +          // operator that writes the final output)
 +          assert work.isFinalMapRed();
 +        }
 +        break;
 +      }
 +
 +      // If it's a move task, get the path the files were moved from, this is what any
 +      // preceding map reduce task inferred information about, and moving does not invalidate
 +      // those assumptions
 +      // This can happen when a conditional merge is added before the final MoveTask, but the
 +      // condition for merging is not met, see GenMRFileSink1.
 +      if (ti.task instanceof MoveTask) {
 +        MoveTask mt = (MoveTask)ti.task;
 +        if (mt.getWork().getLoadFileWork() != null) {
 +          ti.path = mt.getWork().getLoadFileWork().getSourcePath().toUri().toString();
 +        }
 +      }
 +    }
 +  }
 +
 +  private void checkFileFormats(Hive db, LoadTableDesc tbd, Table table)
 +      throws HiveException {
 +    if (work.getCheckFileFormat()) {
 +      // Get all files from the src directory
 +      FileStatus[] dirs;
 +      ArrayList<FileStatus> files;
 +      FileSystem srcFs; // source filesystem
 +      try {
 +        srcFs = tbd.getSourcePath().getFileSystem(conf);
 +        dirs = srcFs.globStatus(tbd.getSourcePath());
 +        files = new ArrayList<FileStatus>();
 +        for (int i = 0; (dirs != null && i < dirs.length); i++) {
 +          files.addAll(Arrays.asList(srcFs.listStatus(dirs[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER)));
 +          // We only check one file, so exit the loop when we have at least
 +          // one.
 +          if (files.size() > 0) {
 +            break;
 +          }
 +        }
 +      } catch (IOException e) {
 +        throw new HiveException(
 +            "addFiles: filesystem error in check phase", e);
 +      }
 +
 +      // handle file format check for table level
 +      if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVECHECKFILEFORMAT)) {
 +        boolean flag = true;
 +        // work.checkFileFormat is set to true only for Load Task, so assumption here is
 +        // dynamic partition context is null
 +        if (tbd.getDPCtx() == null) {
 +          if (tbd.getPartitionSpec() == null || tbd.getPartitionSpec().isEmpty()) {
 +            // Check if the file format of the file matches that of the table.
 +            flag = HiveFileFormatUtils.checkInputFormat(
 +                srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
 +          } else {
 +            // Check if the file format of the file matches that of the partition
 +            Partition oldPart = db.getPartition(table, tbd.getPartitionSpec(), false);
 +            if (oldPart == null) {
 +              // this means we have just created a table and are specifying partition in the
 +              // load statement (without pre-creating the partition), in which case lets use
 +              // table input format class. inheritTableSpecs defaults to true so when a new
 +              // partition is created later it will automatically inherit input format
 +              // from table object
 +              flag = HiveFileFormatUtils.checkInputFormat(
 +                  srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
 +            } else {
 +              flag = HiveFileFormatUtils.checkInputFormat(
 +                  srcFs, conf, oldPart.getInputFormatClass(), files);
 +            }
 +          }
 +          if (!flag) {
-             throw new HiveException(
-                 "Wrong file format. Please check the file's format.");
++            throw new HiveException(ErrorMsg.WRONG_FILE_FORMAT);
 +          }
 +        } else {
 +          LOG.warn("Skipping file format check as dpCtx is not null");
 +        }
 +      }
 +    }
 +  }
 +
 +
    /**
     * so to make sure we crate WriteEntity with the right WriteType.  This is (at this point) only
     * for consistency since LockManager (which is the only thing that pays attention to WriteType)

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index ea0a0fd,aca99f2..88f5a0d
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@@ -1445,14 -1447,21 +1471,21 @@@ public final class Utilities 
          // create empty buckets if necessary
          if (emptyBuckets.size() > 0) {
            perfLogger.PerfLogBegin("FileSinkOperator", "CreateEmptyBuckets");
 -          createEmptyBuckets(hconf, emptyBuckets, conf, reporter);
 +          createEmptyBuckets(
 +              hconf, emptyBuckets, conf.getCompressed(), conf.getTableInfo(), reporter);
+           filesKept.addAll(emptyBuckets);
            perfLogger.PerfLogEnd("FileSinkOperator", "CreateEmptyBuckets");
          }
 -
          // move to the file destination
 -        log.info("Moving tmp dir: " + tmpPath + " to: " + specPath);
 +        Utilities.LOG14535.info("Moving tmp dir: " + tmpPath + " to: " + specPath);
          perfLogger.PerfLogBegin("FileSinkOperator", "RenameOrMoveFiles");
-         Utilities.renameOrMoveFiles(fs, tmpPath, specPath);
+         if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVE_EXEC_MOVE_FILES_FROM_SOURCE_DIR)) {
+           // HIVE-17113 - avoid copying files that may have been written to the temp dir by runaway tasks,
+           // by moving just the files we've tracked from removeTempOrDuplicateFiles().
+           Utilities.moveSpecifiedFiles(fs, tmpPath, specPath, filesKept);
+         } else {
+           Utilities.renameOrMoveFiles(fs, tmpPath, specPath);
+         }
          perfLogger.PerfLogEnd("FileSinkOperator", "RenameOrMoveFiles");
        }
      } else {
@@@ -1534,27 -1552,7 +1578,27 @@@
     * @return a list of path names corresponding to should-be-created empty buckets.
     */
    public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
-       DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf) throws IOException {
+       DynamicPartitionCtx dpCtx, FileSinkDesc conf, Configuration hconf, Set<Path> filesKept) throws IOException {
 +    int dpLevels = dpCtx == null ? 0 : dpCtx.getNumDPCols(),
 +        numBuckets = (conf != null && conf.getTable() != null)
 +          ? conf.getTable().getNumBuckets() : 0;
 +    return removeTempOrDuplicateFiles(fs, fileStats, dpLevels, numBuckets, hconf, null, 0, false);
 +  }
 +  
 +  private static boolean removeEmptyDpDirectory(FileSystem fs, Path path) throws IOException {
 +    FileStatus[] items = fs.listStatus(path);
 +    // remove empty directory since DP insert should not generate empty partitions.
 +    // empty directories could be generated by crashed Task/ScriptOperator
 +    if (items.length != 0) return false;
 +    if (!fs.delete(path, true)) {
 +      LOG.error("Cannot delete empty directory " + path);
 +      throw new IOException("Cannot delete empty directory " + path);
 +    }
 +    return true;
 +  }
 +
 +  public static List<Path> removeTempOrDuplicateFiles(FileSystem fs, FileStatus[] fileStats,
 +      int dpLevels, int numBuckets, Configuration hconf, Long txnId, int stmtId, boolean isMmTable) throws IOException {
      if (fileStats == null) {
        return null;
      }
@@@ -3145,21 -3150,14 +3235,21 @@@
        LOG.info("Processing alias " + alias);
  
        // The alias may not have any path
 +      Collection<Map.Entry<Path, ArrayList<String>>> pathToAliases =
 +          work.getPathToAliases().entrySet();
 +      if (!skipDummy) {
 +        // ConcurrentModification otherwise if adding dummy.
 +        pathToAliases = new ArrayList<>(pathToAliases);
 +      }
        boolean isEmptyTable = true;
        boolean hasLogged = false;
 -      // Note: this copies the list because createDummyFileForEmptyPartition may modify the map.
 -      for (Path file : new LinkedList<Path>(work.getPathToAliases().keySet())) {
 +      Path path = null;
 +      for (Map.Entry<Path, ArrayList<String>> e : pathToAliases) {
          if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT)
-           throw new IOException("Operation is Canceled. ");
+           throw new IOException("Operation is Canceled.");
  
 -        List<String> aliases = work.getPathToAliases().get(file);
 +        Path file = e.getKey();
 +        List<String> aliases = e.getValue();
          if (aliases.contains(alias)) {
            if (file != null) {
              isEmptyTable = false;

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
index 0000000,c944a13..2ae18cf
mode 000000,100644..100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
@@@ -1,0 -1,314 +1,316 @@@
+ /*
+   Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+ 
+       http://www.apache.org/licenses/LICENSE-2.0
+ 
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+  */
+ package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table;
+ 
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+ import org.apache.hadoop.hive.metastore.Warehouse;
+ import org.apache.hadoop.hive.metastore.api.Database;
+ import org.apache.hadoop.hive.metastore.api.MetaException;
+ import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
+ import org.apache.hadoop.hive.ql.exec.Task;
+ import org.apache.hadoop.hive.ql.exec.TaskFactory;
+ import org.apache.hadoop.hive.ql.exec.Utilities;
+ import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.TableEvent;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadTask;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.TaskTracker;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.util.Context;
+ import org.apache.hadoop.hive.ql.metadata.HiveException;
+ import org.apache.hadoop.hive.ql.metadata.Partition;
+ import org.apache.hadoop.hive.ql.metadata.Table;
+ import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer;
+ import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+ import org.apache.hadoop.hive.ql.parse.SemanticException;
+ import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+ import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
+ import org.apache.hadoop.hive.ql.plan.DDLWork;
+ import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
+ import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+ import org.apache.hadoop.hive.ql.plan.MoveWork;
++import org.apache.hadoop.hive.ql.session.SessionState;
++import org.mortbay.jetty.servlet.AbstractSessionManager;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ 
+ import java.io.IOException;
+ import java.io.Serializable;
+ import java.util.*;
+ 
+ import static org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState.PartitionState;
+ import static org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer.isPartitioned;
+ import static org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer.partSpecToString;
+ 
+ public class LoadPartitions {
+   private static Logger LOG = LoggerFactory.getLogger(LoadPartitions.class);
+ 
+   private final Context context;
+   private final ReplLogger replLogger;
+   private final TableContext tableContext;
+   private final TableEvent event;
+   private final TaskTracker tracker;
+   private final AddPartitionDesc lastReplicatedPartition;
+ 
+   private final ImportTableDesc tableDesc;
+   private Table table;
+ 
+   public LoadPartitions(Context context, ReplLogger replLogger, TaskTracker tableTracker,
+                         TableEvent event, String dbNameToLoadIn,
+                         TableContext tableContext) throws HiveException, IOException {
+     this(context, replLogger, tableContext, tableTracker, event, dbNameToLoadIn, null);
+   }
+ 
+   public LoadPartitions(Context context, ReplLogger replLogger, TableContext tableContext,
+                         TaskTracker limiter, TableEvent event, String dbNameToLoadIn,
+                         AddPartitionDesc lastReplicatedPartition) throws HiveException, IOException {
+     this.tracker = new TaskTracker(limiter);
+     this.event = event;
+     this.context = context;
+     this.replLogger = replLogger;
+     this.lastReplicatedPartition = lastReplicatedPartition;
+     this.tableContext = tableContext;
+ 
+     this.tableDesc = tableContext.overrideProperties(event.tableDesc(dbNameToLoadIn));
+     this.table = ImportSemanticAnalyzer.tableIfExists(tableDesc, context.hiveDb);
+   }
+ 
+   private String location() throws MetaException, HiveException {
+     Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName());
+     if (!tableContext.waitOnPrecursor()) {
+       return context.warehouse.getDefaultTablePath(parentDb, tableDesc.getTableName()).toString();
+     } else {
+       Path tablePath = new Path(
+           context.warehouse.getDefaultDatabasePath(tableDesc.getDatabaseName()),
+           MetaStoreUtils.encodeTableName(tableDesc.getTableName().toLowerCase())
+       );
+       return context.warehouse.getDnsPath(tablePath).toString();
+     }
+   }
+ 
+   private void createTableReplLogTask() throws SemanticException {
+     ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger,
+                                             tableDesc.getTableName(), tableDesc.tableType());
+     Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf);
+ 
+     if (tracker.tasks().isEmpty()) {
+       tracker.addTask(replLogTask);
+     } else {
+       ReplLoadTask.dependency(tracker.tasks(), replLogTask);
+ 
+       List<Task<? extends Serializable>> visited = new ArrayList<>();
+       tracker.updateTaskCount(replLogTask, visited);
+     }
+   }
+ 
+   public TaskTracker tasks() throws SemanticException {
+     try {
+       /*
+       We are doing this both in load table and load partitions
+        */
+       if (tableDesc.getLocation() == null) {
+         tableDesc.setLocation(location());
+       }
+ 
+       if (table == null) {
+         //new table
+ 
+         table = new Table(tableDesc.getDatabaseName(), tableDesc.getTableName());
+         if (isPartitioned(tableDesc)) {
+           updateReplicationState(initialReplicationState());
+           if (!forNewTable().hasReplicationState()) {
+             // Add ReplStateLogTask only if no pending table load tasks left for next cycle
+             createTableReplLogTask();
+           }
+           return tracker;
+         }
+       } else {
+         // existing
+ 
+         if (table.isPartitioned()) {
+           List<AddPartitionDesc> partitionDescs = event.partitionDescriptions(tableDesc);
+           if (!event.replicationSpec().isMetadataOnly() && !partitionDescs.isEmpty()) {
+             updateReplicationState(initialReplicationState());
+             if (!forExistingTable(lastReplicatedPartition).hasReplicationState()) {
+               // Add ReplStateLogTask only if no pending table load tasks left for next cycle
+               createTableReplLogTask();
+             }
+             return tracker;
+           }
+         }
+       }
+       return tracker;
+     } catch (Exception e) {
+       throw new SemanticException(e);
+     }
+   }
+ 
+   private void updateReplicationState(ReplicationState replicationState) throws SemanticException {
+     if (!tracker.canAddMoreTasks()) {
+       tracker.setReplicationState(replicationState);
+     }
+   }
+ 
+   private ReplicationState initialReplicationState() throws SemanticException {
+     return new ReplicationState(
+         new PartitionState(tableDesc.getTableName(), lastReplicatedPartition)
+     );
+   }
+ 
+   private TaskTracker forNewTable() throws Exception {
+     Iterator<AddPartitionDesc> iterator = event.partitionDescriptions(tableDesc).iterator();
+     while (iterator.hasNext() && tracker.canAddMoreTasks()) {
+       AddPartitionDesc addPartitionDesc = iterator.next();
+       tracker.addTask(addSinglePartition(table, addPartitionDesc));
+       if (iterator.hasNext() && !tracker.canAddMoreTasks()) {
+         ReplicationState currentReplicationState =
+                 new ReplicationState(new PartitionState(table.getTableName(), addPartitionDesc));
+         updateReplicationState(currentReplicationState);
+       }
+     }
+     return tracker;
+   }
+ 
+   /**
+    * returns the root task for adding a partition
+    */
+   private Task<? extends Serializable> addSinglePartition(Table table,
+       AddPartitionDesc addPartitionDesc) throws MetaException, IOException, HiveException {
+     AddPartitionDesc.OnePartitionDesc partSpec = addPartitionDesc.getPartition(0);
+     Path sourceWarehousePartitionLocation = new Path(partSpec.getLocation());
+     Path replicaWarehousePartitionLocation = locationOnReplicaWarehouse(table, partSpec);
+     partSpec.setLocation(replicaWarehousePartitionLocation.toString());
+     LOG.debug("adding dependent CopyWork/AddPart/MoveWork for partition "
+         + partSpecToString(partSpec.getPartSpec()) + " with source location: "
+         + partSpec.getLocation());
+     Path tmpPath = context.utils.getExternalTmpPath(replicaWarehousePartitionLocation);
+ 
+     Task<?> copyTask = ReplCopyTask.getLoadCopyTask(
+         event.replicationSpec(),
+         sourceWarehousePartitionLocation,
+         tmpPath,
+         context.hiveConf
+     );
+ 
+     Task<?> addPartTask = TaskFactory.get(
+         new DDLWork(new HashSet<>(), new HashSet<>(), addPartitionDesc),
+         context.hiveConf
+     );
+ 
+     Task<?> movePartitionTask = movePartitionTask(table, partSpec, tmpPath);
+ 
+     copyTask.addDependentTask(addPartTask);
+     addPartTask.addDependentTask(movePartitionTask);
+     return copyTask;
+   }
+ 
+   /**
+    * This will create the move of partition data from temp path to actual path
+    */
+   private Task<?> movePartitionTask(Table table, AddPartitionDesc.OnePartitionDesc partSpec,
+       Path tmpPath) {
+     LoadTableDesc loadTableWork = new LoadTableDesc(
+         tmpPath, Utilities.getTableDesc(table), partSpec.getPartSpec(),
 -        event.replicationSpec().isReplace()
++        event.replicationSpec().isReplace(), SessionState.get().getTxnMgr().getCurrentTxnId()
+     );
+     loadTableWork.setInheritTableSpecs(false);
+     MoveWork work = new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false);
+     return TaskFactory.get(work, context.hiveConf);
+   }
+ 
+   private Path locationOnReplicaWarehouse(Table table, AddPartitionDesc.OnePartitionDesc partSpec)
+       throws MetaException, HiveException, IOException {
+     String child = Warehouse.makePartPath(partSpec.getPartSpec());
+     if (tableDesc.getLocation() == null) {
+       if (table.getDataLocation() == null) {
+         Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName());
+         return new Path(
+             context.warehouse.getDefaultTablePath(parentDb, tableDesc.getTableName()), child);
+       } else {
+         return new Path(table.getDataLocation().toString(), child);
+       }
+     } else {
+       return new Path(tableDesc.getLocation(), child);
+     }
+   }
+ 
+   private Task<? extends Serializable> alterSinglePartition(AddPartitionDesc desc,
+       ReplicationSpec replicationSpec, Partition ptn) {
+     desc.setReplaceMode(true);
+     if ((replicationSpec != null) && (replicationSpec.isInReplicationScope())) {
+       desc.setReplicationSpec(replicationSpec);
+     }
+     desc.getPartition(0).setLocation(ptn.getLocation()); // use existing location
+     return TaskFactory.get(
+         new DDLWork(new HashSet<>(), new HashSet<>(), desc),
+         context.hiveConf
+     );
+   }
+ 
+   private TaskTracker forExistingTable(AddPartitionDesc lastPartitionReplicated) throws Exception {
+     boolean encounteredTheLastReplicatedPartition = (lastPartitionReplicated == null);
+     ReplicationSpec replicationSpec = event.replicationSpec();
+     LOG.debug("table partitioned");
+ 
+     Iterator<AddPartitionDesc> iterator = event.partitionDescriptions(tableDesc).iterator();
+     while (iterator.hasNext()) {
+       /*
+       encounteredTheLastReplicatedPartition will be set, when we break creation of partition tasks
+       for a table, as we have reached the limit of number of tasks we should create for execution.
+       in this case on the next run we have to iterate over the partitions desc to reach the last replicated
+       partition so that we can start replicating partitions after that.
+        */
+       AddPartitionDesc addPartitionDesc = iterator.next();
+       if (encounteredTheLastReplicatedPartition && tracker.canAddMoreTasks()) {
+         Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
+         Partition ptn;
+ 
+         if ((ptn = context.hiveDb.getPartition(table, partSpec, false)) == null) {
+           if (!replicationSpec.isMetadataOnly()) {
+             forNewTable();
+           }
+         } else {
+           // If replicating, then the partition already existing means we need to replace, maybe, if
+           // the destination ptn's repl.last.id is older than the replacement's.
+           if (replicationSpec.allowReplacementInto(ptn.getParameters())) {
+             if (replicationSpec.isMetadataOnly()) {
+               tracker.addTask(alterSinglePartition(addPartitionDesc, replicationSpec, ptn));
+               if (iterator.hasNext() && !tracker.canAddMoreTasks()) {
+                 tracker.setReplicationState(
+                     new ReplicationState(new PartitionState(table.getTableName(), addPartitionDesc)
+                     )
+                 );
+               }
+             } else {
+               forNewTable();
+             }
+           } else {
+             // ignore this ptn, do nothing, not an error.
+           }
+         }
+       } else {
+         Map<String, String> currentSpec = addPartitionDesc.getPartition(0).getPartSpec();
+         Map<String, String> lastReplicatedPartSpec =
+             lastPartitionReplicated.getPartition(0).getPartSpec();
+         encounteredTheLastReplicatedPartition = lastReplicatedPartSpec.equals(currentSpec);
+       }
+     }
+     return tracker;
+   }
+ }
+

[20/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/tez/explainuser_3.q.out
index c9ccf23,d26a9a3..9f764b8
--- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
@@@ -509,13 -509,13 +509,13 @@@ Stage-
                    Conditional Operator
                      Stage-1
                        Map 1 vectorized
 -                      File Output Operator [FS_10]
 +                      File Output Operator [FS_8]
                          table:{"name:":"default.orc_merge5"}
-                         Select Operator [SEL_7] (rows=306 width=268)
+                         Select Operator [SEL_9] (rows=306 width=335)
                            Output:["_col0","_col1","_col2","_col3","_col4"]
-                           Filter Operator [FIL_6] (rows=306 width=268)
+                           Filter Operator [FIL_8] (rows=306 width=335)
                              predicate:(userid <= 13)
-                             TableScan [TS_0] (rows=919 width=268)
+                             TableScan [TS_0] (rows=919 width=335)
                                default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"]
              Stage-4(CONDITIONAL)
                File Merge

[44/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 9754039,a5b69ec..df5c6aa
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@@ -6873,22 -6889,16 +6851,24 @@@ public class SemanticAnalyzer extends B
            acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
            checkAcidConstraints(qb, table_desc, dest_tab);
          }
 -        Long currentTransactionId = acidOp == Operation.NOT_ACID ? null :
 +        if (AcidUtils.isInsertOnlyTable(table_desc.getProperties())) {
 +          acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
 +        }
 +        if (isMmTable) {
 +          txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
 +        } else {
 +          txnId = acidOp == Operation.NOT_ACID ? null :
              SessionState.get().getTxnMgr().getCurrentTxnId();
 -        ltd = new LoadTableDesc(queryTmpdir, table_desc, dpCtx, acidOp,
 -            currentTransactionId);
 +        }
 +        boolean isReplace = !qb.getParseInfo().isInsertIntoTable(
 +            dest_tab.getDbName(), dest_tab.getTableName());
 +        ltd = new LoadTableDesc(queryTmpdir, table_desc, dpCtx, acidOp, isReplace, txnId);
          // For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old
          // deltas and base and leave them up to the cleaner to clean up
-         ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
-             dest_tab.getTableName()) && !destTableIsAcid);
+         LoadFileType loadType = (!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
+                 dest_tab.getTableName()) && !destTableIsAcid)
+                 ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING;
+         ltd.setLoadFileType(loadType);
          ltd.setLbCtx(lbCtx);
          loadTableWork.add(ltd);
        } else {
@@@ -6947,20 -7009,16 +6927,22 @@@
          acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
          checkAcidConstraints(qb, table_desc, dest_tab);
        }
 -      Long currentTransactionId = (acidOp == Operation.NOT_ACID) ? null :
 +      if (AcidUtils.isInsertOnlyTable(dest_part.getTable().getParameters())) {
 +        acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
 +      }
 +      if (isMmTable) {
 +        txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
 +      } else {
 +        txnId = (acidOp == Operation.NOT_ACID) ? null :
            SessionState.get().getTxnMgr().getCurrentTxnId();
 -      ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp,
 -          currentTransactionId);
 +      }
 +      ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp, txnId);
        // For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old
        // deltas and base and leave them up to the cleaner to clean up
-       ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
-           dest_tab.getTableName()) && !destTableIsAcid);
+       LoadFileType loadType = (!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
+               dest_tab.getTableName()) && !destTableIsAcid)
+               ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING;
+       ltd.setLoadFileType(loadType);
        ltd.setLbCtx(lbCtx);
  
        loadTableWork.add(ltd);

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
index 1c220b8,7ff321f..6f21cae
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
@@@ -30,10 -29,7 +29,8 @@@ import org.apache.hadoop.hive.conf.Hive
  import org.apache.hadoop.hive.ql.exec.TableScanOperator;
  import org.apache.hadoop.hive.ql.exec.Task;
  import org.apache.hadoop.hive.ql.exec.TaskFactory;
 +import org.apache.hadoop.hive.ql.exec.Utilities;
  import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
- import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork;
  import org.apache.hadoop.hive.ql.lib.Node;
  import org.apache.hadoop.hive.ql.lib.NodeProcessor;
  import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
@@@ -170,44 -160,5 +161,4 @@@ public class SparkProcessAnalyzeTable i
  
      return null;
    }
- 
-   /**
-    * handle partial scan command.
-    *
-    * It is composed of PartialScanTask followed by StatsTask.
-    */
-   private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext,
-       StatsWork statsWork, GenSparkProcContext context, Task<StatsWork> statsTask)
-               throws SemanticException {
-     String aggregationKey = tableScan.getConf().getStatsAggPrefix();
-     StringBuilder aggregationKeyBuffer = new StringBuilder(aggregationKey);
-     List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, aggregationKeyBuffer);
-     aggregationKey = aggregationKeyBuffer.toString();
- 
-     // scan work
-     PartialScanWork scanWork = new PartialScanWork(inputPaths,
-         Utilities.getTableDesc(tableScan.getConf().getTableMetadata()));
-     scanWork.setMapperCannotSpanPartns(true);
-     scanWork.setAggKey(aggregationKey);
-     scanWork.setStatsTmpDir(tableScan.getConf().getTmpStatsDir(), parseContext.getConf());
- 
-     // stats work
-     statsWork.setPartialScanAnalyzeCommand(true);
- 
-     // partial scan task
-     DriverContext driverCxt = new DriverContext();
- 
-     @SuppressWarnings("unchecked")
-     Task<PartialScanWork> partialScanTask = TaskFactory.get(scanWork, parseContext.getConf());
-     partialScanTask.initialize(parseContext.getQueryState(), null, driverCxt,
-         tableScan.getCompilationOpContext());
-     partialScanTask.setWork(scanWork);
-     statsWork.setSourceTask(partialScanTask);
- 
-     // task dependency
-     context.rootTasks.remove(context.currentTask);
-     context.rootTasks.add(partialScanTask);
-     partialScanTask.addDependentTask(statsTask);
-   }
--
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
index f246115,e15f59c..1fa7b40
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
@@@ -61,14 -66,11 +66,14 @@@ public class LoadTableDesc extends Load
    public LoadTableDesc(final Path sourcePath,
        final TableDesc table,
        final Map<String, String> partitionSpec,
-       final boolean replace,
+       final LoadFileType loadFileType,
        final AcidUtils.Operation writeType, Long currentTransactionId) {
      super(sourcePath, writeType);
 -    this.currentTransactionId = currentTransactionId;
 -    init(table, partitionSpec, loadFileType);
 +    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
 +      Utilities.FILE_OP_LOGGER.trace("creating part LTD from " + sourcePath + " to "
 +        + ((table.getProperties() == null) ? "null" : table.getTableName()));
 +    }
-     init(table, partitionSpec, replace, currentTransactionId);
++    init(table, partitionSpec, loadFileType, currentTransactionId);
    }
  
    /**
@@@ -76,14 -78,14 +81,14 @@@
     * @param sourcePath
     * @param table
     * @param partitionSpec
-    * @param replace
+    * @param loadFileType
     */
    public LoadTableDesc(final Path sourcePath,
 -      final TableDesc table,
 -      final Map<String, String> partitionSpec,
 -      final LoadFileType loadFileType) {
 -    this(sourcePath, table, partitionSpec, loadFileType, AcidUtils.Operation.NOT_ACID,
 -        null);
 +                       final TableDesc table,
 +                       final Map<String, String> partitionSpec,
-                        final boolean replace,
++                       final LoadFileType loadFileType,
 +                       final Long txnId) {
-     this(sourcePath, table, partitionSpec, replace, AcidUtils.Operation.NOT_ACID, txnId);
++    this(sourcePath, table, partitionSpec, loadFileType, AcidUtils.Operation.NOT_ACID, txnId);
    }
  
    public LoadTableDesc(final Path sourcePath,
@@@ -100,37 -103,33 +106,39 @@@
     * @param partitionSpec
     */
    public LoadTableDesc(final Path sourcePath,
 -      final TableDesc table,
 -      final Map<String, String> partitionSpec) {
 +                       final org.apache.hadoop.hive.ql.plan.TableDesc table,
 +                       final Map<String, String> partitionSpec, Long txnId) {
-     this(sourcePath, table, partitionSpec, true, AcidUtils.Operation.NOT_ACID, txnId);
+     this(sourcePath, table, partitionSpec, LoadFileType.REPLACE_ALL,
 -            AcidUtils.Operation.NOT_ACID, null);
++      AcidUtils.Operation.NOT_ACID, txnId);
    }
  
    public LoadTableDesc(final Path sourcePath,
        final TableDesc table,
        final DynamicPartitionCtx dpCtx,
 -      final AcidUtils.Operation writeType, Long currentTransactionId) {
 +      final AcidUtils.Operation writeType,
 +      boolean isReplace, Long txnId) {
      super(sourcePath, writeType);
 +    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
 +      Utilities.FILE_OP_LOGGER.trace("creating LTD from " + sourcePath + " to " + table.getTableName());
 +    }
      this.dpCtx = dpCtx;
 -    this.currentTransactionId = currentTransactionId;
++    LoadFileType lft = isReplace ?  LoadFileType.REPLACE_ALL :  LoadFileType.OVERWRITE_EXISTING;
      if (dpCtx != null && dpCtx.getPartSpec() != null && partitionSpec == null) {
-       init(table, dpCtx.getPartSpec(), isReplace, txnId);
 -      init(table, dpCtx.getPartSpec(), LoadFileType.REPLACE_ALL);
++      init(table, dpCtx.getPartSpec(), lft, txnId);
      } else {
-       init(table, new LinkedHashMap<>(), isReplace, txnId);
 -      init(table, new LinkedHashMap<>(), LoadFileType.REPLACE_ALL);
++      init(table, new LinkedHashMap<String, String>(), lft, txnId);
      }
    }
  
    private void init(
        final org.apache.hadoop.hive.ql.plan.TableDesc table,
        final Map<String, String> partitionSpec,
-       final boolean replace,
 -      final LoadFileType loadFileType) {
++      final LoadFileType loadFileType,
 +      Long txnId) {
      this.table = table;
      this.partitionSpec = partitionSpec;
-     this.replace = replace;
+     this.loadFileType = loadFileType;
 +    this.currentTransactionId = txnId;
    }
  
    @Explain(displayName = "table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
@@@ -153,20 -152,15 +161,24 @@@
  
    @Explain(displayName = "replace")
    public boolean getReplace() {
-     return replace;
+     return (loadFileType == LoadFileType.REPLACE_ALL);
+   }
+ 
+   public LoadFileType getLoadFileType() {
+     return loadFileType;
    }
  
 +  @Explain(displayName = "micromanaged table")
 +  public Boolean isMmTableExplain() {
 +    return isMmTable() ? true : null;
 +  }
 +
 +  public boolean isMmTable() {
 +    return AcidUtils.isInsertOnlyTable(table.getProperties());
 +  }
 +
-   public void setReplace(boolean replace) {
-     this.replace = replace;
+   public void setLoadFileType(LoadFileType loadFileType) {
+     this.loadFileType = loadFileType;
    }
  
    public DynamicPartitionCtx getDPCtx() {

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
index 75ea16f,e523049..d5ab079
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
@@@ -140,7 -141,8 +141,8 @@@ public class TestExecDriver extends Tes
          db.dropTable(Warehouse.DEFAULT_DATABASE_NAME, src, true, true);
          db.createTable(src, cols, null, TextInputFormat.class,
              HiveIgnoreKeyTextOutputFormat.class);
-         db.loadTable(hadoopDataFile[i], src, false, true, false, false, false, null, 0, false);
+         db.loadTable(hadoopDataFile[i], src, LoadFileType.KEEP_EXISTING,
 -                true, false, false, false);
++           true, false, false, false, null, 0, false);
          i++;
        }
  

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/standalone-metastore/src/main/resources/package.jdo
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/standalone-metastore/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------

[07/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
----------------------------------------------------------------------
diff --cc standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
index 0000000,6cc7157..3759348
mode 000000,100644..100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java
@@@ -1,0 -1,256 +1,256 @@@
+ /**
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.hadoop.hive.metastore.model;
+ 
+ import java.util.List;
+ import java.util.Map;
+ 
+ public class MTable {
+   
+   private String tableName;
+   private MDatabase database;
+   private MStorageDescriptor sd;
+   private String owner;
+   private int createTime;
+   private int lastAccessTime;
+   private int retention;
+   private List<MFieldSchema> partitionKeys;
+   private Map<String, String> parameters;
+   private String viewOriginalText;
+   private String viewExpandedText;
+   private boolean rewriteEnabled;
+   private String tableType;
+ 
+   public MTable() {}
+ 
+   /**
+    * @param tableName
+    * @param database
+    * @param sd
+    * @param owner
+    * @param createTime
+    * @param lastAccessTime
+    * @param retention
+    * @param partitionKeys
+    * @param parameters
+    * @param viewOriginalText
+    * @param viewExpandedText
+    * @param tableType
+    */
+   public MTable(String tableName, MDatabase database, MStorageDescriptor sd, String owner,
+       int createTime, int lastAccessTime, int retention, List<MFieldSchema> partitionKeys,
 -      Map<String, String> parameters, String viewOriginalText, String viewExpandedText,
 -      boolean rewriteEnabled, String tableType) {
++      Map<String, String> parameters,
++      String viewOriginalText, String viewExpandedText, boolean rewriteEnabled, String tableType) {
+     this.tableName = tableName;
+     this.database = database;
+     this.sd = sd;
+     this.owner = owner;
+     this.createTime = createTime;
+     this.setLastAccessTime(lastAccessTime);
+     this.retention = retention;
+     this.partitionKeys = partitionKeys;
+     this.parameters = parameters;
+     this.viewOriginalText = viewOriginalText;
+     this.viewExpandedText = viewExpandedText;
+     this.rewriteEnabled = rewriteEnabled;
+     this.tableType = tableType;
+   }
+ 
+   /**
+    * @return the tableName
+    */
+   public String getTableName() {
+     return tableName;
+   }
+ 
+   /**
+    * @param tableName the tableName to set
+    */
+   public void setTableName(String tableName) {
+     this.tableName = tableName;
+   }
+ 
+   /**
+    * @return the sd
+    */
+   public MStorageDescriptor getSd() {
+     return sd;
+   }
+ 
+   /**
+    * @param sd the sd to set
+    */
+   public void setSd(MStorageDescriptor sd) {
+     this.sd = sd;
+   }
+ 
+   /**
+    * @return the partKeys
+    */
+   public List<MFieldSchema> getPartitionKeys() {
+     return partitionKeys;
+   }
+ 
+   /**
+    * @param partKeys the partKeys to set
+    */
+   public void setPartitionKeys(List<MFieldSchema> partKeys) {
+     this.partitionKeys = partKeys;
+   }
+ 
+   /**
+    * @return the parameters
+    */
+   public Map<String, String> getParameters() {
+     return parameters;
+   }
+ 
+   /**
+    * @param parameters the parameters to set
+    */
+   public void setParameters(Map<String, String> parameters) {
+     this.parameters = parameters;
+   }
+ 
+   /**
+    * @return the original view text, or null if this table is not a view
+    */
+   public String getViewOriginalText() {
+     return viewOriginalText;
+   }
+ 
+   /**
+    * @param viewOriginalText the original view text to set
+    */
+   public void setViewOriginalText(String viewOriginalText) {
+     this.viewOriginalText = viewOriginalText;
+   }
+ 
+   /**
+    * @return the expanded view text, or null if this table is not a view
+    */
+   public String getViewExpandedText() {
+     return viewExpandedText;
+   }
+ 
+   /**
+    * @param viewExpandedText the expanded view text to set
+    */
+   public void setViewExpandedText(String viewExpandedText) {
+     this.viewExpandedText = viewExpandedText;
+   }
+ 
+   /**
+    * @return whether the view can be used for rewriting queries
+    */
+   public boolean isRewriteEnabled() {
+     return rewriteEnabled;
+   }
+ 
+   /**
+    * @param rewriteEnabled whether the view can be used for rewriting queries
+    */
+   public void setRewriteEnabled(boolean rewriteEnabled) {
+     this.rewriteEnabled = rewriteEnabled;
+   }
+ 
+   /**
+    * @return the owner
+    */
+   public String getOwner() {
+     return owner;
+   }
+ 
+   /**
+    * @param owner the owner to set
+    */
+   public void setOwner(String owner) {
+     this.owner = owner;
+   }
+ 
+   /**
+    * @return the createTime
+    */
+   public int getCreateTime() {
+     return createTime;
+   }
+ 
+   /**
+    * @param createTime the createTime to set
+    */
+   public void setCreateTime(int createTime) {
+     this.createTime = createTime;
+   }
+ 
+   /**
+    * @return the database
+    */
+   public MDatabase getDatabase() {
+     return database;
+   }
+ 
+   /**
+    * @param database the database to set
+    */
+   public void setDatabase(MDatabase database) {
+     this.database = database;
+   }
+ 
+   /**
+    * @return the retention
+    */
+   public int getRetention() {
+     return retention;
+   }
+ 
+   /**
+    * @param retention the retention to set
+    */
+   public void setRetention(int retention) {
+     this.retention = retention;
+   }
+ 
+   /**
+    * @param lastAccessTime the lastAccessTime to set
+    */
+   public void setLastAccessTime(int lastAccessTime) {
+     this.lastAccessTime = lastAccessTime;
+   }
+ 
+   /**
+    * @return the lastAccessTime
+    */
+   public int getLastAccessTime() {
+     return lastAccessTime;
+   }
+ 
+   /**
+    * @param tableType the tableType to set
+    */
+   public void setTableType(String tableType) {
+     this.tableType = tableType;
+   }
+ 
+   /**
+    * @return the tableType
+    */
+   public String getTableType() {
+     return tableType;
+   }
+ }

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/main/resources/package.jdo
----------------------------------------------------------------------
diff --cc standalone-metastore/src/main/resources/package.jdo
index 0000000,570fd44..bbb8f2d
mode 000000,100644..100644
--- a/standalone-metastore/src/main/resources/package.jdo
+++ b/standalone-metastore/src/main/resources/package.jdo
@@@ -1,0 -1,1088 +1,1087 @@@
+ <?xml version="1.0"?>
+ <!--
+   Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+  
+       http://www.apache.org/licenses/LICENSE-2.0
+  
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+ --> 
+ <!DOCTYPE jdo PUBLIC "-//Sun Microsystems, Inc.//DTD Java Data Objects Metadata 2.0//EN"
+   "http://java.sun.com/dtd/jdo_2_0.dtd">
+ <!--
+   Size Limitations:
+ 
+   Indexed VARCHAR: 767 bytes (MySQL running on InnoDB Engine http://bugs.mysql.com/bug.php?id=13315)
+   Non-indexed VARCHAR: 4000 bytes (max length on Oracle 9i/10g/11g)
+ 
+ -->
+ <jdo>  
+   <package name="org.apache.hadoop.hive.metastore.model">  
+     <class name="MDatabase" identity-type="datastore" table="DBS" detachable="true">  
+       <datastore-identity>
+         <column name="DB_ID"/>
+       </datastore-identity>
+       <field name="name">  
+         <column name="NAME" length="128" jdbc-type="VARCHAR"/>
+         <index name="UniqueDatabase" unique="true"/>
+       </field>
+       <field name="description">
+         <column name="DESC" length="4000" jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+       <field name="locationUri">
+         <column name="DB_LOCATION_URI" length="4000" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="parameters" table="DATABASE_PARAMS">
+         <map key-type="java.lang.String" value-type="java.lang.String"/>
+         <join>
+           <column name="DB_ID"/>
+         </join>
+         <key>
+            <column name="PARAM_KEY" length="180" jdbc-type="VARCHAR"/>
+         </key>
+         <value>
+            <column name="PARAM_VALUE" length="4000" jdbc-type="VARCHAR"/>
+         </value>
+       </field>
 -      <field name="ownerName">	
++      <field name="ownerName">    
+         <column name="OWNER_NAME" length="128" jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+        <field name="ownerType">
+         <column name="OWNER_TYPE" length="10" jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+     </class>
+ 
+     <class name="MFieldSchema" embedded-only="true" table="TYPE_FIELDS" detachable="true">
+       <field name="name">
+         <column name="FNAME" length="767" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="type" >
+         <column name="FTYPE" length="32672" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="comment" >
+         <column name="FCOMMENT" length="4000" jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+     </class>
+ 
+     <class name="MType" table="TYPES" detachable="true">  
+       <field name="name" >  
+         <column name="TYPE_NAME" length="128" jdbc-type="VARCHAR"/>  
+         <index name="UniqueType" unique="true"/>
+       </field>
+       <field name="type1">  
+         <column name="TYPE1" length="767" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="type2">  
+         <column name="TYPE2" length="767" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="fields" table="TYPE_FIELDS" >
+         <collection element-type="MFieldSchema"/>
+         <join>
+           <primary-key name="TYPE_FIELDS_PK">
+             <column name="TYPE_NAME"/>
+             <column name="FIELD_NAME"/>
+           </primary-key>
+           <column name="TYPE_NAME"/>
+         </join>
+         <element>
+           <embedded>
+             <field name="name">
+               <column name="FIELD_NAME" length="128" jdbc-type="VARCHAR"/>
+             </field>
+             <field name="type">
+               <column name="FIELD_TYPE" length="767" jdbc-type="VARCHAR"  allows-null="false"/>
+             </field>
+             <field name="comment" >
+               <column name="COMMENT" length="256" jdbc-type="VARCHAR" allows-null="true"/>
+             </field>
+           </embedded>
+         </element>
+       </field>
+     </class>
+ 
+     <class name="MTable" table="TBLS" identity-type="datastore" detachable="true">
+       <datastore-identity>
+         <column name="TBL_ID"/>
+       </datastore-identity>
+       <index name="UniqueTable" unique="true">
+         <column name="TBL_NAME"/>
+         <column name="DB_ID"/>
+       </index>
+       <field name="tableName">
+         <column name="TBL_NAME" length="256" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="database">
+         <column name="DB_ID"/>
+       </field>
+       <field name="partitionKeys" table="PARTITION_KEYS" >
+         <collection element-type="MFieldSchema"/>
+         <join>
+           <primary-key name="PARTITION_KEY_PK">
+             <column name="TBL_ID"/>
+             <column name="PKEY_NAME"/>
+           </primary-key>
+           <column name="TBL_ID"/>
+         </join>
+         <element>
+           <embedded>
+             <field name="name">
+               <column name="PKEY_NAME" length="128" jdbc-type="VARCHAR"/>
+               </field>
+             <field name="type">
+               <column name="PKEY_TYPE" length="767" jdbc-type="VARCHAR" allows-null="false"/>
+             </field>
+             <field name="comment" >
+               <column name="PKEY_COMMENT" length="4000" jdbc-type="VARCHAR" allows-null="true"/>
+             </field>
+           </embedded>
+         </element>
+       </field>
+       <field name="sd" dependent="true">
+         <column name="SD_ID"/>
+       </field>
+       <field name="owner">
+         <column name="OWNER" length="767" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="lastAccessTime">
+         <column name="LAST_ACCESS_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="retention">
+         <column name="RETENTION" jdbc-type="integer"/>
+       </field>
+       <field name="parameters" table="TABLE_PARAMS">
+         <map key-type="java.lang.String" value-type="java.lang.String"/>
+         <join>
+           <column name="TBL_ID"/>
+         </join>
+         <key>
+            <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+         </key>
+         <value>
+            <column name="PARAM_VALUE" length="32672" jdbc-type="VARCHAR"/>
+         </value>
+       </field>
+       <field name="viewOriginalText" default-fetch-group="false">
+         <column name="VIEW_ORIGINAL_TEXT" jdbc-type="LONGVARCHAR"/>
+       </field>
+       <field name="viewExpandedText" default-fetch-group="false">
+         <column name="VIEW_EXPANDED_TEXT" jdbc-type="LONGVARCHAR"/>
+       </field>
+       <field name="rewriteEnabled">
+         <column name="IS_REWRITE_ENABLED"/>
+       </field>
+       <field name="tableType">
+         <column name="TBL_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+     </class>
+ 
+     <class name="MConstraint" identity-type="application" table="KEY_CONSTRAINTS" detachable="true" objectid-class="MConstraint$PK">
+       <field name="constraintName" primary-key="true">
+         <column name="CONSTRAINT_NAME"/>
+       </field>
+       <field name="position" primary-key="true">
+         <column name="POSITION"/>
+       </field>
+       <field name="childColumn">
+         <column name="CHILD_CD_ID"/>
+       </field>
+       <field name="childIntegerIndex">
+         <column name="CHILD_INTEGER_IDX"/>
+       </field>
+       <field name="childTable">
+         <column name="CHILD_TBL_ID"/>
+       </field>
+       <field name="parentColumn">
+         <column name="PARENT_CD_ID"/>
+       </field>
+       <field name="parentIntegerIndex">
 -	<column name="PARENT_INTEGER_IDX"/>
++    <column name="PARENT_INTEGER_IDX"/>
+       </field>
+       <field name="parentTable">
+         <column name="PARENT_TBL_ID"/>
+       </field>
+       <field name="constraintType">
+         <column name="CONSTRAINT_TYPE"/>
+       </field>
+       <field name="deleteRule">
 -	<column name="DELETE_RULE"/>
++        <column name="DELETE_RULE"/>
+       </field>
+       <field name="updateRule">
+         <column name="UPDATE_RULE"/>
+       </field>
+       <field name="enableValidateRely">
+         <column name="ENABLE_VALIDATE_RELY"/>
+       </field>
+     </class>
+ 
+     <class name="MSerDeInfo" identity-type="datastore" table="SERDES" detachable="true">
+       <datastore-identity>
+         <column name="SERDE_ID"/>
+       </datastore-identity>
+       <field name="name">
+         <column name="NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="serializationLib">
+         <column name="SLIB" length="4000" jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+       <field name="parameters" table="SERDE_PARAMS">
+         <map key-type="java.lang.String" value-type="java.lang.String"/>
+         <join>
+           <column name="SERDE_ID"/>
+         </join>
+         <key>
+            <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+         </key>
+         <value>
+            <column name="PARAM_VALUE" length="32672" jdbc-type="VARCHAR"/>
+         </value>
+       </field>
+     </class>
+ 
+     <class name="MOrder" embedded-only="true" table="SORT_ORDER" detachable="true">
+       <field name="col">
+         <column name="COL_NAME" length="767" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="order">
+         <column name="ORDER" jdbc-type="INTEGER"  allows-null="false"/>
+       </field>
+     </class>
+ 
+     <class name="MColumnDescriptor" identity-type="datastore" table="CDS" detachable="true">
+       <datastore-identity>
+         <column name="CD_ID"/>
+       </datastore-identity>
+       <field name="cols" table="COLUMNS_V2" >
+         <collection element-type="MFieldSchema"/>
+         <join>
+           <primary-key name="COLUMNS_PK">
+             <column name="COLUMN_NAME"/>
+           </primary-key>
+           <column name="CD_ID"/>
+         </join>
+         <element>
+           <embedded>
+             <field name="name">
+               <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR"/>
+               </field>
+             <field name="type">
+               <column name="TYPE_NAME" length="32672" jdbc-type="VARCHAR" allows-null="false"/>
+             </field>
+             <field name="comment">
+               <column name="COMMENT" length="256" jdbc-type="VARCHAR" allows-null="true"/>
+             </field>
+           </embedded>
+         </element>
+       </field>
 -	</class>
++    </class>
+ 
+    <class name="MStringList" identity-type="datastore" table="Skewed_STRING_LIST" detachable="true">
+      <datastore-identity>
+        <column name="STRING_LIST_ID"/>
+      </datastore-identity>
+      <field name="internalList" table="Skewed_STRING_LIST_VALUES">
+        <collection element-type="java.lang.String"/>
+        <join>
+          <column name="STRING_LIST_ID"/>
+        </join>
+        <element column="STRING_LIST_VALUE"/>
+      </field>
+    </class>
+ 
+     <class name="MStorageDescriptor" identity-type="datastore" table="SDS" detachable="true">
+       <datastore-identity>
+         <column name="SD_ID"/>
+       </datastore-identity>
+       <field name="cd">
 -      	<column name="CD_ID"/>
++          <column name="CD_ID"/>
+       </field>
+       <field name="location">
+         <column name="LOCATION" length="4000" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="inputFormat">
+         <column name="INPUT_FORMAT" length="4000" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="outputFormat">
+         <column name="OUTPUT_FORMAT" length="4000" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="isCompressed">
+         <column name="IS_COMPRESSED"/>
+       </field>
+       <field name="isStoredAsSubDirectories">
+         <column name="IS_STOREDASSUBDIRECTORIES"/>
+       </field>
+       <field name="numBuckets">
+         <column name="NUM_BUCKETS" jdbc-type="integer"/>
+       </field>
+       <field name="serDeInfo" dependent="true">
+         <column name="SERDE_ID"/>
+       </field>
+       <field name="bucketCols" table="BUCKETING_COLS">
+         <collection element-type="java.lang.String"/>
+         <join>
+           <column name="SD_ID"/>
+         </join>
+         <element column="BUCKET_COL_NAME"/>
+       </field>
+       <field name="sortCols" table="SORT_COLS">
+         <collection element-type="MOrder"/>
+         <join>
+           <column name="SD_ID"/>
+         </join>
+         <element>
+           <embedded>
+             <field name="col">
+               <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR"/>
+               </field>
+             <field name="order">
+               <column name="ORDER" jdbc-type="INTEGER"  allows-null="false"/>
+             </field>
+           </embedded>
+         </element>
+       </field>
+       <field name="parameters" table="SD_PARAMS">
+         <map key-type="java.lang.String" value-type="java.lang.String"/>
+         <join>
+           <column name="SD_ID"/>
+         </join>
+         <key>
+            <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+         </key>
+         <value>
+            <column name="PARAM_VALUE" length="32672" jdbc-type="VARCHAR"/>
+         </value>
+       </field>
+       <field name="skewedColNames" table="SKEWED_COL_NAMES">
+         <collection element-type="java.lang.String"/>
+         <join>
+           <column name="SD_ID"/>
+         </join>
+         <element column="SKEWED_COL_NAME"/>
+       </field>
+       <field name="skewedColValues" table="SKEWED_VALUES">
+         <collection element-type="MStringList"/>
+         <join/>
+       </field>
+       <field name="skewedColValueLocationMaps" table="SKEWED_COL_VALUE_LOC_MAP">
+         <map key-type="MStringList" value-type="java.lang.String"/>
+         <join>
+           <column name="SD_ID"/>
+         </join>
+         <value>
+           <column name="location" length="4000" jdbc-type="VARCHAR"/>
+         </value>
+       </field>
+     </class>
+ 
+     <class name="MPartition" table="PARTITIONS" identity-type="datastore" detachable="true">
+       <index name="UniquePartition" unique="true">
+         <column name="PART_NAME"/>
+         <column name="TBL_ID"/>
+       </index>
+       <datastore-identity>
+         <column name="PART_ID"/>
+       </datastore-identity>
+       <field name="partitionName">
+         <column name="PART_NAME" length="767" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="table">
+         <column name="TBL_ID"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="lastAccessTime">
+         <column name="LAST_ACCESS_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="values" table="PARTITION_KEY_VALS">
+         <collection element-type="java.lang.String"/>
+         <join>
+           <column name="PART_ID"/>
+         </join>
+         <element column="PART_KEY_VAL"/>
+       </field>
+       <field name="sd" dependent="true">
+         <column name="SD_ID"/>
+       </field>
+       <field name="parameters" table="PARTITION_PARAMS">
+         <map key-type="java.lang.String" value-type="java.lang.String"/>
+         <join>
+           <column name="PART_ID"/>
+         </join>
+         <key>
+            <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+         </key>
+         <value>
+            <column name="PARAM_VALUE" length="4000" jdbc-type="VARCHAR"/>
+         </value>
+       </field>
+     </class>
+     
+     <class name="MIndex" table="IDXS" identity-type="datastore" detachable="true">
+       <index name="UniqueINDEX" unique="true">
+         <column name="INDEX_NAME"/>
+         <column name="ORIG_TBL_ID"/>
+       </index>
+       
+       <datastore-identity>
+         <column name="INDEX_ID"/>
+       </datastore-identity>
+       <field name="indexName">
+         <column name="INDEX_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="origTable">
+         <column name="ORIG_TBL_ID"/>
+       </field>
+       <field name="indexTable">
+         <column name="INDEX_TBL_ID"/>
+       </field>
+       <field name="indexHandlerClass">
+         <column name="INDEX_HANDLER_CLASS" length="4000" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="deferredRebuild">
+         <column name="DEFERRED_REBUILD"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="lastAccessTime">
+         <column name="LAST_ACCESS_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="sd" dependent="true">
+         <column name="SD_ID"/>
+       </field>
+       <field name="parameters" table="INDEX_PARAMS">
+         <map key-type="java.lang.String" value-type="java.lang.String"/>
+         <join>
+           <column name="INDEX_ID"/>
+         </join>
+         <key>
+            <column name="PARAM_KEY" length="256" jdbc-type="VARCHAR"/>
+         </key>
+         <value>
+            <column name="PARAM_VALUE" length="4000" jdbc-type="VARCHAR"/>
+         </value>
+       </field>
+     </class>
+ 
+     <class name="MRole" table="ROLES" identity-type="datastore" detachable="true">
+ 
+       <index name="RoleEntityINDEX" unique="true">
+         <column name="ROLE_NAME"/>
+       </index>
+ 
+       <datastore-identity>
+         <column name="ROLE_ID"/>
+       </datastore-identity>
+ 
+       <field name="roleName">
+         <column name="ROLE_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="ownerName">
+         <column name="OWNER_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       
+     </class>
+ 
+     <class name="MRoleMap" table="ROLE_MAP" identity-type="datastore" detachable="true">
+       <index name="UserRoleMapINDEX" unique="true">
+         <column name="PRINCIPAL_NAME"/>
+         <column name="ROLE_ID"/>
+         <column name="GRANTOR"/>
+         <column name="GRANTOR_TYPE"/>
+       </index>
+ 
+       <datastore-identity>
+         <column name="ROLE_GRANT_ID"/>
+       </datastore-identity>
+ 
+       <field name="role">
+         <column name="ROLE_ID" />
+       </field>
+       <field name="principalName">
+         <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="principalType">
+         <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="addTime">
+         <column name="ADD_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="grantor">
+         <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantorType">
+         <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantOption">
+         <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+       </field>
+     </class>
+ 
+     <class name="MGlobalPrivilege" table="GLOBAL_PRIVS" identity-type="datastore" detachable="true">
+       
+       <index name="GlobalPrivilegeIndex" unique="true">
+         <column name="PRINCIPAL_NAME"/>
+         <column name="PRINCIPAL_TYPE"/>
+         <column name="USER_PRIV"/>
+         <column name="GRANTOR"/>
+         <column name="GRANTOR_TYPE"/>
+       </index>
+     
+       <datastore-identity>
+         <column name="USER_GRANT_ID"/>
+       </datastore-identity>
+ 
+       <field name="principalName">
+         <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="principalType">
+         <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="privilege">
+         <column name="USER_PRIV" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="grantor">
+         <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantorType">
+         <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantOption">
+         <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+       </field>
+     </class>
+ 
+     <class name="MDBPrivilege" table="DB_PRIVS" identity-type="datastore" detachable="true">
+       
+       <index name="DBPrivilegeIndex" unique="true">
+         <column name="DB_ID"/>
+         <column name="PRINCIPAL_NAME"/>
+         <column name="PRINCIPAL_TYPE"/>
+         <column name="DB_PRIV"/>
+         <column name="GRANTOR"/>
+         <column name="GRANTOR_TYPE"/>
+       </index>
+ 
+       <datastore-identity>
+         <column name="DB_GRANT_ID"/>
+       </datastore-identity>
+ 
+       <field name="principalName">
+         <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="principalType">
+         <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="database">
+         <column name="DB_ID" />
+       </field>
+       <field name="privilege">
+         <column name="DB_PRIV" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="grantor">
+         <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantorType">
+         <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantOption">
+         <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+       </field>
+     </class>
+ 
+     <class name="MTablePrivilege" table="TBL_PRIVS" identity-type="datastore" detachable="true">
+     
+       <index name="TablePrivilegeIndex" unique="false">
+         <column name="TBL_ID"/>
+         <column name="PRINCIPAL_NAME"/>
+         <column name="PRINCIPAL_TYPE"/>
+         <column name="TBL_PRIV"/>
+         <column name="GRANTOR"/>
+         <column name="GRANTOR_TYPE"/>
+       </index>
+ 
+       <datastore-identity>
+         <column name="TBL_GRANT_ID"/>
+       </datastore-identity>
+ 
+       <field name="principalName">
+         <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="principalType">
+         <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="table">
+         <column name="TBL_ID"/>
+       </field>
+       <field name="privilege">
+         <column name="TBL_PRIV" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="grantor">
+         <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantorType">
+         <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantOption">
+         <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+       </field>
+     </class>
+ 
+     <class name="MPartitionPrivilege" table="PART_PRIVS" identity-type="datastore" detachable="true">
+     
+       <index name="PartPrivilegeIndex" unique="false">
+         <column name="PART_ID"/>
+         <column name="PRINCIPAL_NAME"/>
+         <column name="PRINCIPAL_TYPE"/>
+         <column name="PART_PRIV"/>
+         <column name="GRANTOR"/>
+         <column name="GRANTOR_TYPE"/>
+       </index>
+ 
+       <datastore-identity>
+         <column name="PART_GRANT_ID"/>
+       </datastore-identity>
+ 
+       <field name="principalName">
+         <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="principalType">
+         <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="partition">
+         <column name="PART_ID" />
+       </field>
+       <field name="privilege">
+         <column name="PART_PRIV" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="grantor">
+         <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantorType">
+         <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantOption">
+         <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+       </field>
+     </class>
+ 
+     <class name="MTableColumnPrivilege" table="TBL_COL_PRIVS" identity-type="datastore" detachable="true">
+     
+      <index name="TableColumnPrivilegeIndex" unique="false">
+         <column name="TBL_ID"/>
+         <column name="COLUMN_NAME"/>
+         <column name="PRINCIPAL_NAME"/>
+         <column name="PRINCIPAL_TYPE"/>
+         <column name="TBL_COL_PRIV"/>
+         <column name="GRANTOR"/>
+         <column name="GRANTOR_TYPE"/>
+       </index>
+ 
+       <datastore-identity>
+         <column name="TBL_COLUMN_GRANT_ID"/>
+       </datastore-identity>
+ 
+       <field name="principalName">
+         <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="principalType">
+         <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="table">
+         <column name="TBL_ID" />
+       </field>
+       <field name="columnName">
+         <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="privilege">
+         <column name="TBL_COL_PRIV" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="grantor">
+         <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantorType">
+         <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantOption">
+         <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+       </field>
+     </class>
+ 
+     <class name="MPartitionColumnPrivilege" table="PART_COL_PRIVS" identity-type="datastore" detachable="true">
+     
+      <index name="PartitionColumnPrivilegeIndex" unique="false">
+         <column name="PART_ID"/>
+         <column name="COLUMN_NAME"/>
+         <column name="PRINCIPAL_NAME"/>
+         <column name="PRINCIPAL_TYPE"/>
+         <column name="PART_COL_PRIV"/>
+         <column name="GRANTOR"/>
+         <column name="GRANTOR_TYPE"/>
+       </index>
+ 
+       <datastore-identity>
+         <column name="PART_COLUMN_GRANT_ID"/>
+       </datastore-identity>
+ 
+       <field name="principalName">
+         <column name="PRINCIPAL_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="principalType">
+         <column name="PRINCIPAL_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="partition">
+         <column name="PART_ID" />
+       </field>
+       <field name="columnName">
+         <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="privilege">
+         <column name="PART_COL_PRIV" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="grantor">
+         <column name="GRANTOR" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantorType">
+         <column name="GRANTOR_TYPE" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="grantOption">
+         <column name="GRANT_OPTION" jdbc-type="SMALLINT"/>
+       </field>
+     </class>
+ 
+     <class name="MPartitionEvent"  table="PARTITION_EVENTS" identity-type="datastore" detachable="true">  
+        
+       <index name="PartitionEventIndex" unique="false">
+         <column name="PARTITION_NAME"/>
+       </index>
+       
+       <datastore-identity>
+         <column name="PART_NAME_ID"/>
+       </datastore-identity>
+       
+       <field name="dbName">  
+         <column name="DB_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="tblName">
+         <column name="TBL_NAME" length="256" jdbc-type="VARCHAR"/>
+       </field>
+        <field name="partName">
+         <column name="PARTITION_NAME" length="767" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="eventType">
+         <column name="EVENT_TYPE"  jdbc-type="integer"/>
+       </field>
+      <field name="eventTime">
+         <column name="EVENT_TIME"  jdbc-type="BIGINT"/>
+       </field>
+ 
+     </class>
+     
+     <class name="MMasterKey" table="MASTER_KEYS" identity-type="application" detachable="true">
+ 
+       <field name="keyId" primary-key="true" value-strategy="native">
+         <column name="KEY_ID" jdbc-type="integer" />
+       </field>
+         
+       <field name="masterKey">
+         <column name="MASTER_KEY" length="767" jdbc-type="VARCHAR" />
+       </field>  
+       
+     </class>
+ 
+     <class name="MDelegationToken" table="DELEGATION_TOKENS" identity-type="application" detachable="true">
+ 
+       <field name="tokenIdentifier" primary-key="true">
+         <column name="TOKEN_IDENT" length="767" jdbc-type="VARCHAR" />
+       </field>  
+ 
+       <field name="tokenStr">
+         <column name="TOKEN" length="767" jdbc-type="VARCHAR" />
+       </field>
+             
+     </class>    
+ 
+     <class name="MTableColumnStatistics" table="TAB_COL_STATS" identity-type="datastore" detachable="true">
+       <datastore-identity>
+         <column name="CS_ID"/>
+       </datastore-identity>
+ 
+       <field name ="dbName">
+         <column name="DB_NAME" length="128" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="tableName">
+         <column name="TABLE_NAME" length="256" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="table">
+         <column name="TBL_ID"/>
+       </field>
+       <field name="colName">
+         <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="colType">
+         <column name="COLUMN_TYPE" length="128" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="longLowValue">
+         <column name="LONG_LOW_VALUE"  jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+       <field name="longHighValue">
+         <column name="LONG_HIGH_VALUE" jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+        <field name="doubleLowValue">
+         <column name="DOUBLE_LOW_VALUE"  jdbc-type="DOUBLE" allows-null="true"/>
+       </field>
+       <field name="doubleHighValue">
+         <column name="DOUBLE_HIGH_VALUE" jdbc-type="DOUBLE" allows-null="true"/>
+       </field>
+       <field name="decimalLowValue">
+         <column name="BIG_DECIMAL_LOW_VALUE"  jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+       <field name="decimalHighValue">
+         <column name="BIG_DECIMAL_HIGH_VALUE" jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+       <field name="numNulls">
+         <column name="NUM_NULLS" jdbc-type="BIGINT" allows-null="false"/>
+       </field>
+       <field name="numDVs">
+         <column name="NUM_DISTINCTS" jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+       <field name="bitVector">
+         <column name="BIT_VECTOR" jdbc-type="BLOB" allows-null="true"/>
+       </field>
+       <field name="avgColLen">
+         <column name="AVG_COL_LEN" jdbc-type="DOUBLE" allows-null="true"/>
+       </field>
+       <field name="maxColLen">
+         <column name="MAX_COL_LEN" jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+       <field name="numTrues">
+         <column name="NUM_TRUES" jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+       <field name="numFalses">
+         <column name="NUM_FALSES" jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+       <field name="lastAnalyzed">
+         <column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/>
+       </field>
+     </class>
+ 
+     <class name="MPartitionColumnStatistics" table="PART_COL_STATS" identity-type="datastore" detachable="true">
+       <datastore-identity>
+         <column name="CS_ID"/>
+       </datastore-identity>
+ 
+       <field name ="dbName">
+         <column name="DB_NAME" length="128" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="tableName">
+         <column name="TABLE_NAME" length="256" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="partitionName">
+         <column name="PARTITION_NAME" length="767" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="partition">
+         <column name="PART_ID"/>
+       </field>
+       <field name="colName">
+         <column name="COLUMN_NAME" length="767" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="colType">
+         <column name="COLUMN_TYPE" length="128" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="longLowValue">
+         <column name="LONG_LOW_VALUE"  jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+       <field name="longHighValue">
+         <column name="LONG_HIGH_VALUE" jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+        <field name="doubleLowValue">
+         <column name="DOUBLE_LOW_VALUE"  jdbc-type="DOUBLE" allows-null="true"/>
+       </field>
+       <field name="doubleHighValue">
+         <column name="DOUBLE_HIGH_VALUE" jdbc-type="DOUBLE" allows-null="true"/>
+       </field>
+       <field name="decimalLowValue">
+         <column name="BIG_DECIMAL_LOW_VALUE"  jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+       <field name="decimalHighValue">
+         <column name="BIG_DECIMAL_HIGH_VALUE" jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+       <field name="numNulls">
+         <column name="NUM_NULLS" jdbc-type="BIGINT" allows-null="false"/>
+       </field>
+       <field name="numDVs">
+         <column name="NUM_DISTINCTS" jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+       <field name="bitVector">
+         <column name="BIT_VECTOR" jdbc-type="BLOB" allows-null="true"/>
+       </field>
+       <field name="avgColLen">
+         <column name="AVG_COL_LEN" jdbc-type="DOUBLE" allows-null="true"/>
+       </field>
+       <field name="maxColLen">
+         <column name="MAX_COL_LEN" jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+       <field name="numTrues">
+         <column name="NUM_TRUES" jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+       <field name="numFalses">
+         <column name="NUM_FALSES" jdbc-type="BIGINT" allows-null="true"/>
+       </field>
+       <field name="lastAnalyzed">
+         <column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/>
+       </field>
+     </class>
+     <class name="MVersionTable" table="VERSION" identity-type="datastore" detachable="true">
+       <datastore-identity>
+         <column name="VER_ID"/>
+       </datastore-identity>
+       <field name ="schemaVersion">
+         <column name="SCHEMA_VERSION" length="127" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name ="versionComment">
+         <column name="VERSION_COMMENT" length="255" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+     </class>
+ 
+     <class name="MMetastoreDBProperties" table="METASTORE_DB_PROPERTIES" identity-type="application" detachable="true">
+       <field name ="propertyKey" primary-key="true">
+         <column name="PROPERTY_KEY" length="255" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name ="propertyValue">
+         <column name="PROPERTY_VALUE" length="1000" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name ="description">
+         <column name="DESCRIPTION" length="1000" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+     </class>
+ 
+     <class name="MResourceUri" embedded-only="true" table="RESOURCE_URI" detachable="true">
+       <field name="resourceType">
+         <column name="RESOURCE_TYPE" jdbc-type="INTEGER"/>
+       </field>
+       <field name="uri">
+         <column name="RESOURCE_URI" length="4000" jdbc-type="VARCHAR"/>
+       </field>
+     </class>
+ 
+     <class name="MFunction" table="FUNCS" identity-type="datastore" detachable="true">
+       <datastore-identity>
+         <column name="FUNC_ID"/>
+       </datastore-identity>
+       <index name="UniqueFunction" unique="true">
+         <column name="FUNC_NAME"/>
+         <column name="DB_ID"/>
+       </index>
+       <field name="functionName">
+         <column name="FUNC_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="database">
+         <column name="DB_ID"/>
+       </field>
+       <field name="functionType">
+         <column name="FUNC_TYPE" jdbc-type="integer"/>
+       </field>
+       <field name="className">
+         <column name="CLASS_NAME" length="4000" jdbc-type="VARCHAR"/>
+       </field>
 -      <field name="ownerName">	
++      <field name="ownerName">    
+         <column name="OWNER_NAME" length="128" jdbc-type="VARCHAR"/>
+       </field>
+        <field name="ownerType">
+         <column name="OWNER_TYPE" length="10" jdbc-type="VARCHAR"/>
+       </field>
+       <field name="createTime">
+         <column name="CREATE_TIME" jdbc-type="integer"/>
+       </field>
+       <field name="resourceUris" table="FUNC_RU">
+         <collection element-type="MResourceUri"/>
+         <join>
+           <column name="FUNC_ID"/>
+         </join>
+         <element>
+           <embedded>
+             <field name="resourceType">
+               <column name="RESOURCE_TYPE" jdbc-type="INTEGER"/>
+             </field>
+             <field name="uri">
+               <column name="RESOURCE_URI" length="4000" jdbc-type="VARCHAR"/>
+             </field>
+           </embedded>
+         </element>
+       </field>
+     </class>
+ 
+     <class name="MNotificationLog" table="NOTIFICATION_LOG" identity-type="datastore" detachable="true">
+       <datastore-identity>
+         <column name="NL_ID"/>
+       </datastore-identity>
+       <field name="eventId">
+         <column name="EVENT_ID" jdbc-type="BIGINT" allows-null="false"/>
+       </field>
+       <field name="eventTime">
+         <column name="EVENT_TIME" jdbc-type="INTEGER" allows-null="false"/>
+       </field>
+       <field name="eventType">
+         <column name="EVENT_TYPE" length="32" jdbc-type="VARCHAR" allows-null="false"/>
+       </field>
+       <field name="dbName">
+         <column name="DB_NAME" length="128" jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+       <field name="tableName">
+         <column name="TBL_NAME" length="256" jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+       <field name="message">
+         <column name="MESSAGE" jdbc-type="LONGVARCHAR"/>
+       </field>
+       <field name="messageFormat">
+         <column name="MESSAGE_FORMAT" length="16" jdbc-type="VARCHAR" allows-null="true"/>
+       </field>
+     </class>
+ 
+     <!-- I tried to use a sequence here but derby didn't handle it well. -->
+     <class name="MNotificationNextId" table="NOTIFICATION_SEQUENCE" identity-type="datastore" detachable="true">
+       <datastore-identity>
+         <column name="NNI_ID"/>
+       </datastore-identity>
+       <field name="nextEventId">
+         <column name="NEXT_EVENT_ID" jdbc-type="BIGINT" allows-null="false"/>
+       </field>
+     </class>
+ 
 -
+   </package>
+ </jdo>
+

[10/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
index 0000000,f0f0a57..71c2cef
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
+++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
@@@ -1,0 -1,21260 +1,21262 @@@
+ <?php
+ namespace metastore;
+ 
+ /**
+  * Autogenerated by Thrift Compiler (0.9.3)
+  *
+  * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+  *  @generated
+  */
+ use Thrift\Base\TBase;
+ use Thrift\Type\TType;
+ use Thrift\Type\TMessageType;
+ use Thrift\Exception\TException;
+ use Thrift\Exception\TProtocolException;
+ use Thrift\Protocol\TProtocol;
+ use Thrift\Protocol\TBinaryProtocolAccelerated;
+ use Thrift\Exception\TApplicationException;
+ 
+ 
+ final class HiveObjectType {
+   const GLOBAL = 1;
+   const DATABASE = 2;
+   const TABLE = 3;
+   const PARTITION = 4;
+   const COLUMN = 5;
+   static public $__names = array(
+     1 => 'GLOBAL',
+     2 => 'DATABASE',
+     3 => 'TABLE',
+     4 => 'PARTITION',
+     5 => 'COLUMN',
+   );
+ }
+ 
+ final class PrincipalType {
+   const USER = 1;
+   const ROLE = 2;
+   const GROUP = 3;
+   static public $__names = array(
+     1 => 'USER',
+     2 => 'ROLE',
+     3 => 'GROUP',
+   );
+ }
+ 
+ final class PartitionEventType {
+   const LOAD_DONE = 1;
+   static public $__names = array(
+     1 => 'LOAD_DONE',
+   );
+ }
+ 
+ final class TxnState {
+   const COMMITTED = 1;
+   const ABORTED = 2;
+   const OPEN = 3;
+   static public $__names = array(
+     1 => 'COMMITTED',
+     2 => 'ABORTED',
+     3 => 'OPEN',
+   );
+ }
+ 
+ final class LockLevel {
+   const DB = 1;
+   const TABLE = 2;
+   const PARTITION = 3;
+   static public $__names = array(
+     1 => 'DB',
+     2 => 'TABLE',
+     3 => 'PARTITION',
+   );
+ }
+ 
+ final class LockState {
+   const ACQUIRED = 1;
+   const WAITING = 2;
+   const ABORT = 3;
+   const NOT_ACQUIRED = 4;
+   static public $__names = array(
+     1 => 'ACQUIRED',
+     2 => 'WAITING',
+     3 => 'ABORT',
+     4 => 'NOT_ACQUIRED',
+   );
+ }
+ 
+ final class LockType {
+   const SHARED_READ = 1;
+   const SHARED_WRITE = 2;
+   const EXCLUSIVE = 3;
+   static public $__names = array(
+     1 => 'SHARED_READ',
+     2 => 'SHARED_WRITE',
+     3 => 'EXCLUSIVE',
+   );
+ }
+ 
+ final class CompactionType {
+   const MINOR = 1;
+   const MAJOR = 2;
+   static public $__names = array(
+     1 => 'MINOR',
+     2 => 'MAJOR',
+   );
+ }
+ 
+ final class GrantRevokeType {
+   const GRANT = 1;
+   const REVOKE = 2;
+   static public $__names = array(
+     1 => 'GRANT',
+     2 => 'REVOKE',
+   );
+ }
+ 
+ final class DataOperationType {
+   const SELECT = 1;
+   const INSERT = 2;
+   const UPDATE = 3;
+   const DELETE = 4;
+   const UNSET = 5;
+   const NO_TXN = 6;
+   static public $__names = array(
+     1 => 'SELECT',
+     2 => 'INSERT',
+     3 => 'UPDATE',
+     4 => 'DELETE',
+     5 => 'UNSET',
+     6 => 'NO_TXN',
+   );
+ }
+ 
+ final class EventRequestType {
+   const INSERT = 1;
+   const UPDATE = 2;
+   const DELETE = 3;
+   static public $__names = array(
+     1 => 'INSERT',
+     2 => 'UPDATE',
+     3 => 'DELETE',
+   );
+ }
+ 
+ final class FunctionType {
+   const JAVA = 1;
+   static public $__names = array(
+     1 => 'JAVA',
+   );
+ }
+ 
+ final class ResourceType {
+   const JAR = 1;
+   const FILE = 2;
+   const ARCHIVE = 3;
+   static public $__names = array(
+     1 => 'JAR',
+     2 => 'FILE',
+     3 => 'ARCHIVE',
+   );
+ }
+ 
+ final class FileMetadataExprType {
+   const ORC_SARG = 1;
+   static public $__names = array(
+     1 => 'ORC_SARG',
+   );
+ }
+ 
+ final class ClientCapability {
+   const TEST_CAPABILITY = 1;
++  const INSERT_ONLY_TABLES = 2;
+   static public $__names = array(
+     1 => 'TEST_CAPABILITY',
++    2 => 'INSERT_ONLY_TABLES',
+   );
+ }
+ 
+ class Version {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $version = null;
+   /**
+    * @var string
+    */
+   public $comments = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'version',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'comments',
+           'type' => TType::STRING,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['version'])) {
+         $this->version = $vals['version'];
+       }
+       if (isset($vals['comments'])) {
+         $this->comments = $vals['comments'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'Version';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->version);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->comments);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('Version');
+     if ($this->version !== null) {
+       $xfer += $output->writeFieldBegin('version', TType::STRING, 1);
+       $xfer += $output->writeString($this->version);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->comments !== null) {
+       $xfer += $output->writeFieldBegin('comments', TType::STRING, 2);
+       $xfer += $output->writeString($this->comments);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class FieldSchema {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $name = null;
+   /**
+    * @var string
+    */
+   public $type = null;
+   /**
+    * @var string
+    */
+   public $comment = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'name',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'type',
+           'type' => TType::STRING,
+           ),
+         3 => array(
+           'var' => 'comment',
+           'type' => TType::STRING,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['name'])) {
+         $this->name = $vals['name'];
+       }
+       if (isset($vals['type'])) {
+         $this->type = $vals['type'];
+       }
+       if (isset($vals['comment'])) {
+         $this->comment = $vals['comment'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'FieldSchema';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->type);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->comment);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('FieldSchema');
+     if ($this->name !== null) {
+       $xfer += $output->writeFieldBegin('name', TType::STRING, 1);
+       $xfer += $output->writeString($this->name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->type !== null) {
+       $xfer += $output->writeFieldBegin('type', TType::STRING, 2);
+       $xfer += $output->writeString($this->type);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->comment !== null) {
+       $xfer += $output->writeFieldBegin('comment', TType::STRING, 3);
+       $xfer += $output->writeString($this->comment);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class SQLPrimaryKey {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $table_db = null;
+   /**
+    * @var string
+    */
+   public $table_name = null;
+   /**
+    * @var string
+    */
+   public $column_name = null;
+   /**
+    * @var int
+    */
+   public $key_seq = null;
+   /**
+    * @var string
+    */
+   public $pk_name = null;
+   /**
+    * @var bool
+    */
+   public $enable_cstr = null;
+   /**
+    * @var bool
+    */
+   public $validate_cstr = null;
+   /**
+    * @var bool
+    */
+   public $rely_cstr = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'table_db',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'table_name',
+           'type' => TType::STRING,
+           ),
+         3 => array(
+           'var' => 'column_name',
+           'type' => TType::STRING,
+           ),
+         4 => array(
+           'var' => 'key_seq',
+           'type' => TType::I32,
+           ),
+         5 => array(
+           'var' => 'pk_name',
+           'type' => TType::STRING,
+           ),
+         6 => array(
+           'var' => 'enable_cstr',
+           'type' => TType::BOOL,
+           ),
+         7 => array(
+           'var' => 'validate_cstr',
+           'type' => TType::BOOL,
+           ),
+         8 => array(
+           'var' => 'rely_cstr',
+           'type' => TType::BOOL,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['table_db'])) {
+         $this->table_db = $vals['table_db'];
+       }
+       if (isset($vals['table_name'])) {
+         $this->table_name = $vals['table_name'];
+       }
+       if (isset($vals['column_name'])) {
+         $this->column_name = $vals['column_name'];
+       }
+       if (isset($vals['key_seq'])) {
+         $this->key_seq = $vals['key_seq'];
+       }
+       if (isset($vals['pk_name'])) {
+         $this->pk_name = $vals['pk_name'];
+       }
+       if (isset($vals['enable_cstr'])) {
+         $this->enable_cstr = $vals['enable_cstr'];
+       }
+       if (isset($vals['validate_cstr'])) {
+         $this->validate_cstr = $vals['validate_cstr'];
+       }
+       if (isset($vals['rely_cstr'])) {
+         $this->rely_cstr = $vals['rely_cstr'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'SQLPrimaryKey';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->table_db);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->table_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->column_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 4:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->key_seq);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 5:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->pk_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 6:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->enable_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 7:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->validate_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 8:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->rely_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('SQLPrimaryKey');
+     if ($this->table_db !== null) {
+       $xfer += $output->writeFieldBegin('table_db', TType::STRING, 1);
+       $xfer += $output->writeString($this->table_db);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->table_name !== null) {
+       $xfer += $output->writeFieldBegin('table_name', TType::STRING, 2);
+       $xfer += $output->writeString($this->table_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->column_name !== null) {
+       $xfer += $output->writeFieldBegin('column_name', TType::STRING, 3);
+       $xfer += $output->writeString($this->column_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->key_seq !== null) {
+       $xfer += $output->writeFieldBegin('key_seq', TType::I32, 4);
+       $xfer += $output->writeI32($this->key_seq);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->pk_name !== null) {
+       $xfer += $output->writeFieldBegin('pk_name', TType::STRING, 5);
+       $xfer += $output->writeString($this->pk_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->enable_cstr !== null) {
+       $xfer += $output->writeFieldBegin('enable_cstr', TType::BOOL, 6);
+       $xfer += $output->writeBool($this->enable_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->validate_cstr !== null) {
+       $xfer += $output->writeFieldBegin('validate_cstr', TType::BOOL, 7);
+       $xfer += $output->writeBool($this->validate_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->rely_cstr !== null) {
+       $xfer += $output->writeFieldBegin('rely_cstr', TType::BOOL, 8);
+       $xfer += $output->writeBool($this->rely_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class SQLForeignKey {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $pktable_db = null;
+   /**
+    * @var string
+    */
+   public $pktable_name = null;
+   /**
+    * @var string
+    */
+   public $pkcolumn_name = null;
+   /**
+    * @var string
+    */
+   public $fktable_db = null;
+   /**
+    * @var string
+    */
+   public $fktable_name = null;
+   /**
+    * @var string
+    */
+   public $fkcolumn_name = null;
+   /**
+    * @var int
+    */
+   public $key_seq = null;
+   /**
+    * @var int
+    */
+   public $update_rule = null;
+   /**
+    * @var int
+    */
+   public $delete_rule = null;
+   /**
+    * @var string
+    */
+   public $fk_name = null;
+   /**
+    * @var string
+    */
+   public $pk_name = null;
+   /**
+    * @var bool
+    */
+   public $enable_cstr = null;
+   /**
+    * @var bool
+    */
+   public $validate_cstr = null;
+   /**
+    * @var bool
+    */
+   public $rely_cstr = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'pktable_db',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'pktable_name',
+           'type' => TType::STRING,
+           ),
+         3 => array(
+           'var' => 'pkcolumn_name',
+           'type' => TType::STRING,
+           ),
+         4 => array(
+           'var' => 'fktable_db',
+           'type' => TType::STRING,
+           ),
+         5 => array(
+           'var' => 'fktable_name',
+           'type' => TType::STRING,
+           ),
+         6 => array(
+           'var' => 'fkcolumn_name',
+           'type' => TType::STRING,
+           ),
+         7 => array(
+           'var' => 'key_seq',
+           'type' => TType::I32,
+           ),
+         8 => array(
+           'var' => 'update_rule',
+           'type' => TType::I32,
+           ),
+         9 => array(
+           'var' => 'delete_rule',
+           'type' => TType::I32,
+           ),
+         10 => array(
+           'var' => 'fk_name',
+           'type' => TType::STRING,
+           ),
+         11 => array(
+           'var' => 'pk_name',
+           'type' => TType::STRING,
+           ),
+         12 => array(
+           'var' => 'enable_cstr',
+           'type' => TType::BOOL,
+           ),
+         13 => array(
+           'var' => 'validate_cstr',
+           'type' => TType::BOOL,
+           ),
+         14 => array(
+           'var' => 'rely_cstr',
+           'type' => TType::BOOL,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['pktable_db'])) {
+         $this->pktable_db = $vals['pktable_db'];
+       }
+       if (isset($vals['pktable_name'])) {
+         $this->pktable_name = $vals['pktable_name'];
+       }
+       if (isset($vals['pkcolumn_name'])) {
+         $this->pkcolumn_name = $vals['pkcolumn_name'];
+       }
+       if (isset($vals['fktable_db'])) {
+         $this->fktable_db = $vals['fktable_db'];
+       }
+       if (isset($vals['fktable_name'])) {
+         $this->fktable_name = $vals['fktable_name'];
+       }
+       if (isset($vals['fkcolumn_name'])) {
+         $this->fkcolumn_name = $vals['fkcolumn_name'];
+       }
+       if (isset($vals['key_seq'])) {
+         $this->key_seq = $vals['key_seq'];
+       }
+       if (isset($vals['update_rule'])) {
+         $this->update_rule = $vals['update_rule'];
+       }
+       if (isset($vals['delete_rule'])) {
+         $this->delete_rule = $vals['delete_rule'];
+       }
+       if (isset($vals['fk_name'])) {
+         $this->fk_name = $vals['fk_name'];
+       }
+       if (isset($vals['pk_name'])) {
+         $this->pk_name = $vals['pk_name'];
+       }
+       if (isset($vals['enable_cstr'])) {
+         $this->enable_cstr = $vals['enable_cstr'];
+       }
+       if (isset($vals['validate_cstr'])) {
+         $this->validate_cstr = $vals['validate_cstr'];
+       }
+       if (isset($vals['rely_cstr'])) {
+         $this->rely_cstr = $vals['rely_cstr'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'SQLForeignKey';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->pktable_db);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->pktable_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->pkcolumn_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 4:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->fktable_db);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 5:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->fktable_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 6:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->fkcolumn_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 7:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->key_seq);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 8:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->update_rule);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 9:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->delete_rule);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 10:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->fk_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 11:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->pk_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 12:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->enable_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 13:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->validate_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 14:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->rely_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('SQLForeignKey');
+     if ($this->pktable_db !== null) {
+       $xfer += $output->writeFieldBegin('pktable_db', TType::STRING, 1);
+       $xfer += $output->writeString($this->pktable_db);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->pktable_name !== null) {
+       $xfer += $output->writeFieldBegin('pktable_name', TType::STRING, 2);
+       $xfer += $output->writeString($this->pktable_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->pkcolumn_name !== null) {
+       $xfer += $output->writeFieldBegin('pkcolumn_name', TType::STRING, 3);
+       $xfer += $output->writeString($this->pkcolumn_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->fktable_db !== null) {
+       $xfer += $output->writeFieldBegin('fktable_db', TType::STRING, 4);
+       $xfer += $output->writeString($this->fktable_db);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->fktable_name !== null) {
+       $xfer += $output->writeFieldBegin('fktable_name', TType::STRING, 5);
+       $xfer += $output->writeString($this->fktable_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->fkcolumn_name !== null) {
+       $xfer += $output->writeFieldBegin('fkcolumn_name', TType::STRING, 6);
+       $xfer += $output->writeString($this->fkcolumn_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->key_seq !== null) {
+       $xfer += $output->writeFieldBegin('key_seq', TType::I32, 7);
+       $xfer += $output->writeI32($this->key_seq);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->update_rule !== null) {
+       $xfer += $output->writeFieldBegin('update_rule', TType::I32, 8);
+       $xfer += $output->writeI32($this->update_rule);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->delete_rule !== null) {
+       $xfer += $output->writeFieldBegin('delete_rule', TType::I32, 9);
+       $xfer += $output->writeI32($this->delete_rule);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->fk_name !== null) {
+       $xfer += $output->writeFieldBegin('fk_name', TType::STRING, 10);
+       $xfer += $output->writeString($this->fk_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->pk_name !== null) {
+       $xfer += $output->writeFieldBegin('pk_name', TType::STRING, 11);
+       $xfer += $output->writeString($this->pk_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->enable_cstr !== null) {
+       $xfer += $output->writeFieldBegin('enable_cstr', TType::BOOL, 12);
+       $xfer += $output->writeBool($this->enable_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->validate_cstr !== null) {
+       $xfer += $output->writeFieldBegin('validate_cstr', TType::BOOL, 13);
+       $xfer += $output->writeBool($this->validate_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->rely_cstr !== null) {
+       $xfer += $output->writeFieldBegin('rely_cstr', TType::BOOL, 14);
+       $xfer += $output->writeBool($this->rely_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class SQLUniqueConstraint {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $table_db = null;
+   /**
+    * @var string
+    */
+   public $table_name = null;
+   /**
+    * @var string
+    */
+   public $column_name = null;
+   /**
+    * @var int
+    */
+   public $key_seq = null;
+   /**
+    * @var string
+    */
+   public $uk_name = null;
+   /**
+    * @var bool
+    */
+   public $enable_cstr = null;
+   /**
+    * @var bool
+    */
+   public $validate_cstr = null;
+   /**
+    * @var bool
+    */
+   public $rely_cstr = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'table_db',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'table_name',
+           'type' => TType::STRING,
+           ),
+         3 => array(
+           'var' => 'column_name',
+           'type' => TType::STRING,
+           ),
+         4 => array(
+           'var' => 'key_seq',
+           'type' => TType::I32,
+           ),
+         5 => array(
+           'var' => 'uk_name',
+           'type' => TType::STRING,
+           ),
+         6 => array(
+           'var' => 'enable_cstr',
+           'type' => TType::BOOL,
+           ),
+         7 => array(
+           'var' => 'validate_cstr',
+           'type' => TType::BOOL,
+           ),
+         8 => array(
+           'var' => 'rely_cstr',
+           'type' => TType::BOOL,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['table_db'])) {
+         $this->table_db = $vals['table_db'];
+       }
+       if (isset($vals['table_name'])) {
+         $this->table_name = $vals['table_name'];
+       }
+       if (isset($vals['column_name'])) {
+         $this->column_name = $vals['column_name'];
+       }
+       if (isset($vals['key_seq'])) {
+         $this->key_seq = $vals['key_seq'];
+       }
+       if (isset($vals['uk_name'])) {
+         $this->uk_name = $vals['uk_name'];
+       }
+       if (isset($vals['enable_cstr'])) {
+         $this->enable_cstr = $vals['enable_cstr'];
+       }
+       if (isset($vals['validate_cstr'])) {
+         $this->validate_cstr = $vals['validate_cstr'];
+       }
+       if (isset($vals['rely_cstr'])) {
+         $this->rely_cstr = $vals['rely_cstr'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'SQLUniqueConstraint';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->table_db);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->table_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->column_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 4:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->key_seq);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 5:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->uk_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 6:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->enable_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 7:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->validate_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 8:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->rely_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('SQLUniqueConstraint');
+     if ($this->table_db !== null) {
+       $xfer += $output->writeFieldBegin('table_db', TType::STRING, 1);
+       $xfer += $output->writeString($this->table_db);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->table_name !== null) {
+       $xfer += $output->writeFieldBegin('table_name', TType::STRING, 2);
+       $xfer += $output->writeString($this->table_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->column_name !== null) {
+       $xfer += $output->writeFieldBegin('column_name', TType::STRING, 3);
+       $xfer += $output->writeString($this->column_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->key_seq !== null) {
+       $xfer += $output->writeFieldBegin('key_seq', TType::I32, 4);
+       $xfer += $output->writeI32($this->key_seq);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->uk_name !== null) {
+       $xfer += $output->writeFieldBegin('uk_name', TType::STRING, 5);
+       $xfer += $output->writeString($this->uk_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->enable_cstr !== null) {
+       $xfer += $output->writeFieldBegin('enable_cstr', TType::BOOL, 6);
+       $xfer += $output->writeBool($this->enable_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->validate_cstr !== null) {
+       $xfer += $output->writeFieldBegin('validate_cstr', TType::BOOL, 7);
+       $xfer += $output->writeBool($this->validate_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->rely_cstr !== null) {
+       $xfer += $output->writeFieldBegin('rely_cstr', TType::BOOL, 8);
+       $xfer += $output->writeBool($this->rely_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class SQLNotNullConstraint {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $table_db = null;
+   /**
+    * @var string
+    */
+   public $table_name = null;
+   /**
+    * @var string
+    */
+   public $column_name = null;
+   /**
+    * @var string
+    */
+   public $nn_name = null;
+   /**
+    * @var bool
+    */
+   public $enable_cstr = null;
+   /**
+    * @var bool
+    */
+   public $validate_cstr = null;
+   /**
+    * @var bool
+    */
+   public $rely_cstr = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'table_db',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'table_name',
+           'type' => TType::STRING,
+           ),
+         3 => array(
+           'var' => 'column_name',
+           'type' => TType::STRING,
+           ),
+         4 => array(
+           'var' => 'nn_name',
+           'type' => TType::STRING,
+           ),
+         5 => array(
+           'var' => 'enable_cstr',
+           'type' => TType::BOOL,
+           ),
+         6 => array(
+           'var' => 'validate_cstr',
+           'type' => TType::BOOL,
+           ),
+         7 => array(
+           'var' => 'rely_cstr',
+           'type' => TType::BOOL,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['table_db'])) {
+         $this->table_db = $vals['table_db'];
+       }
+       if (isset($vals['table_name'])) {
+         $this->table_name = $vals['table_name'];
+       }
+       if (isset($vals['column_name'])) {
+         $this->column_name = $vals['column_name'];
+       }
+       if (isset($vals['nn_name'])) {
+         $this->nn_name = $vals['nn_name'];
+       }
+       if (isset($vals['enable_cstr'])) {
+         $this->enable_cstr = $vals['enable_cstr'];
+       }
+       if (isset($vals['validate_cstr'])) {
+         $this->validate_cstr = $vals['validate_cstr'];
+       }
+       if (isset($vals['rely_cstr'])) {
+         $this->rely_cstr = $vals['rely_cstr'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'SQLNotNullConstraint';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->table_db);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->table_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->column_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 4:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->nn_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 5:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->enable_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 6:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->validate_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 7:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->rely_cstr);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('SQLNotNullConstraint');
+     if ($this->table_db !== null) {
+       $xfer += $output->writeFieldBegin('table_db', TType::STRING, 1);
+       $xfer += $output->writeString($this->table_db);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->table_name !== null) {
+       $xfer += $output->writeFieldBegin('table_name', TType::STRING, 2);
+       $xfer += $output->writeString($this->table_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->column_name !== null) {
+       $xfer += $output->writeFieldBegin('column_name', TType::STRING, 3);
+       $xfer += $output->writeString($this->column_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->nn_name !== null) {
+       $xfer += $output->writeFieldBegin('nn_name', TType::STRING, 4);
+       $xfer += $output->writeString($this->nn_name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->enable_cstr !== null) {
+       $xfer += $output->writeFieldBegin('enable_cstr', TType::BOOL, 5);
+       $xfer += $output->writeBool($this->enable_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->validate_cstr !== null) {
+       $xfer += $output->writeFieldBegin('validate_cstr', TType::BOOL, 6);
+       $xfer += $output->writeBool($this->validate_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->rely_cstr !== null) {
+       $xfer += $output->writeFieldBegin('rely_cstr', TType::BOOL, 7);
+       $xfer += $output->writeBool($this->rely_cstr);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class Type {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $name = null;
+   /**
+    * @var string
+    */
+   public $type1 = null;
+   /**
+    * @var string
+    */
+   public $type2 = null;
+   /**
+    * @var \metastore\FieldSchema[]
+    */
+   public $fields = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'name',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'type1',
+           'type' => TType::STRING,
+           ),
+         3 => array(
+           'var' => 'type2',
+           'type' => TType::STRING,
+           ),
+         4 => array(
+           'var' => 'fields',
+           'type' => TType::LST,
+           'etype' => TType::STRUCT,
+           'elem' => array(
+             'type' => TType::STRUCT,
+             'class' => '\metastore\FieldSchema',
+             ),
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['name'])) {
+         $this->name = $vals['name'];
+       }
+       if (isset($vals['type1'])) {
+         $this->type1 = $vals['type1'];
+       }
+       if (isset($vals['type2'])) {
+         $this->type2 = $vals['type2'];
+       }
+       if (isset($vals['fields'])) {
+         $this->fields = $vals['fields'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'Type';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->type1);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->type2);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 4:
+           if ($ftype == TType::LST) {
+             $this->fields = array();
+             $_size0 = 0;
+             $_etype3 = 0;
+             $xfer += $input->readListBegin($_etype3, $_size0);
+             for ($_i4 = 0; $_i4 < $_size0; ++$_i4)
+             {
+               $elem5 = null;
+               $elem5 = new \metastore\FieldSchema();
+               $xfer += $elem5->read($input);
+               $this->fields []= $elem5;
+             }
+             $xfer += $input->readListEnd();
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('Type');
+     if ($this->name !== null) {
+       $xfer += $output->writeFieldBegin('name', TType::STRING, 1);
+       $xfer += $output->writeString($this->name);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->type1 !== null) {
+       $xfer += $output->writeFieldBegin('type1', TType::STRING, 2);
+       $xfer += $output->writeString($this->type1);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->type2 !== null) {
+       $xfer += $output->writeFieldBegin('type2', TType::STRING, 3);
+       $xfer += $output->writeString($this->type2);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->fields !== null) {
+       if (!is_array($this->fields)) {
+         throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+       }
+       $xfer += $output->writeFieldBegin('fields', TType::LST, 4);
+       {
+         $output->writeListBegin(TType::STRUCT, count($this->fields));
+         {
+           foreach ($this->fields as $iter6)
+           {
+             $xfer += $iter6->write($output);
+           }
+         }
+         $output->writeListEnd();
+       }
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class HiveObjectRef {
+   static $_TSPEC;
+ 
+   /**
+    * @var int
+    */
+   public $objectType = null;
+   /**
+    * @var string
+    */
+   public $dbName = null;
+   /**
+    * @var string
+    */
+   public $objectName = null;
+   /**
+    * @var string[]
+    */
+   public $partValues = null;
+   /**
+    * @var string
+    */
+   public $columnName = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'objectType',
+           'type' => TType::I32,
+           ),
+         2 => array(
+           'var' => 'dbName',
+           'type' => TType::STRING,
+           ),
+         3 => array(
+           'var' => 'objectName',
+           'type' => TType::STRING,
+           ),
+         4 => array(
+           'var' => 'partValues',
+           'type' => TType::LST,
+           'etype' => TType::STRING,
+           'elem' => array(
+             'type' => TType::STRING,
+             ),
+           ),
+         5 => array(
+           'var' => 'columnName',
+           'type' => TType::STRING,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['objectType'])) {
+         $this->objectType = $vals['objectType'];
+       }
+       if (isset($vals['dbName'])) {
+         $this->dbName = $vals['dbName'];
+       }
+       if (isset($vals['objectName'])) {
+         $this->objectName = $vals['objectName'];
+       }
+       if (isset($vals['partValues'])) {
+         $this->partValues = $vals['partValues'];
+       }
+       if (isset($vals['columnName'])) {
+         $this->columnName = $vals['columnName'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'HiveObjectRef';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->objectType);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->dbName);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->objectName);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 4:
+           if ($ftype == TType::LST) {
+             $this->partValues = array();
+             $_size7 = 0;
+             $_etype10 = 0;
+             $xfer += $input->readListBegin($_etype10, $_size7);
+             for ($_i11 = 0; $_i11 < $_size7; ++$_i11)
+             {
+               $elem12 = null;
+               $xfer += $input->readString($elem12);
+               $this->partValues []= $elem12;
+             }
+             $xfer += $input->readListEnd();
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 5:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->columnName);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('HiveObjectRef');
+     if ($this->objectType !== null) {
+       $xfer += $output->writeFieldBegin('objectType', TType::I32, 1);
+       $xfer += $output->writeI32($this->objectType);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->dbName !== null) {
+       $xfer += $output->writeFieldBegin('dbName', TType::STRING, 2);
+       $xfer += $output->writeString($this->dbName);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->objectName !== null) {
+       $xfer += $output->writeFieldBegin('objectName', TType::STRING, 3);
+       $xfer += $output->writeString($this->objectName);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->partValues !== null) {
+       if (!is_array($this->partValues)) {
+         throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+       }
+       $xfer += $output->writeFieldBegin('partValues', TType::LST, 4);
+       {
+         $output->writeListBegin(TType::STRING, count($this->partValues));
+         {
+           foreach ($this->partValues as $iter13)
+           {
+             $xfer += $output->writeString($iter13);
+           }
+         }
+         $output->writeListEnd();
+       }
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->columnName !== null) {
+       $xfer += $output->writeFieldBegin('columnName', TType::STRING, 5);
+       $xfer += $output->writeString($this->columnName);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class PrivilegeGrantInfo {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $privilege = null;
+   /**
+    * @var int
+    */
+   public $createTime = null;
+   /**
+    * @var string
+    */
+   public $grantor = null;
+   /**
+    * @var int
+    */
+   public $grantorType = null;
+   /**
+    * @var bool
+    */
+   public $grantOption = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'privilege',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'createTime',
+           'type' => TType::I32,
+           ),
+         3 => array(
+           'var' => 'grantor',
+           'type' => TType::STRING,
+           ),
+         4 => array(
+           'var' => 'grantorType',
+           'type' => TType::I32,
+           ),
+         5 => array(
+           'var' => 'grantOption',
+           'type' => TType::BOOL,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['privilege'])) {
+         $this->privilege = $vals['privilege'];
+       }
+       if (isset($vals['createTime'])) {
+         $this->createTime = $vals['createTime'];
+       }
+       if (isset($vals['grantor'])) {
+         $this->grantor = $vals['grantor'];
+       }
+       if (isset($vals['grantorType'])) {
+         $this->grantorType = $vals['grantorType'];
+       }
+       if (isset($vals['grantOption'])) {
+         $this->grantOption = $vals['grantOption'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'PrivilegeGrantInfo';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->privilege);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->createTime);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->grantor);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 4:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->grantorType);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 5:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->grantOption);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('PrivilegeGrantInfo');
+     if ($this->privilege !== null) {
+       $xfer += $output->writeFieldBegin('privilege', TType::STRING, 1);
+       $xfer += $output->writeString($this->privilege);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->createTime !== null) {
+       $xfer += $output->writeFieldBegin('createTime', TType::I32, 2);
+       $xfer += $output->writeI32($this->createTime);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->grantor !== null) {
+       $xfer += $output->writeFieldBegin('grantor', TType::STRING, 3);
+       $xfer += $output->writeString($this->grantor);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->grantorType !== null) {
+       $xfer += $output->writeFieldBegin('grantorType', TType::I32, 4);
+       $xfer += $output->writeI32($this->grantorType);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->grantOption !== null) {
+       $xfer += $output->writeFieldBegin('grantOption', TType::BOOL, 5);
+       $xfer += $output->writeBool($this->grantOption);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class HiveObjectPrivilege {
+   static $_TSPEC;
+ 
+   /**
+    * @var \metastore\HiveObjectRef
+    */
+   public $hiveObject = null;
+   /**
+    * @var string
+    */
+   public $principalName = null;
+   /**
+    * @var int
+    */
+   public $principalType = null;
+   /**
+    * @var \metastore\PrivilegeGrantInfo
+    */
+   public $grantInfo = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'hiveObject',
+           'type' => TType::STRUCT,
+           'class' => '\metastore\HiveObjectRef',
+           ),
+         2 => array(
+           'var' => 'principalName',
+           'type' => TType::STRING,
+           ),
+         3 => array(
+           'var' => 'principalType',
+           'type' => TType::I32,
+           ),
+         4 => array(
+           'var' => 'grantInfo',
+           'type' => TType::STRUCT,
+           'class' => '\metastore\PrivilegeGrantInfo',
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['hiveObject'])) {
+         $this->hiveObject = $vals['hiveObject'];
+       }
+       if (isset($vals['principalName'])) {
+         $this->principalName = $vals['principalName'];
+       }
+       if (isset($vals['principalType'])) {
+         $this->principalType = $vals['principalType'];
+       }
+       if (isset($vals['grantInfo'])) {
+         $this->grantInfo = $vals['grantInfo'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'HiveObjectPrivilege';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRUCT) {
+             $this->hiveObject = new \metastore\HiveObjectRef();
+             $xfer += $this->hiveObject->read($input);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->principalName);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->principalType);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 4:
+           if ($ftype == TType::STRUCT) {
+             $this->grantInfo = new \metastore\PrivilegeGrantInfo();
+             $xfer += $this->grantInfo->read($input);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('HiveObjectPrivilege');
+     if ($this->hiveObject !== null) {
+       if (!is_object($this->hiveObject)) {
+         throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+       }
+       $xfer += $output->writeFieldBegin('hiveObject', TType::STRUCT, 1);
+       $xfer += $this->hiveObject->write($output);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->principalName !== null) {
+       $xfer += $output->writeFieldBegin('principalName', TType::STRING, 2);
+       $xfer += $output->writeString($this->principalName);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->principalType !== null) {
+       $xfer += $output->writeFieldBegin('principalType', TType::I32, 3);
+       $xfer += $output->writeI32($this->principalType);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->grantInfo !== null) {
+       if (!is_object($this->grantInfo)) {
+         throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+       }
+       $xfer += $output->writeFieldBegin('grantInfo', TType::STRUCT, 4);
+       $xfer += $this->grantInfo->write($output);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class PrivilegeBag {
+   static $_TSPEC;
+ 
+   /**
+    * @var \metastore\HiveObjectPrivilege[]
+    */
+   public $privileges = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'privileges',
+           'type' => TType::LST,
+           'etype' => TType::STRUCT,
+           'elem' => array(
+             'type' => TType::STRUCT,
+             'class' => '\metastore\HiveObjectPrivilege',
+             ),
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['privileges'])) {
+         $this->privileges = $vals['privileges'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'PrivilegeBag';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::LST) {
+             $this->privileges = array();
+             $_size14 = 0;
+             $_etype17 = 0;
+             $xfer += $input->readListBegin($_etype17, $_size14);
+             for ($_i18 = 0; $_i18 < $_size14; ++$_i18)
+             {
+               $elem19 = null;
+               $elem19 = new \metastore\HiveObjectPrivilege();
+               $xfer += $elem19->read($input);
+               $this->privileges []= $elem19;
+             }
+             $xfer += $input->readListEnd();
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('PrivilegeBag');
+     if ($this->privileges !== null) {
+       if (!is_array($this->privileges)) {
+         throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+       }
+       $xfer += $output->writeFieldBegin('privileges', TType::LST, 1);
+       {
+         $output->writeListBegin(TType::STRUCT, count($this->privileges));
+         {
+           foreach ($this->privileges as $iter20)
+           {
+             $xfer += $iter20->write($output);
+           }
+         }
+         $output->writeListEnd();
+       }
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class PrincipalPrivilegeSet {
+   static $_TSPEC;
+ 
+   /**
+    * @var array
+    */
+   public $userPrivileges = null;
+   /**
+    * @var array
+    */
+   public $groupPrivileges = null;
+   /**
+    * @var array
+    */
+   public $rolePrivileges = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'userPrivileges',
+           'type' => TType::MAP,
+           'ktype' => TType::STRING,
+           'vtype' => TType::LST,
+           'key' => array(
+             'type' => TType::STRING,
+           ),
+           'val' => array(
+             'type' => TType::LST,
+             'etype' => TType::STRUCT,
+             'elem' => array(
+               'type' => TType::STRUCT,
+               'class' => '\metastore\PrivilegeGrantInfo',
+               ),
+             ),
+           ),
+         2 => array(
+           'var' => 'groupPrivileges',
+           'type' => TType::MAP,
+           'ktype' => TType::STRING,
+           'vtype' => TType::LST,
+           'key' => array(
+             'type' => TType::STRING,
+           ),
+           'val' => array(
+             'type' => TType::LST,
+             'etype' => TType::STRUCT,
+             'elem' => array(
+               'type' => TType::STRUCT,
+               'class' => '\metastore\PrivilegeGrantInfo',
+               ),
+             ),
+           ),
+         3 => array(
+           'var' => 'rolePrivileges',
+           'type' => TType::MAP,
+           'ktype' => TType::STRING,
+           'vtype' => TType::LST,
+           'key' => array(
+             'type' => TType::STRING,
+           ),
+           'val' => array(
+             'type' => TType::LST,
+             'etype' => TType::STRUCT,
+             'elem' => array(
+               'type' => TType::STRUCT,
+               'class' => '\metastore\PrivilegeGrantInfo',
+               ),
+             ),
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['userPrivileges'])) {
+         $this->userPrivileges = $vals['userPrivileges'];
+       }
+       if (isset($vals['groupPrivileges'])) {
+         $this->groupPrivileges = $vals['groupPrivileges'];
+       }
+       if (isset($vals['rolePrivileges'])) {
+         $this->rolePrivileges = $vals['rolePrivileges'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'PrincipalPrivilegeSet';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::MAP) {
+             $this->userPrivileges = array();
+             $_size21 = 0;
+             $_ktype22 = 0;
+             $_vtype23 = 0;
+             $xfer += $input->readMapBegin($_ktype22, $_vtype23, $_size21);
+             for ($_i25 = 0; $_i25 < $_size21; ++$_i25)
+             {
+               $key26 = '';
+               $val27 = array();
+               $xfer += $input->readString($key26);
+               $val27 = array();
+               $_size28 = 0;
+               $_etype31 = 0;
+               $xfer += $input->readListBegin($_etype31, $_size28);
+               for ($_i32 = 0; $_i32 < $_size28; ++$_i32)
+               {
+                 $elem33 = null;
+                 $elem33 = new \metastore\PrivilegeGrantInfo();
+                 $xfer += $elem33->read($input);
+                 $val27 []= $elem33;
+               }
+               $xfer += $input->readListEnd();
+               $this->userPrivileges[$key26] = $val27;
+             }
+             $xfer += $input->readMapEnd();
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::MAP) {
+             $this->groupPrivileges = array();
+             $_size34 = 0;
+             $_ktype35 = 0;
+             $_vtype36 = 0;
+             $xfer += $input->readMapBegin($_ktype35, $_vtype36, $_size34);
+             for ($_i38 = 0; $_i38 < $_size34; ++$_i38)
+             {
+               $key39 = '';
+               $val40 = array();
+               $xfer += $input->readString($key39);
+               $val40 = array();
+               $_size41 = 0;
+               $_etype44 = 0;
+               $xfer += $input->readListBegin($_etype44, $_size41);
+               for ($_i45 = 0; $_i45 < $_size41; ++$_i45)
+               {
+                 $elem46 = null;
+                 $elem46 = new \metastore\PrivilegeGrantInfo();
+                 $xfer += $elem46->read($input);
+                 $val40 []= $elem46;
+               }
+               $xfer += $input->readListEnd();
+               $this->groupPrivileges[$key39] = $val40;
+             }
+             $xfer += $input->readMapEnd();
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::MAP) {
+             $this->rolePrivileges = array();
+             $_size47 = 0;
+             $_ktype48 = 0;
+             $_vtype49 = 0;
+             $xfer += $input->readMapBegin($_ktype48, $_vtype49, $_size47);
+             for ($_i51 = 0; $_i51 < $_size47; ++$_i51)
+             {
+               $key52 = '';
+               $val53 = array();
+               $xfer += $input->readString($key52);
+               $val53 = array();
+               $_size54 = 0;
+               $_etype57 = 0;
+               $xfer += $input->readListBegin($_etype57, $_size54);
+               for ($_i58 = 0; $_i58 < $_size54; ++$_i58)
+               {
+                 $elem59 = null;
+                 $elem59 = new \metastore\PrivilegeGrantInfo();
+                 $xfer += $elem59->read($input);
+                 $val53 []= $elem59;
+               }
+               $xfer += $input->readListEnd();
+               $this->rolePrivileges[$key52] = $val53;
+             }
+             $xfer += $input->readMapEnd();
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('PrincipalPrivilegeSet');
+     if ($this->userPrivileges !== null) {
+       if (!is_array($this->userPrivileges)) {
+         throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+       }
+       $xfer += $output->writeFieldBegin('userPrivileges', TType::MAP, 1);
+       {
+         $output->writeMapBegin(TType::STRING, TType::LST, count($this->userPrivileges));
+         {
+           foreach ($this->userPrivileges as $kiter60 => $viter61)
+           {
+             $xfer += $output->writeString($kiter60);
+             {
+               $output->writeListBegin(TType::STRUCT, count($viter61));
+               {
+                 foreach ($viter61 as $iter62)
+                 {
+                   $xfer += $iter62->write($output);
+                 }
+               }
+               $output->writeListEnd();
+             }
+           }
+         }
+         $output->writeMapEnd();
+       }
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->groupPrivileges !== null) {
+       if (!is_array($this->groupPrivileges)) {
+         throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+       }
+       $xfer += $output->writeFieldBegin('groupPrivileges', TType::MAP, 2);
+       {
+         $output->writeMapBegin(TType::STRING, TType::LST, count($this->groupPrivileges));
+         {
+           foreach ($this->groupPrivileges as $kiter63 => $viter64)
+           {
+             $xfer += $output->writeString($kiter63);
+             {
+               $output->writeListBegin(TType::STRUCT, count($viter64));
+               {
+                 foreach ($viter64 as $iter65)
+                 {
+                   $xfer += $iter65->write($output);
+                 }
+               }
+               $output->writeListEnd();
+             }
+           }
+         }
+         $output->writeMapEnd();
+       }
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->rolePrivileges !== null) {
+       if (!is_array($this->rolePrivileges)) {
+         throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+       }
+       $xfer += $output->writeFieldBegin('rolePrivileges', TType::MAP, 3);
+       {
+         $output->writeMapBegin(TType::STRING, TType::LST, count($this->rolePrivileges));
+         {
+           foreach ($this->rolePrivileges as $kiter66 => $viter67)
+           {
+             $xfer += $output->writeString($kiter66);
+             {
+               $output->writeListBegin(TType::STRUCT, count($viter67));
+               {
+                 foreach ($viter67 as $iter68)
+                 {
+                   $xfer += $iter68->write($output);
+                 }
+               }
+               $output->writeListEnd();
+             }
+           }
+         }
+         $output->writeMapEnd();
+       }
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class GrantRevokePrivilegeRequest {
+   static $_TSPEC;
+ 
+   /**
+    * @var int
+    */
+   public $requestType = null;
+   /**
+    * @var \metastore\PrivilegeBag
+    */
+   public $privileges = null;
+   /**
+    * @var bool
+    */
+   public $revokeGrantOption = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'requestType',
+           'type' => TType::I32,
+           ),
+         2 => array(
+           'var' => 'privileges',
+           'type' => TType::STRUCT,
+           'class' => '\metastore\PrivilegeBag',
+           ),
+         3 => array(
+           'var' => 'revokeGrantOption',
+           'type' => TType::BOOL,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['requestType'])) {
+         $this->requestType = $vals['requestType'];
+       }
+       if (isset($vals['privileges'])) {
+         $this->privileges = $vals['privileges'];
+       }
+       if (isset($vals['revokeGrantOption'])) {
+         $this->revokeGrantOption = $vals['revokeGrantOption'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'GrantRevokePrivilegeRequest';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->requestType);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRUCT) {
+             $this->privileges = new \metastore\PrivilegeBag();
+             $xfer += $this->privileges->read($input);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->revokeGrantOption);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('GrantRevokePrivilegeRequest');
+     if ($this->requestType !== null) {
+       $xfer += $output->writeFieldBegin('requestType', TType::I32, 1);
+       $xfer += $output->writeI32($this->requestType);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->privileges !== null) {
+       if (!is_object($this->privileges)) {
+         throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA);
+       }
+       $xfer += $output->writeFieldBegin('privileges', TType::STRUCT, 2);
+       $xfer += $this->privileges->write($output);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->revokeGrantOption !== null) {
+       $xfer += $output->writeFieldBegin('revokeGrantOption', TType::BOOL, 3);
+       $xfer += $output->writeBool($this->revokeGrantOption);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class GrantRevokePrivilegeResponse {
+   static $_TSPEC;
+ 
+   /**
+    * @var bool
+    */
+   public $success = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'success',
+           'type' => TType::BOOL,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['success'])) {
+         $this->success = $vals['success'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'GrantRevokePrivilegeResponse';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->success);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('GrantRevokePrivilegeResponse');
+     if ($this->success !== null) {
+       $xfer += $output->writeFieldBegin('success', TType::BOOL, 1);
+       $xfer += $output->writeBool($this->success);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class Role {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $roleName = null;
+   /**
+    * @var int
+    */
+   public $createTime = null;
+   /**
+    * @var string
+    */
+   public $ownerName = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'roleName',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'createTime',
+           'type' => TType::I32,
+           ),
+         3 => array(
+           'var' => 'ownerName',
+           'type' => TType::STRING,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['roleName'])) {
+         $this->roleName = $vals['roleName'];
+       }
+       if (isset($vals['createTime'])) {
+         $this->createTime = $vals['createTime'];
+       }
+       if (isset($vals['ownerName'])) {
+         $this->ownerName = $vals['ownerName'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'Role';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->roleName);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->createTime);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->ownerName);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('Role');
+     if ($this->roleName !== null) {
+       $xfer += $output->writeFieldBegin('roleName', TType::STRING, 1);
+       $xfer += $output->writeString($this->roleName);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->createTime !== null) {
+       $xfer += $output->writeFieldBegin('createTime', TType::I32, 2);
+       $xfer += $output->writeI32($this->createTime);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->ownerName !== null) {
+       $xfer += $output->writeFieldBegin('ownerName', TType::STRING, 3);
+       $xfer += $output->writeString($this->ownerName);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class RolePrincipalGrant {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $roleName = null;
+   /**
+    * @var string
+    */
+   public $principalName = null;
+   /**
+    * @var int
+    */
+   public $principalType = null;
+   /**
+    * @var bool
+    */
+   public $grantOption = null;
+   /**
+    * @var int
+    */
+   public $grantTime = null;
+   /**
+    * @var string
+    */
+   public $grantorName = null;
+   /**
+    * @var int
+    */
+   public $grantorPrincipalType = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'roleName',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'principalName',
+           'type' => TType::STRING,
+           ),
+         3 => array(
+           'var' => 'principalType',
+           'type' => TType::I32,
+           ),
+         4 => array(
+           'var' => 'grantOption',
+           'type' => TType::BOOL,
+           ),
+         5 => array(
+           'var' => 'grantTime',
+           'type' => TType::I32,
+           ),
+         6 => array(
+           'var' => 'grantorName',
+           'type' => TType::STRING,
+           ),
+         7 => array(
+           'var' => 'grantorPrincipalType',
+           'type' => TType::I32,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['roleName'])) {
+         $this->roleName = $vals['roleName'];
+       }
+       if (isset($vals['principalName'])) {
+         $this->principalName = $vals['principalName'];
+       }
+       if (isset($vals['principalType'])) {
+         $this->principalType = $vals['principalType'];
+       }
+       if (isset($vals['grantOption'])) {
+         $this->grantOption = $vals['grantOption'];
+       }
+       if (isset($vals['grantTime'])) {
+         $this->grantTime = $vals['grantTime'];
+       }
+       if (isset($vals['grantorName'])) {
+         $this->grantorName = $vals['grantorName'];
+       }
+       if (isset($vals['grantorPrincipalType'])) {
+         $this->grantorPrincipalType = $vals['grantorPrincipalType'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'RolePrincipalGrant';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->roleName);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->principalName);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 3:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->principalType);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 4:
+           if ($ftype == TType::BOOL) {
+             $xfer += $input->readBool($this->grantOption);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 5:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->grantTime);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 6:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->grantorName);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 7:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->grantorPrincipalType);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStructBegin('RolePrincipalGrant');
+     if ($this->roleName !== null) {
+       $xfer += $output->writeFieldBegin('roleName', TType::STRING, 1);
+       $xfer += $output->writeString($this->roleName);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->principalName !== null) {
+       $xfer += $output->writeFieldBegin('principalName', TType::STRING, 2);
+       $xfer += $output->writeString($this->principalName);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->principalType !== null) {
+       $xfer += $output->writeFieldBegin('principalType', TType::I32, 3);
+       $xfer += $output->writeI32($this->principalType);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->grantOption !== null) {
+       $xfer += $output->writeFieldBegin('grantOption', TType::BOOL, 4);
+       $xfer += $output->writeBool($this->grantOption);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->grantTime !== null) {
+       $xfer += $output->writeFieldBegin('grantTime', TType::I32, 5);
+       $xfer += $output->writeI32($this->grantTime);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->grantorName !== null) {
+       $xfer += $output->writeFieldBegin('grantorName', TType::STRING, 6);
+       $xfer += $output->writeString($this->grantorName);
+       $xfer += $output->writeFieldEnd();
+     }
+     if ($this->grantorPrincipalType !== null) {
+       $xfer += $output->writeFieldBegin('grantorPrincipalType', TType::I32, 7);
+       $xfer += $output->writeI32($this->grantorPrincipalType);
+       $xfer += $output->writeFieldEnd();
+     }
+     $xfer += $output->writeFieldStop();
+     $xfer += $output->writeStructEnd();
+     return $xfer;
+   }
+ 
+ }
+ 
+ class GetRoleGrantsForPrincipalRequest {
+   static $_TSPEC;
+ 
+   /**
+    * @var string
+    */
+   public $principal_name = null;
+   /**
+    * @var int
+    */
+   public $principal_type = null;
+ 
+   public function __construct($vals=null) {
+     if (!isset(self::$_TSPEC)) {
+       self::$_TSPEC = array(
+         1 => array(
+           'var' => 'principal_name',
+           'type' => TType::STRING,
+           ),
+         2 => array(
+           'var' => 'principal_type',
+           'type' => TType::I32,
+           ),
+         );
+     }
+     if (is_array($vals)) {
+       if (isset($vals['principal_name'])) {
+         $this->principal_name = $vals['principal_name'];
+       }
+       if (isset($vals['principal_type'])) {
+         $this->principal_type = $vals['principal_type'];
+       }
+     }
+   }
+ 
+   public function getName() {
+     return 'GetRoleGrantsForPrincipalRequest';
+   }
+ 
+   public function read($input)
+   {
+     $xfer = 0;
+     $fname = null;
+     $ftype = 0;
+     $fid = 0;
+     $xfer += $input->readStructBegin($fname);
+     while (true)
+     {
+       $xfer += $input->readFieldBegin($fname, $ftype, $fid);
+       if ($ftype == TType::STOP) {
+         break;
+       }
+       switch ($fid)
+       {
+         case 1:
+           if ($ftype == TType::STRING) {
+             $xfer += $input->readString($this->principal_name);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         case 2:
+           if ($ftype == TType::I32) {
+             $xfer += $input->readI32($this->principal_type);
+           } else {
+             $xfer += $input->skip($ftype);
+           }
+           break;
+         default:
+           $xfer += $input->skip($ftype);
+           break;
+       }
+       $xfer += $input->readFieldEnd();
+     }
+     $xfer += $input->readStructEnd();
+     return $xfer;
+   }
+ 
+   public function write($output) {
+     $xfer = 0;
+     $xfer += $output->writeStru

<TRUNCATED>

[02/50] [abbrv] hive git commit: HIVE-14671 : Merge branch 'master' into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

HIVE-14671 : Merge branch 'master' into hive-14535 (Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/78f28c88
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/78f28c88
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/78f28c88

Branch: refs/heads/master
Commit: 78f28c8855f9a2867c81f911cf56684c09a35ea9
Parents: d9e4ae0 ee3a3ed
Author: Wei Zheng <we...@apache.org>
Authored: Tue Jun 6 11:34:43 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Tue Jun 6 11:34:43 2017 -0700

----------------------------------------------------------------------
 .../apache/hive/beeline/HiveSchemaHelper.java   |  581 ------
 .../org/apache/hive/beeline/HiveSchemaTool.java |   79 +-
 .../beeline/SeparatedValuesOutputFormat.java    |   11 +-
 .../apache/hive/beeline/TestHiveSchemaTool.java |    1 +
 bin/ext/version.sh                              |    2 +-
 bin/hive                                        |   20 +-
 .../hadoop/hive/common/StatsSetupConst.java     |    7 +-
 .../org/apache/hadoop/hive/conf/HiveConf.java   |    6 +
 .../hive/http/Log4j2ConfiguratorServlet.java    |   18 +-
 .../hadoop/hive/druid/serde/DruidSerDe.java     |   95 +-
 .../hadoop/hive/druid/TestDruidSerDe.java       |  169 +-
 .../results/positive/external_table_ppd.q.out   |    2 +-
 .../positive/hbase_binary_storage_queries.q.out |    4 +-
 .../src/test/results/positive/hbasestats.q.out  |    2 +-
 .../hcatalog/templeton/tool/TempletonUtils.java |   28 +-
 .../templeton/tool/TestTempletonUtils.java      |   11 +-
 .../write_final_output_blobstore.q.out          |   16 +-
 itests/hive-unit/pom.xml                        |    6 +
 .../hive/metastore/TestReplChangeManager.java   |   89 +-
 ...TestReplicationScenariosAcrossInstances.java |  166 +-
 .../hadoop/hive/ql/parse/WarehouseInstance.java |  176 +-
 .../org/apache/hive/beeline/TestSchemaTool.java |    6 +-
 .../test/resources/testconfiguration.properties |    6 +-
 .../hadoop/hive/cli/control/CliConfigs.java     |    1 +
 .../java/org/apache/hive/beeline/QFile.java     |    1 +
 .../llap/io/encoded/OrcEncodedDataReader.java   |    2 +-
 .../llap/io/metadata/OrcStripeMetadata.java     |    4 +-
 .../hadoop/hive/metastore/HiveAlterHandler.java |    4 +-
 .../hadoop/hive/metastore/HiveMetaStore.java    |   14 +-
 .../hive/metastore/IMetaStoreSchemaInfo.java    |   15 +
 .../hive/metastore/MetaStoreSchemaInfo.java     |   33 +
 .../hadoop/hive/metastore/MetaStoreUtils.java   |   22 +-
 .../hadoop/hive/metastore/ObjectStore.java      |   12 +-
 .../hive/metastore/ReplChangeManager.java       |   51 +-
 .../apache/hadoop/hive/metastore/Warehouse.java |    4 +
 .../hive/metastore/tools/HiveSchemaHelper.java  |  640 +++++++
 .../hive/metastore/TestMetaStoreUtils.java      |   16 +-
 .../hadoop/hive/metastore/TestObjectStore.java  |   48 +-
 pom.xml                                         |    4 +-
 ql/pom.xml                                      |   13 +
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |    4 +-
 .../apache/hadoop/hive/ql/exec/Operator.java    |   10 +-
 .../hadoop/hive/ql/exec/ReplCopyTask.java       |   22 +-
 .../hadoop/hive/ql/exec/spark/SparkTask.java    |    2 +-
 .../spark/status/RemoteSparkJobMonitor.java     |   14 +
 .../ql/exec/spark/status/SparkJobMonitor.java   |   12 +
 .../hive/ql/exec/tez/TezSessionPoolManager.java |    9 +-
 .../apache/hadoop/hive/ql/exec/tez/TezTask.java |    2 +-
 .../ql/exec/tez/monitoring/TezJobMonitor.java   |    8 +-
 .../exec/tez/monitoring/TezProgressMonitor.java |   42 +-
 .../fast/VectorMapJoinFastHashTable.java        |    3 +-
 .../apache/hadoop/hive/ql/metadata/Hive.java    |    2 +-
 .../ql/optimizer/CountDistinctRewriteProc.java  |  504 +++++
 .../hadoop/hive/ql/optimizer/MergeJoinProc.java |   23 +-
 .../hadoop/hive/ql/optimizer/Optimizer.java     |    4 +
 .../calcite/cost/HiveAlgorithmsUtil.java        |    4 +-
 .../optimizer/calcite/cost/HiveCostModel.java   |    7 +-
 .../calcite/cost/HiveDefaultCostModel.java      |    2 +-
 .../calcite/cost/HiveOnTezCostModel.java        |    4 +-
 .../optimizer/calcite/cost/HiveRelMdCost.java   |    2 +-
 .../rules/HiveFilterAggregateTransposeRule.java |    6 +-
 .../calcite/rules/HiveRelDecorrelator.java      |   46 +-
 .../stats/FilterSelectivityEstimator.java       |    9 +-
 .../stats/HiveRelMdDistinctRowCount.java        |    2 +-
 .../calcite/stats/HiveRelMdRowCount.java        |    3 +-
 .../calcite/stats/HiveRelMdSelectivity.java     |    4 +-
 .../calcite/stats/HiveRelMdUniqueKeys.java      |    2 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |   88 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java      |    4 +-
 .../ql/parse/ReplicationSemanticAnalyzer.java   |   50 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |   36 +-
 .../hadoop/hive/ql/parse/repl/PathBuilder.java  |   66 +
 .../repl/dump/events/CreateFunctionHandler.java |    9 +-
 .../parse/repl/dump/io/FunctionSerializer.java  |   40 +-
 .../hive/ql/parse/repl/load/MetaData.java       |    2 +-
 .../load/message/CreateFunctionHandler.java     |  156 +-
 .../hadoop/hive/ql/plan/CreateTableDesc.java    |    7 +-
 .../apache/hadoop/hive/ql/plan/GroupByDesc.java |   15 +
 .../hive/ql/exec/tez/TestTezSessionPool.java    |   43 +-
 .../tez/monitoring/TestTezProgressMonitor.java  |    3 +-
 .../PrimaryToReplicaResourceFunctionTest.java   |   87 +
 .../orderby_position_unsupported.q              |    3 +-
 .../clientnegative/spark_stage_max_tasks.q      |    6 +
 .../queries/clientpositive/alterColumnStats.q   |   21 +
 .../clientpositive/alterColumnStatsPart.q       |   47 +
 .../queries/clientpositive/count_dist_rewrite.q |   65 +
 .../test/queries/clientpositive/druid_basic2.q  |    4 +
 .../test/queries/clientpositive/filter_aggr.q   |   18 +
 .../test/queries/clientpositive/order_by_pos.q  |   20 +
 .../test/queries/clientpositive/perf/query4.q   |  111 ++
 .../test/queries/clientpositive/perf/query47.q  |   53 +
 .../test/queries/clientpositive/perf/query57.q  |   50 +
 .../test/queries/clientpositive/perf/query74.q  |   63 +
 ql/src/test/queries/clientpositive/tez-tag.q    |   68 +
 .../clientnegative/alter_file_format.q.out      |    2 +-
 .../orderby_position_unsupported.q.out          |    4 +-
 .../spark/spark_stage_max_tasks.q.out           |   77 +
 .../clientnegative/unset_table_property.q.out   |    2 +-
 .../clientpositive/alterColumnStats.q.out       |  158 ++
 .../clientpositive/alterColumnStatsPart.q.out   |  507 +++++
 .../clientpositive/alter_file_format.q.out      |   26 +-
 .../alter_partition_clusterby_sortby.q.out      |    6 +-
 .../clientpositive/alter_skewed_table.q.out     |   12 +-
 .../alter_table_add_partition.q.out             |    2 +-
 .../clientpositive/alter_table_not_sorted.q.out |    4 +-
 .../clientpositive/autoColumnStats_3.q.out      |    2 +-
 .../clientpositive/autoColumnStats_4.q.out      |    2 +-
 .../insert_overwrite_local_directory_1.q.out    | 1769 ++++++++++++++++++
 .../clientpositive/binary_output_format.q.out   |   16 +-
 .../test/results/clientpositive/bucket1.q.out   |    4 +-
 .../test/results/clientpositive/bucket2.q.out   |    4 +-
 .../clientpositive/bucket_map_join_spark1.q.out |    4 +-
 .../clientpositive/bucket_map_join_spark2.q.out |    4 +-
 .../clientpositive/bucket_map_join_spark3.q.out |    4 +-
 .../results/clientpositive/bucketmapjoin5.q.out |   16 +-
 .../clientpositive/bucketmapjoin_negative.q.out |   16 +-
 .../bucketmapjoin_negative2.q.out               |   16 +-
 ...names_with_leading_and_trailing_spaces.q.out |    4 +-
 .../results/clientpositive/compustat_avro.q.out |    4 +-
 .../clientpositive/count_dist_rewrite.q.out     | 1151 ++++++++++++
 .../create_alter_list_bucketing_table1.q.out    |   14 +-
 .../results/clientpositive/create_like.q.out    |   18 +-
 .../results/clientpositive/create_like2.q.out   |    2 +-
 .../clientpositive/create_like_tbl_props.q.out  |   10 +-
 .../clientpositive/create_like_view.q.out       |    6 +-
 .../clientpositive/create_skewed_table1.q.out   |    6 +-
 .../create_table_like_stats.q.out               |    2 +-
 .../create_with_constraints.q.out               |   72 +-
 .../clientpositive/database_location.q.out      |    4 +-
 .../clientpositive/default_file_format.q.out    |   16 +-
 .../describe_comment_indent.q.out               |    2 +-
 .../describe_comment_nonascii.q.out             |    2 +-
 .../describe_formatted_view_partitioned.q.out   |    2 +-
 .../clientpositive/describe_syntax.q.out        |    4 +-
 .../results/clientpositive/describe_table.q.out |   24 +-
 .../display_colstats_tbllvl.q.out               |    2 +-
 .../results/clientpositive/druid_basic1.q.out   |    4 +-
 .../results/clientpositive/druid_basic2.q.out   |   58 +-
 .../clientpositive/druid_intervals.q.out        |    2 +-
 .../clientpositive/druid_timeseries.q.out       |    2 +-
 .../results/clientpositive/druid_topn.q.out     |    2 +-
 .../results/clientpositive/filter_aggr.q.out    |  680 +++++++
 .../clientpositive/groupby_map_ppr.q.out        |    4 +-
 .../groupby_map_ppr_multi_distinct.q.out        |    4 +-
 .../results/clientpositive/groupby_ppr.q.out    |    4 +-
 .../groupby_ppr_multi_distinct.q.out            |    4 +-
 .../clientpositive/groupby_sort_11.q.out        |   39 +-
 .../clientpositive/groupby_sort_1_23.q.out      |   56 +-
 .../results/clientpositive/groupby_sort_6.q.out |    4 +-
 .../clientpositive/groupby_sort_skew_1_23.q.out |   56 +-
 .../results/clientpositive/input_part1.q.out    |   16 +-
 .../results/clientpositive/input_part2.q.out    |   32 +-
 .../insert_values_orig_table_use_metadata.q.out |    4 +-
 ql/src/test/results/clientpositive/join17.q.out |    4 +-
 ql/src/test/results/clientpositive/join26.q.out |    4 +-
 ql/src/test/results/clientpositive/join32.q.out |    4 +-
 ql/src/test/results/clientpositive/join33.q.out |    4 +-
 ql/src/test/results/clientpositive/join34.q.out |    6 +-
 ql/src/test/results/clientpositive/join35.q.out |    6 +-
 ql/src/test/results/clientpositive/join9.q.out  |    4 +-
 .../results/clientpositive/join_map_ppr.q.out   |   16 +-
 .../clientpositive/list_bucket_dml_14.q.out     |    4 +-
 .../clientpositive/llap/autoColumnStats_2.q.out |    2 +-
 .../results/clientpositive/llap/bucket2.q.out   |    4 +-
 .../results/clientpositive/llap/bucket4.q.out   |    4 +-
 .../results/clientpositive/llap/bucket5.q.out   |   20 +-
 .../clientpositive/llap/bucket_many.q.out       |    4 +-
 .../clientpositive/llap/bucketmapjoin1.q.out    |    4 +-
 .../clientpositive/llap/bucketmapjoin2.q.out    |    4 +-
 .../clientpositive/llap/bucketmapjoin3.q.out    |    4 +-
 .../clientpositive/llap/bucketmapjoin4.q.out    |    4 +-
 ...names_with_leading_and_trailing_spaces.q.out |    4 +-
 .../llap/count_dist_rewrite.q.out               | 1169 ++++++++++++
 .../llap/disable_merge_for_bucketing.q.out      |    4 +-
 .../llap/dynamic_semijoin_reduction.q.out       |    2 +-
 .../llap/dynamic_semijoin_user_level.q.out      |    2 +-
 .../clientpositive/llap/join32_lessSize.q.out   |    8 +-
 .../clientpositive/llap/metadataonly1.q.out     |   40 +-
 .../clientpositive/llap/orc_create.q.out        |    4 +-
 .../clientpositive/llap/orc_llap_counters.q.out |    2 +-
 .../clientpositive/llap/orc_ppd_basic.q.out     |  230 ---
 .../llap/orc_ppd_schema_evol_3a.q.out           |    6 +-
 .../llap/reduce_deduplicate.q.out               |    4 +-
 .../results/clientpositive/llap/sample1.q.out   |    4 +-
 .../results/clientpositive/llap/stats11.q.out   |    4 +-
 .../clientpositive/llap/subquery_scalar.q.out   |    2 +-
 .../results/clientpositive/llap/sysdb.q.out     |   10 +-
 ql/src/test/results/clientpositive/mm_all.q.out |   35 +-
 .../results/clientpositive/nullgroup4.q.out     |   41 +-
 .../results/clientpositive/order_by_pos.q.out   |  129 ++
 .../parquet_array_null_element.q.out            |    2 +-
 .../results/clientpositive/parquet_create.q.out |    2 +-
 .../parquet_int96_timestamp.q.out               |    4 +-
 .../clientpositive/part_inherit_tbl_props.q.out |    2 +-
 .../part_inherit_tbl_props_empty.q.out          |    2 +-
 .../part_inherit_tbl_props_with_star.q.out      |    2 +-
 .../clientpositive/partial_column_stats.q.out   |    2 +-
 .../partition_coltype_literals.q.out            |    8 +-
 ql/src/test/results/clientpositive/pcr.q.out    |   32 +-
 .../results/clientpositive/perf/query16.q.out   |  318 ++--
 .../results/clientpositive/perf/query28.q.out   |   58 +-
 .../results/clientpositive/perf/query4.q.out    | 1128 +++++++++++
 .../results/clientpositive/perf/query47.q.out   |  357 ++++
 .../results/clientpositive/perf/query57.q.out   |  351 ++++
 .../results/clientpositive/perf/query74.q.out   |  529 ++++++
 .../results/clientpositive/perf/query94.q.out   |  318 ++--
 .../results/clientpositive/perf/query95.q.out   |  302 +--
 .../results/clientpositive/quotedid_stats.q.out |    2 +-
 .../clientpositive/rand_partitionpruner2.q.out  |   16 +-
 .../clientpositive/rcfile_default_format.q.out  |    6 +-
 .../test/results/clientpositive/sample1.q.out   |   16 +-
 .../test/results/clientpositive/sample2.q.out   |   16 +-
 .../test/results/clientpositive/sample4.q.out   |   16 +-
 .../test/results/clientpositive/sample5.q.out   |   16 +-
 .../test/results/clientpositive/sample6.q.out   |   16 +-
 .../test/results/clientpositive/sample7.q.out   |   16 +-
 .../clientpositive/show_tblproperties.q.out     |   10 +-
 .../results/clientpositive/spark/bucket2.q.out  |    4 +-
 .../results/clientpositive/spark/bucket4.q.out  |    4 +-
 .../results/clientpositive/spark/bucket5.q.out  |    8 +-
 .../spark/bucket_map_join_spark1.q.out          |    4 +-
 .../spark/bucket_map_join_spark2.q.out          |    4 +-
 .../spark/bucket_map_join_spark3.q.out          |    4 +-
 .../clientpositive/spark/bucketmapjoin1.q.out   |    4 +-
 .../clientpositive/spark/bucketmapjoin2.q.out   |    4 +-
 .../clientpositive/spark/bucketmapjoin3.q.out   |    4 +-
 .../clientpositive/spark/bucketmapjoin4.q.out   |    4 +-
 .../clientpositive/spark/bucketmapjoin5.q.out   |    4 +-
 .../spark/bucketmapjoin_negative.q.out          |    4 +-
 .../spark/bucketmapjoin_negative2.q.out         |    4 +-
 .../spark/disable_merge_for_bucketing.q.out     |    4 +-
 .../clientpositive/spark/groupby_map_ppr.q.out  |    4 +-
 .../spark/groupby_map_ppr_multi_distinct.q.out  |    4 +-
 .../clientpositive/spark/groupby_ppr.q.out      |    4 +-
 .../spark/groupby_ppr_multi_distinct.q.out      |    4 +-
 .../spark/groupby_sort_1_23.q.out               |   20 +-
 .../spark/groupby_sort_skew_1_23.q.out          |   20 +-
 .../clientpositive/spark/input_part2.q.out      |    8 +-
 .../results/clientpositive/spark/join17.q.out   |    4 +-
 .../results/clientpositive/spark/join26.q.out   |    4 +-
 .../results/clientpositive/spark/join32.q.out   |    4 +-
 .../clientpositive/spark/join32_lessSize.q.out  |    8 +-
 .../results/clientpositive/spark/join33.q.out   |    4 +-
 .../results/clientpositive/spark/join34.q.out   |    4 +-
 .../results/clientpositive/spark/join35.q.out   |    4 +-
 .../results/clientpositive/spark/join9.q.out    |    4 +-
 .../clientpositive/spark/join_map_ppr.q.out     |    4 +-
 .../clientpositive/spark/nullgroup4.q.out       |   31 +-
 .../test/results/clientpositive/spark/pcr.q.out |    8 +-
 .../spark/reduce_deduplicate.q.out              |    4 +-
 .../results/clientpositive/spark/sample1.q.out  |    4 +-
 .../results/clientpositive/spark/sample2.q.out  |    4 +-
 .../results/clientpositive/spark/sample4.q.out  |    4 +-
 .../results/clientpositive/spark/sample5.q.out  |    4 +-
 .../results/clientpositive/spark/sample6.q.out  |    4 +-
 .../results/clientpositive/spark/sample7.q.out  |    4 +-
 .../results/clientpositive/spark/stats0.q.out   |    8 +-
 .../results/clientpositive/spark/stats3.q.out   |    2 +-
 ql/src/test/results/clientpositive/stats0.q.out |   20 +-
 ql/src/test/results/clientpositive/stats3.q.out |    2 +-
 .../temp_table_display_colstats_tbllvl.q.out    |    6 +-
 .../results/clientpositive/tez/tez-tag.q.out    |  301 +++
 .../test/results/clientpositive/udf_count.q.out |   39 +-
 .../clientpositive/unicode_notation.q.out       |    6 +-
 .../unset_table_view_property.q.out             |   18 +-
 .../clientpositive/vector_empty_where.q.out     |  300 ++-
 266 files changed, 13050 insertions(+), 2477 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index b13b0a4,ff8412c..e7bbd62
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@@ -1964,60 -1969,13 +1972,68 @@@ public class MetaStoreUtils 
      }
      return metaException;
    }
+   
+   public static List<String> getColumnNames(List<FieldSchema> schema) {
+     List<String> cols = new ArrayList<>();
+     for (FieldSchema fs : schema) {
+       cols.add(fs.getName());
+     }
+     return cols;
+   }
  
 +  // TODO The following two utility methods can be moved to AcidUtils once no class in metastore is relying on them,
 +  // right now ObjectStore.getAllMmTablesForCleanup is calling these method
 +  /**
 +   * Checks if a table is an ACID table that only supports INSERT, but not UPDATE/DELETE
 +   * @param params table properties
 +   * @return true if table is an INSERT_ONLY table, false otherwise
 +   */
 +  // TODO# also check that transactional is true
 +  public static boolean isInsertOnlyTable(Map<String, String> params) {
 +    return isInsertOnlyTable(params, false);
 +  }
 +
 +  public static boolean isInsertOnlyTable(Map<String, String> params, boolean isCtas) {
 +    String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
 +    return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
 +  }
 +
 +   public static boolean isInsertOnlyTable(Properties params) {
 +    // TODO#  redirect for now - fix before merge
 +    HashMap<String, String> testMap = new HashMap<String, String>();
 +    for (String n  : params.stringPropertyNames()) {
 +      testMap.put(n, params.getProperty(n));
 +    }
 +    return isInsertOnlyTable(testMap);
 +  }
 +
 +   /** The method for altering table props; may set the table to MM, non-MM, or not affect MM. */
 +  public static Boolean isToInsertOnlyTable(Map<String, String> props) {
 +    // TODO# Setting these separately is a very hairy issue in certain combinations, since we
 +    //       cannot decide what type of table this becomes without taking both into account, and
 +    //       in many cases the conversion might be illegal.
 +    //       The only thing we allow is tx = true w/o tx-props, for backward compat.
 +    String transactional = props.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
 +    String transactionalProp = props.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
 +    if (transactional == null && transactionalProp == null) return null; // Not affected.
 +    boolean isSetToTxn = "true".equalsIgnoreCase(transactional);
 +    if (transactionalProp == null) {
 +      if (isSetToTxn) return false; // Assume the full ACID table.
 +      throw new RuntimeException("Cannot change '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL
 +          + "' without '" + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "'");
 +    }
 +    if (!"insert_only".equalsIgnoreCase(transactionalProp)) return false; // Not MM.
 +    if (!isSetToTxn) {
 +      throw new RuntimeException("Cannot set '"
 +          + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "' to 'insert_only' without "
 +          + "setting '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL + "' to 'true'");
 +    }
 +    return true;
 +  }
 +
 +  public static boolean isRemovedInsertOnlyTable(Set<String> removedSet) {
 +    boolean hasTxn = removedSet.contains(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL),
 +        hasProps = removedSet.contains(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
 +    return hasTxn || hasProps;
 +  }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index e0302fb,b28ea73..9542990
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@@ -63,7 -63,8 +63,8 @@@ import org.mockito.Mockito
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
 +import com.google.common.base.Supplier;
+ import com.google.common.collect.ImmutableList;
 -
  import javax.jdo.Query;
  
  public class TestObjectStore {

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/78f28c88/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
----------------------------------------------------------------------

[40/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/47178498
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/47178498
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/47178498

Branch: refs/heads/master
Commit: 471784980adb079f6610449dd3ca855a5605121c
Parents: f159b5a 23478cf
Author: sergey <se...@apache.org>
Authored: Fri Sep 29 14:01:30 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Sep 29 14:01:30 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |    2 +-
 .../hadoop/hive/druid/io/DruidOutputFormat.java |   15 +-
 .../hadoop/hive/druid/serde/DruidSerDe.java     |    9 +
 .../apache/hive/hcatalog/common/HCatUtil.java   |  141 ++-
 .../apache/hadoop/hive/ql/TestAcidOnTez.java    |   87 ++
 .../hive/llap/cli/LlapOptionsProcessor.java     |   16 +
 .../hadoop/hive/llap/cli/LlapSliderUtils.java   |   17 +-
 .../llap/cli/LlapStatusOptionsProcessor.java    |   33 +-
 .../hive/llap/cli/LlapStatusServiceDriver.java  |  206 ++--
 .../hive/llap/io/api/impl/LlapRecordReader.java |    3 +-
 llap-server/src/main/resources/package.py       |    8 +-
 llap-server/src/main/resources/templates.py     |    5 +-
 .../hive/metastore/MetaStoreDirectSql.java      |   48 +-
 ql/pom.xml                                      |    4 +
 .../hive/llap/LlapOutputFormatService.java      |    3 +-
 .../java/org/apache/hadoop/hive/ql/Driver.java  |  111 +-
 .../org/apache/hadoop/hive/ql/QueryState.java   |   14 +
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   18 +-
 .../apache/hadoop/hive/ql/exec/Utilities.java   |   10 -
 .../hadoop/hive/ql/exec/mr/MapredLocalTask.java |    7 +-
 .../hive/ql/exec/tez/TezSessionPoolManager.java |   19 +-
 .../hive/ql/exec/tez/TezSessionPoolSession.java |    4 +-
 .../hive/ql/exec/tez/TezSessionState.java       |    4 +-
 .../apache/hadoop/hive/ql/exec/tez/TezTask.java |    3 +
 .../hive/ql/exec/tez/WorkloadManager.java       |   11 +-
 .../ql/io/orc/encoded/EncodedReaderImpl.java    |  210 ++--
 .../apache/hadoop/hive/ql/metadata/Hive.java    |    6 +-
 .../hive/ql/parse/BaseSemanticAnalyzer.java     |   13 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |   10 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |   19 +-
 .../ql/parse/UpdateDeleteSemanticAnalyzer.java  |    2 +-
 .../hadoop/hive/ql/plan/CreateTableDesc.java    |    2 +-
 .../hadoop/hive/ql/session/SessionState.java    |   15 +
 .../apache/hadoop/hive/ql/stats/StatsUtils.java |    3 +-
 .../ql/udf/generic/GenericUDTFGetSplits.java    |   90 +-
 .../hive/ql/exec/tez/TestTezSessionPool.java    |   16 +-
 .../queries/clientpositive/mapjoin_addjar.q     |   13 +-
 .../test/results/clientnegative/external2.q.out |    2 +-
 .../clientpositive/annotate_stats_select.q.out  |   16 +-
 .../clientpositive/llap/lateral_view.q.out      |   32 +-
 .../clientpositive/llap/lvj_mapjoin.q.out       |   60 +-
 ...chema_evol_orc_nonvec_part_all_complex.q.out |   18 +-
 .../schema_evol_orc_vec_part_all_complex.q.out  |   18 +-
 ...hema_evol_text_nonvec_part_all_complex.q.out |   18 +-
 .../schema_evol_text_vec_part_all_complex.q.out |   18 +-
 ...hema_evol_text_vecrow_part_all_complex.q.out |   18 +-
 .../llap/tez_join_result_complex.q.out          |  212 ++--
 .../clientpositive/llap/unionDistinct_1.q.out   |    4 +-
 .../llap/vector_complex_all.q.out               |   86 +-
 .../llap/vector_complex_join.q.out              |   14 +-
 .../results/clientpositive/mapjoin_addjar.q.out | 1045 +++++++++++++++++-
 .../clientpositive/spark/mapjoin_addjar.q.out   | 1045 +++++++++++++++++-
 52 files changed, 3124 insertions(+), 679 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/47178498/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/47178498/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 3a0f228,f01edf8..6bfec11
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@@ -1205,17 -1231,17 +1231,17 @@@ public class Driver implements CommandP
          //so this makes (file name -> data) mapping stable
          acidSinks.sort((FileSinkDesc fsd1, FileSinkDesc fsd2) -> fsd1.getDirName().compareTo(fsd2.getDirName()));
          for (FileSinkDesc desc : acidSinks) {
-           desc.setTransactionId(txnMgr.getCurrentTxnId());
+           desc.setTransactionId(queryTxnMgr.getCurrentTxnId());
            //it's possible to have > 1 FileSink writing to the same table/partition
            //e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes
-           desc.setStatementId(txnMgr.getWriteIdAndIncrement());
+           desc.setStatementId(queryTxnMgr.getWriteIdAndIncrement());
          }
        }
 -      /*It's imperative that {@code acquireLocks()} is called for all commands so that 
 +      /*It's imperative that {@code acquireLocks()} is called for all commands so that
        HiveTxnManager can transition its state machine correctly*/
-       txnMgr.acquireLocks(plan, ctx, userFromUGI, lDrvState);
-       if(txnMgr.recordSnapshot(plan)) {
-         recordValidTxns();
+       queryTxnMgr.acquireLocks(plan, ctx, userFromUGI, lDrvState);
+       if(queryTxnMgr.recordSnapshot(plan)) {
+         recordValidTxns(queryTxnMgr);
        }
        return 0;
      } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/hive/blob/47178498/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/47178498/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/47178498/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/47178498/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index fb5d566,06e00d7..41fbb0c
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@@ -120,10 -122,11 +122,12 @@@ public abstract class BaseSemanticAnaly
     */
    protected Set<FileSinkDesc> acidFileSinks = new HashSet<FileSinkDesc>();
  
 -  // whether any ACID table is involved in a query
 +  // whether any ACID table or Insert-only (mm) table is involved in a query
 +  // They both require DbTxnManager and both need to recordValidTxns when acquiring locks in Driver
    protected boolean acidInQuery;
  
+   protected HiveTxnManager txnManager;
+ 
    public static final int HIVE_COLUMN_ORDER_ASC = 1;
    public static final int HIVE_COLUMN_ORDER_DESC = 0;
    public static final int HIVE_COLUMN_NULLS_FIRST = 0;

http://git-wip-us.apache.org/repos/asf/hive/blob/47178498/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/47178498/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
----------------------------------------------------------------------

[31/50] [abbrv] hive git commit: cherry picked HIVE-15899 from master

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
index a0581f8..7a5f8b4 100644
--- a/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
+++ b/ql/src/test/results/clientpositive/autoColumnStats_4.q.out
@@ -136,6 +136,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_dtt
+          Write Type: INSERT
 
   Stage: Stage-3
     Stats-Aggr Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
index 34dd487..7905194 100644
--- a/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
+++ b/ql/src/test/results/clientpositive/llap/acid_no_buckets.q.out
@@ -166,6 +166,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.srcpart_acid
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -344,6 +345,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.srcpart_acid
+          Write Type: DELETE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -660,6 +662,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.srcpart_acidb
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -839,6 +842,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.srcpart_acidb
+          Write Type: DELETE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -1154,6 +1158,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.srcpart_acidv
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -1332,6 +1337,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.srcpart_acidv
+          Write Type: DELETE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -1648,6 +1654,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.srcpart_acidvb
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -1827,6 +1834,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.srcpart_acidvb
+          Write Type: DELETE
 
   Stage: Stage-3
     Stats-Aggr Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
index 8aee753..095ae9d 100644
--- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_3.q.out
@@ -256,6 +256,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: DELETE
 
   Stage: Stage-6
     Stats-Aggr Operator
@@ -269,6 +270,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: UPDATE
 
   Stage: Stage-7
     Stats-Aggr Operator
@@ -295,6 +297,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: INSERT
 
   Stage: Stage-9
     Stats-Aggr Operator
@@ -427,6 +430,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: INSERT
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -727,6 +731,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: DELETE
 
   Stage: Stage-6
     Stats-Aggr Operator
@@ -740,6 +745,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: UPDATE
 
   Stage: Stage-7
     Stats-Aggr Operator
@@ -766,6 +772,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: INSERT
 
   Stage: Stage-9
     Stats-Aggr Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
index 6488e9e..16c0b29 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out
@@ -138,6 +138,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_part
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -232,6 +233,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_part
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -418,6 +420,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_part_sdpo
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -512,6 +515,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_part_sdpo
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -708,6 +712,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_2l_part
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -803,6 +808,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_2l_part
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -927,6 +933,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_2l_part
+          Write Type: DELETE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -1120,6 +1127,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_2l_part_sdpo
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -1215,6 +1223,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_2l_part_sdpo
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -1339,6 +1348,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_2l_part_sdpo
+          Write Type: DELETE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -1534,6 +1544,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_2l_part_sdpo_no_cp
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator
@@ -1630,6 +1641,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acid_2l_part_sdpo_no_cp
+          Write Type: UPDATE
 
   Stage: Stage-3
     Stats-Aggr Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/test/results/clientpositive/llap/mm_all.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_all.q.out b/ql/src/test/results/clientpositive/llap/mm_all.q.out
index f184ba6..0adbddb 100644
--- a/ql/src/test/results/clientpositive/llap/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_all.q.out
@@ -99,6 +99,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.part_mm
+          Write Type: INSERT
           micromanaged table: true
 
   Stage: Stage-3

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/test/results/clientpositive/llap/sqlmerge.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/sqlmerge.q.out b/ql/src/test/results/clientpositive/llap/sqlmerge.q.out
index 61349f8..01fe124 100644
--- a/ql/src/test/results/clientpositive/llap/sqlmerge.q.out
+++ b/ql/src/test/results/clientpositive/llap/sqlmerge.q.out
@@ -226,6 +226,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: DELETE
 
   Stage: Stage-6
     Stats-Aggr Operator
@@ -239,6 +240,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: UPDATE
 
   Stage: Stage-7
     Stats-Aggr Operator
@@ -265,6 +267,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: INSERT
 
   Stage: Stage-9
     Stats-Aggr Operator
@@ -367,6 +370,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.acidtbl
+          Write Type: INSERT
 
   Stage: Stage-3
     Stats-Aggr Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/test/results/clientpositive/mm_all.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/mm_all.q.out b/ql/src/test/results/clientpositive/mm_all.q.out
index 4162ce9..fa9e4a7 100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@ -104,6 +104,7 @@ STAGE PLANS:
               output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
               serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
               name: default.part_mm
+          Write Type: INSERT
           micromanaged table: true
 
   Stage: Stage-2

[35/50] [abbrv] hive git commit: HIVE-15212 : merge branch into master - clean up some pre-merge TODOs (Sergey Shelukhin)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
index 85d1324..0b7fe01 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
@@ -45,7 +45,6 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
   // TODO: the below seems like they should just be combined into partitionDesc
   private org.apache.hadoop.hive.ql.plan.TableDesc table;
   private Map<String, String> partitionSpec; // NOTE: this partitionSpec has to be ordered map
-  private boolean commitMmWriteId = true;
 
   public LoadTableDesc(final LoadTableDesc o) {
     super(o.getSourcePath(), o.getWriteType());
@@ -65,8 +64,10 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
       final boolean replace,
       final AcidUtils.Operation writeType, Long currentTransactionId) {
     super(sourcePath, writeType);
-    Utilities.LOG14535.info("creating part LTD from " + sourcePath + " to "
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("creating part LTD from " + sourcePath + " to "
         + ((table.getProperties() == null) ? "null" : table.getTableName()));
+    }
     init(table, partitionSpec, replace, currentTransactionId);
   }
 
@@ -110,7 +111,9 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
       final AcidUtils.Operation writeType,
       boolean isReplace, Long txnId) {
     super(sourcePath, writeType);
-    Utilities.LOG14535.info("creating LTD from " + sourcePath + " to " + table.getTableName()/*, new Exception()*/);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("creating LTD from " + sourcePath + " to " + table.getTableName());
+    }
     this.dpCtx = dpCtx;
     if (dpCtx != null && dpCtx.getPartSpec() != null && partitionSpec == null) {
       init(table, dpCtx.getPartSpec(), isReplace, txnId);
@@ -211,12 +214,4 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
   public void setStmtId(int stmtId) {
     this.stmtId = stmtId;
   }
-
-  public void setIntermediateInMmWrite(boolean b) {
-    this.commitMmWriteId = !b;
-  }
-
-  public boolean isCommitMmWrite() {
-    return commitMmWriteId;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
index f6303ba..17d2f31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
@@ -80,8 +80,10 @@ public class MoveWork implements Serializable {
       final LoadTableDesc loadTableWork, final LoadFileDesc loadFileWork,
       boolean checkFileFormat, boolean srcLocal, LineageState lineageState) {
     this(inputs, outputs, lineageState);
-    Utilities.LOG14535.info("Creating MoveWork " + System.identityHashCode(this)
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("Creating MoveWork " + System.identityHashCode(this)
         + " with " + loadTableWork + "; " + loadFileWork);
+    }
     this.loadTableWork = loadTableWork;
     this.loadFileWork = loadFileWork;
     this.checkFileFormat = checkFileFormat;

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java
index 03444c0..74e4387 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsAggregator.java
@@ -50,7 +50,9 @@ public class FSStatsAggregator implements StatsAggregator {
     List<String> statsDirs = scc.getStatsTmpDirs();
     assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs;
     Path statsDir = new Path(statsDirs.get(0));
-    Utilities.LOG14535.info("About to read stats from : " + statsDir);
+    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+      Utilities.FILE_OP_LOGGER.trace("About to read stats from : " + statsDir);
+    }
     statsMap  = new HashMap<String, Map<String,String>>();
 
     try {
@@ -70,13 +72,12 @@ public class FSStatsAggregator implements StatsAggregator {
         } finally {
           SerializationUtilities.releaseKryo(kryo);
         }
-        Utilities.LOG14535.info("Read stats : " +statsMap);
         statsList.add(statsMap);
         in.close();
       }
       return true;
     } catch (IOException e) {
-      Utilities.LOG14535.error("Failed to read stats from filesystem ", e);
+      Utilities.FILE_OP_LOGGER.error("Failed to read stats from filesystem ", e);
       return false;
     }
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/55d9ceee/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
index a92465f..e8b878c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/fs/FSStatsPublisher.java
@@ -77,7 +77,6 @@ public class FSStatsPublisher implements StatsPublisher {
 
   @Override
   public boolean publishStat(String partKV, Map<String, String> stats) {
-    Utilities.LOG14535.info("Putting in map : " + partKV + "\t" + stats);
     // we need to do new hashmap, since stats object is reused across calls.
     Map<String,String> cpy = new HashMap<String, String>(stats);
     Map<String,String> statMap = statsMap.get(partKV);
@@ -106,7 +105,9 @@ public class FSStatsPublisher implements StatsPublisher {
         statsFile = new Path(statsDir, StatsSetupConst.STATS_FILE_PREFIX
             + conf.getInt("mapred.task.partition", 0));
       }
-      Utilities.LOG14535.info("About to create stats file for this task : " + statsFile);
+      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
+        Utilities.FILE_OP_LOGGER.trace("About to create stats file for this task : " + statsFile);
+      }
       Output output = new Output(statsFile.getFileSystem(conf).create(statsFile,true));
       LOG.debug("Created file : " + statsFile);
       LOG.debug("Writing stats in it : " + statsMap);
@@ -119,7 +120,7 @@ public class FSStatsPublisher implements StatsPublisher {
       output.close();
       return true;
     } catch (IOException e) {
-      Utilities.LOG14535.error("Failed to persist stats on filesystem",e);
+      Utilities.FILE_OP_LOGGER.error("Failed to persist stats on filesystem",e);
       return false;
     }
   }

[13/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 0000000,5c09fdd..3e4c754
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@@ -1,0 -1,21817 +1,21819 @@@
+ /**
+  * Autogenerated by Thrift Compiler (0.9.3)
+  *
+  * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+  *  @generated
+  */
+ #include "hive_metastore_types.h"
+ 
+ #include <algorithm>
+ #include <ostream>
+ 
+ #include <thrift/TToString.h>
+ 
+ namespace Apache { namespace Hadoop { namespace Hive {
+ 
+ int _kHiveObjectTypeValues[] = {
+   HiveObjectType::GLOBAL,
+   HiveObjectType::DATABASE,
+   HiveObjectType::TABLE,
+   HiveObjectType::PARTITION,
+   HiveObjectType::COLUMN
+ };
+ const char* _kHiveObjectTypeNames[] = {
+   "GLOBAL",
+   "DATABASE",
+   "TABLE",
+   "PARTITION",
+   "COLUMN"
+ };
+ const std::map<int, const char*> _HiveObjectType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(5, _kHiveObjectTypeValues, _kHiveObjectTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kPrincipalTypeValues[] = {
+   PrincipalType::USER,
+   PrincipalType::ROLE,
+   PrincipalType::GROUP
+ };
+ const char* _kPrincipalTypeNames[] = {
+   "USER",
+   "ROLE",
+   "GROUP"
+ };
+ const std::map<int, const char*> _PrincipalType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kPrincipalTypeValues, _kPrincipalTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kPartitionEventTypeValues[] = {
+   PartitionEventType::LOAD_DONE
+ };
+ const char* _kPartitionEventTypeNames[] = {
+   "LOAD_DONE"
+ };
+ const std::map<int, const char*> _PartitionEventType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(1, _kPartitionEventTypeValues, _kPartitionEventTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kTxnStateValues[] = {
+   TxnState::COMMITTED,
+   TxnState::ABORTED,
+   TxnState::OPEN
+ };
+ const char* _kTxnStateNames[] = {
+   "COMMITTED",
+   "ABORTED",
+   "OPEN"
+ };
+ const std::map<int, const char*> _TxnState_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kTxnStateValues, _kTxnStateNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kLockLevelValues[] = {
+   LockLevel::DB,
+   LockLevel::TABLE,
+   LockLevel::PARTITION
+ };
+ const char* _kLockLevelNames[] = {
+   "DB",
+   "TABLE",
+   "PARTITION"
+ };
+ const std::map<int, const char*> _LockLevel_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kLockLevelValues, _kLockLevelNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kLockStateValues[] = {
+   LockState::ACQUIRED,
+   LockState::WAITING,
+   LockState::ABORT,
+   LockState::NOT_ACQUIRED
+ };
+ const char* _kLockStateNames[] = {
+   "ACQUIRED",
+   "WAITING",
+   "ABORT",
+   "NOT_ACQUIRED"
+ };
+ const std::map<int, const char*> _LockState_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kLockStateValues, _kLockStateNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kLockTypeValues[] = {
+   LockType::SHARED_READ,
+   LockType::SHARED_WRITE,
+   LockType::EXCLUSIVE
+ };
+ const char* _kLockTypeNames[] = {
+   "SHARED_READ",
+   "SHARED_WRITE",
+   "EXCLUSIVE"
+ };
+ const std::map<int, const char*> _LockType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kLockTypeValues, _kLockTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kCompactionTypeValues[] = {
+   CompactionType::MINOR,
+   CompactionType::MAJOR
+ };
+ const char* _kCompactionTypeNames[] = {
+   "MINOR",
+   "MAJOR"
+ };
+ const std::map<int, const char*> _CompactionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(2, _kCompactionTypeValues, _kCompactionTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kGrantRevokeTypeValues[] = {
+   GrantRevokeType::GRANT,
+   GrantRevokeType::REVOKE
+ };
+ const char* _kGrantRevokeTypeNames[] = {
+   "GRANT",
+   "REVOKE"
+ };
+ const std::map<int, const char*> _GrantRevokeType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(2, _kGrantRevokeTypeValues, _kGrantRevokeTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kDataOperationTypeValues[] = {
+   DataOperationType::SELECT,
+   DataOperationType::INSERT,
+   DataOperationType::UPDATE,
+   DataOperationType::DELETE,
+   DataOperationType::UNSET,
+   DataOperationType::NO_TXN
+ };
+ const char* _kDataOperationTypeNames[] = {
+   "SELECT",
+   "INSERT",
+   "UPDATE",
+   "DELETE",
+   "UNSET",
+   "NO_TXN"
+ };
+ const std::map<int, const char*> _DataOperationType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(6, _kDataOperationTypeValues, _kDataOperationTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kEventRequestTypeValues[] = {
+   EventRequestType::INSERT,
+   EventRequestType::UPDATE,
+   EventRequestType::DELETE
+ };
+ const char* _kEventRequestTypeNames[] = {
+   "INSERT",
+   "UPDATE",
+   "DELETE"
+ };
+ const std::map<int, const char*> _EventRequestType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kEventRequestTypeValues, _kEventRequestTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kFunctionTypeValues[] = {
+   FunctionType::JAVA
+ };
+ const char* _kFunctionTypeNames[] = {
+   "JAVA"
+ };
+ const std::map<int, const char*> _FunctionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(1, _kFunctionTypeValues, _kFunctionTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kResourceTypeValues[] = {
+   ResourceType::JAR,
+   ResourceType::FILE,
+   ResourceType::ARCHIVE
+ };
+ const char* _kResourceTypeNames[] = {
+   "JAR",
+   "FILE",
+   "ARCHIVE"
+ };
+ const std::map<int, const char*> _ResourceType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3, _kResourceTypeValues, _kResourceTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kFileMetadataExprTypeValues[] = {
+   FileMetadataExprType::ORC_SARG
+ };
+ const char* _kFileMetadataExprTypeNames[] = {
+   "ORC_SARG"
+ };
+ const std::map<int, const char*> _FileMetadataExprType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(1, _kFileMetadataExprTypeValues, _kFileMetadataExprTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ int _kClientCapabilityValues[] = {
 -  ClientCapability::TEST_CAPABILITY
++  ClientCapability::TEST_CAPABILITY,
++  ClientCapability::INSERT_ONLY_TABLES
+ };
+ const char* _kClientCapabilityNames[] = {
 -  "TEST_CAPABILITY"
++  "TEST_CAPABILITY",
++  "INSERT_ONLY_TABLES"
+ };
 -const std::map<int, const char*> _ClientCapability_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(1, _kClientCapabilityValues, _kClientCapabilityNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
++const std::map<int, const char*> _ClientCapability_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(2, _kClientCapabilityValues, _kClientCapabilityNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+ 
+ 
+ Version::~Version() throw() {
+ }
+ 
+ 
+ void Version::__set_version(const std::string& val) {
+   this->version = val;
+ }
+ 
+ void Version::__set_comments(const std::string& val) {
+   this->comments = val;
+ }
+ 
+ uint32_t Version::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->version);
+           this->__isset.version = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->comments);
+           this->__isset.comments = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t Version::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("Version");
+ 
+   xfer += oprot->writeFieldBegin("version", ::apache::thrift::protocol::T_STRING, 1);
+   xfer += oprot->writeString(this->version);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("comments", ::apache::thrift::protocol::T_STRING, 2);
+   xfer += oprot->writeString(this->comments);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(Version &a, Version &b) {
+   using ::std::swap;
+   swap(a.version, b.version);
+   swap(a.comments, b.comments);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ Version::Version(const Version& other0) {
+   version = other0.version;
+   comments = other0.comments;
+   __isset = other0.__isset;
+ }
+ Version& Version::operator=(const Version& other1) {
+   version = other1.version;
+   comments = other1.comments;
+   __isset = other1.__isset;
+   return *this;
+ }
+ void Version::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "Version(";
+   out << "version=" << to_string(version);
+   out << ", " << "comments=" << to_string(comments);
+   out << ")";
+ }
+ 
+ 
+ FieldSchema::~FieldSchema() throw() {
+ }
+ 
+ 
+ void FieldSchema::__set_name(const std::string& val) {
+   this->name = val;
+ }
+ 
+ void FieldSchema::__set_type(const std::string& val) {
+   this->type = val;
+ }
+ 
+ void FieldSchema::__set_comment(const std::string& val) {
+   this->comment = val;
+ }
+ 
+ uint32_t FieldSchema::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->name);
+           this->__isset.name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->type);
+           this->__isset.type = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->comment);
+           this->__isset.comment = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t FieldSchema::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("FieldSchema");
+ 
+   xfer += oprot->writeFieldBegin("name", ::apache::thrift::protocol::T_STRING, 1);
+   xfer += oprot->writeString(this->name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("type", ::apache::thrift::protocol::T_STRING, 2);
+   xfer += oprot->writeString(this->type);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("comment", ::apache::thrift::protocol::T_STRING, 3);
+   xfer += oprot->writeString(this->comment);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(FieldSchema &a, FieldSchema &b) {
+   using ::std::swap;
+   swap(a.name, b.name);
+   swap(a.type, b.type);
+   swap(a.comment, b.comment);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ FieldSchema::FieldSchema(const FieldSchema& other2) {
+   name = other2.name;
+   type = other2.type;
+   comment = other2.comment;
+   __isset = other2.__isset;
+ }
+ FieldSchema& FieldSchema::operator=(const FieldSchema& other3) {
+   name = other3.name;
+   type = other3.type;
+   comment = other3.comment;
+   __isset = other3.__isset;
+   return *this;
+ }
+ void FieldSchema::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "FieldSchema(";
+   out << "name=" << to_string(name);
+   out << ", " << "type=" << to_string(type);
+   out << ", " << "comment=" << to_string(comment);
+   out << ")";
+ }
+ 
+ 
+ SQLPrimaryKey::~SQLPrimaryKey() throw() {
+ }
+ 
+ 
+ void SQLPrimaryKey::__set_table_db(const std::string& val) {
+   this->table_db = val;
+ }
+ 
+ void SQLPrimaryKey::__set_table_name(const std::string& val) {
+   this->table_name = val;
+ }
+ 
+ void SQLPrimaryKey::__set_column_name(const std::string& val) {
+   this->column_name = val;
+ }
+ 
+ void SQLPrimaryKey::__set_key_seq(const int32_t val) {
+   this->key_seq = val;
+ }
+ 
+ void SQLPrimaryKey::__set_pk_name(const std::string& val) {
+   this->pk_name = val;
+ }
+ 
+ void SQLPrimaryKey::__set_enable_cstr(const bool val) {
+   this->enable_cstr = val;
+ }
+ 
+ void SQLPrimaryKey::__set_validate_cstr(const bool val) {
+   this->validate_cstr = val;
+ }
+ 
+ void SQLPrimaryKey::__set_rely_cstr(const bool val) {
+   this->rely_cstr = val;
+ }
+ 
+ uint32_t SQLPrimaryKey::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->table_db);
+           this->__isset.table_db = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->table_name);
+           this->__isset.table_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->column_name);
+           this->__isset.column_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 4:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           xfer += iprot->readI32(this->key_seq);
+           this->__isset.key_seq = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 5:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->pk_name);
+           this->__isset.pk_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 6:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->enable_cstr);
+           this->__isset.enable_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 7:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->validate_cstr);
+           this->__isset.validate_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 8:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->rely_cstr);
+           this->__isset.rely_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t SQLPrimaryKey::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("SQLPrimaryKey");
+ 
+   xfer += oprot->writeFieldBegin("table_db", ::apache::thrift::protocol::T_STRING, 1);
+   xfer += oprot->writeString(this->table_db);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2);
+   xfer += oprot->writeString(this->table_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("column_name", ::apache::thrift::protocol::T_STRING, 3);
+   xfer += oprot->writeString(this->column_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("key_seq", ::apache::thrift::protocol::T_I32, 4);
+   xfer += oprot->writeI32(this->key_seq);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("pk_name", ::apache::thrift::protocol::T_STRING, 5);
+   xfer += oprot->writeString(this->pk_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("enable_cstr", ::apache::thrift::protocol::T_BOOL, 6);
+   xfer += oprot->writeBool(this->enable_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("validate_cstr", ::apache::thrift::protocol::T_BOOL, 7);
+   xfer += oprot->writeBool(this->validate_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("rely_cstr", ::apache::thrift::protocol::T_BOOL, 8);
+   xfer += oprot->writeBool(this->rely_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(SQLPrimaryKey &a, SQLPrimaryKey &b) {
+   using ::std::swap;
+   swap(a.table_db, b.table_db);
+   swap(a.table_name, b.table_name);
+   swap(a.column_name, b.column_name);
+   swap(a.key_seq, b.key_seq);
+   swap(a.pk_name, b.pk_name);
+   swap(a.enable_cstr, b.enable_cstr);
+   swap(a.validate_cstr, b.validate_cstr);
+   swap(a.rely_cstr, b.rely_cstr);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ SQLPrimaryKey::SQLPrimaryKey(const SQLPrimaryKey& other4) {
+   table_db = other4.table_db;
+   table_name = other4.table_name;
+   column_name = other4.column_name;
+   key_seq = other4.key_seq;
+   pk_name = other4.pk_name;
+   enable_cstr = other4.enable_cstr;
+   validate_cstr = other4.validate_cstr;
+   rely_cstr = other4.rely_cstr;
+   __isset = other4.__isset;
+ }
+ SQLPrimaryKey& SQLPrimaryKey::operator=(const SQLPrimaryKey& other5) {
+   table_db = other5.table_db;
+   table_name = other5.table_name;
+   column_name = other5.column_name;
+   key_seq = other5.key_seq;
+   pk_name = other5.pk_name;
+   enable_cstr = other5.enable_cstr;
+   validate_cstr = other5.validate_cstr;
+   rely_cstr = other5.rely_cstr;
+   __isset = other5.__isset;
+   return *this;
+ }
+ void SQLPrimaryKey::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "SQLPrimaryKey(";
+   out << "table_db=" << to_string(table_db);
+   out << ", " << "table_name=" << to_string(table_name);
+   out << ", " << "column_name=" << to_string(column_name);
+   out << ", " << "key_seq=" << to_string(key_seq);
+   out << ", " << "pk_name=" << to_string(pk_name);
+   out << ", " << "enable_cstr=" << to_string(enable_cstr);
+   out << ", " << "validate_cstr=" << to_string(validate_cstr);
+   out << ", " << "rely_cstr=" << to_string(rely_cstr);
+   out << ")";
+ }
+ 
+ 
+ SQLForeignKey::~SQLForeignKey() throw() {
+ }
+ 
+ 
+ void SQLForeignKey::__set_pktable_db(const std::string& val) {
+   this->pktable_db = val;
+ }
+ 
+ void SQLForeignKey::__set_pktable_name(const std::string& val) {
+   this->pktable_name = val;
+ }
+ 
+ void SQLForeignKey::__set_pkcolumn_name(const std::string& val) {
+   this->pkcolumn_name = val;
+ }
+ 
+ void SQLForeignKey::__set_fktable_db(const std::string& val) {
+   this->fktable_db = val;
+ }
+ 
+ void SQLForeignKey::__set_fktable_name(const std::string& val) {
+   this->fktable_name = val;
+ }
+ 
+ void SQLForeignKey::__set_fkcolumn_name(const std::string& val) {
+   this->fkcolumn_name = val;
+ }
+ 
+ void SQLForeignKey::__set_key_seq(const int32_t val) {
+   this->key_seq = val;
+ }
+ 
+ void SQLForeignKey::__set_update_rule(const int32_t val) {
+   this->update_rule = val;
+ }
+ 
+ void SQLForeignKey::__set_delete_rule(const int32_t val) {
+   this->delete_rule = val;
+ }
+ 
+ void SQLForeignKey::__set_fk_name(const std::string& val) {
+   this->fk_name = val;
+ }
+ 
+ void SQLForeignKey::__set_pk_name(const std::string& val) {
+   this->pk_name = val;
+ }
+ 
+ void SQLForeignKey::__set_enable_cstr(const bool val) {
+   this->enable_cstr = val;
+ }
+ 
+ void SQLForeignKey::__set_validate_cstr(const bool val) {
+   this->validate_cstr = val;
+ }
+ 
+ void SQLForeignKey::__set_rely_cstr(const bool val) {
+   this->rely_cstr = val;
+ }
+ 
+ uint32_t SQLForeignKey::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->pktable_db);
+           this->__isset.pktable_db = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->pktable_name);
+           this->__isset.pktable_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->pkcolumn_name);
+           this->__isset.pkcolumn_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 4:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->fktable_db);
+           this->__isset.fktable_db = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 5:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->fktable_name);
+           this->__isset.fktable_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 6:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->fkcolumn_name);
+           this->__isset.fkcolumn_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 7:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           xfer += iprot->readI32(this->key_seq);
+           this->__isset.key_seq = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 8:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           xfer += iprot->readI32(this->update_rule);
+           this->__isset.update_rule = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 9:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           xfer += iprot->readI32(this->delete_rule);
+           this->__isset.delete_rule = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 10:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->fk_name);
+           this->__isset.fk_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 11:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->pk_name);
+           this->__isset.pk_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 12:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->enable_cstr);
+           this->__isset.enable_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 13:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->validate_cstr);
+           this->__isset.validate_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 14:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->rely_cstr);
+           this->__isset.rely_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t SQLForeignKey::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("SQLForeignKey");
+ 
+   xfer += oprot->writeFieldBegin("pktable_db", ::apache::thrift::protocol::T_STRING, 1);
+   xfer += oprot->writeString(this->pktable_db);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("pktable_name", ::apache::thrift::protocol::T_STRING, 2);
+   xfer += oprot->writeString(this->pktable_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("pkcolumn_name", ::apache::thrift::protocol::T_STRING, 3);
+   xfer += oprot->writeString(this->pkcolumn_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("fktable_db", ::apache::thrift::protocol::T_STRING, 4);
+   xfer += oprot->writeString(this->fktable_db);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("fktable_name", ::apache::thrift::protocol::T_STRING, 5);
+   xfer += oprot->writeString(this->fktable_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("fkcolumn_name", ::apache::thrift::protocol::T_STRING, 6);
+   xfer += oprot->writeString(this->fkcolumn_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("key_seq", ::apache::thrift::protocol::T_I32, 7);
+   xfer += oprot->writeI32(this->key_seq);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("update_rule", ::apache::thrift::protocol::T_I32, 8);
+   xfer += oprot->writeI32(this->update_rule);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("delete_rule", ::apache::thrift::protocol::T_I32, 9);
+   xfer += oprot->writeI32(this->delete_rule);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("fk_name", ::apache::thrift::protocol::T_STRING, 10);
+   xfer += oprot->writeString(this->fk_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("pk_name", ::apache::thrift::protocol::T_STRING, 11);
+   xfer += oprot->writeString(this->pk_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("enable_cstr", ::apache::thrift::protocol::T_BOOL, 12);
+   xfer += oprot->writeBool(this->enable_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("validate_cstr", ::apache::thrift::protocol::T_BOOL, 13);
+   xfer += oprot->writeBool(this->validate_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("rely_cstr", ::apache::thrift::protocol::T_BOOL, 14);
+   xfer += oprot->writeBool(this->rely_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(SQLForeignKey &a, SQLForeignKey &b) {
+   using ::std::swap;
+   swap(a.pktable_db, b.pktable_db);
+   swap(a.pktable_name, b.pktable_name);
+   swap(a.pkcolumn_name, b.pkcolumn_name);
+   swap(a.fktable_db, b.fktable_db);
+   swap(a.fktable_name, b.fktable_name);
+   swap(a.fkcolumn_name, b.fkcolumn_name);
+   swap(a.key_seq, b.key_seq);
+   swap(a.update_rule, b.update_rule);
+   swap(a.delete_rule, b.delete_rule);
+   swap(a.fk_name, b.fk_name);
+   swap(a.pk_name, b.pk_name);
+   swap(a.enable_cstr, b.enable_cstr);
+   swap(a.validate_cstr, b.validate_cstr);
+   swap(a.rely_cstr, b.rely_cstr);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ SQLForeignKey::SQLForeignKey(const SQLForeignKey& other6) {
+   pktable_db = other6.pktable_db;
+   pktable_name = other6.pktable_name;
+   pkcolumn_name = other6.pkcolumn_name;
+   fktable_db = other6.fktable_db;
+   fktable_name = other6.fktable_name;
+   fkcolumn_name = other6.fkcolumn_name;
+   key_seq = other6.key_seq;
+   update_rule = other6.update_rule;
+   delete_rule = other6.delete_rule;
+   fk_name = other6.fk_name;
+   pk_name = other6.pk_name;
+   enable_cstr = other6.enable_cstr;
+   validate_cstr = other6.validate_cstr;
+   rely_cstr = other6.rely_cstr;
+   __isset = other6.__isset;
+ }
+ SQLForeignKey& SQLForeignKey::operator=(const SQLForeignKey& other7) {
+   pktable_db = other7.pktable_db;
+   pktable_name = other7.pktable_name;
+   pkcolumn_name = other7.pkcolumn_name;
+   fktable_db = other7.fktable_db;
+   fktable_name = other7.fktable_name;
+   fkcolumn_name = other7.fkcolumn_name;
+   key_seq = other7.key_seq;
+   update_rule = other7.update_rule;
+   delete_rule = other7.delete_rule;
+   fk_name = other7.fk_name;
+   pk_name = other7.pk_name;
+   enable_cstr = other7.enable_cstr;
+   validate_cstr = other7.validate_cstr;
+   rely_cstr = other7.rely_cstr;
+   __isset = other7.__isset;
+   return *this;
+ }
+ void SQLForeignKey::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "SQLForeignKey(";
+   out << "pktable_db=" << to_string(pktable_db);
+   out << ", " << "pktable_name=" << to_string(pktable_name);
+   out << ", " << "pkcolumn_name=" << to_string(pkcolumn_name);
+   out << ", " << "fktable_db=" << to_string(fktable_db);
+   out << ", " << "fktable_name=" << to_string(fktable_name);
+   out << ", " << "fkcolumn_name=" << to_string(fkcolumn_name);
+   out << ", " << "key_seq=" << to_string(key_seq);
+   out << ", " << "update_rule=" << to_string(update_rule);
+   out << ", " << "delete_rule=" << to_string(delete_rule);
+   out << ", " << "fk_name=" << to_string(fk_name);
+   out << ", " << "pk_name=" << to_string(pk_name);
+   out << ", " << "enable_cstr=" << to_string(enable_cstr);
+   out << ", " << "validate_cstr=" << to_string(validate_cstr);
+   out << ", " << "rely_cstr=" << to_string(rely_cstr);
+   out << ")";
+ }
+ 
+ 
+ SQLUniqueConstraint::~SQLUniqueConstraint() throw() {
+ }
+ 
+ 
+ void SQLUniqueConstraint::__set_table_db(const std::string& val) {
+   this->table_db = val;
+ }
+ 
+ void SQLUniqueConstraint::__set_table_name(const std::string& val) {
+   this->table_name = val;
+ }
+ 
+ void SQLUniqueConstraint::__set_column_name(const std::string& val) {
+   this->column_name = val;
+ }
+ 
+ void SQLUniqueConstraint::__set_key_seq(const int32_t val) {
+   this->key_seq = val;
+ }
+ 
+ void SQLUniqueConstraint::__set_uk_name(const std::string& val) {
+   this->uk_name = val;
+ }
+ 
+ void SQLUniqueConstraint::__set_enable_cstr(const bool val) {
+   this->enable_cstr = val;
+ }
+ 
+ void SQLUniqueConstraint::__set_validate_cstr(const bool val) {
+   this->validate_cstr = val;
+ }
+ 
+ void SQLUniqueConstraint::__set_rely_cstr(const bool val) {
+   this->rely_cstr = val;
+ }
+ 
+ uint32_t SQLUniqueConstraint::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->table_db);
+           this->__isset.table_db = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->table_name);
+           this->__isset.table_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->column_name);
+           this->__isset.column_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 4:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           xfer += iprot->readI32(this->key_seq);
+           this->__isset.key_seq = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 5:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->uk_name);
+           this->__isset.uk_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 6:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->enable_cstr);
+           this->__isset.enable_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 7:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->validate_cstr);
+           this->__isset.validate_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 8:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->rely_cstr);
+           this->__isset.rely_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t SQLUniqueConstraint::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("SQLUniqueConstraint");
+ 
+   xfer += oprot->writeFieldBegin("table_db", ::apache::thrift::protocol::T_STRING, 1);
+   xfer += oprot->writeString(this->table_db);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2);
+   xfer += oprot->writeString(this->table_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("column_name", ::apache::thrift::protocol::T_STRING, 3);
+   xfer += oprot->writeString(this->column_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("key_seq", ::apache::thrift::protocol::T_I32, 4);
+   xfer += oprot->writeI32(this->key_seq);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("uk_name", ::apache::thrift::protocol::T_STRING, 5);
+   xfer += oprot->writeString(this->uk_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("enable_cstr", ::apache::thrift::protocol::T_BOOL, 6);
+   xfer += oprot->writeBool(this->enable_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("validate_cstr", ::apache::thrift::protocol::T_BOOL, 7);
+   xfer += oprot->writeBool(this->validate_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("rely_cstr", ::apache::thrift::protocol::T_BOOL, 8);
+   xfer += oprot->writeBool(this->rely_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(SQLUniqueConstraint &a, SQLUniqueConstraint &b) {
+   using ::std::swap;
+   swap(a.table_db, b.table_db);
+   swap(a.table_name, b.table_name);
+   swap(a.column_name, b.column_name);
+   swap(a.key_seq, b.key_seq);
+   swap(a.uk_name, b.uk_name);
+   swap(a.enable_cstr, b.enable_cstr);
+   swap(a.validate_cstr, b.validate_cstr);
+   swap(a.rely_cstr, b.rely_cstr);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ SQLUniqueConstraint::SQLUniqueConstraint(const SQLUniqueConstraint& other8) {
+   table_db = other8.table_db;
+   table_name = other8.table_name;
+   column_name = other8.column_name;
+   key_seq = other8.key_seq;
+   uk_name = other8.uk_name;
+   enable_cstr = other8.enable_cstr;
+   validate_cstr = other8.validate_cstr;
+   rely_cstr = other8.rely_cstr;
+   __isset = other8.__isset;
+ }
+ SQLUniqueConstraint& SQLUniqueConstraint::operator=(const SQLUniqueConstraint& other9) {
+   table_db = other9.table_db;
+   table_name = other9.table_name;
+   column_name = other9.column_name;
+   key_seq = other9.key_seq;
+   uk_name = other9.uk_name;
+   enable_cstr = other9.enable_cstr;
+   validate_cstr = other9.validate_cstr;
+   rely_cstr = other9.rely_cstr;
+   __isset = other9.__isset;
+   return *this;
+ }
+ void SQLUniqueConstraint::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "SQLUniqueConstraint(";
+   out << "table_db=" << to_string(table_db);
+   out << ", " << "table_name=" << to_string(table_name);
+   out << ", " << "column_name=" << to_string(column_name);
+   out << ", " << "key_seq=" << to_string(key_seq);
+   out << ", " << "uk_name=" << to_string(uk_name);
+   out << ", " << "enable_cstr=" << to_string(enable_cstr);
+   out << ", " << "validate_cstr=" << to_string(validate_cstr);
+   out << ", " << "rely_cstr=" << to_string(rely_cstr);
+   out << ")";
+ }
+ 
+ 
+ SQLNotNullConstraint::~SQLNotNullConstraint() throw() {
+ }
+ 
+ 
+ void SQLNotNullConstraint::__set_table_db(const std::string& val) {
+   this->table_db = val;
+ }
+ 
+ void SQLNotNullConstraint::__set_table_name(const std::string& val) {
+   this->table_name = val;
+ }
+ 
+ void SQLNotNullConstraint::__set_column_name(const std::string& val) {
+   this->column_name = val;
+ }
+ 
+ void SQLNotNullConstraint::__set_nn_name(const std::string& val) {
+   this->nn_name = val;
+ }
+ 
+ void SQLNotNullConstraint::__set_enable_cstr(const bool val) {
+   this->enable_cstr = val;
+ }
+ 
+ void SQLNotNullConstraint::__set_validate_cstr(const bool val) {
+   this->validate_cstr = val;
+ }
+ 
+ void SQLNotNullConstraint::__set_rely_cstr(const bool val) {
+   this->rely_cstr = val;
+ }
+ 
+ uint32_t SQLNotNullConstraint::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->table_db);
+           this->__isset.table_db = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->table_name);
+           this->__isset.table_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->column_name);
+           this->__isset.column_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 4:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->nn_name);
+           this->__isset.nn_name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 5:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->enable_cstr);
+           this->__isset.enable_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 6:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->validate_cstr);
+           this->__isset.validate_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 7:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->rely_cstr);
+           this->__isset.rely_cstr = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t SQLNotNullConstraint::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("SQLNotNullConstraint");
+ 
+   xfer += oprot->writeFieldBegin("table_db", ::apache::thrift::protocol::T_STRING, 1);
+   xfer += oprot->writeString(this->table_db);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2);
+   xfer += oprot->writeString(this->table_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("column_name", ::apache::thrift::protocol::T_STRING, 3);
+   xfer += oprot->writeString(this->column_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("nn_name", ::apache::thrift::protocol::T_STRING, 4);
+   xfer += oprot->writeString(this->nn_name);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("enable_cstr", ::apache::thrift::protocol::T_BOOL, 5);
+   xfer += oprot->writeBool(this->enable_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("validate_cstr", ::apache::thrift::protocol::T_BOOL, 6);
+   xfer += oprot->writeBool(this->validate_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("rely_cstr", ::apache::thrift::protocol::T_BOOL, 7);
+   xfer += oprot->writeBool(this->rely_cstr);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(SQLNotNullConstraint &a, SQLNotNullConstraint &b) {
+   using ::std::swap;
+   swap(a.table_db, b.table_db);
+   swap(a.table_name, b.table_name);
+   swap(a.column_name, b.column_name);
+   swap(a.nn_name, b.nn_name);
+   swap(a.enable_cstr, b.enable_cstr);
+   swap(a.validate_cstr, b.validate_cstr);
+   swap(a.rely_cstr, b.rely_cstr);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ SQLNotNullConstraint::SQLNotNullConstraint(const SQLNotNullConstraint& other10) {
+   table_db = other10.table_db;
+   table_name = other10.table_name;
+   column_name = other10.column_name;
+   nn_name = other10.nn_name;
+   enable_cstr = other10.enable_cstr;
+   validate_cstr = other10.validate_cstr;
+   rely_cstr = other10.rely_cstr;
+   __isset = other10.__isset;
+ }
+ SQLNotNullConstraint& SQLNotNullConstraint::operator=(const SQLNotNullConstraint& other11) {
+   table_db = other11.table_db;
+   table_name = other11.table_name;
+   column_name = other11.column_name;
+   nn_name = other11.nn_name;
+   enable_cstr = other11.enable_cstr;
+   validate_cstr = other11.validate_cstr;
+   rely_cstr = other11.rely_cstr;
+   __isset = other11.__isset;
+   return *this;
+ }
+ void SQLNotNullConstraint::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "SQLNotNullConstraint(";
+   out << "table_db=" << to_string(table_db);
+   out << ", " << "table_name=" << to_string(table_name);
+   out << ", " << "column_name=" << to_string(column_name);
+   out << ", " << "nn_name=" << to_string(nn_name);
+   out << ", " << "enable_cstr=" << to_string(enable_cstr);
+   out << ", " << "validate_cstr=" << to_string(validate_cstr);
+   out << ", " << "rely_cstr=" << to_string(rely_cstr);
+   out << ")";
+ }
+ 
+ 
+ Type::~Type() throw() {
+ }
+ 
+ 
+ void Type::__set_name(const std::string& val) {
+   this->name = val;
+ }
+ 
+ void Type::__set_type1(const std::string& val) {
+   this->type1 = val;
+ __isset.type1 = true;
+ }
+ 
+ void Type::__set_type2(const std::string& val) {
+   this->type2 = val;
+ __isset.type2 = true;
+ }
+ 
+ void Type::__set_fields(const std::vector<FieldSchema> & val) {
+   this->fields = val;
+ __isset.fields = true;
+ }
+ 
+ uint32_t Type::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->name);
+           this->__isset.name = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->type1);
+           this->__isset.type1 = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->type2);
+           this->__isset.type2 = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 4:
+         if (ftype == ::apache::thrift::protocol::T_LIST) {
+           {
+             this->fields.clear();
+             uint32_t _size12;
+             ::apache::thrift::protocol::TType _etype15;
+             xfer += iprot->readListBegin(_etype15, _size12);
+             this->fields.resize(_size12);
+             uint32_t _i16;
+             for (_i16 = 0; _i16 < _size12; ++_i16)
+             {
+               xfer += this->fields[_i16].read(iprot);
+             }
+             xfer += iprot->readListEnd();
+           }
+           this->__isset.fields = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t Type::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("Type");
+ 
+   xfer += oprot->writeFieldBegin("name", ::apache::thrift::protocol::T_STRING, 1);
+   xfer += oprot->writeString(this->name);
+   xfer += oprot->writeFieldEnd();
+ 
+   if (this->__isset.type1) {
+     xfer += oprot->writeFieldBegin("type1", ::apache::thrift::protocol::T_STRING, 2);
+     xfer += oprot->writeString(this->type1);
+     xfer += oprot->writeFieldEnd();
+   }
+   if (this->__isset.type2) {
+     xfer += oprot->writeFieldBegin("type2", ::apache::thrift::protocol::T_STRING, 3);
+     xfer += oprot->writeString(this->type2);
+     xfer += oprot->writeFieldEnd();
+   }
+   if (this->__isset.fields) {
+     xfer += oprot->writeFieldBegin("fields", ::apache::thrift::protocol::T_LIST, 4);
+     {
+       xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->fields.size()));
+       std::vector<FieldSchema> ::const_iterator _iter17;
+       for (_iter17 = this->fields.begin(); _iter17 != this->fields.end(); ++_iter17)
+       {
+         xfer += (*_iter17).write(oprot);
+       }
+       xfer += oprot->writeListEnd();
+     }
+     xfer += oprot->writeFieldEnd();
+   }
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(Type &a, Type &b) {
+   using ::std::swap;
+   swap(a.name, b.name);
+   swap(a.type1, b.type1);
+   swap(a.type2, b.type2);
+   swap(a.fields, b.fields);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ Type::Type(const Type& other18) {
+   name = other18.name;
+   type1 = other18.type1;
+   type2 = other18.type2;
+   fields = other18.fields;
+   __isset = other18.__isset;
+ }
+ Type& Type::operator=(const Type& other19) {
+   name = other19.name;
+   type1 = other19.type1;
+   type2 = other19.type2;
+   fields = other19.fields;
+   __isset = other19.__isset;
+   return *this;
+ }
+ void Type::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "Type(";
+   out << "name=" << to_string(name);
+   out << ", " << "type1="; (__isset.type1 ? (out << to_string(type1)) : (out << "<null>"));
+   out << ", " << "type2="; (__isset.type2 ? (out << to_string(type2)) : (out << "<null>"));
+   out << ", " << "fields="; (__isset.fields ? (out << to_string(fields)) : (out << "<null>"));
+   out << ")";
+ }
+ 
+ 
+ HiveObjectRef::~HiveObjectRef() throw() {
+ }
+ 
+ 
+ void HiveObjectRef::__set_objectType(const HiveObjectType::type val) {
+   this->objectType = val;
+ }
+ 
+ void HiveObjectRef::__set_dbName(const std::string& val) {
+   this->dbName = val;
+ }
+ 
+ void HiveObjectRef::__set_objectName(const std::string& val) {
+   this->objectName = val;
+ }
+ 
+ void HiveObjectRef::__set_partValues(const std::vector<std::string> & val) {
+   this->partValues = val;
+ }
+ 
+ void HiveObjectRef::__set_columnName(const std::string& val) {
+   this->columnName = val;
+ }
+ 
+ uint32_t HiveObjectRef::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           int32_t ecast20;
+           xfer += iprot->readI32(ecast20);
+           this->objectType = (HiveObjectType::type)ecast20;
+           this->__isset.objectType = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->dbName);
+           this->__isset.dbName = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->objectName);
+           this->__isset.objectName = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 4:
+         if (ftype == ::apache::thrift::protocol::T_LIST) {
+           {
+             this->partValues.clear();
+             uint32_t _size21;
+             ::apache::thrift::protocol::TType _etype24;
+             xfer += iprot->readListBegin(_etype24, _size21);
+             this->partValues.resize(_size21);
+             uint32_t _i25;
+             for (_i25 = 0; _i25 < _size21; ++_i25)
+             {
+               xfer += iprot->readString(this->partValues[_i25]);
+             }
+             xfer += iprot->readListEnd();
+           }
+           this->__isset.partValues = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 5:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->columnName);
+           this->__isset.columnName = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t HiveObjectRef::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("HiveObjectRef");
+ 
+   xfer += oprot->writeFieldBegin("objectType", ::apache::thrift::protocol::T_I32, 1);
+   xfer += oprot->writeI32((int32_t)this->objectType);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("dbName", ::apache::thrift::protocol::T_STRING, 2);
+   xfer += oprot->writeString(this->dbName);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("objectName", ::apache::thrift::protocol::T_STRING, 3);
+   xfer += oprot->writeString(this->objectName);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("partValues", ::apache::thrift::protocol::T_LIST, 4);
+   {
+     xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast<uint32_t>(this->partValues.size()));
+     std::vector<std::string> ::const_iterator _iter26;
+     for (_iter26 = this->partValues.begin(); _iter26 != this->partValues.end(); ++_iter26)
+     {
+       xfer += oprot->writeString((*_iter26));
+     }
+     xfer += oprot->writeListEnd();
+   }
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("columnName", ::apache::thrift::protocol::T_STRING, 5);
+   xfer += oprot->writeString(this->columnName);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(HiveObjectRef &a, HiveObjectRef &b) {
+   using ::std::swap;
+   swap(a.objectType, b.objectType);
+   swap(a.dbName, b.dbName);
+   swap(a.objectName, b.objectName);
+   swap(a.partValues, b.partValues);
+   swap(a.columnName, b.columnName);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ HiveObjectRef::HiveObjectRef(const HiveObjectRef& other27) {
+   objectType = other27.objectType;
+   dbName = other27.dbName;
+   objectName = other27.objectName;
+   partValues = other27.partValues;
+   columnName = other27.columnName;
+   __isset = other27.__isset;
+ }
+ HiveObjectRef& HiveObjectRef::operator=(const HiveObjectRef& other28) {
+   objectType = other28.objectType;
+   dbName = other28.dbName;
+   objectName = other28.objectName;
+   partValues = other28.partValues;
+   columnName = other28.columnName;
+   __isset = other28.__isset;
+   return *this;
+ }
+ void HiveObjectRef::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "HiveObjectRef(";
+   out << "objectType=" << to_string(objectType);
+   out << ", " << "dbName=" << to_string(dbName);
+   out << ", " << "objectName=" << to_string(objectName);
+   out << ", " << "partValues=" << to_string(partValues);
+   out << ", " << "columnName=" << to_string(columnName);
+   out << ")";
+ }
+ 
+ 
+ PrivilegeGrantInfo::~PrivilegeGrantInfo() throw() {
+ }
+ 
+ 
+ void PrivilegeGrantInfo::__set_privilege(const std::string& val) {
+   this->privilege = val;
+ }
+ 
+ void PrivilegeGrantInfo::__set_createTime(const int32_t val) {
+   this->createTime = val;
+ }
+ 
+ void PrivilegeGrantInfo::__set_grantor(const std::string& val) {
+   this->grantor = val;
+ }
+ 
+ void PrivilegeGrantInfo::__set_grantorType(const PrincipalType::type val) {
+   this->grantorType = val;
+ }
+ 
+ void PrivilegeGrantInfo::__set_grantOption(const bool val) {
+   this->grantOption = val;
+ }
+ 
+ uint32_t PrivilegeGrantInfo::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->privilege);
+           this->__isset.privilege = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           xfer += iprot->readI32(this->createTime);
+           this->__isset.createTime = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->grantor);
+           this->__isset.grantor = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 4:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           int32_t ecast29;
+           xfer += iprot->readI32(ecast29);
+           this->grantorType = (PrincipalType::type)ecast29;
+           this->__isset.grantorType = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 5:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->grantOption);
+           this->__isset.grantOption = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t PrivilegeGrantInfo::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("PrivilegeGrantInfo");
+ 
+   xfer += oprot->writeFieldBegin("privilege", ::apache::thrift::protocol::T_STRING, 1);
+   xfer += oprot->writeString(this->privilege);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("createTime", ::apache::thrift::protocol::T_I32, 2);
+   xfer += oprot->writeI32(this->createTime);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("grantor", ::apache::thrift::protocol::T_STRING, 3);
+   xfer += oprot->writeString(this->grantor);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("grantorType", ::apache::thrift::protocol::T_I32, 4);
+   xfer += oprot->writeI32((int32_t)this->grantorType);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("grantOption", ::apache::thrift::protocol::T_BOOL, 5);
+   xfer += oprot->writeBool(this->grantOption);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(PrivilegeGrantInfo &a, PrivilegeGrantInfo &b) {
+   using ::std::swap;
+   swap(a.privilege, b.privilege);
+   swap(a.createTime, b.createTime);
+   swap(a.grantor, b.grantor);
+   swap(a.grantorType, b.grantorType);
+   swap(a.grantOption, b.grantOption);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ PrivilegeGrantInfo::PrivilegeGrantInfo(const PrivilegeGrantInfo& other30) {
+   privilege = other30.privilege;
+   createTime = other30.createTime;
+   grantor = other30.grantor;
+   grantorType = other30.grantorType;
+   grantOption = other30.grantOption;
+   __isset = other30.__isset;
+ }
+ PrivilegeGrantInfo& PrivilegeGrantInfo::operator=(const PrivilegeGrantInfo& other31) {
+   privilege = other31.privilege;
+   createTime = other31.createTime;
+   grantor = other31.grantor;
+   grantorType = other31.grantorType;
+   grantOption = other31.grantOption;
+   __isset = other31.__isset;
+   return *this;
+ }
+ void PrivilegeGrantInfo::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "PrivilegeGrantInfo(";
+   out << "privilege=" << to_string(privilege);
+   out << ", " << "createTime=" << to_string(createTime);
+   out << ", " << "grantor=" << to_string(grantor);
+   out << ", " << "grantorType=" << to_string(grantorType);
+   out << ", " << "grantOption=" << to_string(grantOption);
+   out << ")";
+ }
+ 
+ 
+ HiveObjectPrivilege::~HiveObjectPrivilege() throw() {
+ }
+ 
+ 
+ void HiveObjectPrivilege::__set_hiveObject(const HiveObjectRef& val) {
+   this->hiveObject = val;
+ }
+ 
+ void HiveObjectPrivilege::__set_principalName(const std::string& val) {
+   this->principalName = val;
+ }
+ 
+ void HiveObjectPrivilege::__set_principalType(const PrincipalType::type val) {
+   this->principalType = val;
+ }
+ 
+ void HiveObjectPrivilege::__set_grantInfo(const PrivilegeGrantInfo& val) {
+   this->grantInfo = val;
+ }
+ 
+ uint32_t HiveObjectPrivilege::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_STRUCT) {
+           xfer += this->hiveObject.read(iprot);
+           this->__isset.hiveObject = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->principalName);
+           this->__isset.principalName = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           int32_t ecast32;
+           xfer += iprot->readI32(ecast32);
+           this->principalType = (PrincipalType::type)ecast32;
+           this->__isset.principalType = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 4:
+         if (ftype == ::apache::thrift::protocol::T_STRUCT) {
+           xfer += this->grantInfo.read(iprot);
+           this->__isset.grantInfo = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t HiveObjectPrivilege::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("HiveObjectPrivilege");
+ 
+   xfer += oprot->writeFieldBegin("hiveObject", ::apache::thrift::protocol::T_STRUCT, 1);
+   xfer += this->hiveObject.write(oprot);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("principalName", ::apache::thrift::protocol::T_STRING, 2);
+   xfer += oprot->writeString(this->principalName);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("principalType", ::apache::thrift::protocol::T_I32, 3);
+   xfer += oprot->writeI32((int32_t)this->principalType);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("grantInfo", ::apache::thrift::protocol::T_STRUCT, 4);
+   xfer += this->grantInfo.write(oprot);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(HiveObjectPrivilege &a, HiveObjectPrivilege &b) {
+   using ::std::swap;
+   swap(a.hiveObject, b.hiveObject);
+   swap(a.principalName, b.principalName);
+   swap(a.principalType, b.principalType);
+   swap(a.grantInfo, b.grantInfo);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ HiveObjectPrivilege::HiveObjectPrivilege(const HiveObjectPrivilege& other33) {
+   hiveObject = other33.hiveObject;
+   principalName = other33.principalName;
+   principalType = other33.principalType;
+   grantInfo = other33.grantInfo;
+   __isset = other33.__isset;
+ }
+ HiveObjectPrivilege& HiveObjectPrivilege::operator=(const HiveObjectPrivilege& other34) {
+   hiveObject = other34.hiveObject;
+   principalName = other34.principalName;
+   principalType = other34.principalType;
+   grantInfo = other34.grantInfo;
+   __isset = other34.__isset;
+   return *this;
+ }
+ void HiveObjectPrivilege::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "HiveObjectPrivilege(";
+   out << "hiveObject=" << to_string(hiveObject);
+   out << ", " << "principalName=" << to_string(principalName);
+   out << ", " << "principalType=" << to_string(principalType);
+   out << ", " << "grantInfo=" << to_string(grantInfo);
+   out << ")";
+ }
+ 
+ 
+ PrivilegeBag::~PrivilegeBag() throw() {
+ }
+ 
+ 
+ void PrivilegeBag::__set_privileges(const std::vector<HiveObjectPrivilege> & val) {
+   this->privileges = val;
+ }
+ 
+ uint32_t PrivilegeBag::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_LIST) {
+           {
+             this->privileges.clear();
+             uint32_t _size35;
+             ::apache::thrift::protocol::TType _etype38;
+             xfer += iprot->readListBegin(_etype38, _size35);
+             this->privileges.resize(_size35);
+             uint32_t _i39;
+             for (_i39 = 0; _i39 < _size35; ++_i39)
+             {
+               xfer += this->privileges[_i39].read(iprot);
+             }
+             xfer += iprot->readListEnd();
+           }
+           this->__isset.privileges = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t PrivilegeBag::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("PrivilegeBag");
+ 
+   xfer += oprot->writeFieldBegin("privileges", ::apache::thrift::protocol::T_LIST, 1);
+   {
+     xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(this->privileges.size()));
+     std::vector<HiveObjectPrivilege> ::const_iterator _iter40;
+     for (_iter40 = this->privileges.begin(); _iter40 != this->privileges.end(); ++_iter40)
+     {
+       xfer += (*_iter40).write(oprot);
+     }
+     xfer += oprot->writeListEnd();
+   }
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(PrivilegeBag &a, PrivilegeBag &b) {
+   using ::std::swap;
+   swap(a.privileges, b.privileges);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ PrivilegeBag::PrivilegeBag(const PrivilegeBag& other41) {
+   privileges = other41.privileges;
+   __isset = other41.__isset;
+ }
+ PrivilegeBag& PrivilegeBag::operator=(const PrivilegeBag& other42) {
+   privileges = other42.privileges;
+   __isset = other42.__isset;
+   return *this;
+ }
+ void PrivilegeBag::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "PrivilegeBag(";
+   out << "privileges=" << to_string(privileges);
+   out << ")";
+ }
+ 
+ 
+ PrincipalPrivilegeSet::~PrincipalPrivilegeSet() throw() {
+ }
+ 
+ 
+ void PrincipalPrivilegeSet::__set_userPrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val) {
+   this->userPrivileges = val;
+ }
+ 
+ void PrincipalPrivilegeSet::__set_groupPrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val) {
+   this->groupPrivileges = val;
+ }
+ 
+ void PrincipalPrivilegeSet::__set_rolePrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val) {
+   this->rolePrivileges = val;
+ }
+ 
+ uint32_t PrincipalPrivilegeSet::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_MAP) {
+           {
+             this->userPrivileges.clear();
+             uint32_t _size43;
+             ::apache::thrift::protocol::TType _ktype44;
+             ::apache::thrift::protocol::TType _vtype45;
+             xfer += iprot->readMapBegin(_ktype44, _vtype45, _size43);
+             uint32_t _i47;
+             for (_i47 = 0; _i47 < _size43; ++_i47)
+             {
+               std::string _key48;
+               xfer += iprot->readString(_key48);
+               std::vector<PrivilegeGrantInfo> & _val49 = this->userPrivileges[_key48];
+               {
+                 _val49.clear();
+                 uint32_t _size50;
+                 ::apache::thrift::protocol::TType _etype53;
+                 xfer += iprot->readListBegin(_etype53, _size50);
+                 _val49.resize(_size50);
+                 uint32_t _i54;
+                 for (_i54 = 0; _i54 < _size50; ++_i54)
+                 {
+                   xfer += _val49[_i54].read(iprot);
+                 }
+                 xfer += iprot->readListEnd();
+               }
+             }
+             xfer += iprot->readMapEnd();
+           }
+           this->__isset.userPrivileges = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_MAP) {
+           {
+             this->groupPrivileges.clear();
+             uint32_t _size55;
+             ::apache::thrift::protocol::TType _ktype56;
+             ::apache::thrift::protocol::TType _vtype57;
+             xfer += iprot->readMapBegin(_ktype56, _vtype57, _size55);
+             uint32_t _i59;
+             for (_i59 = 0; _i59 < _size55; ++_i59)
+             {
+               std::string _key60;
+               xfer += iprot->readString(_key60);
+               std::vector<PrivilegeGrantInfo> & _val61 = this->groupPrivileges[_key60];
+               {
+                 _val61.clear();
+                 uint32_t _size62;
+                 ::apache::thrift::protocol::TType _etype65;
+                 xfer += iprot->readListBegin(_etype65, _size62);
+                 _val61.resize(_size62);
+                 uint32_t _i66;
+                 for (_i66 = 0; _i66 < _size62; ++_i66)
+                 {
+                   xfer += _val61[_i66].read(iprot);
+                 }
+                 xfer += iprot->readListEnd();
+               }
+             }
+             xfer += iprot->readMapEnd();
+           }
+           this->__isset.groupPrivileges = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_MAP) {
+           {
+             this->rolePrivileges.clear();
+             uint32_t _size67;
+             ::apache::thrift::protocol::TType _ktype68;
+             ::apache::thrift::protocol::TType _vtype69;
+             xfer += iprot->readMapBegin(_ktype68, _vtype69, _size67);
+             uint32_t _i71;
+             for (_i71 = 0; _i71 < _size67; ++_i71)
+             {
+               std::string _key72;
+               xfer += iprot->readString(_key72);
+               std::vector<PrivilegeGrantInfo> & _val73 = this->rolePrivileges[_key72];
+               {
+                 _val73.clear();
+                 uint32_t _size74;
+                 ::apache::thrift::protocol::TType _etype77;
+                 xfer += iprot->readListBegin(_etype77, _size74);
+                 _val73.resize(_size74);
+                 uint32_t _i78;
+                 for (_i78 = 0; _i78 < _size74; ++_i78)
+                 {
+                   xfer += _val73[_i78].read(iprot);
+                 }
+                 xfer += iprot->readListEnd();
+               }
+             }
+             xfer += iprot->readMapEnd();
+           }
+           this->__isset.rolePrivileges = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t PrincipalPrivilegeSet::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("PrincipalPrivilegeSet");
+ 
+   xfer += oprot->writeFieldBegin("userPrivileges", ::apache::thrift::protocol::T_MAP, 1);
+   {
+     xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_LIST, static_cast<uint32_t>(this->userPrivileges.size()));
+     std::map<std::string, std::vector<PrivilegeGrantInfo> > ::const_iterator _iter79;
+     for (_iter79 = this->userPrivileges.begin(); _iter79 != this->userPrivileges.end(); ++_iter79)
+     {
+       xfer += oprot->writeString(_iter79->first);
+       {
+         xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(_iter79->second.size()));
+         std::vector<PrivilegeGrantInfo> ::const_iterator _iter80;
+         for (_iter80 = _iter79->second.begin(); _iter80 != _iter79->second.end(); ++_iter80)
+         {
+           xfer += (*_iter80).write(oprot);
+         }
+         xfer += oprot->writeListEnd();
+       }
+     }
+     xfer += oprot->writeMapEnd();
+   }
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("groupPrivileges", ::apache::thrift::protocol::T_MAP, 2);
+   {
+     xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_LIST, static_cast<uint32_t>(this->groupPrivileges.size()));
+     std::map<std::string, std::vector<PrivilegeGrantInfo> > ::const_iterator _iter81;
+     for (_iter81 = this->groupPrivileges.begin(); _iter81 != this->groupPrivileges.end(); ++_iter81)
+     {
+       xfer += oprot->writeString(_iter81->first);
+       {
+         xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(_iter81->second.size()));
+         std::vector<PrivilegeGrantInfo> ::const_iterator _iter82;
+         for (_iter82 = _iter81->second.begin(); _iter82 != _iter81->second.end(); ++_iter82)
+         {
+           xfer += (*_iter82).write(oprot);
+         }
+         xfer += oprot->writeListEnd();
+       }
+     }
+     xfer += oprot->writeMapEnd();
+   }
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("rolePrivileges", ::apache::thrift::protocol::T_MAP, 3);
+   {
+     xfer += oprot->writeMapBegin(::apache::thrift::protocol::T_STRING, ::apache::thrift::protocol::T_LIST, static_cast<uint32_t>(this->rolePrivileges.size()));
+     std::map<std::string, std::vector<PrivilegeGrantInfo> > ::const_iterator _iter83;
+     for (_iter83 = this->rolePrivileges.begin(); _iter83 != this->rolePrivileges.end(); ++_iter83)
+     {
+       xfer += oprot->writeString(_iter83->first);
+       {
+         xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast<uint32_t>(_iter83->second.size()));
+         std::vector<PrivilegeGrantInfo> ::const_iterator _iter84;
+         for (_iter84 = _iter83->second.begin(); _iter84 != _iter83->second.end(); ++_iter84)
+         {
+           xfer += (*_iter84).write(oprot);
+         }
+         xfer += oprot->writeListEnd();
+       }
+     }
+     xfer += oprot->writeMapEnd();
+   }
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(PrincipalPrivilegeSet &a, PrincipalPrivilegeSet &b) {
+   using ::std::swap;
+   swap(a.userPrivileges, b.userPrivileges);
+   swap(a.groupPrivileges, b.groupPrivileges);
+   swap(a.rolePrivileges, b.rolePrivileges);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ PrincipalPrivilegeSet::PrincipalPrivilegeSet(const PrincipalPrivilegeSet& other85) {
+   userPrivileges = other85.userPrivileges;
+   groupPrivileges = other85.groupPrivileges;
+   rolePrivileges = other85.rolePrivileges;
+   __isset = other85.__isset;
+ }
+ PrincipalPrivilegeSet& PrincipalPrivilegeSet::operator=(const PrincipalPrivilegeSet& other86) {
+   userPrivileges = other86.userPrivileges;
+   groupPrivileges = other86.groupPrivileges;
+   rolePrivileges = other86.rolePrivileges;
+   __isset = other86.__isset;
+   return *this;
+ }
+ void PrincipalPrivilegeSet::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "PrincipalPrivilegeSet(";
+   out << "userPrivileges=" << to_string(userPrivileges);
+   out << ", " << "groupPrivileges=" << to_string(groupPrivileges);
+   out << ", " << "rolePrivileges=" << to_string(rolePrivileges);
+   out << ")";
+ }
+ 
+ 
+ GrantRevokePrivilegeRequest::~GrantRevokePrivilegeRequest() throw() {
+ }
+ 
+ 
+ void GrantRevokePrivilegeRequest::__set_requestType(const GrantRevokeType::type val) {
+   this->requestType = val;
+ }
+ 
+ void GrantRevokePrivilegeRequest::__set_privileges(const PrivilegeBag& val) {
+   this->privileges = val;
+ }
+ 
+ void GrantRevokePrivilegeRequest::__set_revokeGrantOption(const bool val) {
+   this->revokeGrantOption = val;
+ __isset.revokeGrantOption = true;
+ }
+ 
+ uint32_t GrantRevokePrivilegeRequest::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           int32_t ecast87;
+           xfer += iprot->readI32(ecast87);
+           this->requestType = (GrantRevokeType::type)ecast87;
+           this->__isset.requestType = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_STRUCT) {
+           xfer += this->privileges.read(iprot);
+           this->__isset.privileges = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->revokeGrantOption);
+           this->__isset.revokeGrantOption = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t GrantRevokePrivilegeRequest::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("GrantRevokePrivilegeRequest");
+ 
+   xfer += oprot->writeFieldBegin("requestType", ::apache::thrift::protocol::T_I32, 1);
+   xfer += oprot->writeI32((int32_t)this->requestType);
+   xfer += oprot->writeFieldEnd();
+ 
+   xfer += oprot->writeFieldBegin("privileges", ::apache::thrift::protocol::T_STRUCT, 2);
+   xfer += this->privileges.write(oprot);
+   xfer += oprot->writeFieldEnd();
+ 
+   if (this->__isset.revokeGrantOption) {
+     xfer += oprot->writeFieldBegin("revokeGrantOption", ::apache::thrift::protocol::T_BOOL, 3);
+     xfer += oprot->writeBool(this->revokeGrantOption);
+     xfer += oprot->writeFieldEnd();
+   }
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(GrantRevokePrivilegeRequest &a, GrantRevokePrivilegeRequest &b) {
+   using ::std::swap;
+   swap(a.requestType, b.requestType);
+   swap(a.privileges, b.privileges);
+   swap(a.revokeGrantOption, b.revokeGrantOption);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ GrantRevokePrivilegeRequest::GrantRevokePrivilegeRequest(const GrantRevokePrivilegeRequest& other88) {
+   requestType = other88.requestType;
+   privileges = other88.privileges;
+   revokeGrantOption = other88.revokeGrantOption;
+   __isset = other88.__isset;
+ }
+ GrantRevokePrivilegeRequest& GrantRevokePrivilegeRequest::operator=(const GrantRevokePrivilegeRequest& other89) {
+   requestType = other89.requestType;
+   privileges = other89.privileges;
+   revokeGrantOption = other89.revokeGrantOption;
+   __isset = other89.__isset;
+   return *this;
+ }
+ void GrantRevokePrivilegeRequest::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "GrantRevokePrivilegeRequest(";
+   out << "requestType=" << to_string(requestType);
+   out << ", " << "privileges=" << to_string(privileges);
+   out << ", " << "revokeGrantOption="; (__isset.revokeGrantOption ? (out << to_string(revokeGrantOption)) : (out << "<null>"));
+   out << ")";
+ }
+ 
+ 
+ GrantRevokePrivilegeResponse::~GrantRevokePrivilegeResponse() throw() {
+ }
+ 
+ 
+ void GrantRevokePrivilegeResponse::__set_success(const bool val) {
+   this->success = val;
+ __isset.success = true;
+ }
+ 
+ uint32_t GrantRevokePrivilegeResponse::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_BOOL) {
+           xfer += iprot->readBool(this->success);
+           this->__isset.success = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       default:
+         xfer += iprot->skip(ftype);
+         break;
+     }
+     xfer += iprot->readFieldEnd();
+   }
+ 
+   xfer += iprot->readStructEnd();
+ 
+   return xfer;
+ }
+ 
+ uint32_t GrantRevokePrivilegeResponse::write(::apache::thrift::protocol::TProtocol* oprot) const {
+   uint32_t xfer = 0;
+   apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot);
+   xfer += oprot->writeStructBegin("GrantRevokePrivilegeResponse");
+ 
+   if (this->__isset.success) {
+     xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_BOOL, 1);
+     xfer += oprot->writeBool(this->success);
+     xfer += oprot->writeFieldEnd();
+   }
+   xfer += oprot->writeFieldStop();
+   xfer += oprot->writeStructEnd();
+   return xfer;
+ }
+ 
+ void swap(GrantRevokePrivilegeResponse &a, GrantRevokePrivilegeResponse &b) {
+   using ::std::swap;
+   swap(a.success, b.success);
+   swap(a.__isset, b.__isset);
+ }
+ 
+ GrantRevokePrivilegeResponse::GrantRevokePrivilegeResponse(const GrantRevokePrivilegeResponse& other90) {
+   success = other90.success;
+   __isset = other90.__isset;
+ }
+ GrantRevokePrivilegeResponse& GrantRevokePrivilegeResponse::operator=(const GrantRevokePrivilegeResponse& other91) {
+   success = other91.success;
+   __isset = other91.__isset;
+   return *this;
+ }
+ void GrantRevokePrivilegeResponse::printTo(std::ostream& out) const {
+   using ::apache::thrift::to_string;
+   out << "GrantRevokePrivilegeResponse(";
+   out << "success="; (__isset.success ? (out << to_string(success)) : (out << "<null>"));
+   out << ")";
+ }
+ 
+ 
+ Role::~Role() throw() {
+ }
+ 
+ 
+ void Role::__set_roleName(const std::string& val) {
+   this->roleName = val;
+ }
+ 
+ void Role::__set_createTime(const int32_t val) {
+   this->createTime = val;
+ }
+ 
+ void Role::__set_ownerName(const std::string& val) {
+   this->ownerName = val;
+ }
+ 
+ uint32_t Role::read(::apache::thrift::protocol::TProtocol* iprot) {
+ 
+   apache::thrift::protocol::TInputRecursionTracker tracker(*iprot);
+   uint32_t xfer = 0;
+   std::string fname;
+   ::apache::thrift::protocol::TType ftype;
+   int16_t fid;
+ 
+   xfer += iprot->readStructBegin(fname);
+ 
+   using ::apache::thrift::protocol::TProtocolException;
+ 
+ 
+   while (true)
+   {
+     xfer += iprot->readFieldBegin(fname, ftype, fid);
+     if (ftype == ::apache::thrift::protocol::T_STOP) {
+       break;
+     }
+     switch (fid)
+     {
+       case 1:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->roleName);
+           this->__isset.roleName = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 2:
+         if (ftype == ::apache::thrift::protocol::T_I32) {
+           xfer += iprot->readI32(this->createTime);
+           this->__isset.createTime = true;
+         } else {
+           xfer += iprot->skip(ftype);
+         }
+         break;
+       case 3:
+         if (ftype == ::apache::thrift::protocol::T_STRING) {
+           xfer += iprot->readString(this->ownerName);
+           this->__isset.ownerName = true;
+         } else {
+           xfer += ip

<TRUNCATED>

[08/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
index 0000000,ea73b34..1dae310
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@@ -1,0 -1,3515 +1,3516 @@@
+ #
+ # Autogenerated by Thrift Compiler (0.9.3)
+ #
+ # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ #
+ 
+ require 'thrift'
+ require 'fb303_types'
+ 
+ 
+ module HiveObjectType
+   GLOBAL = 1
+   DATABASE = 2
+   TABLE = 3
+   PARTITION = 4
+   COLUMN = 5
+   VALUE_MAP = {1 => "GLOBAL", 2 => "DATABASE", 3 => "TABLE", 4 => "PARTITION", 5 => "COLUMN"}
+   VALID_VALUES = Set.new([GLOBAL, DATABASE, TABLE, PARTITION, COLUMN]).freeze
+ end
+ 
+ module PrincipalType
+   USER = 1
+   ROLE = 2
+   GROUP = 3
+   VALUE_MAP = {1 => "USER", 2 => "ROLE", 3 => "GROUP"}
+   VALID_VALUES = Set.new([USER, ROLE, GROUP]).freeze
+ end
+ 
+ module PartitionEventType
+   LOAD_DONE = 1
+   VALUE_MAP = {1 => "LOAD_DONE"}
+   VALID_VALUES = Set.new([LOAD_DONE]).freeze
+ end
+ 
+ module TxnState
+   COMMITTED = 1
+   ABORTED = 2
+   OPEN = 3
+   VALUE_MAP = {1 => "COMMITTED", 2 => "ABORTED", 3 => "OPEN"}
+   VALID_VALUES = Set.new([COMMITTED, ABORTED, OPEN]).freeze
+ end
+ 
+ module LockLevel
+   DB = 1
+   TABLE = 2
+   PARTITION = 3
+   VALUE_MAP = {1 => "DB", 2 => "TABLE", 3 => "PARTITION"}
+   VALID_VALUES = Set.new([DB, TABLE, PARTITION]).freeze
+ end
+ 
+ module LockState
+   ACQUIRED = 1
+   WAITING = 2
+   ABORT = 3
+   NOT_ACQUIRED = 4
+   VALUE_MAP = {1 => "ACQUIRED", 2 => "WAITING", 3 => "ABORT", 4 => "NOT_ACQUIRED"}
+   VALID_VALUES = Set.new([ACQUIRED, WAITING, ABORT, NOT_ACQUIRED]).freeze
+ end
+ 
+ module LockType
+   SHARED_READ = 1
+   SHARED_WRITE = 2
+   EXCLUSIVE = 3
+   VALUE_MAP = {1 => "SHARED_READ", 2 => "SHARED_WRITE", 3 => "EXCLUSIVE"}
+   VALID_VALUES = Set.new([SHARED_READ, SHARED_WRITE, EXCLUSIVE]).freeze
+ end
+ 
+ module CompactionType
+   MINOR = 1
+   MAJOR = 2
+   VALUE_MAP = {1 => "MINOR", 2 => "MAJOR"}
+   VALID_VALUES = Set.new([MINOR, MAJOR]).freeze
+ end
+ 
+ module GrantRevokeType
+   GRANT = 1
+   REVOKE = 2
+   VALUE_MAP = {1 => "GRANT", 2 => "REVOKE"}
+   VALID_VALUES = Set.new([GRANT, REVOKE]).freeze
+ end
+ 
+ module DataOperationType
+   SELECT = 1
+   INSERT = 2
+   UPDATE = 3
+   DELETE = 4
+   UNSET = 5
+   NO_TXN = 6
+   VALUE_MAP = {1 => "SELECT", 2 => "INSERT", 3 => "UPDATE", 4 => "DELETE", 5 => "UNSET", 6 => "NO_TXN"}
+   VALID_VALUES = Set.new([SELECT, INSERT, UPDATE, DELETE, UNSET, NO_TXN]).freeze
+ end
+ 
+ module EventRequestType
+   INSERT = 1
+   UPDATE = 2
+   DELETE = 3
+   VALUE_MAP = {1 => "INSERT", 2 => "UPDATE", 3 => "DELETE"}
+   VALID_VALUES = Set.new([INSERT, UPDATE, DELETE]).freeze
+ end
+ 
+ module FunctionType
+   JAVA = 1
+   VALUE_MAP = {1 => "JAVA"}
+   VALID_VALUES = Set.new([JAVA]).freeze
+ end
+ 
+ module ResourceType
+   JAR = 1
+   FILE = 2
+   ARCHIVE = 3
+   VALUE_MAP = {1 => "JAR", 2 => "FILE", 3 => "ARCHIVE"}
+   VALID_VALUES = Set.new([JAR, FILE, ARCHIVE]).freeze
+ end
+ 
+ module FileMetadataExprType
+   ORC_SARG = 1
+   VALUE_MAP = {1 => "ORC_SARG"}
+   VALID_VALUES = Set.new([ORC_SARG]).freeze
+ end
+ 
+ module ClientCapability
+   TEST_CAPABILITY = 1
 -  VALUE_MAP = {1 => "TEST_CAPABILITY"}
 -  VALID_VALUES = Set.new([TEST_CAPABILITY]).freeze
++  INSERT_ONLY_TABLES = 2
++  VALUE_MAP = {1 => "TEST_CAPABILITY", 2 => "INSERT_ONLY_TABLES"}
++  VALID_VALUES = Set.new([TEST_CAPABILITY, INSERT_ONLY_TABLES]).freeze
+ end
+ 
+ class Version
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   VERSION = 1
+   COMMENTS = 2
+ 
+   FIELDS = {
+     VERSION => {:type => ::Thrift::Types::STRING, :name => 'version'},
+     COMMENTS => {:type => ::Thrift::Types::STRING, :name => 'comments'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class FieldSchema
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   NAME = 1
+   TYPE = 2
+   COMMENT = 3
+ 
+   FIELDS = {
+     NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+     TYPE => {:type => ::Thrift::Types::STRING, :name => 'type'},
+     COMMENT => {:type => ::Thrift::Types::STRING, :name => 'comment'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class SQLPrimaryKey
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TABLE_DB = 1
+   TABLE_NAME = 2
+   COLUMN_NAME = 3
+   KEY_SEQ = 4
+   PK_NAME = 5
+   ENABLE_CSTR = 6
+   VALIDATE_CSTR = 7
+   RELY_CSTR = 8
+ 
+   FIELDS = {
+     TABLE_DB => {:type => ::Thrift::Types::STRING, :name => 'table_db'},
+     TABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'table_name'},
+     COLUMN_NAME => {:type => ::Thrift::Types::STRING, :name => 'column_name'},
+     KEY_SEQ => {:type => ::Thrift::Types::I32, :name => 'key_seq'},
+     PK_NAME => {:type => ::Thrift::Types::STRING, :name => 'pk_name'},
+     ENABLE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'enable_cstr'},
+     VALIDATE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'validate_cstr'},
+     RELY_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'rely_cstr'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class SQLForeignKey
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PKTABLE_DB = 1
+   PKTABLE_NAME = 2
+   PKCOLUMN_NAME = 3
+   FKTABLE_DB = 4
+   FKTABLE_NAME = 5
+   FKCOLUMN_NAME = 6
+   KEY_SEQ = 7
+   UPDATE_RULE = 8
+   DELETE_RULE = 9
+   FK_NAME = 10
+   PK_NAME = 11
+   ENABLE_CSTR = 12
+   VALIDATE_CSTR = 13
+   RELY_CSTR = 14
+ 
+   FIELDS = {
+     PKTABLE_DB => {:type => ::Thrift::Types::STRING, :name => 'pktable_db'},
+     PKTABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'pktable_name'},
+     PKCOLUMN_NAME => {:type => ::Thrift::Types::STRING, :name => 'pkcolumn_name'},
+     FKTABLE_DB => {:type => ::Thrift::Types::STRING, :name => 'fktable_db'},
+     FKTABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'fktable_name'},
+     FKCOLUMN_NAME => {:type => ::Thrift::Types::STRING, :name => 'fkcolumn_name'},
+     KEY_SEQ => {:type => ::Thrift::Types::I32, :name => 'key_seq'},
+     UPDATE_RULE => {:type => ::Thrift::Types::I32, :name => 'update_rule'},
+     DELETE_RULE => {:type => ::Thrift::Types::I32, :name => 'delete_rule'},
+     FK_NAME => {:type => ::Thrift::Types::STRING, :name => 'fk_name'},
+     PK_NAME => {:type => ::Thrift::Types::STRING, :name => 'pk_name'},
+     ENABLE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'enable_cstr'},
+     VALIDATE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'validate_cstr'},
+     RELY_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'rely_cstr'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class SQLUniqueConstraint
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TABLE_DB = 1
+   TABLE_NAME = 2
+   COLUMN_NAME = 3
+   KEY_SEQ = 4
+   UK_NAME = 5
+   ENABLE_CSTR = 6
+   VALIDATE_CSTR = 7
+   RELY_CSTR = 8
+ 
+   FIELDS = {
+     TABLE_DB => {:type => ::Thrift::Types::STRING, :name => 'table_db'},
+     TABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'table_name'},
+     COLUMN_NAME => {:type => ::Thrift::Types::STRING, :name => 'column_name'},
+     KEY_SEQ => {:type => ::Thrift::Types::I32, :name => 'key_seq'},
+     UK_NAME => {:type => ::Thrift::Types::STRING, :name => 'uk_name'},
+     ENABLE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'enable_cstr'},
+     VALIDATE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'validate_cstr'},
+     RELY_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'rely_cstr'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class SQLNotNullConstraint
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TABLE_DB = 1
+   TABLE_NAME = 2
+   COLUMN_NAME = 3
+   NN_NAME = 4
+   ENABLE_CSTR = 5
+   VALIDATE_CSTR = 6
+   RELY_CSTR = 7
+ 
+   FIELDS = {
+     TABLE_DB => {:type => ::Thrift::Types::STRING, :name => 'table_db'},
+     TABLE_NAME => {:type => ::Thrift::Types::STRING, :name => 'table_name'},
+     COLUMN_NAME => {:type => ::Thrift::Types::STRING, :name => 'column_name'},
+     NN_NAME => {:type => ::Thrift::Types::STRING, :name => 'nn_name'},
+     ENABLE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'enable_cstr'},
+     VALIDATE_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'validate_cstr'},
+     RELY_CSTR => {:type => ::Thrift::Types::BOOL, :name => 'rely_cstr'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Type
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   NAME = 1
+   TYPE1 = 2
+   TYPE2 = 3
+   FIELDS = 4
+ 
+   FIELDS = {
+     NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+     TYPE1 => {:type => ::Thrift::Types::STRING, :name => 'type1', :optional => true},
+     TYPE2 => {:type => ::Thrift::Types::STRING, :name => 'type2', :optional => true},
+     FIELDS => {:type => ::Thrift::Types::LIST, :name => 'fields', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class HiveObjectRef
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   OBJECTTYPE = 1
+   DBNAME = 2
+   OBJECTNAME = 3
+   PARTVALUES = 4
+   COLUMNNAME = 5
+ 
+   FIELDS = {
+     OBJECTTYPE => {:type => ::Thrift::Types::I32, :name => 'objectType', :enum_class => ::HiveObjectType},
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     OBJECTNAME => {:type => ::Thrift::Types::STRING, :name => 'objectName'},
+     PARTVALUES => {:type => ::Thrift::Types::LIST, :name => 'partValues', :element => {:type => ::Thrift::Types::STRING}},
+     COLUMNNAME => {:type => ::Thrift::Types::STRING, :name => 'columnName'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     unless @objectType.nil? || ::HiveObjectType::VALID_VALUES.include?(@objectType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field objectType!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PrivilegeGrantInfo
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PRIVILEGE = 1
+   CREATETIME = 2
+   GRANTOR = 3
+   GRANTORTYPE = 4
+   GRANTOPTION = 5
+ 
+   FIELDS = {
+     PRIVILEGE => {:type => ::Thrift::Types::STRING, :name => 'privilege'},
+     CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+     GRANTOR => {:type => ::Thrift::Types::STRING, :name => 'grantor'},
+     GRANTORTYPE => {:type => ::Thrift::Types::I32, :name => 'grantorType', :enum_class => ::PrincipalType},
+     GRANTOPTION => {:type => ::Thrift::Types::BOOL, :name => 'grantOption'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     unless @grantorType.nil? || ::PrincipalType::VALID_VALUES.include?(@grantorType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field grantorType!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class HiveObjectPrivilege
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   HIVEOBJECT = 1
+   PRINCIPALNAME = 2
+   PRINCIPALTYPE = 3
+   GRANTINFO = 4
+ 
+   FIELDS = {
+     HIVEOBJECT => {:type => ::Thrift::Types::STRUCT, :name => 'hiveObject', :class => ::HiveObjectRef},
+     PRINCIPALNAME => {:type => ::Thrift::Types::STRING, :name => 'principalName'},
+     PRINCIPALTYPE => {:type => ::Thrift::Types::I32, :name => 'principalType', :enum_class => ::PrincipalType},
+     GRANTINFO => {:type => ::Thrift::Types::STRUCT, :name => 'grantInfo', :class => ::PrivilegeGrantInfo}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     unless @principalType.nil? || ::PrincipalType::VALID_VALUES.include?(@principalType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field principalType!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PrivilegeBag
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PRIVILEGES = 1
+ 
+   FIELDS = {
+     PRIVILEGES => {:type => ::Thrift::Types::LIST, :name => 'privileges', :element => {:type => ::Thrift::Types::STRUCT, :class => ::HiveObjectPrivilege}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PrincipalPrivilegeSet
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   USERPRIVILEGES = 1
+   GROUPPRIVILEGES = 2
+   ROLEPRIVILEGES = 3
+ 
+   FIELDS = {
+     USERPRIVILEGES => {:type => ::Thrift::Types::MAP, :name => 'userPrivileges', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRUCT, :class => ::PrivilegeGrantInfo}}},
+     GROUPPRIVILEGES => {:type => ::Thrift::Types::MAP, :name => 'groupPrivileges', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRUCT, :class => ::PrivilegeGrantInfo}}},
+     ROLEPRIVILEGES => {:type => ::Thrift::Types::MAP, :name => 'rolePrivileges', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRUCT, :class => ::PrivilegeGrantInfo}}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class GrantRevokePrivilegeRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   REQUESTTYPE = 1
+   PRIVILEGES = 2
+   REVOKEGRANTOPTION = 3
+ 
+   FIELDS = {
+     REQUESTTYPE => {:type => ::Thrift::Types::I32, :name => 'requestType', :enum_class => ::GrantRevokeType},
+     PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrivilegeBag},
+     REVOKEGRANTOPTION => {:type => ::Thrift::Types::BOOL, :name => 'revokeGrantOption', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     unless @requestType.nil? || ::GrantRevokeType::VALID_VALUES.include?(@requestType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field requestType!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class GrantRevokePrivilegeResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   SUCCESS = 1
+ 
+   FIELDS = {
+     SUCCESS => {:type => ::Thrift::Types::BOOL, :name => 'success', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Role
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   ROLENAME = 1
+   CREATETIME = 2
+   OWNERNAME = 3
+ 
+   FIELDS = {
+     ROLENAME => {:type => ::Thrift::Types::STRING, :name => 'roleName'},
+     CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+     OWNERNAME => {:type => ::Thrift::Types::STRING, :name => 'ownerName'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class RolePrincipalGrant
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   ROLENAME = 1
+   PRINCIPALNAME = 2
+   PRINCIPALTYPE = 3
+   GRANTOPTION = 4
+   GRANTTIME = 5
+   GRANTORNAME = 6
+   GRANTORPRINCIPALTYPE = 7
+ 
+   FIELDS = {
+     ROLENAME => {:type => ::Thrift::Types::STRING, :name => 'roleName'},
+     PRINCIPALNAME => {:type => ::Thrift::Types::STRING, :name => 'principalName'},
+     PRINCIPALTYPE => {:type => ::Thrift::Types::I32, :name => 'principalType', :enum_class => ::PrincipalType},
+     GRANTOPTION => {:type => ::Thrift::Types::BOOL, :name => 'grantOption'},
+     GRANTTIME => {:type => ::Thrift::Types::I32, :name => 'grantTime'},
+     GRANTORNAME => {:type => ::Thrift::Types::STRING, :name => 'grantorName'},
+     GRANTORPRINCIPALTYPE => {:type => ::Thrift::Types::I32, :name => 'grantorPrincipalType', :enum_class => ::PrincipalType}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     unless @principalType.nil? || ::PrincipalType::VALID_VALUES.include?(@principalType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field principalType!')
+     end
+     unless @grantorPrincipalType.nil? || ::PrincipalType::VALID_VALUES.include?(@grantorPrincipalType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field grantorPrincipalType!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class GetRoleGrantsForPrincipalRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PRINCIPAL_NAME = 1
+   PRINCIPAL_TYPE = 2
+ 
+   FIELDS = {
+     PRINCIPAL_NAME => {:type => ::Thrift::Types::STRING, :name => 'principal_name'},
+     PRINCIPAL_TYPE => {:type => ::Thrift::Types::I32, :name => 'principal_type', :enum_class => ::PrincipalType}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field principal_name is unset!') unless @principal_name
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field principal_type is unset!') unless @principal_type
+     unless @principal_type.nil? || ::PrincipalType::VALID_VALUES.include?(@principal_type)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field principal_type!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class GetRoleGrantsForPrincipalResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PRINCIPALGRANTS = 1
+ 
+   FIELDS = {
+     PRINCIPALGRANTS => {:type => ::Thrift::Types::LIST, :name => 'principalGrants', :element => {:type => ::Thrift::Types::STRUCT, :class => ::RolePrincipalGrant}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field principalGrants is unset!') unless @principalGrants
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class GetPrincipalsInRoleRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   ROLENAME = 1
+ 
+   FIELDS = {
+     ROLENAME => {:type => ::Thrift::Types::STRING, :name => 'roleName'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field roleName is unset!') unless @roleName
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class GetPrincipalsInRoleResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PRINCIPALGRANTS = 1
+ 
+   FIELDS = {
+     PRINCIPALGRANTS => {:type => ::Thrift::Types::LIST, :name => 'principalGrants', :element => {:type => ::Thrift::Types::STRUCT, :class => ::RolePrincipalGrant}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field principalGrants is unset!') unless @principalGrants
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class GrantRevokeRoleRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   REQUESTTYPE = 1
+   ROLENAME = 2
+   PRINCIPALNAME = 3
+   PRINCIPALTYPE = 4
+   GRANTOR = 5
+   GRANTORTYPE = 6
+   GRANTOPTION = 7
+ 
+   FIELDS = {
+     REQUESTTYPE => {:type => ::Thrift::Types::I32, :name => 'requestType', :enum_class => ::GrantRevokeType},
+     ROLENAME => {:type => ::Thrift::Types::STRING, :name => 'roleName'},
+     PRINCIPALNAME => {:type => ::Thrift::Types::STRING, :name => 'principalName'},
+     PRINCIPALTYPE => {:type => ::Thrift::Types::I32, :name => 'principalType', :enum_class => ::PrincipalType},
+     GRANTOR => {:type => ::Thrift::Types::STRING, :name => 'grantor', :optional => true},
+     GRANTORTYPE => {:type => ::Thrift::Types::I32, :name => 'grantorType', :optional => true, :enum_class => ::PrincipalType},
+     GRANTOPTION => {:type => ::Thrift::Types::BOOL, :name => 'grantOption', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     unless @requestType.nil? || ::GrantRevokeType::VALID_VALUES.include?(@requestType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field requestType!')
+     end
+     unless @principalType.nil? || ::PrincipalType::VALID_VALUES.include?(@principalType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field principalType!')
+     end
+     unless @grantorType.nil? || ::PrincipalType::VALID_VALUES.include?(@grantorType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field grantorType!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class GrantRevokeRoleResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   SUCCESS = 1
+ 
+   FIELDS = {
+     SUCCESS => {:type => ::Thrift::Types::BOOL, :name => 'success', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Database
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   NAME = 1
+   DESCRIPTION = 2
+   LOCATIONURI = 3
+   PARAMETERS = 4
+   PRIVILEGES = 5
+   OWNERNAME = 6
+   OWNERTYPE = 7
+ 
+   FIELDS = {
+     NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+     DESCRIPTION => {:type => ::Thrift::Types::STRING, :name => 'description'},
+     LOCATIONURI => {:type => ::Thrift::Types::STRING, :name => 'locationUri'},
+     PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+     PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true},
+     OWNERNAME => {:type => ::Thrift::Types::STRING, :name => 'ownerName', :optional => true},
+     OWNERTYPE => {:type => ::Thrift::Types::I32, :name => 'ownerType', :optional => true, :enum_class => ::PrincipalType}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     unless @ownerType.nil? || ::PrincipalType::VALID_VALUES.include?(@ownerType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field ownerType!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class SerDeInfo
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   NAME = 1
+   SERIALIZATIONLIB = 2
+   PARAMETERS = 3
+ 
+   FIELDS = {
+     NAME => {:type => ::Thrift::Types::STRING, :name => 'name'},
+     SERIALIZATIONLIB => {:type => ::Thrift::Types::STRING, :name => 'serializationLib'},
+     PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Order
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   COL = 1
+   ORDER = 2
+ 
+   FIELDS = {
+     COL => {:type => ::Thrift::Types::STRING, :name => 'col'},
+     ORDER => {:type => ::Thrift::Types::I32, :name => 'order'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class SkewedInfo
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   SKEWEDCOLNAMES = 1
+   SKEWEDCOLVALUES = 2
+   SKEWEDCOLVALUELOCATIONMAPS = 3
+ 
+   FIELDS = {
+     SKEWEDCOLNAMES => {:type => ::Thrift::Types::LIST, :name => 'skewedColNames', :element => {:type => ::Thrift::Types::STRING}},
+     SKEWEDCOLVALUES => {:type => ::Thrift::Types::LIST, :name => 'skewedColValues', :element => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}},
+     SKEWEDCOLVALUELOCATIONMAPS => {:type => ::Thrift::Types::MAP, :name => 'skewedColValueLocationMaps', :key => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRING}}, :value => {:type => ::Thrift::Types::STRING}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class StorageDescriptor
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   COLS = 1
+   LOCATION = 2
+   INPUTFORMAT = 3
+   OUTPUTFORMAT = 4
+   COMPRESSED = 5
+   NUMBUCKETS = 6
+   SERDEINFO = 7
+   BUCKETCOLS = 8
+   SORTCOLS = 9
+   PARAMETERS = 10
+   SKEWEDINFO = 11
+   STOREDASSUBDIRECTORIES = 12
+ 
+   FIELDS = {
+     COLS => {:type => ::Thrift::Types::LIST, :name => 'cols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}},
+     LOCATION => {:type => ::Thrift::Types::STRING, :name => 'location'},
+     INPUTFORMAT => {:type => ::Thrift::Types::STRING, :name => 'inputFormat'},
+     OUTPUTFORMAT => {:type => ::Thrift::Types::STRING, :name => 'outputFormat'},
+     COMPRESSED => {:type => ::Thrift::Types::BOOL, :name => 'compressed'},
+     NUMBUCKETS => {:type => ::Thrift::Types::I32, :name => 'numBuckets'},
+     SERDEINFO => {:type => ::Thrift::Types::STRUCT, :name => 'serdeInfo', :class => ::SerDeInfo},
+     BUCKETCOLS => {:type => ::Thrift::Types::LIST, :name => 'bucketCols', :element => {:type => ::Thrift::Types::STRING}},
+     SORTCOLS => {:type => ::Thrift::Types::LIST, :name => 'sortCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Order}},
+     PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+     SKEWEDINFO => {:type => ::Thrift::Types::STRUCT, :name => 'skewedInfo', :class => ::SkewedInfo, :optional => true},
+     STOREDASSUBDIRECTORIES => {:type => ::Thrift::Types::BOOL, :name => 'storedAsSubDirectories', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Table
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TABLENAME = 1
+   DBNAME = 2
+   OWNER = 3
+   CREATETIME = 4
+   LASTACCESSTIME = 5
+   RETENTION = 6
+   SD = 7
+   PARTITIONKEYS = 8
+   PARAMETERS = 9
+   VIEWORIGINALTEXT = 10
+   VIEWEXPANDEDTEXT = 11
+   TABLETYPE = 12
+   PRIVILEGES = 13
+   TEMPORARY = 14
+   REWRITEENABLED = 15
+ 
+   FIELDS = {
+     TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     OWNER => {:type => ::Thrift::Types::STRING, :name => 'owner'},
+     CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+     LASTACCESSTIME => {:type => ::Thrift::Types::I32, :name => 'lastAccessTime'},
+     RETENTION => {:type => ::Thrift::Types::I32, :name => 'retention'},
+     SD => {:type => ::Thrift::Types::STRUCT, :name => 'sd', :class => ::StorageDescriptor},
+     PARTITIONKEYS => {:type => ::Thrift::Types::LIST, :name => 'partitionKeys', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}},
+     PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+     VIEWORIGINALTEXT => {:type => ::Thrift::Types::STRING, :name => 'viewOriginalText'},
+     VIEWEXPANDEDTEXT => {:type => ::Thrift::Types::STRING, :name => 'viewExpandedText'},
+     TABLETYPE => {:type => ::Thrift::Types::STRING, :name => 'tableType'},
+     PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true},
+     TEMPORARY => {:type => ::Thrift::Types::BOOL, :name => 'temporary', :default => false, :optional => true},
+     REWRITEENABLED => {:type => ::Thrift::Types::BOOL, :name => 'rewriteEnabled', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Partition
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   VALUES = 1
+   DBNAME = 2
+   TABLENAME = 3
+   CREATETIME = 4
+   LASTACCESSTIME = 5
+   SD = 6
+   PARAMETERS = 7
+   PRIVILEGES = 8
+ 
+   FIELDS = {
+     VALUES => {:type => ::Thrift::Types::LIST, :name => 'values', :element => {:type => ::Thrift::Types::STRING}},
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
+     CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+     LASTACCESSTIME => {:type => ::Thrift::Types::I32, :name => 'lastAccessTime'},
+     SD => {:type => ::Thrift::Types::STRUCT, :name => 'sd', :class => ::StorageDescriptor},
+     PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+     PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PartitionWithoutSD
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   VALUES = 1
+   CREATETIME = 2
+   LASTACCESSTIME = 3
+   RELATIVEPATH = 4
+   PARAMETERS = 5
+   PRIVILEGES = 6
+ 
+   FIELDS = {
+     VALUES => {:type => ::Thrift::Types::LIST, :name => 'values', :element => {:type => ::Thrift::Types::STRING}},
+     CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+     LASTACCESSTIME => {:type => ::Thrift::Types::I32, :name => 'lastAccessTime'},
+     RELATIVEPATH => {:type => ::Thrift::Types::STRING, :name => 'relativePath'},
+     PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+     PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PartitionSpecWithSharedSD
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PARTITIONS = 1
+   SD = 2
+ 
+   FIELDS = {
+     PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::PartitionWithoutSD}},
+     SD => {:type => ::Thrift::Types::STRUCT, :name => 'sd', :class => ::StorageDescriptor}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PartitionListComposingSpec
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PARTITIONS = 1
+ 
+   FIELDS = {
+     PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PartitionSpec
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DBNAME = 1
+   TABLENAME = 2
+   ROOTPATH = 3
+   SHAREDSDPARTITIONSPEC = 4
+   PARTITIONLIST = 5
+ 
+   FIELDS = {
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
+     ROOTPATH => {:type => ::Thrift::Types::STRING, :name => 'rootPath'},
+     SHAREDSDPARTITIONSPEC => {:type => ::Thrift::Types::STRUCT, :name => 'sharedSDPartitionSpec', :class => ::PartitionSpecWithSharedSD, :optional => true},
+     PARTITIONLIST => {:type => ::Thrift::Types::STRUCT, :name => 'partitionList', :class => ::PartitionListComposingSpec, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Index
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   INDEXNAME = 1
+   INDEXHANDLERCLASS = 2
+   DBNAME = 3
+   ORIGTABLENAME = 4
+   CREATETIME = 5
+   LASTACCESSTIME = 6
+   INDEXTABLENAME = 7
+   SD = 8
+   PARAMETERS = 9
+   DEFERREDREBUILD = 10
+ 
+   FIELDS = {
+     INDEXNAME => {:type => ::Thrift::Types::STRING, :name => 'indexName'},
+     INDEXHANDLERCLASS => {:type => ::Thrift::Types::STRING, :name => 'indexHandlerClass'},
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     ORIGTABLENAME => {:type => ::Thrift::Types::STRING, :name => 'origTableName'},
+     CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+     LASTACCESSTIME => {:type => ::Thrift::Types::I32, :name => 'lastAccessTime'},
+     INDEXTABLENAME => {:type => ::Thrift::Types::STRING, :name => 'indexTableName'},
+     SD => {:type => ::Thrift::Types::STRUCT, :name => 'sd', :class => ::StorageDescriptor},
+     PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+     DEFERREDREBUILD => {:type => ::Thrift::Types::BOOL, :name => 'deferredRebuild'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class BooleanColumnStatsData
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   NUMTRUES = 1
+   NUMFALSES = 2
+   NUMNULLS = 3
+   BITVECTORS = 4
+ 
+   FIELDS = {
+     NUMTRUES => {:type => ::Thrift::Types::I64, :name => 'numTrues'},
+     NUMFALSES => {:type => ::Thrift::Types::I64, :name => 'numFalses'},
+     NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+     BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numTrues is unset!') unless @numTrues
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numFalses is unset!') unless @numFalses
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class DoubleColumnStatsData
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   LOWVALUE = 1
+   HIGHVALUE = 2
+   NUMNULLS = 3
+   NUMDVS = 4
+   BITVECTORS = 5
+ 
+   FIELDS = {
+     LOWVALUE => {:type => ::Thrift::Types::DOUBLE, :name => 'lowValue', :optional => true},
+     HIGHVALUE => {:type => ::Thrift::Types::DOUBLE, :name => 'highValue', :optional => true},
+     NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+     NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
+     BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numDVs is unset!') unless @numDVs
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class LongColumnStatsData
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   LOWVALUE = 1
+   HIGHVALUE = 2
+   NUMNULLS = 3
+   NUMDVS = 4
+   BITVECTORS = 5
+ 
+   FIELDS = {
+     LOWVALUE => {:type => ::Thrift::Types::I64, :name => 'lowValue', :optional => true},
+     HIGHVALUE => {:type => ::Thrift::Types::I64, :name => 'highValue', :optional => true},
+     NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+     NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
+     BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numDVs is unset!') unless @numDVs
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class StringColumnStatsData
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   MAXCOLLEN = 1
+   AVGCOLLEN = 2
+   NUMNULLS = 3
+   NUMDVS = 4
+   BITVECTORS = 5
+ 
+   FIELDS = {
+     MAXCOLLEN => {:type => ::Thrift::Types::I64, :name => 'maxColLen'},
+     AVGCOLLEN => {:type => ::Thrift::Types::DOUBLE, :name => 'avgColLen'},
+     NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+     NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
+     BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field maxColLen is unset!') unless @maxColLen
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field avgColLen is unset!') unless @avgColLen
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numDVs is unset!') unless @numDVs
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class BinaryColumnStatsData
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   MAXCOLLEN = 1
+   AVGCOLLEN = 2
+   NUMNULLS = 3
+   BITVECTORS = 4
+ 
+   FIELDS = {
+     MAXCOLLEN => {:type => ::Thrift::Types::I64, :name => 'maxColLen'},
+     AVGCOLLEN => {:type => ::Thrift::Types::DOUBLE, :name => 'avgColLen'},
+     NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+     BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field maxColLen is unset!') unless @maxColLen
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field avgColLen is unset!') unless @avgColLen
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Decimal
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   UNSCALED = 1
+   SCALE = 3
+ 
+   FIELDS = {
+     UNSCALED => {:type => ::Thrift::Types::STRING, :name => 'unscaled', :binary => true},
+     SCALE => {:type => ::Thrift::Types::I16, :name => 'scale'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field unscaled is unset!') unless @unscaled
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field scale is unset!') unless @scale
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class DecimalColumnStatsData
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   LOWVALUE = 1
+   HIGHVALUE = 2
+   NUMNULLS = 3
+   NUMDVS = 4
+   BITVECTORS = 5
+ 
+   FIELDS = {
+     LOWVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'lowValue', :class => ::Decimal, :optional => true},
+     HIGHVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'highValue', :class => ::Decimal, :optional => true},
+     NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+     NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
+     BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numDVs is unset!') unless @numDVs
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Date
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DAYSSINCEEPOCH = 1
+ 
+   FIELDS = {
+     DAYSSINCEEPOCH => {:type => ::Thrift::Types::I64, :name => 'daysSinceEpoch'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field daysSinceEpoch is unset!') unless @daysSinceEpoch
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class DateColumnStatsData
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   LOWVALUE = 1
+   HIGHVALUE = 2
+   NUMNULLS = 3
+   NUMDVS = 4
+   BITVECTORS = 5
+ 
+   FIELDS = {
+     LOWVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'lowValue', :class => ::Date, :optional => true},
+     HIGHVALUE => {:type => ::Thrift::Types::STRUCT, :name => 'highValue', :class => ::Date, :optional => true},
+     NUMNULLS => {:type => ::Thrift::Types::I64, :name => 'numNulls'},
+     NUMDVS => {:type => ::Thrift::Types::I64, :name => 'numDVs'},
+     BITVECTORS => {:type => ::Thrift::Types::STRING, :name => 'bitVectors', :binary => true, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numNulls is unset!') unless @numNulls
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field numDVs is unset!') unless @numDVs
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class ColumnStatisticsData < ::Thrift::Union
+   include ::Thrift::Struct_Union
+   class << self
+     def booleanStats(val)
+       ColumnStatisticsData.new(:booleanStats, val)
+     end
+ 
+     def longStats(val)
+       ColumnStatisticsData.new(:longStats, val)
+     end
+ 
+     def doubleStats(val)
+       ColumnStatisticsData.new(:doubleStats, val)
+     end
+ 
+     def stringStats(val)
+       ColumnStatisticsData.new(:stringStats, val)
+     end
+ 
+     def binaryStats(val)
+       ColumnStatisticsData.new(:binaryStats, val)
+     end
+ 
+     def decimalStats(val)
+       ColumnStatisticsData.new(:decimalStats, val)
+     end
+ 
+     def dateStats(val)
+       ColumnStatisticsData.new(:dateStats, val)
+     end
+   end
+ 
+   BOOLEANSTATS = 1
+   LONGSTATS = 2
+   DOUBLESTATS = 3
+   STRINGSTATS = 4
+   BINARYSTATS = 5
+   DECIMALSTATS = 6
+   DATESTATS = 7
+ 
+   FIELDS = {
+     BOOLEANSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'booleanStats', :class => ::BooleanColumnStatsData},
+     LONGSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'longStats', :class => ::LongColumnStatsData},
+     DOUBLESTATS => {:type => ::Thrift::Types::STRUCT, :name => 'doubleStats', :class => ::DoubleColumnStatsData},
+     STRINGSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'stringStats', :class => ::StringColumnStatsData},
+     BINARYSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'binaryStats', :class => ::BinaryColumnStatsData},
+     DECIMALSTATS => {:type => ::Thrift::Types::STRUCT, :name => 'decimalStats', :class => ::DecimalColumnStatsData},
+     DATESTATS => {:type => ::Thrift::Types::STRUCT, :name => 'dateStats', :class => ::DateColumnStatsData}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise(StandardError, 'Union fields are not set.') if get_set_field.nil? || get_value.nil?
+   end
+ 
+   ::Thrift::Union.generate_accessors self
+ end
+ 
+ class ColumnStatisticsObj
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   COLNAME = 1
+   COLTYPE = 2
+   STATSDATA = 3
+ 
+   FIELDS = {
+     COLNAME => {:type => ::Thrift::Types::STRING, :name => 'colName'},
+     COLTYPE => {:type => ::Thrift::Types::STRING, :name => 'colType'},
+     STATSDATA => {:type => ::Thrift::Types::STRUCT, :name => 'statsData', :class => ::ColumnStatisticsData}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colName is unset!') unless @colName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colType is unset!') unless @colType
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field statsData is unset!') unless @statsData
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class ColumnStatisticsDesc
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   ISTBLLEVEL = 1
+   DBNAME = 2
+   TABLENAME = 3
+   PARTNAME = 4
+   LASTANALYZED = 5
+ 
+   FIELDS = {
+     ISTBLLEVEL => {:type => ::Thrift::Types::BOOL, :name => 'isTblLevel'},
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
+     PARTNAME => {:type => ::Thrift::Types::STRING, :name => 'partName', :optional => true},
+     LASTANALYZED => {:type => ::Thrift::Types::I64, :name => 'lastAnalyzed', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field isTblLevel is unset!') if @isTblLevel.nil?
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tableName is unset!') unless @tableName
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class ColumnStatistics
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   STATSDESC = 1
+   STATSOBJ = 2
+ 
+   FIELDS = {
+     STATSDESC => {:type => ::Thrift::Types::STRUCT, :name => 'statsDesc', :class => ::ColumnStatisticsDesc},
+     STATSOBJ => {:type => ::Thrift::Types::LIST, :name => 'statsObj', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatisticsObj}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field statsDesc is unset!') unless @statsDesc
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field statsObj is unset!') unless @statsObj
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class AggrStats
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   COLSTATS = 1
+   PARTSFOUND = 2
+ 
+   FIELDS = {
+     COLSTATS => {:type => ::Thrift::Types::LIST, :name => 'colStats', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatisticsObj}},
+     PARTSFOUND => {:type => ::Thrift::Types::I64, :name => 'partsFound'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colStats is unset!') unless @colStats
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partsFound is unset!') unless @partsFound
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class SetPartitionsStatsRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   COLSTATS = 1
+   NEEDMERGE = 2
+ 
+   FIELDS = {
+     COLSTATS => {:type => ::Thrift::Types::LIST, :name => 'colStats', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatistics}},
+     NEEDMERGE => {:type => ::Thrift::Types::BOOL, :name => 'needMerge', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colStats is unset!') unless @colStats
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Schema
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   FIELDSCHEMAS = 1
+   PROPERTIES = 2
+ 
+   FIELDS = {
+     FIELDSCHEMAS => {:type => ::Thrift::Types::LIST, :name => 'fieldSchemas', :element => {:type => ::Thrift::Types::STRUCT, :class => ::FieldSchema}},
+     PROPERTIES => {:type => ::Thrift::Types::MAP, :name => 'properties', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class EnvironmentContext
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PROPERTIES = 1
+ 
+   FIELDS = {
+     PROPERTIES => {:type => ::Thrift::Types::MAP, :name => 'properties', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PrimaryKeysRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DB_NAME = 1
+   TBL_NAME = 2
+ 
+   FIELDS = {
+     DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'},
+     TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field db_name is unset!') unless @db_name
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tbl_name is unset!') unless @tbl_name
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PrimaryKeysResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PRIMARYKEYS = 1
+ 
+   FIELDS = {
+     PRIMARYKEYS => {:type => ::Thrift::Types::LIST, :name => 'primaryKeys', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLPrimaryKey}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field primaryKeys is unset!') unless @primaryKeys
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class ForeignKeysRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PARENT_DB_NAME = 1
+   PARENT_TBL_NAME = 2
+   FOREIGN_DB_NAME = 3
+   FOREIGN_TBL_NAME = 4
+ 
+   FIELDS = {
+     PARENT_DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'parent_db_name'},
+     PARENT_TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'parent_tbl_name'},
+     FOREIGN_DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'foreign_db_name'},
+     FOREIGN_TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'foreign_tbl_name'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class ForeignKeysResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   FOREIGNKEYS = 1
+ 
+   FIELDS = {
+     FOREIGNKEYS => {:type => ::Thrift::Types::LIST, :name => 'foreignKeys', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLForeignKey}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field foreignKeys is unset!') unless @foreignKeys
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class UniqueConstraintsRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DB_NAME = 1
+   TBL_NAME = 2
+ 
+   FIELDS = {
+     DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'},
+     TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field db_name is unset!') unless @db_name
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tbl_name is unset!') unless @tbl_name
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class UniqueConstraintsResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   UNIQUECONSTRAINTS = 1
+ 
+   FIELDS = {
+     UNIQUECONSTRAINTS => {:type => ::Thrift::Types::LIST, :name => 'uniqueConstraints', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLUniqueConstraint}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field uniqueConstraints is unset!') unless @uniqueConstraints
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class NotNullConstraintsRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DB_NAME = 1
+   TBL_NAME = 2
+ 
+   FIELDS = {
+     DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'},
+     TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field db_name is unset!') unless @db_name
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tbl_name is unset!') unless @tbl_name
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class NotNullConstraintsResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   NOTNULLCONSTRAINTS = 1
+ 
+   FIELDS = {
+     NOTNULLCONSTRAINTS => {:type => ::Thrift::Types::LIST, :name => 'notNullConstraints', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLNotNullConstraint}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field notNullConstraints is unset!') unless @notNullConstraints
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class DropConstraintRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DBNAME = 1
+   TABLENAME = 2
+   CONSTRAINTNAME = 3
+ 
+   FIELDS = {
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
+     TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename'},
+     CONSTRAINTNAME => {:type => ::Thrift::Types::STRING, :name => 'constraintname'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbname is unset!') unless @dbname
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tablename is unset!') unless @tablename
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field constraintname is unset!') unless @constraintname
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class AddPrimaryKeyRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PRIMARYKEYCOLS = 1
+ 
+   FIELDS = {
+     PRIMARYKEYCOLS => {:type => ::Thrift::Types::LIST, :name => 'primaryKeyCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLPrimaryKey}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field primaryKeyCols is unset!') unless @primaryKeyCols
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class AddForeignKeyRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   FOREIGNKEYCOLS = 1
+ 
+   FIELDS = {
+     FOREIGNKEYCOLS => {:type => ::Thrift::Types::LIST, :name => 'foreignKeyCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLForeignKey}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field foreignKeyCols is unset!') unless @foreignKeyCols
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class AddUniqueConstraintRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   UNIQUECONSTRAINTCOLS = 1
+ 
+   FIELDS = {
+     UNIQUECONSTRAINTCOLS => {:type => ::Thrift::Types::LIST, :name => 'uniqueConstraintCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLUniqueConstraint}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field uniqueConstraintCols is unset!') unless @uniqueConstraintCols
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class AddNotNullConstraintRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   NOTNULLCONSTRAINTCOLS = 1
+ 
+   FIELDS = {
+     NOTNULLCONSTRAINTCOLS => {:type => ::Thrift::Types::LIST, :name => 'notNullConstraintCols', :element => {:type => ::Thrift::Types::STRUCT, :class => ::SQLNotNullConstraint}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field notNullConstraintCols is unset!') unless @notNullConstraintCols
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PartitionsByExprResult
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PARTITIONS = 1
+   HASUNKNOWNPARTITIONS = 2
+ 
+   FIELDS = {
+     PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}},
+     HASUNKNOWNPARTITIONS => {:type => ::Thrift::Types::BOOL, :name => 'hasUnknownPartitions'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partitions is unset!') unless @partitions
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hasUnknownPartitions is unset!') if @hasUnknownPartitions.nil?
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PartitionsByExprRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DBNAME = 1
+   TBLNAME = 2
+   EXPR = 3
+   DEFAULTPARTITIONNAME = 4
+   MAXPARTS = 5
+ 
+   FIELDS = {
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+     EXPR => {:type => ::Thrift::Types::STRING, :name => 'expr', :binary => true},
+     DEFAULTPARTITIONNAME => {:type => ::Thrift::Types::STRING, :name => 'defaultPartitionName', :optional => true},
+     MAXPARTS => {:type => ::Thrift::Types::I16, :name => 'maxParts', :default => -1, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field expr is unset!') unless @expr
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class TableStatsResult
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TABLESTATS = 1
+ 
+   FIELDS = {
+     TABLESTATS => {:type => ::Thrift::Types::LIST, :name => 'tableStats', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatisticsObj}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tableStats is unset!') unless @tableStats
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PartitionsStatsResult
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PARTSTATS = 1
+ 
+   FIELDS = {
+     PARTSTATS => {:type => ::Thrift::Types::MAP, :name => 'partStats', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::LIST, :element => {:type => ::Thrift::Types::STRUCT, :class => ::ColumnStatisticsObj}}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partStats is unset!') unless @partStats
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class TableStatsRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DBNAME = 1
+   TBLNAME = 2
+   COLNAMES = 3
+ 
+   FIELDS = {
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+     COLNAMES => {:type => ::Thrift::Types::LIST, :name => 'colNames', :element => {:type => ::Thrift::Types::STRING}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colNames is unset!') unless @colNames
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class PartitionsStatsRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DBNAME = 1
+   TBLNAME = 2
+   COLNAMES = 3
+   PARTNAMES = 4
+ 
+   FIELDS = {
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+     COLNAMES => {:type => ::Thrift::Types::LIST, :name => 'colNames', :element => {:type => ::Thrift::Types::STRING}},
+     PARTNAMES => {:type => ::Thrift::Types::LIST, :name => 'partNames', :element => {:type => ::Thrift::Types::STRING}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field colNames is unset!') unless @colNames
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field partNames is unset!') unless @partNames
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class AddPartitionsResult
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PARTITIONS = 1
+ 
+   FIELDS = {
+     PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class AddPartitionsRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DBNAME = 1
+   TBLNAME = 2
+   PARTS = 3
+   IFNOTEXISTS = 4
+   NEEDRESULT = 5
+ 
+   FIELDS = {
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+     PARTS => {:type => ::Thrift::Types::LIST, :name => 'parts', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}},
+     IFNOTEXISTS => {:type => ::Thrift::Types::BOOL, :name => 'ifNotExists'},
+     NEEDRESULT => {:type => ::Thrift::Types::BOOL, :name => 'needResult', :default => true, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field parts is unset!') unless @parts
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field ifNotExists is unset!') if @ifNotExists.nil?
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class DropPartitionsResult
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   PARTITIONS = 1
+ 
+   FIELDS = {
+     PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class DropPartitionsExpr
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   EXPR = 1
+   PARTARCHIVELEVEL = 2
+ 
+   FIELDS = {
+     EXPR => {:type => ::Thrift::Types::STRING, :name => 'expr', :binary => true},
+     PARTARCHIVELEVEL => {:type => ::Thrift::Types::I32, :name => 'partArchiveLevel', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field expr is unset!') unless @expr
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class RequestPartsSpec < ::Thrift::Union
+   include ::Thrift::Struct_Union
+   class << self
+     def names(val)
+       RequestPartsSpec.new(:names, val)
+     end
+ 
+     def exprs(val)
+       RequestPartsSpec.new(:exprs, val)
+     end
+   end
+ 
+   NAMES = 1
+   EXPRS = 2
+ 
+   FIELDS = {
+     NAMES => {:type => ::Thrift::Types::LIST, :name => 'names', :element => {:type => ::Thrift::Types::STRING}},
+     EXPRS => {:type => ::Thrift::Types::LIST, :name => 'exprs', :element => {:type => ::Thrift::Types::STRUCT, :class => ::DropPartitionsExpr}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise(StandardError, 'Union fields are not set.') if get_set_field.nil? || get_value.nil?
+   end
+ 
+   ::Thrift::Union.generate_accessors self
+ end
+ 
+ class DropPartitionsRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DBNAME = 1
+   TBLNAME = 2
+   PARTS = 3
+   DELETEDATA = 4
+   IFEXISTS = 5
+   IGNOREPROTECTION = 6
+   ENVIRONMENTCONTEXT = 7
+   NEEDRESULT = 8
+ 
+   FIELDS = {
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     TBLNAME => {:type => ::Thrift::Types::STRING, :name => 'tblName'},
+     PARTS => {:type => ::Thrift::Types::STRUCT, :name => 'parts', :class => ::RequestPartsSpec},
+     DELETEDATA => {:type => ::Thrift::Types::BOOL, :name => 'deleteData', :optional => true},
+     IFEXISTS => {:type => ::Thrift::Types::BOOL, :name => 'ifExists', :default => true, :optional => true},
+     IGNOREPROTECTION => {:type => ::Thrift::Types::BOOL, :name => 'ignoreProtection', :optional => true},
+     ENVIRONMENTCONTEXT => {:type => ::Thrift::Types::STRUCT, :name => 'environmentContext', :class => ::EnvironmentContext, :optional => true},
+     NEEDRESULT => {:type => ::Thrift::Types::BOOL, :name => 'needResult', :default => true, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbName is unset!') unless @dbName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tblName is unset!') unless @tblName
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field parts is unset!') unless @parts
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class ResourceUri
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   RESOURCETYPE = 1
+   URI = 2
+ 
+   FIELDS = {
+     RESOURCETYPE => {:type => ::Thrift::Types::I32, :name => 'resourceType', :enum_class => ::ResourceType},
+     URI => {:type => ::Thrift::Types::STRING, :name => 'uri'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     unless @resourceType.nil? || ::ResourceType::VALID_VALUES.include?(@resourceType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field resourceType!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class Function
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   FUNCTIONNAME = 1
+   DBNAME = 2
+   CLASSNAME = 3
+   OWNERNAME = 4
+   OWNERTYPE = 5
+   CREATETIME = 6
+   FUNCTIONTYPE = 7
+   RESOURCEURIS = 8
+ 
+   FIELDS = {
+     FUNCTIONNAME => {:type => ::Thrift::Types::STRING, :name => 'functionName'},
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'},
+     CLASSNAME => {:type => ::Thrift::Types::STRING, :name => 'className'},
+     OWNERNAME => {:type => ::Thrift::Types::STRING, :name => 'ownerName'},
+     OWNERTYPE => {:type => ::Thrift::Types::I32, :name => 'ownerType', :enum_class => ::PrincipalType},
+     CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'},
+     FUNCTIONTYPE => {:type => ::Thrift::Types::I32, :name => 'functionType', :enum_class => ::FunctionType},
+     RESOURCEURIS => {:type => ::Thrift::Types::LIST, :name => 'resourceUris', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ResourceUri}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     unless @ownerType.nil? || ::PrincipalType::VALID_VALUES.include?(@ownerType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field ownerType!')
+     end
+     unless @functionType.nil? || ::FunctionType::VALID_VALUES.include?(@functionType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field functionType!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class TxnInfo
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   ID = 1
+   STATE = 2
+   USER = 3
+   HOSTNAME = 4
+   AGENTINFO = 5
+   HEARTBEATCOUNT = 6
+   METAINFO = 7
+   STARTEDTIME = 8
+   LASTHEARTBEATTIME = 9
+ 
+   FIELDS = {
+     ID => {:type => ::Thrift::Types::I64, :name => 'id'},
+     STATE => {:type => ::Thrift::Types::I32, :name => 'state', :enum_class => ::TxnState},
+     USER => {:type => ::Thrift::Types::STRING, :name => 'user'},
+     HOSTNAME => {:type => ::Thrift::Types::STRING, :name => 'hostname'},
+     AGENTINFO => {:type => ::Thrift::Types::STRING, :name => 'agentInfo', :default => %q"Unknown", :optional => true},
+     HEARTBEATCOUNT => {:type => ::Thrift::Types::I32, :name => 'heartbeatCount', :default => 0, :optional => true},
+     METAINFO => {:type => ::Thrift::Types::STRING, :name => 'metaInfo', :optional => true},
+     STARTEDTIME => {:type => ::Thrift::Types::I64, :name => 'startedTime', :optional => true},
+     LASTHEARTBEATTIME => {:type => ::Thrift::Types::I64, :name => 'lastHeartbeatTime', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field id is unset!') unless @id
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field state is unset!') unless @state
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field user is unset!') unless @user
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname
+     unless @state.nil? || ::TxnState::VALID_VALUES.include?(@state)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field state!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class GetOpenTxnsInfoResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TXN_HIGH_WATER_MARK = 1
+   OPEN_TXNS = 2
+ 
+   FIELDS = {
+     TXN_HIGH_WATER_MARK => {:type => ::Thrift::Types::I64, :name => 'txn_high_water_mark'},
+     OPEN_TXNS => {:type => ::Thrift::Types::LIST, :name => 'open_txns', :element => {:type => ::Thrift::Types::STRUCT, :class => ::TxnInfo}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txn_high_water_mark is unset!') unless @txn_high_water_mark
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field open_txns is unset!') unless @open_txns
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class GetOpenTxnsResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TXN_HIGH_WATER_MARK = 1
+   OPEN_TXNS = 2
+   MIN_OPEN_TXN = 3
+   ABORTEDBITS = 4
+ 
+   FIELDS = {
+     TXN_HIGH_WATER_MARK => {:type => ::Thrift::Types::I64, :name => 'txn_high_water_mark'},
+     OPEN_TXNS => {:type => ::Thrift::Types::LIST, :name => 'open_txns', :element => {:type => ::Thrift::Types::I64}},
+     MIN_OPEN_TXN => {:type => ::Thrift::Types::I64, :name => 'min_open_txn', :optional => true},
+     ABORTEDBITS => {:type => ::Thrift::Types::STRING, :name => 'abortedBits', :binary => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txn_high_water_mark is unset!') unless @txn_high_water_mark
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field open_txns is unset!') unless @open_txns
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field abortedBits is unset!') unless @abortedBits
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class OpenTxnRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   NUM_TXNS = 1
+   USER = 2
+   HOSTNAME = 3
+   AGENTINFO = 4
+ 
+   FIELDS = {
+     NUM_TXNS => {:type => ::Thrift::Types::I32, :name => 'num_txns'},
+     USER => {:type => ::Thrift::Types::STRING, :name => 'user'},
+     HOSTNAME => {:type => ::Thrift::Types::STRING, :name => 'hostname'},
+     AGENTINFO => {:type => ::Thrift::Types::STRING, :name => 'agentInfo', :default => %q"Unknown", :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field num_txns is unset!') unless @num_txns
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field user is unset!') unless @user
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class OpenTxnsResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TXN_IDS = 1
+ 
+   FIELDS = {
+     TXN_IDS => {:type => ::Thrift::Types::LIST, :name => 'txn_ids', :element => {:type => ::Thrift::Types::I64}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txn_ids is unset!') unless @txn_ids
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class AbortTxnRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TXNID = 1
+ 
+   FIELDS = {
+     TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txnid is unset!') unless @txnid
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class AbortTxnsRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TXN_IDS = 1
+ 
+   FIELDS = {
+     TXN_IDS => {:type => ::Thrift::Types::LIST, :name => 'txn_ids', :element => {:type => ::Thrift::Types::I64}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txn_ids is unset!') unless @txn_ids
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class CommitTxnRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TXNID = 1
+ 
+   FIELDS = {
+     TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field txnid is unset!') unless @txnid
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class LockComponent
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   TYPE = 1
+   LEVEL = 2
+   DBNAME = 3
+   TABLENAME = 4
+   PARTITIONNAME = 5
+   OPERATIONTYPE = 6
+   ISACID = 7
+   ISDYNAMICPARTITIONWRITE = 8
+ 
+   FIELDS = {
+     TYPE => {:type => ::Thrift::Types::I32, :name => 'type', :enum_class => ::LockType},
+     LEVEL => {:type => ::Thrift::Types::I32, :name => 'level', :enum_class => ::LockLevel},
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
+     TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename', :optional => true},
+     PARTITIONNAME => {:type => ::Thrift::Types::STRING, :name => 'partitionname', :optional => true},
+     OPERATIONTYPE => {:type => ::Thrift::Types::I32, :name => 'operationType', :default =>     5, :optional => true, :enum_class => ::DataOperationType},
+     ISACID => {:type => ::Thrift::Types::BOOL, :name => 'isAcid', :default => false, :optional => true},
+     ISDYNAMICPARTITIONWRITE => {:type => ::Thrift::Types::BOOL, :name => 'isDynamicPartitionWrite', :default => false, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field type is unset!') unless @type
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field level is unset!') unless @level
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbname is unset!') unless @dbname
+     unless @type.nil? || ::LockType::VALID_VALUES.include?(@type)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field type!')
+     end
+     unless @level.nil? || ::LockLevel::VALID_VALUES.include?(@level)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field level!')
+     end
+     unless @operationType.nil? || ::DataOperationType::VALID_VALUES.include?(@operationType)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field operationType!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class LockRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   COMPONENT = 1
+   TXNID = 2
+   USER = 3
+   HOSTNAME = 4
+   AGENTINFO = 5
+ 
+   FIELDS = {
+     COMPONENT => {:type => ::Thrift::Types::LIST, :name => 'component', :element => {:type => ::Thrift::Types::STRUCT, :class => ::LockComponent}},
+     TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid', :optional => true},
+     USER => {:type => ::Thrift::Types::STRING, :name => 'user'},
+     HOSTNAME => {:type => ::Thrift::Types::STRING, :name => 'hostname'},
+     AGENTINFO => {:type => ::Thrift::Types::STRING, :name => 'agentInfo', :default => %q"Unknown", :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field component is unset!') unless @component
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field user is unset!') unless @user
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class LockResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   LOCKID = 1
+   STATE = 2
+ 
+   FIELDS = {
+     LOCKID => {:type => ::Thrift::Types::I64, :name => 'lockid'},
+     STATE => {:type => ::Thrift::Types::I32, :name => 'state', :enum_class => ::LockState}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lockid is unset!') unless @lockid
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field state is unset!') unless @state
+     unless @state.nil? || ::LockState::VALID_VALUES.include?(@state)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field state!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class CheckLockRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   LOCKID = 1
+   TXNID = 2
+   ELAPSED_MS = 3
+ 
+   FIELDS = {
+     LOCKID => {:type => ::Thrift::Types::I64, :name => 'lockid'},
+     TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid', :optional => true},
+     ELAPSED_MS => {:type => ::Thrift::Types::I64, :name => 'elapsed_ms', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lockid is unset!') unless @lockid
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class UnlockRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   LOCKID = 1
+ 
+   FIELDS = {
+     LOCKID => {:type => ::Thrift::Types::I64, :name => 'lockid'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lockid is unset!') unless @lockid
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class ShowLocksRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DBNAME = 1
+   TABLENAME = 2
+   PARTNAME = 3
+   ISEXTENDED = 4
+ 
+   FIELDS = {
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname', :optional => true},
+     TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename', :optional => true},
+     PARTNAME => {:type => ::Thrift::Types::STRING, :name => 'partname', :optional => true},
+     ISEXTENDED => {:type => ::Thrift::Types::BOOL, :name => 'isExtended', :default => false, :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class ShowLocksResponseElement
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   LOCKID = 1
+   DBNAME = 2
+   TABLENAME = 3
+   PARTNAME = 4
+   STATE = 5
+   TYPE = 6
+   TXNID = 7
+   LASTHEARTBEAT = 8
+   ACQUIREDAT = 9
+   USER = 10
+   HOSTNAME = 11
+   HEARTBEATCOUNT = 12
+   AGENTINFO = 13
+   BLOCKEDBYEXTID = 14
+   BLOCKEDBYINTID = 15
+   LOCKIDINTERNAL = 16
+ 
+   FIELDS = {
+     LOCKID => {:type => ::Thrift::Types::I64, :name => 'lockid'},
+     DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
+     TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename', :optional => true},
+     PARTNAME => {:type => ::Thrift::Types::STRING, :name => 'partname', :optional => true},
+     STATE => {:type => ::Thrift::Types::I32, :name => 'state', :enum_class => ::LockState},
+     TYPE => {:type => ::Thrift::Types::I32, :name => 'type', :enum_class => ::LockType},
+     TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid', :optional => true},
+     LASTHEARTBEAT => {:type => ::Thrift::Types::I64, :name => 'lastheartbeat'},
+     ACQUIREDAT => {:type => ::Thrift::Types::I64, :name => 'acquiredat', :optional => true},
+     USER => {:type => ::Thrift::Types::STRING, :name => 'user'},
+     HOSTNAME => {:type => ::Thrift::Types::STRING, :name => 'hostname'},
+     HEARTBEATCOUNT => {:type => ::Thrift::Types::I32, :name => 'heartbeatCount', :default => 0, :optional => true},
+     AGENTINFO => {:type => ::Thrift::Types::STRING, :name => 'agentInfo', :optional => true},
+     BLOCKEDBYEXTID => {:type => ::Thrift::Types::I64, :name => 'blockedByExtId', :optional => true},
+     BLOCKEDBYINTID => {:type => ::Thrift::Types::I64, :name => 'blockedByIntId', :optional => true},
+     LOCKIDINTERNAL => {:type => ::Thrift::Types::I64, :name => 'lockIdInternal', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lockid is unset!') unless @lockid
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbname is unset!') unless @dbname
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field state is unset!') unless @state
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field type is unset!') unless @type
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field lastheartbeat is unset!') unless @lastheartbeat
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field user is unset!') unless @user
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname
+     unless @state.nil? || ::LockState::VALID_VALUES.include?(@state)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field state!')
+     end
+     unless @type.nil? || ::LockType::VALID_VALUES.include?(@type)
+       raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field type!')
+     end
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class ShowLocksResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   LOCKS = 1
+ 
+   FIELDS = {
+     LOCKS => {:type => ::Thrift::Types::LIST, :name => 'locks', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ShowLocksResponseElement}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class HeartbeatRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   LOCKID = 1
+   TXNID = 2
+ 
+   FIELDS = {
+     LOCKID => {:type => ::Thrift::Types::I64, :name => 'lockid', :optional => true},
+     TXNID => {:type => ::Thrift::Types::I64, :name => 'txnid', :optional => true}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class HeartbeatTxnRangeRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   MIN = 1
+   MAX = 2
+ 
+   FIELDS = {
+     MIN => {:type => ::Thrift::Types::I64, :name => 'min'},
+     MAX => {:type => ::Thrift::Types::I64, :name => 'max'}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field min is unset!') unless @min
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field max is unset!') unless @max
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class HeartbeatTxnRangeResponse
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   ABORTED = 1
+   NOSUCH = 2
+ 
+   FIELDS = {
+     ABORTED => {:type => ::Thrift::Types::SET, :name => 'aborted', :element => {:type => ::Thrift::Types::I64}},
+     NOSUCH => {:type => ::Thrift::Types::SET, :name => 'nosuch', :element => {:type => ::Thrift::Types::I64}}
+   }
+ 
+   def struct_fields; FIELDS; end
+ 
+   def validate
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field aborted is unset!') unless @aborted
+     raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field nosuch is unset!') unless @nosuch
+   end
+ 
+   ::Thrift::Struct.generate_accessors self
+ end
+ 
+ class CompactionRequest
+   include ::Thrift::Struct, ::Thrift::Struct_Union
+   DBNAME = 1
+   TABLENAME = 2
+   PARTITIONNAME = 3
+   TYPE = 4
+   RUNAS = 5
+   PROPERTIES = 6
+ 
+

<TRUNCATED>

[03/50] [abbrv] hive git commit: HIVE-16850 : Only open a new transaction when there's no currently opened transaction (Wei Zheng)

Posted by se...@apache.org.

HIVE-16850 : Only open a new transaction when there's no currently opened transaction (Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fae25566
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fae25566
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fae25566

Branch: refs/heads/master
Commit: fae25566beae8ae43e694b3a312fb47dbaf4186b
Parents: 78f28c8
Author: Wei Zheng <we...@apache.org>
Authored: Wed Jun 7 15:28:19 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Wed Jun 7 15:28:19 2017 -0700

----------------------------------------------------------------------
 ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java      | 8 ++++++--
 ql/src/test/results/clientpositive/llap/mm_conversions.q.out | 2 --
 ql/src/test/results/clientpositive/mm_conversions.q.out      | 2 --
 3 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/fae25566/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index e9b8f6e..82a9fad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4147,8 +4147,12 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
     long mmWriteId = 0;
     try {
       HiveTxnManager txnManager = SessionState.get().getTxnMgr();
-      mmWriteId = txnManager.openTxn(new Context(conf), conf.getUser());
-      txnManager.commitTxn();
+      if (txnManager.isTxnOpen()) {
+        mmWriteId = txnManager.getCurrentTxnId();
+      } else {
+        mmWriteId = txnManager.openTxn(new Context(conf), conf.getUser());
+        txnManager.commitTxn();
+      }
     } catch (Exception e) {
       String errorMessage = "FAILED: Error in acquiring locks: " + e.getMessage();
       console.printError(errorMessage, "\n"

http://git-wip-us.apache.org/repos/asf/hive/blob/fae25566/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_conversions.q.out b/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
index 1610672..d95a70e 100644
--- a/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
@@ -315,7 +315,6 @@ PREHOOK: query: alter table simple_to_mm set tblproperties("transactional"="true
 PREHOOK: type: ALTERTABLE_PROPERTIES
 PREHOOK: Input: default@simple_to_mm
 PREHOOK: Output: default@simple_to_mm
-FAILED: Error in acquiring locks: Transaction already opened. txnid:30
 POSTHOOK: query: alter table simple_to_mm set tblproperties("transactional"="true", "transactional_properties"="insert_only")
 POSTHOOK: type: ALTERTABLE_PROPERTIES
 POSTHOOK: Input: default@simple_to_mm
@@ -758,7 +757,6 @@ PREHOOK: query: alter table part_to_mm set tblproperties("transactional"="true",
 PREHOOK: type: ALTERTABLE_PROPERTIES
 PREHOOK: Input: default@part_to_mm
 PREHOOK: Output: default@part_to_mm
-FAILED: Error in acquiring locks: Transaction already opened. txnid:63
 POSTHOOK: query: alter table part_to_mm set tblproperties("transactional"="true", "transactional_properties"="insert_only")
 POSTHOOK: type: ALTERTABLE_PROPERTIES
 POSTHOOK: Input: default@part_to_mm

http://git-wip-us.apache.org/repos/asf/hive/blob/fae25566/ql/src/test/results/clientpositive/mm_conversions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/mm_conversions.q.out b/ql/src/test/results/clientpositive/mm_conversions.q.out
index 1610672..d95a70e 100644
--- a/ql/src/test/results/clientpositive/mm_conversions.q.out
+++ b/ql/src/test/results/clientpositive/mm_conversions.q.out
@@ -315,7 +315,6 @@ PREHOOK: query: alter table simple_to_mm set tblproperties("transactional"="true
 PREHOOK: type: ALTERTABLE_PROPERTIES
 PREHOOK: Input: default@simple_to_mm
 PREHOOK: Output: default@simple_to_mm
-FAILED: Error in acquiring locks: Transaction already opened. txnid:30
 POSTHOOK: query: alter table simple_to_mm set tblproperties("transactional"="true", "transactional_properties"="insert_only")
 POSTHOOK: type: ALTERTABLE_PROPERTIES
 POSTHOOK: Input: default@simple_to_mm
@@ -758,7 +757,6 @@ PREHOOK: query: alter table part_to_mm set tblproperties("transactional"="true",
 PREHOOK: type: ALTERTABLE_PROPERTIES
 PREHOOK: Input: default@part_to_mm
 PREHOOK: Output: default@part_to_mm
-FAILED: Error in acquiring locks: Transaction already opened. txnid:63
 POSTHOOK: query: alter table part_to_mm set tblproperties("transactional"="true", "transactional_properties"="insert_only")
 POSTHOOK: type: ALTERTABLE_PROPERTIES
 POSTHOOK: Input: default@part_to_mm

[39/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f159b5a1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f159b5a1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f159b5a1

Branch: refs/heads/master
Commit: f159b5a198aa04e34c78ebc8159d2525146d8c9b
Parents: a02a61b b8aad36
Author: sergey <se...@apache.org>
Authored: Thu Sep 28 20:35:00 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Sep 28 20:35:00 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |    3 +
 data/conf/perf-reg/spark/hive-site.xml          |  268 ++
 .../hadoop/hive/druid/DruidStorageHandler.java  |   59 +-
 .../hive/druid/DruidStorageHandlerUtils.java    |  488 ++-
 .../druid/io/DruidQueryBasedInputFormat.java    |   66 +-
 .../serde/DruidGroupByQueryRecordReader.java    |  225 +-
 .../druid/serde/DruidQueryRecordReader.java     |  215 +-
 .../serde/DruidSelectQueryRecordReader.java     |   26 +-
 .../hadoop/hive/druid/serde/DruidSerDe.java     |   18 +-
 .../hive/druid/serde/DruidSerDeUtils.java       |   27 +-
 .../serde/DruidTimeseriesQueryRecordReader.java |   22 +-
 .../druid/serde/DruidTopNQueryRecordReader.java |   38 +-
 .../hadoop/hive/druid/TestDruidSerDe.java       |  918 -----
 .../hive/druid/TestDruidStorageHandler.java     |  153 +-
 .../hadoop/hive/druid/serde/TestDruidSerDe.java |  991 ++++++
 .../apache/hive/hcatalog/common/HCatUtil.java   |   24 +-
 .../hcatalog/mapreduce/HCatBaseInputFormat.java |    8 +-
 .../hive/hcatalog/streaming/HiveEndPoint.java   |   26 +-
 .../hcatalog/streaming/TransactionBatch.java    |   12 +-
 .../hive/hcatalog/streaming/TestStreaming.java  |   13 +-
 .../hive/hcatalog/api/TestHCatClient.java       |    7 +
 .../hadoop/hive/ql/parse/TestCopyUtils.java     |    8 +-
 .../hadoop/hive/ql/parse/TestExportImport.java  |    2 +
 .../hive/ql/parse/TestReplicationScenarios.java |   39 +
 ...TestReplicationScenariosAcrossInstances.java |    2 +
 .../hadoop/hive/cli/TestSparkPerfCliDriver.java |   62 +
 .../hadoop/hive/cli/TestPerfCliDriver.java      |   62 -
 .../hadoop/hive/cli/TestTezPerfCliDriver.java   |   62 +
 .../test/resources/testconfiguration.properties |    2 +
 .../hadoop/hive/cli/control/CliConfigs.java     |   32 +-
 .../hadoop/hive/llap/LlapRowRecordReader.java   |    7 +-
 .../hadoop/hive/metastore/HiveMetaStore.java    |   41 +
 .../hadoop/hive/metastore/MetaStoreUtils.java   |   26 +
 .../physical/SparkCrossProductCheck.java        |    2 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |    3 +-
 .../apache/hadoop/hive/ql/plan/SparkWork.java   |    4 +-
 .../queries/clientpositive/input_lazyserde2.q   |   53 +
 .../clientpositive/input_lazyserde2.q.out       |  844 +++++
 .../clientpositive/perf/spark/query1.q.out      |  340 ++
 .../clientpositive/perf/spark/query10.q.out     |  533 +++
 .../clientpositive/perf/spark/query11.q.out     |  683 ++++
 .../clientpositive/perf/spark/query12.q.out     |  249 ++
 .../clientpositive/perf/spark/query13.q.out     |  339 ++
 .../clientpositive/perf/spark/query14.q.out     | 3237 ++++++++++++++++++
 .../clientpositive/perf/spark/query15.q.out     |  223 ++
 .../clientpositive/perf/spark/query16.q.out     |  363 ++
 .../clientpositive/perf/spark/query17.q.out     |  409 +++
 .../clientpositive/perf/spark/query18.q.out     |  360 ++
 .../clientpositive/perf/spark/query19.q.out     |  306 ++
 .../clientpositive/perf/spark/query2.q.out      |  429 +++
 .../clientpositive/perf/spark/query20.q.out     |  241 ++
 .../clientpositive/perf/spark/query21.q.out     |  245 ++
 .../clientpositive/perf/spark/query22.q.out     |  218 ++
 .../clientpositive/perf/spark/query23.q.out     | 1110 ++++++
 .../clientpositive/perf/spark/query24.q.out     |  610 ++++
 .../clientpositive/perf/spark/query25.q.out     |  415 +++
 .../clientpositive/perf/spark/query26.q.out     |  253 ++
 .../clientpositive/perf/spark/query27.q.out     |  265 ++
 .../clientpositive/perf/spark/query28.q.out     |  517 +++
 .../clientpositive/perf/spark/query29.q.out     |  415 +++
 .../clientpositive/perf/spark/query3.q.out      |  183 +
 .../clientpositive/perf/spark/query30.q.out     |  424 +++
 .../clientpositive/perf/spark/query31.q.out     |  815 +++++
 .../clientpositive/perf/spark/query32.q.out     |  284 ++
 .../clientpositive/perf/spark/query33.q.out     |  683 ++++
 .../clientpositive/perf/spark/query34.q.out     |  273 ++
 .../clientpositive/perf/spark/query35.q.out     |  524 +++
 .../clientpositive/perf/spark/query36.q.out     |  282 ++
 .../clientpositive/perf/spark/query37.q.out     |  192 ++
 .../clientpositive/perf/spark/query38.q.out     |  458 +++
 .../clientpositive/perf/spark/query39.q.out     |  473 +++
 .../clientpositive/perf/spark/query4.q.out      |  986 ++++++
 .../clientpositive/perf/spark/query40.q.out     |  274 ++
 .../clientpositive/perf/spark/query42.q.out     |  191 ++
 .../clientpositive/perf/spark/query43.q.out     |  184 +
 .../clientpositive/perf/spark/query44.q.out     |  486 +++
 .../clientpositive/perf/spark/query45.q.out     |  374 ++
 .../clientpositive/perf/spark/query46.q.out     |  355 ++
 .../clientpositive/perf/spark/query47.q.out     |  771 +++++
 .../clientpositive/perf/spark/query48.q.out     |  329 ++
 .../clientpositive/perf/spark/query49.q.out     |  876 +++++
 .../clientpositive/perf/spark/query5.q.out      |  740 ++++
 .../clientpositive/perf/spark/query50.q.out     |  333 ++
 .../clientpositive/perf/spark/query51.q.out     |  383 +++
 .../clientpositive/perf/spark/query52.q.out     |  188 +
 .../clientpositive/perf/spark/query53.q.out     |  267 ++
 .../clientpositive/perf/spark/query54.q.out     |  742 ++++
 .../clientpositive/perf/spark/query55.q.out     |  168 +
 .../clientpositive/perf/spark/query56.q.out     |  669 ++++
 .../clientpositive/perf/spark/query57.q.out     |  765 +++++
 .../clientpositive/perf/spark/query58.q.out     |  944 +++++
 .../clientpositive/perf/spark/query59.q.out     |  445 +++
 .../clientpositive/perf/spark/query6.q.out      |  459 +++
 .../clientpositive/perf/spark/query60.q.out     |  691 ++++
 .../clientpositive/perf/spark/query61.q.out     |  586 ++++
 .../clientpositive/perf/spark/query63.q.out     |  269 ++
 .../clientpositive/perf/spark/query65.q.out     |  333 ++
 .../clientpositive/perf/spark/query66.q.out     |  873 +++++
 .../clientpositive/perf/spark/query67.q.out     |  315 ++
 .../clientpositive/perf/spark/query68.q.out     |  363 ++
 .../clientpositive/perf/spark/query69.q.out     |  516 +++
 .../clientpositive/perf/spark/query7.q.out      |  253 ++
 .../clientpositive/perf/spark/query70.q.out     |  429 +++
 .../clientpositive/perf/spark/query71.q.out     |  343 ++
 .../clientpositive/perf/spark/query72.q.out     |  489 +++
 .../clientpositive/perf/spark/query73.q.out     |  261 ++
 .../clientpositive/perf/spark/query74.q.out     |  639 ++++
 .../clientpositive/perf/spark/query75.q.out     |  898 +++++
 .../clientpositive/perf/spark/query76.q.out     |  383 +++
 .../clientpositive/perf/spark/query77.q.out     |  912 +++++
 .../clientpositive/perf/spark/query78.q.out     |  523 +++
 .../clientpositive/perf/spark/query79.q.out     |  258 ++
 .../clientpositive/perf/spark/query8.q.out      |  549 +++
 .../clientpositive/perf/spark/query80.q.out     |  903 +++++
 .../clientpositive/perf/spark/query81.q.out     |  429 +++
 .../clientpositive/perf/spark/query82.q.out     |  192 ++
 .../clientpositive/perf/spark/query83.q.out     |  742 ++++
 .../clientpositive/perf/spark/query84.q.out     |  253 ++
 .../clientpositive/perf/spark/query85.q.out     |  493 +++
 .../clientpositive/perf/spark/query86.q.out     |  240 ++
 .../clientpositive/perf/spark/query87.q.out     |  500 +++
 .../clientpositive/perf/spark/query88.q.out     | 1294 +++++++
 .../clientpositive/perf/spark/query89.q.out     |  281 ++
 .../clientpositive/perf/spark/query9.q.out      |  824 +++++
 .../clientpositive/perf/spark/query90.q.out     |  351 ++
 .../clientpositive/perf/spark/query91.q.out     |  347 ++
 .../clientpositive/perf/spark/query92.q.out     |  303 ++
 .../clientpositive/perf/spark/query93.q.out     |  181 +
 .../clientpositive/perf/spark/query94.q.out     |  359 ++
 .../clientpositive/perf/spark/query95.q.out     |  468 +++
 .../clientpositive/perf/spark/query96.q.out     |  193 ++
 .../clientpositive/perf/spark/query97.q.out     |  250 ++
 .../clientpositive/perf/spark/query98.q.out     |  243 ++
 .../clientpositive/perf/spark/query99.q.out     |  270 ++
 .../hive/serde2/lazybinary/LazyBinarySerDe.java |   14 +-
 .../serde2/lazybinary/LazyBinarySerDe2.java     |  623 ++++
 .../serde2/lazybinary/TestLazyBinaryFast.java   |    7 +-
 .../serde2/lazybinary/TestLazyBinarySerDe.java  |    2 +-
 .../serde2/lazybinary/TestLazyBinarySerDe2.java |   56 +
 139 files changed, 50191 insertions(+), 1648 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f159b5a1/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/f159b5a1/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/f159b5a1/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/f159b5a1/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/f159b5a1/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------

[11/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java
index 0000000,f53bd82..8fc8311
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java
+++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapability.java
@@@ -1,0 -1,42 +1,45 @@@
+ /**
+  * Autogenerated by Thrift Compiler (0.9.3)
+  *
+  * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+  *  @generated
+  */
+ package org.apache.hadoop.hive.metastore.api;
+ 
+ 
+ import java.util.Map;
+ import java.util.HashMap;
+ import org.apache.thrift.TEnum;
+ 
+ public enum ClientCapability implements org.apache.thrift.TEnum {
 -  TEST_CAPABILITY(1);
++  TEST_CAPABILITY(1),
++  INSERT_ONLY_TABLES(2);
+ 
+   private final int value;
+ 
+   private ClientCapability(int value) {
+     this.value = value;
+   }
+ 
+   /**
+    * Get the integer value of this enum value, as defined in the Thrift IDL.
+    */
+   public int getValue() {
+     return value;
+   }
+ 
+   /**
+    * Find a the enum type by its integer value, as defined in the Thrift IDL.
+    * @return null if the value is not found.
+    */
+   public static ClientCapability findByValue(int value) { 
+     switch (value) {
+       case 1:
+         return TEST_CAPABILITY;
++      case 2:
++        return INSERT_ONLY_TABLES;
+       default:
+         return null;
+     }
+   }
+ }

[49/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ba9cd794
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ba9cd794
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ba9cd794

Branch: refs/heads/master
Commit: ba9cd7947f4530f84f0a2ccb7a636fe12231640b
Parents: d7b6272 9375cf3
Author: sergey <se...@apache.org>
Authored: Thu Oct 12 17:06:36 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Oct 12 17:06:36 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |  12 +-
 druid-handler/pom.xml                           |   4 +
 .../hadoop/hive/druid/DruidStorageHandler.java  |  32 +-
 .../hive/druid/DruidStorageHandlerUtils.java    |  50 +-
 .../druid/io/DruidQueryBasedInputFormat.java    | 181 +----
 .../hive/druid/TestDruidStorageHandler.java     | 195 ++++-
 .../TestHiveDruidQueryBasedInputFormat.java     | 104 ---
 .../listener/TestDbNotificationListener.java    |   6 +
 .../hive/minikdc/TestJdbcWithDBTokenStore.java  |   5 +
 .../minikdc/TestJdbcWithDBTokenStoreNoDoAs.java |  50 ++
 .../hive/minikdc/TestJdbcWithMiniKdc.java       |  27 +-
 itests/pom.xml                                  |   2 +
 .../test/resources/testconfiguration.properties |   1 -
 .../hadoop/hive/metastore/HiveMetaStore.java    |   3 +-
 .../hadoop/hive/metastore/ObjectStore.java      |   3 +
 .../metastore/messaging/DropTableMessage.java   |   4 +
 .../messaging/json/JSONDropTableMessage.java    |  21 +-
 .../messaging/json/JSONMessageFactory.java      |   3 +-
 .../hadoop/hive/metastore/TestObjectStore.java  |   8 +
 pom.xml                                         |   2 +-
 .../hadoop/hive/ql/exec/ReplCopyTask.java       |   4 +-
 .../org/apache/hadoop/hive/ql/exec/Task.java    |   8 +-
 .../apache/hadoop/hive/ql/exec/TaskFactory.java |  13 +-
 .../ql/exec/repl/bootstrap/ReplLoadTask.java    |   2 +-
 .../ql/exec/vector/VectorizationContext.java    |   7 +-
 .../vector/expressions/IfExprColumnNull.java    |  22 +-
 .../expressions/IfExprConditionalFilter.java    | 192 +++++
 .../IfExprDoubleColumnDoubleColumn.java         |  45 +-
 .../IfExprIntervalDayTimeColumnColumn.java      |  17 +-
 .../expressions/IfExprLongColumnLongColumn.java |  45 +-
 .../vector/expressions/IfExprNullColumn.java    |  16 +-
 ...fExprStringGroupColumnStringGroupColumn.java |  17 +-
 .../IfExprTimestampColumnColumnBase.java        |  18 +-
 .../ql/exec/vector/udf/VectorUDFAdaptor.java    |  14 +-
 .../hive/ql/io/sarg/ConvertAstToSearchArg.java  |  59 +-
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   |   4 +-
 .../calcite/rules/HiveRelDecorrelator.java      | 300 ++-----
 .../calcite/translator/ASTConverter.java        |  57 +-
 .../apache/hadoop/hive/ql/parse/HiveParser.g    |  20 +-
 .../ql/parse/ReplicationSemanticAnalyzer.java   |  42 +-
 .../parse/TestReplicationSemanticAnalyzer.java  |  39 +-
 .../queries/clientnegative/druid_datasource2.q  |   3 +
 .../queries/clientpositive/subquery_exists.q    |  24 +
 .../test/queries/clientpositive/subquery_in.q   |  50 +-
 .../queries/clientpositive/vectorized_case.q    |  53 +-
 .../clientnegative/druid_datasource2.q.out      |   7 +
 .../clientpositive/constprog_partitioner.q.out  |  10 +-
 .../clientpositive/llap/explainuser_1.q.out     |  98 +--
 .../clientpositive/llap/subquery_exists.q.out   | 385 ++++++++-
 .../clientpositive/llap/subquery_in.q.out       | 657 ++-------------
 .../clientpositive/llap/subquery_multi.q.out    |  48 +-
 .../clientpositive/llap/subquery_notin.q.out    | 170 ++--
 .../clientpositive/llap/subquery_scalar.q.out   |  30 +-
 .../clientpositive/llap/subquery_select.q.out   |  42 +-
 .../clientpositive/llap/subquery_views.q.out    |  82 +-
 .../clientpositive/llap/vector_coalesce.q.out   |   2 +-
 .../vector_groupby_grouping_sets_grouping.q.out |   2 +-
 .../llap/vector_mapjoin_reduce.q.out            |  12 +-
 .../llap/vector_ptf_part_simple.q.out           |  16 +-
 .../clientpositive/llap/vectorized_case.q.out   | 354 +++++++-
 .../clientpositive/min_structvalue.q.out        |  45 ++
 .../clientpositive/perf/spark/query16.q.out     | 209 ++++-
 .../clientpositive/perf/spark/query36.q.out     |   6 +-
 .../clientpositive/perf/spark/query70.q.out     |   6 +-
 .../clientpositive/perf/spark/query86.q.out     |   6 +-
 .../clientpositive/perf/spark/query94.q.out     | 209 ++++-
 .../clientpositive/perf/tez/query14.q.out       |   4 +-
 .../clientpositive/perf/tez/query16.q.out       | 263 +++---
 .../clientpositive/perf/tez/query23.q.out       |   2 +-
 .../clientpositive/perf/tez/query36.q.out       |   4 +-
 .../clientpositive/perf/tez/query70.q.out       |   4 +-
 .../clientpositive/perf/tez/query86.q.out       |   4 +-
 .../clientpositive/perf/tez/query94.q.out       | 263 +++---
 .../spark/constprog_partitioner.q.out           |  10 +-
 .../spark/spark_explainuser_1.q.out             |  98 +--
 .../clientpositive/spark/subquery_exists.q.out  | 390 ++++++++-
 .../clientpositive/spark/subquery_in.q.out      | 611 ++------------
 .../spark/vector_mapjoin_reduce.q.out           |  10 +-
 .../clientpositive/spark/vectorized_case.q.out  | 348 +++++++-
 .../clientpositive/subquery_exists.q.out        | 420 +++++++++-
 .../clientpositive/subquery_exists_having.q.out |  28 +-
 .../clientpositive/subquery_in_having.q.out     | 803 ++++++++++---------
 .../clientpositive/subquery_notexists.q.out     |  32 +-
 .../subquery_notexists_having.q.out             |  32 +-
 .../clientpositive/subquery_notin_having.q.out  |  26 +-
 .../subquery_unqualcolumnrefs.q.out             |  82 +-
 .../clientpositive/vector_coalesce.q.out        |   2 +-
 .../clientpositive/vector_mapjoin_reduce.q.out  |   8 +-
 .../clientpositive/vectorized_case.q.out        | 330 +++++++-
 .../apache/hadoop/hive/thrift/DBTokenStore.java |   3 +-
 90 files changed, 4830 insertions(+), 3127 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ba9cd794/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/ba9cd794/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/ba9cd794/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/ba9cd794/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/ba9cd794/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index e6661f9,64b2c70..67b798f
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@@ -19,9 -19,9 +19,10 @@@ package org.apache.hadoop.hive.metastor
  
  
  import com.codahale.metrics.Counter;
 +import com.google.common.base.Supplier;
  import com.google.common.collect.ImmutableList;
  import org.apache.hadoop.hive.conf.HiveConf;
+ import org.apache.hadoop.hive.metastore.ObjectStore.RetryingExecutor;
  import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
  import org.apache.hadoop.hive.metastore.api.Database;
  import org.apache.hadoop.hive.metastore.api.FieldSchema;

http://git-wip-us.apache.org/repos/asf/hive/blob/ba9cd794/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/ba9cd794/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
----------------------------------------------------------------------

[09/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index 0000000,7570895..c21dfb0
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@@ -1,0 -1,15160 +1,15163 @@@
+ #
+ # Autogenerated by Thrift Compiler (0.9.3)
+ #
+ # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ #
+ #  options string: py
+ #
+ 
+ from thrift.Thrift import TType, TMessageType, TException, TApplicationException
+ import fb303.ttypes
+ 
+ 
+ from thrift.transport import TTransport
+ from thrift.protocol import TBinaryProtocol, TProtocol
+ try:
+   from thrift.protocol import fastbinary
+ except:
+   fastbinary = None
+ 
+ 
+ class HiveObjectType:
+   GLOBAL = 1
+   DATABASE = 2
+   TABLE = 3
+   PARTITION = 4
+   COLUMN = 5
+ 
+   _VALUES_TO_NAMES = {
+     1: "GLOBAL",
+     2: "DATABASE",
+     3: "TABLE",
+     4: "PARTITION",
+     5: "COLUMN",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "GLOBAL": 1,
+     "DATABASE": 2,
+     "TABLE": 3,
+     "PARTITION": 4,
+     "COLUMN": 5,
+   }
+ 
+ class PrincipalType:
+   USER = 1
+   ROLE = 2
+   GROUP = 3
+ 
+   _VALUES_TO_NAMES = {
+     1: "USER",
+     2: "ROLE",
+     3: "GROUP",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "USER": 1,
+     "ROLE": 2,
+     "GROUP": 3,
+   }
+ 
+ class PartitionEventType:
+   LOAD_DONE = 1
+ 
+   _VALUES_TO_NAMES = {
+     1: "LOAD_DONE",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "LOAD_DONE": 1,
+   }
+ 
+ class TxnState:
+   COMMITTED = 1
+   ABORTED = 2
+   OPEN = 3
+ 
+   _VALUES_TO_NAMES = {
+     1: "COMMITTED",
+     2: "ABORTED",
+     3: "OPEN",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "COMMITTED": 1,
+     "ABORTED": 2,
+     "OPEN": 3,
+   }
+ 
+ class LockLevel:
+   DB = 1
+   TABLE = 2
+   PARTITION = 3
+ 
+   _VALUES_TO_NAMES = {
+     1: "DB",
+     2: "TABLE",
+     3: "PARTITION",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "DB": 1,
+     "TABLE": 2,
+     "PARTITION": 3,
+   }
+ 
+ class LockState:
+   ACQUIRED = 1
+   WAITING = 2
+   ABORT = 3
+   NOT_ACQUIRED = 4
+ 
+   _VALUES_TO_NAMES = {
+     1: "ACQUIRED",
+     2: "WAITING",
+     3: "ABORT",
+     4: "NOT_ACQUIRED",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "ACQUIRED": 1,
+     "WAITING": 2,
+     "ABORT": 3,
+     "NOT_ACQUIRED": 4,
+   }
+ 
+ class LockType:
+   SHARED_READ = 1
+   SHARED_WRITE = 2
+   EXCLUSIVE = 3
+ 
+   _VALUES_TO_NAMES = {
+     1: "SHARED_READ",
+     2: "SHARED_WRITE",
+     3: "EXCLUSIVE",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "SHARED_READ": 1,
+     "SHARED_WRITE": 2,
+     "EXCLUSIVE": 3,
+   }
+ 
+ class CompactionType:
+   MINOR = 1
+   MAJOR = 2
+ 
+   _VALUES_TO_NAMES = {
+     1: "MINOR",
+     2: "MAJOR",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "MINOR": 1,
+     "MAJOR": 2,
+   }
+ 
+ class GrantRevokeType:
+   GRANT = 1
+   REVOKE = 2
+ 
+   _VALUES_TO_NAMES = {
+     1: "GRANT",
+     2: "REVOKE",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "GRANT": 1,
+     "REVOKE": 2,
+   }
+ 
+ class DataOperationType:
+   SELECT = 1
+   INSERT = 2
+   UPDATE = 3
+   DELETE = 4
+   UNSET = 5
+   NO_TXN = 6
+ 
+   _VALUES_TO_NAMES = {
+     1: "SELECT",
+     2: "INSERT",
+     3: "UPDATE",
+     4: "DELETE",
+     5: "UNSET",
+     6: "NO_TXN",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "SELECT": 1,
+     "INSERT": 2,
+     "UPDATE": 3,
+     "DELETE": 4,
+     "UNSET": 5,
+     "NO_TXN": 6,
+   }
+ 
+ class EventRequestType:
+   INSERT = 1
+   UPDATE = 2
+   DELETE = 3
+ 
+   _VALUES_TO_NAMES = {
+     1: "INSERT",
+     2: "UPDATE",
+     3: "DELETE",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "INSERT": 1,
+     "UPDATE": 2,
+     "DELETE": 3,
+   }
+ 
+ class FunctionType:
+   JAVA = 1
+ 
+   _VALUES_TO_NAMES = {
+     1: "JAVA",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "JAVA": 1,
+   }
+ 
+ class ResourceType:
+   JAR = 1
+   FILE = 2
+   ARCHIVE = 3
+ 
+   _VALUES_TO_NAMES = {
+     1: "JAR",
+     2: "FILE",
+     3: "ARCHIVE",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "JAR": 1,
+     "FILE": 2,
+     "ARCHIVE": 3,
+   }
+ 
+ class FileMetadataExprType:
+   ORC_SARG = 1
+ 
+   _VALUES_TO_NAMES = {
+     1: "ORC_SARG",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "ORC_SARG": 1,
+   }
+ 
+ class ClientCapability:
+   TEST_CAPABILITY = 1
++  INSERT_ONLY_TABLES = 2
+ 
+   _VALUES_TO_NAMES = {
+     1: "TEST_CAPABILITY",
++    2: "INSERT_ONLY_TABLES",
+   }
+ 
+   _NAMES_TO_VALUES = {
+     "TEST_CAPABILITY": 1,
++    "INSERT_ONLY_TABLES": 2,
+   }
+ 
+ 
+ class Version:
+   """
+   Attributes:
+    - version
+    - comments
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'version', None, None, ), # 1
+     (2, TType.STRING, 'comments', None, None, ), # 2
+   )
+ 
+   def __init__(self, version=None, comments=None,):
+     self.version = version
+     self.comments = comments
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.version = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.comments = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('Version')
+     if self.version is not None:
+       oprot.writeFieldBegin('version', TType.STRING, 1)
+       oprot.writeString(self.version)
+       oprot.writeFieldEnd()
+     if self.comments is not None:
+       oprot.writeFieldBegin('comments', TType.STRING, 2)
+       oprot.writeString(self.comments)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.version)
+     value = (value * 31) ^ hash(self.comments)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class FieldSchema:
+   """
+   Attributes:
+    - name
+    - type
+    - comment
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'name', None, None, ), # 1
+     (2, TType.STRING, 'type', None, None, ), # 2
+     (3, TType.STRING, 'comment', None, None, ), # 3
+   )
+ 
+   def __init__(self, name=None, type=None, comment=None,):
+     self.name = name
+     self.type = type
+     self.comment = comment
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.type = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.STRING:
+           self.comment = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('FieldSchema')
+     if self.name is not None:
+       oprot.writeFieldBegin('name', TType.STRING, 1)
+       oprot.writeString(self.name)
+       oprot.writeFieldEnd()
+     if self.type is not None:
+       oprot.writeFieldBegin('type', TType.STRING, 2)
+       oprot.writeString(self.type)
+       oprot.writeFieldEnd()
+     if self.comment is not None:
+       oprot.writeFieldBegin('comment', TType.STRING, 3)
+       oprot.writeString(self.comment)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.name)
+     value = (value * 31) ^ hash(self.type)
+     value = (value * 31) ^ hash(self.comment)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class SQLPrimaryKey:
+   """
+   Attributes:
+    - table_db
+    - table_name
+    - column_name
+    - key_seq
+    - pk_name
+    - enable_cstr
+    - validate_cstr
+    - rely_cstr
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'table_db', None, None, ), # 1
+     (2, TType.STRING, 'table_name', None, None, ), # 2
+     (3, TType.STRING, 'column_name', None, None, ), # 3
+     (4, TType.I32, 'key_seq', None, None, ), # 4
+     (5, TType.STRING, 'pk_name', None, None, ), # 5
+     (6, TType.BOOL, 'enable_cstr', None, None, ), # 6
+     (7, TType.BOOL, 'validate_cstr', None, None, ), # 7
+     (8, TType.BOOL, 'rely_cstr', None, None, ), # 8
+   )
+ 
+   def __init__(self, table_db=None, table_name=None, column_name=None, key_seq=None, pk_name=None, enable_cstr=None, validate_cstr=None, rely_cstr=None,):
+     self.table_db = table_db
+     self.table_name = table_name
+     self.column_name = column_name
+     self.key_seq = key_seq
+     self.pk_name = pk_name
+     self.enable_cstr = enable_cstr
+     self.validate_cstr = validate_cstr
+     self.rely_cstr = rely_cstr
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.table_db = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.table_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.STRING:
+           self.column_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 4:
+         if ftype == TType.I32:
+           self.key_seq = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 5:
+         if ftype == TType.STRING:
+           self.pk_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 6:
+         if ftype == TType.BOOL:
+           self.enable_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       elif fid == 7:
+         if ftype == TType.BOOL:
+           self.validate_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       elif fid == 8:
+         if ftype == TType.BOOL:
+           self.rely_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('SQLPrimaryKey')
+     if self.table_db is not None:
+       oprot.writeFieldBegin('table_db', TType.STRING, 1)
+       oprot.writeString(self.table_db)
+       oprot.writeFieldEnd()
+     if self.table_name is not None:
+       oprot.writeFieldBegin('table_name', TType.STRING, 2)
+       oprot.writeString(self.table_name)
+       oprot.writeFieldEnd()
+     if self.column_name is not None:
+       oprot.writeFieldBegin('column_name', TType.STRING, 3)
+       oprot.writeString(self.column_name)
+       oprot.writeFieldEnd()
+     if self.key_seq is not None:
+       oprot.writeFieldBegin('key_seq', TType.I32, 4)
+       oprot.writeI32(self.key_seq)
+       oprot.writeFieldEnd()
+     if self.pk_name is not None:
+       oprot.writeFieldBegin('pk_name', TType.STRING, 5)
+       oprot.writeString(self.pk_name)
+       oprot.writeFieldEnd()
+     if self.enable_cstr is not None:
+       oprot.writeFieldBegin('enable_cstr', TType.BOOL, 6)
+       oprot.writeBool(self.enable_cstr)
+       oprot.writeFieldEnd()
+     if self.validate_cstr is not None:
+       oprot.writeFieldBegin('validate_cstr', TType.BOOL, 7)
+       oprot.writeBool(self.validate_cstr)
+       oprot.writeFieldEnd()
+     if self.rely_cstr is not None:
+       oprot.writeFieldBegin('rely_cstr', TType.BOOL, 8)
+       oprot.writeBool(self.rely_cstr)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.table_db)
+     value = (value * 31) ^ hash(self.table_name)
+     value = (value * 31) ^ hash(self.column_name)
+     value = (value * 31) ^ hash(self.key_seq)
+     value = (value * 31) ^ hash(self.pk_name)
+     value = (value * 31) ^ hash(self.enable_cstr)
+     value = (value * 31) ^ hash(self.validate_cstr)
+     value = (value * 31) ^ hash(self.rely_cstr)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class SQLForeignKey:
+   """
+   Attributes:
+    - pktable_db
+    - pktable_name
+    - pkcolumn_name
+    - fktable_db
+    - fktable_name
+    - fkcolumn_name
+    - key_seq
+    - update_rule
+    - delete_rule
+    - fk_name
+    - pk_name
+    - enable_cstr
+    - validate_cstr
+    - rely_cstr
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'pktable_db', None, None, ), # 1
+     (2, TType.STRING, 'pktable_name', None, None, ), # 2
+     (3, TType.STRING, 'pkcolumn_name', None, None, ), # 3
+     (4, TType.STRING, 'fktable_db', None, None, ), # 4
+     (5, TType.STRING, 'fktable_name', None, None, ), # 5
+     (6, TType.STRING, 'fkcolumn_name', None, None, ), # 6
+     (7, TType.I32, 'key_seq', None, None, ), # 7
+     (8, TType.I32, 'update_rule', None, None, ), # 8
+     (9, TType.I32, 'delete_rule', None, None, ), # 9
+     (10, TType.STRING, 'fk_name', None, None, ), # 10
+     (11, TType.STRING, 'pk_name', None, None, ), # 11
+     (12, TType.BOOL, 'enable_cstr', None, None, ), # 12
+     (13, TType.BOOL, 'validate_cstr', None, None, ), # 13
+     (14, TType.BOOL, 'rely_cstr', None, None, ), # 14
+   )
+ 
+   def __init__(self, pktable_db=None, pktable_name=None, pkcolumn_name=None, fktable_db=None, fktable_name=None, fkcolumn_name=None, key_seq=None, update_rule=None, delete_rule=None, fk_name=None, pk_name=None, enable_cstr=None, validate_cstr=None, rely_cstr=None,):
+     self.pktable_db = pktable_db
+     self.pktable_name = pktable_name
+     self.pkcolumn_name = pkcolumn_name
+     self.fktable_db = fktable_db
+     self.fktable_name = fktable_name
+     self.fkcolumn_name = fkcolumn_name
+     self.key_seq = key_seq
+     self.update_rule = update_rule
+     self.delete_rule = delete_rule
+     self.fk_name = fk_name
+     self.pk_name = pk_name
+     self.enable_cstr = enable_cstr
+     self.validate_cstr = validate_cstr
+     self.rely_cstr = rely_cstr
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.pktable_db = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.pktable_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.STRING:
+           self.pkcolumn_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 4:
+         if ftype == TType.STRING:
+           self.fktable_db = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 5:
+         if ftype == TType.STRING:
+           self.fktable_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 6:
+         if ftype == TType.STRING:
+           self.fkcolumn_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 7:
+         if ftype == TType.I32:
+           self.key_seq = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 8:
+         if ftype == TType.I32:
+           self.update_rule = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 9:
+         if ftype == TType.I32:
+           self.delete_rule = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 10:
+         if ftype == TType.STRING:
+           self.fk_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 11:
+         if ftype == TType.STRING:
+           self.pk_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 12:
+         if ftype == TType.BOOL:
+           self.enable_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       elif fid == 13:
+         if ftype == TType.BOOL:
+           self.validate_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       elif fid == 14:
+         if ftype == TType.BOOL:
+           self.rely_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('SQLForeignKey')
+     if self.pktable_db is not None:
+       oprot.writeFieldBegin('pktable_db', TType.STRING, 1)
+       oprot.writeString(self.pktable_db)
+       oprot.writeFieldEnd()
+     if self.pktable_name is not None:
+       oprot.writeFieldBegin('pktable_name', TType.STRING, 2)
+       oprot.writeString(self.pktable_name)
+       oprot.writeFieldEnd()
+     if self.pkcolumn_name is not None:
+       oprot.writeFieldBegin('pkcolumn_name', TType.STRING, 3)
+       oprot.writeString(self.pkcolumn_name)
+       oprot.writeFieldEnd()
+     if self.fktable_db is not None:
+       oprot.writeFieldBegin('fktable_db', TType.STRING, 4)
+       oprot.writeString(self.fktable_db)
+       oprot.writeFieldEnd()
+     if self.fktable_name is not None:
+       oprot.writeFieldBegin('fktable_name', TType.STRING, 5)
+       oprot.writeString(self.fktable_name)
+       oprot.writeFieldEnd()
+     if self.fkcolumn_name is not None:
+       oprot.writeFieldBegin('fkcolumn_name', TType.STRING, 6)
+       oprot.writeString(self.fkcolumn_name)
+       oprot.writeFieldEnd()
+     if self.key_seq is not None:
+       oprot.writeFieldBegin('key_seq', TType.I32, 7)
+       oprot.writeI32(self.key_seq)
+       oprot.writeFieldEnd()
+     if self.update_rule is not None:
+       oprot.writeFieldBegin('update_rule', TType.I32, 8)
+       oprot.writeI32(self.update_rule)
+       oprot.writeFieldEnd()
+     if self.delete_rule is not None:
+       oprot.writeFieldBegin('delete_rule', TType.I32, 9)
+       oprot.writeI32(self.delete_rule)
+       oprot.writeFieldEnd()
+     if self.fk_name is not None:
+       oprot.writeFieldBegin('fk_name', TType.STRING, 10)
+       oprot.writeString(self.fk_name)
+       oprot.writeFieldEnd()
+     if self.pk_name is not None:
+       oprot.writeFieldBegin('pk_name', TType.STRING, 11)
+       oprot.writeString(self.pk_name)
+       oprot.writeFieldEnd()
+     if self.enable_cstr is not None:
+       oprot.writeFieldBegin('enable_cstr', TType.BOOL, 12)
+       oprot.writeBool(self.enable_cstr)
+       oprot.writeFieldEnd()
+     if self.validate_cstr is not None:
+       oprot.writeFieldBegin('validate_cstr', TType.BOOL, 13)
+       oprot.writeBool(self.validate_cstr)
+       oprot.writeFieldEnd()
+     if self.rely_cstr is not None:
+       oprot.writeFieldBegin('rely_cstr', TType.BOOL, 14)
+       oprot.writeBool(self.rely_cstr)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.pktable_db)
+     value = (value * 31) ^ hash(self.pktable_name)
+     value = (value * 31) ^ hash(self.pkcolumn_name)
+     value = (value * 31) ^ hash(self.fktable_db)
+     value = (value * 31) ^ hash(self.fktable_name)
+     value = (value * 31) ^ hash(self.fkcolumn_name)
+     value = (value * 31) ^ hash(self.key_seq)
+     value = (value * 31) ^ hash(self.update_rule)
+     value = (value * 31) ^ hash(self.delete_rule)
+     value = (value * 31) ^ hash(self.fk_name)
+     value = (value * 31) ^ hash(self.pk_name)
+     value = (value * 31) ^ hash(self.enable_cstr)
+     value = (value * 31) ^ hash(self.validate_cstr)
+     value = (value * 31) ^ hash(self.rely_cstr)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class SQLUniqueConstraint:
+   """
+   Attributes:
+    - table_db
+    - table_name
+    - column_name
+    - key_seq
+    - uk_name
+    - enable_cstr
+    - validate_cstr
+    - rely_cstr
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'table_db', None, None, ), # 1
+     (2, TType.STRING, 'table_name', None, None, ), # 2
+     (3, TType.STRING, 'column_name', None, None, ), # 3
+     (4, TType.I32, 'key_seq', None, None, ), # 4
+     (5, TType.STRING, 'uk_name', None, None, ), # 5
+     (6, TType.BOOL, 'enable_cstr', None, None, ), # 6
+     (7, TType.BOOL, 'validate_cstr', None, None, ), # 7
+     (8, TType.BOOL, 'rely_cstr', None, None, ), # 8
+   )
+ 
+   def __init__(self, table_db=None, table_name=None, column_name=None, key_seq=None, uk_name=None, enable_cstr=None, validate_cstr=None, rely_cstr=None,):
+     self.table_db = table_db
+     self.table_name = table_name
+     self.column_name = column_name
+     self.key_seq = key_seq
+     self.uk_name = uk_name
+     self.enable_cstr = enable_cstr
+     self.validate_cstr = validate_cstr
+     self.rely_cstr = rely_cstr
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.table_db = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.table_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.STRING:
+           self.column_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 4:
+         if ftype == TType.I32:
+           self.key_seq = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 5:
+         if ftype == TType.STRING:
+           self.uk_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 6:
+         if ftype == TType.BOOL:
+           self.enable_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       elif fid == 7:
+         if ftype == TType.BOOL:
+           self.validate_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       elif fid == 8:
+         if ftype == TType.BOOL:
+           self.rely_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('SQLUniqueConstraint')
+     if self.table_db is not None:
+       oprot.writeFieldBegin('table_db', TType.STRING, 1)
+       oprot.writeString(self.table_db)
+       oprot.writeFieldEnd()
+     if self.table_name is not None:
+       oprot.writeFieldBegin('table_name', TType.STRING, 2)
+       oprot.writeString(self.table_name)
+       oprot.writeFieldEnd()
+     if self.column_name is not None:
+       oprot.writeFieldBegin('column_name', TType.STRING, 3)
+       oprot.writeString(self.column_name)
+       oprot.writeFieldEnd()
+     if self.key_seq is not None:
+       oprot.writeFieldBegin('key_seq', TType.I32, 4)
+       oprot.writeI32(self.key_seq)
+       oprot.writeFieldEnd()
+     if self.uk_name is not None:
+       oprot.writeFieldBegin('uk_name', TType.STRING, 5)
+       oprot.writeString(self.uk_name)
+       oprot.writeFieldEnd()
+     if self.enable_cstr is not None:
+       oprot.writeFieldBegin('enable_cstr', TType.BOOL, 6)
+       oprot.writeBool(self.enable_cstr)
+       oprot.writeFieldEnd()
+     if self.validate_cstr is not None:
+       oprot.writeFieldBegin('validate_cstr', TType.BOOL, 7)
+       oprot.writeBool(self.validate_cstr)
+       oprot.writeFieldEnd()
+     if self.rely_cstr is not None:
+       oprot.writeFieldBegin('rely_cstr', TType.BOOL, 8)
+       oprot.writeBool(self.rely_cstr)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.table_db)
+     value = (value * 31) ^ hash(self.table_name)
+     value = (value * 31) ^ hash(self.column_name)
+     value = (value * 31) ^ hash(self.key_seq)
+     value = (value * 31) ^ hash(self.uk_name)
+     value = (value * 31) ^ hash(self.enable_cstr)
+     value = (value * 31) ^ hash(self.validate_cstr)
+     value = (value * 31) ^ hash(self.rely_cstr)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class SQLNotNullConstraint:
+   """
+   Attributes:
+    - table_db
+    - table_name
+    - column_name
+    - nn_name
+    - enable_cstr
+    - validate_cstr
+    - rely_cstr
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'table_db', None, None, ), # 1
+     (2, TType.STRING, 'table_name', None, None, ), # 2
+     (3, TType.STRING, 'column_name', None, None, ), # 3
+     (4, TType.STRING, 'nn_name', None, None, ), # 4
+     (5, TType.BOOL, 'enable_cstr', None, None, ), # 5
+     (6, TType.BOOL, 'validate_cstr', None, None, ), # 6
+     (7, TType.BOOL, 'rely_cstr', None, None, ), # 7
+   )
+ 
+   def __init__(self, table_db=None, table_name=None, column_name=None, nn_name=None, enable_cstr=None, validate_cstr=None, rely_cstr=None,):
+     self.table_db = table_db
+     self.table_name = table_name
+     self.column_name = column_name
+     self.nn_name = nn_name
+     self.enable_cstr = enable_cstr
+     self.validate_cstr = validate_cstr
+     self.rely_cstr = rely_cstr
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.table_db = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.table_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.STRING:
+           self.column_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 4:
+         if ftype == TType.STRING:
+           self.nn_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 5:
+         if ftype == TType.BOOL:
+           self.enable_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       elif fid == 6:
+         if ftype == TType.BOOL:
+           self.validate_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       elif fid == 7:
+         if ftype == TType.BOOL:
+           self.rely_cstr = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('SQLNotNullConstraint')
+     if self.table_db is not None:
+       oprot.writeFieldBegin('table_db', TType.STRING, 1)
+       oprot.writeString(self.table_db)
+       oprot.writeFieldEnd()
+     if self.table_name is not None:
+       oprot.writeFieldBegin('table_name', TType.STRING, 2)
+       oprot.writeString(self.table_name)
+       oprot.writeFieldEnd()
+     if self.column_name is not None:
+       oprot.writeFieldBegin('column_name', TType.STRING, 3)
+       oprot.writeString(self.column_name)
+       oprot.writeFieldEnd()
+     if self.nn_name is not None:
+       oprot.writeFieldBegin('nn_name', TType.STRING, 4)
+       oprot.writeString(self.nn_name)
+       oprot.writeFieldEnd()
+     if self.enable_cstr is not None:
+       oprot.writeFieldBegin('enable_cstr', TType.BOOL, 5)
+       oprot.writeBool(self.enable_cstr)
+       oprot.writeFieldEnd()
+     if self.validate_cstr is not None:
+       oprot.writeFieldBegin('validate_cstr', TType.BOOL, 6)
+       oprot.writeBool(self.validate_cstr)
+       oprot.writeFieldEnd()
+     if self.rely_cstr is not None:
+       oprot.writeFieldBegin('rely_cstr', TType.BOOL, 7)
+       oprot.writeBool(self.rely_cstr)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.table_db)
+     value = (value * 31) ^ hash(self.table_name)
+     value = (value * 31) ^ hash(self.column_name)
+     value = (value * 31) ^ hash(self.nn_name)
+     value = (value * 31) ^ hash(self.enable_cstr)
+     value = (value * 31) ^ hash(self.validate_cstr)
+     value = (value * 31) ^ hash(self.rely_cstr)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class Type:
+   """
+   Attributes:
+    - name
+    - type1
+    - type2
+    - fields
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'name', None, None, ), # 1
+     (2, TType.STRING, 'type1', None, None, ), # 2
+     (3, TType.STRING, 'type2', None, None, ), # 3
+     (4, TType.LIST, 'fields', (TType.STRUCT,(FieldSchema, FieldSchema.thrift_spec)), None, ), # 4
+   )
+ 
+   def __init__(self, name=None, type1=None, type2=None, fields=None,):
+     self.name = name
+     self.type1 = type1
+     self.type2 = type2
+     self.fields = fields
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.type1 = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.STRING:
+           self.type2 = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 4:
+         if ftype == TType.LIST:
+           self.fields = []
+           (_etype3, _size0) = iprot.readListBegin()
+           for _i4 in xrange(_size0):
+             _elem5 = FieldSchema()
+             _elem5.read(iprot)
+             self.fields.append(_elem5)
+           iprot.readListEnd()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('Type')
+     if self.name is not None:
+       oprot.writeFieldBegin('name', TType.STRING, 1)
+       oprot.writeString(self.name)
+       oprot.writeFieldEnd()
+     if self.type1 is not None:
+       oprot.writeFieldBegin('type1', TType.STRING, 2)
+       oprot.writeString(self.type1)
+       oprot.writeFieldEnd()
+     if self.type2 is not None:
+       oprot.writeFieldBegin('type2', TType.STRING, 3)
+       oprot.writeString(self.type2)
+       oprot.writeFieldEnd()
+     if self.fields is not None:
+       oprot.writeFieldBegin('fields', TType.LIST, 4)
+       oprot.writeListBegin(TType.STRUCT, len(self.fields))
+       for iter6 in self.fields:
+         iter6.write(oprot)
+       oprot.writeListEnd()
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.name)
+     value = (value * 31) ^ hash(self.type1)
+     value = (value * 31) ^ hash(self.type2)
+     value = (value * 31) ^ hash(self.fields)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class HiveObjectRef:
+   """
+   Attributes:
+    - objectType
+    - dbName
+    - objectName
+    - partValues
+    - columnName
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.I32, 'objectType', None, None, ), # 1
+     (2, TType.STRING, 'dbName', None, None, ), # 2
+     (3, TType.STRING, 'objectName', None, None, ), # 3
+     (4, TType.LIST, 'partValues', (TType.STRING,None), None, ), # 4
+     (5, TType.STRING, 'columnName', None, None, ), # 5
+   )
+ 
+   def __init__(self, objectType=None, dbName=None, objectName=None, partValues=None, columnName=None,):
+     self.objectType = objectType
+     self.dbName = dbName
+     self.objectName = objectName
+     self.partValues = partValues
+     self.columnName = columnName
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.I32:
+           self.objectType = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.dbName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.STRING:
+           self.objectName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 4:
+         if ftype == TType.LIST:
+           self.partValues = []
+           (_etype10, _size7) = iprot.readListBegin()
+           for _i11 in xrange(_size7):
+             _elem12 = iprot.readString()
+             self.partValues.append(_elem12)
+           iprot.readListEnd()
+         else:
+           iprot.skip(ftype)
+       elif fid == 5:
+         if ftype == TType.STRING:
+           self.columnName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('HiveObjectRef')
+     if self.objectType is not None:
+       oprot.writeFieldBegin('objectType', TType.I32, 1)
+       oprot.writeI32(self.objectType)
+       oprot.writeFieldEnd()
+     if self.dbName is not None:
+       oprot.writeFieldBegin('dbName', TType.STRING, 2)
+       oprot.writeString(self.dbName)
+       oprot.writeFieldEnd()
+     if self.objectName is not None:
+       oprot.writeFieldBegin('objectName', TType.STRING, 3)
+       oprot.writeString(self.objectName)
+       oprot.writeFieldEnd()
+     if self.partValues is not None:
+       oprot.writeFieldBegin('partValues', TType.LIST, 4)
+       oprot.writeListBegin(TType.STRING, len(self.partValues))
+       for iter13 in self.partValues:
+         oprot.writeString(iter13)
+       oprot.writeListEnd()
+       oprot.writeFieldEnd()
+     if self.columnName is not None:
+       oprot.writeFieldBegin('columnName', TType.STRING, 5)
+       oprot.writeString(self.columnName)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.objectType)
+     value = (value * 31) ^ hash(self.dbName)
+     value = (value * 31) ^ hash(self.objectName)
+     value = (value * 31) ^ hash(self.partValues)
+     value = (value * 31) ^ hash(self.columnName)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class PrivilegeGrantInfo:
+   """
+   Attributes:
+    - privilege
+    - createTime
+    - grantor
+    - grantorType
+    - grantOption
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'privilege', None, None, ), # 1
+     (2, TType.I32, 'createTime', None, None, ), # 2
+     (3, TType.STRING, 'grantor', None, None, ), # 3
+     (4, TType.I32, 'grantorType', None, None, ), # 4
+     (5, TType.BOOL, 'grantOption', None, None, ), # 5
+   )
+ 
+   def __init__(self, privilege=None, createTime=None, grantor=None, grantorType=None, grantOption=None,):
+     self.privilege = privilege
+     self.createTime = createTime
+     self.grantor = grantor
+     self.grantorType = grantorType
+     self.grantOption = grantOption
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.privilege = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.I32:
+           self.createTime = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.STRING:
+           self.grantor = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 4:
+         if ftype == TType.I32:
+           self.grantorType = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 5:
+         if ftype == TType.BOOL:
+           self.grantOption = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('PrivilegeGrantInfo')
+     if self.privilege is not None:
+       oprot.writeFieldBegin('privilege', TType.STRING, 1)
+       oprot.writeString(self.privilege)
+       oprot.writeFieldEnd()
+     if self.createTime is not None:
+       oprot.writeFieldBegin('createTime', TType.I32, 2)
+       oprot.writeI32(self.createTime)
+       oprot.writeFieldEnd()
+     if self.grantor is not None:
+       oprot.writeFieldBegin('grantor', TType.STRING, 3)
+       oprot.writeString(self.grantor)
+       oprot.writeFieldEnd()
+     if self.grantorType is not None:
+       oprot.writeFieldBegin('grantorType', TType.I32, 4)
+       oprot.writeI32(self.grantorType)
+       oprot.writeFieldEnd()
+     if self.grantOption is not None:
+       oprot.writeFieldBegin('grantOption', TType.BOOL, 5)
+       oprot.writeBool(self.grantOption)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.privilege)
+     value = (value * 31) ^ hash(self.createTime)
+     value = (value * 31) ^ hash(self.grantor)
+     value = (value * 31) ^ hash(self.grantorType)
+     value = (value * 31) ^ hash(self.grantOption)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class HiveObjectPrivilege:
+   """
+   Attributes:
+    - hiveObject
+    - principalName
+    - principalType
+    - grantInfo
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRUCT, 'hiveObject', (HiveObjectRef, HiveObjectRef.thrift_spec), None, ), # 1
+     (2, TType.STRING, 'principalName', None, None, ), # 2
+     (3, TType.I32, 'principalType', None, None, ), # 3
+     (4, TType.STRUCT, 'grantInfo', (PrivilegeGrantInfo, PrivilegeGrantInfo.thrift_spec), None, ), # 4
+   )
+ 
+   def __init__(self, hiveObject=None, principalName=None, principalType=None, grantInfo=None,):
+     self.hiveObject = hiveObject
+     self.principalName = principalName
+     self.principalType = principalType
+     self.grantInfo = grantInfo
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRUCT:
+           self.hiveObject = HiveObjectRef()
+           self.hiveObject.read(iprot)
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.principalName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.I32:
+           self.principalType = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 4:
+         if ftype == TType.STRUCT:
+           self.grantInfo = PrivilegeGrantInfo()
+           self.grantInfo.read(iprot)
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('HiveObjectPrivilege')
+     if self.hiveObject is not None:
+       oprot.writeFieldBegin('hiveObject', TType.STRUCT, 1)
+       self.hiveObject.write(oprot)
+       oprot.writeFieldEnd()
+     if self.principalName is not None:
+       oprot.writeFieldBegin('principalName', TType.STRING, 2)
+       oprot.writeString(self.principalName)
+       oprot.writeFieldEnd()
+     if self.principalType is not None:
+       oprot.writeFieldBegin('principalType', TType.I32, 3)
+       oprot.writeI32(self.principalType)
+       oprot.writeFieldEnd()
+     if self.grantInfo is not None:
+       oprot.writeFieldBegin('grantInfo', TType.STRUCT, 4)
+       self.grantInfo.write(oprot)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.hiveObject)
+     value = (value * 31) ^ hash(self.principalName)
+     value = (value * 31) ^ hash(self.principalType)
+     value = (value * 31) ^ hash(self.grantInfo)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class PrivilegeBag:
+   """
+   Attributes:
+    - privileges
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.LIST, 'privileges', (TType.STRUCT,(HiveObjectPrivilege, HiveObjectPrivilege.thrift_spec)), None, ), # 1
+   )
+ 
+   def __init__(self, privileges=None,):
+     self.privileges = privileges
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.LIST:
+           self.privileges = []
+           (_etype17, _size14) = iprot.readListBegin()
+           for _i18 in xrange(_size14):
+             _elem19 = HiveObjectPrivilege()
+             _elem19.read(iprot)
+             self.privileges.append(_elem19)
+           iprot.readListEnd()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('PrivilegeBag')
+     if self.privileges is not None:
+       oprot.writeFieldBegin('privileges', TType.LIST, 1)
+       oprot.writeListBegin(TType.STRUCT, len(self.privileges))
+       for iter20 in self.privileges:
+         iter20.write(oprot)
+       oprot.writeListEnd()
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.privileges)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class PrincipalPrivilegeSet:
+   """
+   Attributes:
+    - userPrivileges
+    - groupPrivileges
+    - rolePrivileges
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.MAP, 'userPrivileges', (TType.STRING,None,TType.LIST,(TType.STRUCT,(PrivilegeGrantInfo, PrivilegeGrantInfo.thrift_spec))), None, ), # 1
+     (2, TType.MAP, 'groupPrivileges', (TType.STRING,None,TType.LIST,(TType.STRUCT,(PrivilegeGrantInfo, PrivilegeGrantInfo.thrift_spec))), None, ), # 2
+     (3, TType.MAP, 'rolePrivileges', (TType.STRING,None,TType.LIST,(TType.STRUCT,(PrivilegeGrantInfo, PrivilegeGrantInfo.thrift_spec))), None, ), # 3
+   )
+ 
+   def __init__(self, userPrivileges=None, groupPrivileges=None, rolePrivileges=None,):
+     self.userPrivileges = userPrivileges
+     self.groupPrivileges = groupPrivileges
+     self.rolePrivileges = rolePrivileges
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.MAP:
+           self.userPrivileges = {}
+           (_ktype22, _vtype23, _size21 ) = iprot.readMapBegin()
+           for _i25 in xrange(_size21):
+             _key26 = iprot.readString()
+             _val27 = []
+             (_etype31, _size28) = iprot.readListBegin()
+             for _i32 in xrange(_size28):
+               _elem33 = PrivilegeGrantInfo()
+               _elem33.read(iprot)
+               _val27.append(_elem33)
+             iprot.readListEnd()
+             self.userPrivileges[_key26] = _val27
+           iprot.readMapEnd()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.MAP:
+           self.groupPrivileges = {}
+           (_ktype35, _vtype36, _size34 ) = iprot.readMapBegin()
+           for _i38 in xrange(_size34):
+             _key39 = iprot.readString()
+             _val40 = []
+             (_etype44, _size41) = iprot.readListBegin()
+             for _i45 in xrange(_size41):
+               _elem46 = PrivilegeGrantInfo()
+               _elem46.read(iprot)
+               _val40.append(_elem46)
+             iprot.readListEnd()
+             self.groupPrivileges[_key39] = _val40
+           iprot.readMapEnd()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.MAP:
+           self.rolePrivileges = {}
+           (_ktype48, _vtype49, _size47 ) = iprot.readMapBegin()
+           for _i51 in xrange(_size47):
+             _key52 = iprot.readString()
+             _val53 = []
+             (_etype57, _size54) = iprot.readListBegin()
+             for _i58 in xrange(_size54):
+               _elem59 = PrivilegeGrantInfo()
+               _elem59.read(iprot)
+               _val53.append(_elem59)
+             iprot.readListEnd()
+             self.rolePrivileges[_key52] = _val53
+           iprot.readMapEnd()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('PrincipalPrivilegeSet')
+     if self.userPrivileges is not None:
+       oprot.writeFieldBegin('userPrivileges', TType.MAP, 1)
+       oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.userPrivileges))
+       for kiter60,viter61 in self.userPrivileges.items():
+         oprot.writeString(kiter60)
+         oprot.writeListBegin(TType.STRUCT, len(viter61))
+         for iter62 in viter61:
+           iter62.write(oprot)
+         oprot.writeListEnd()
+       oprot.writeMapEnd()
+       oprot.writeFieldEnd()
+     if self.groupPrivileges is not None:
+       oprot.writeFieldBegin('groupPrivileges', TType.MAP, 2)
+       oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.groupPrivileges))
+       for kiter63,viter64 in self.groupPrivileges.items():
+         oprot.writeString(kiter63)
+         oprot.writeListBegin(TType.STRUCT, len(viter64))
+         for iter65 in viter64:
+           iter65.write(oprot)
+         oprot.writeListEnd()
+       oprot.writeMapEnd()
+       oprot.writeFieldEnd()
+     if self.rolePrivileges is not None:
+       oprot.writeFieldBegin('rolePrivileges', TType.MAP, 3)
+       oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.rolePrivileges))
+       for kiter66,viter67 in self.rolePrivileges.items():
+         oprot.writeString(kiter66)
+         oprot.writeListBegin(TType.STRUCT, len(viter67))
+         for iter68 in viter67:
+           iter68.write(oprot)
+         oprot.writeListEnd()
+       oprot.writeMapEnd()
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.userPrivileges)
+     value = (value * 31) ^ hash(self.groupPrivileges)
+     value = (value * 31) ^ hash(self.rolePrivileges)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class GrantRevokePrivilegeRequest:
+   """
+   Attributes:
+    - requestType
+    - privileges
+    - revokeGrantOption
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.I32, 'requestType', None, None, ), # 1
+     (2, TType.STRUCT, 'privileges', (PrivilegeBag, PrivilegeBag.thrift_spec), None, ), # 2
+     (3, TType.BOOL, 'revokeGrantOption', None, None, ), # 3
+   )
+ 
+   def __init__(self, requestType=None, privileges=None, revokeGrantOption=None,):
+     self.requestType = requestType
+     self.privileges = privileges
+     self.revokeGrantOption = revokeGrantOption
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.I32:
+           self.requestType = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRUCT:
+           self.privileges = PrivilegeBag()
+           self.privileges.read(iprot)
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.BOOL:
+           self.revokeGrantOption = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('GrantRevokePrivilegeRequest')
+     if self.requestType is not None:
+       oprot.writeFieldBegin('requestType', TType.I32, 1)
+       oprot.writeI32(self.requestType)
+       oprot.writeFieldEnd()
+     if self.privileges is not None:
+       oprot.writeFieldBegin('privileges', TType.STRUCT, 2)
+       self.privileges.write(oprot)
+       oprot.writeFieldEnd()
+     if self.revokeGrantOption is not None:
+       oprot.writeFieldBegin('revokeGrantOption', TType.BOOL, 3)
+       oprot.writeBool(self.revokeGrantOption)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.requestType)
+     value = (value * 31) ^ hash(self.privileges)
+     value = (value * 31) ^ hash(self.revokeGrantOption)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class GrantRevokePrivilegeResponse:
+   """
+   Attributes:
+    - success
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.BOOL, 'success', None, None, ), # 1
+   )
+ 
+   def __init__(self, success=None,):
+     self.success = success
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.BOOL:
+           self.success = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('GrantRevokePrivilegeResponse')
+     if self.success is not None:
+       oprot.writeFieldBegin('success', TType.BOOL, 1)
+       oprot.writeBool(self.success)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.success)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class Role:
+   """
+   Attributes:
+    - roleName
+    - createTime
+    - ownerName
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'roleName', None, None, ), # 1
+     (2, TType.I32, 'createTime', None, None, ), # 2
+     (3, TType.STRING, 'ownerName', None, None, ), # 3
+   )
+ 
+   def __init__(self, roleName=None, createTime=None, ownerName=None,):
+     self.roleName = roleName
+     self.createTime = createTime
+     self.ownerName = ownerName
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.roleName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.I32:
+           self.createTime = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.STRING:
+           self.ownerName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('Role')
+     if self.roleName is not None:
+       oprot.writeFieldBegin('roleName', TType.STRING, 1)
+       oprot.writeString(self.roleName)
+       oprot.writeFieldEnd()
+     if self.createTime is not None:
+       oprot.writeFieldBegin('createTime', TType.I32, 2)
+       oprot.writeI32(self.createTime)
+       oprot.writeFieldEnd()
+     if self.ownerName is not None:
+       oprot.writeFieldBegin('ownerName', TType.STRING, 3)
+       oprot.writeString(self.ownerName)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.roleName)
+     value = (value * 31) ^ hash(self.createTime)
+     value = (value * 31) ^ hash(self.ownerName)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class RolePrincipalGrant:
+   """
+   Attributes:
+    - roleName
+    - principalName
+    - principalType
+    - grantOption
+    - grantTime
+    - grantorName
+    - grantorPrincipalType
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'roleName', None, None, ), # 1
+     (2, TType.STRING, 'principalName', None, None, ), # 2
+     (3, TType.I32, 'principalType', None, None, ), # 3
+     (4, TType.BOOL, 'grantOption', None, None, ), # 4
+     (5, TType.I32, 'grantTime', None, None, ), # 5
+     (6, TType.STRING, 'grantorName', None, None, ), # 6
+     (7, TType.I32, 'grantorPrincipalType', None, None, ), # 7
+   )
+ 
+   def __init__(self, roleName=None, principalName=None, principalType=None, grantOption=None, grantTime=None, grantorName=None, grantorPrincipalType=None,):
+     self.roleName = roleName
+     self.principalName = principalName
+     self.principalType = principalType
+     self.grantOption = grantOption
+     self.grantTime = grantTime
+     self.grantorName = grantorName
+     self.grantorPrincipalType = grantorPrincipalType
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.roleName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.principalName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.I32:
+           self.principalType = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 4:
+         if ftype == TType.BOOL:
+           self.grantOption = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       elif fid == 5:
+         if ftype == TType.I32:
+           self.grantTime = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 6:
+         if ftype == TType.STRING:
+           self.grantorName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 7:
+         if ftype == TType.I32:
+           self.grantorPrincipalType = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('RolePrincipalGrant')
+     if self.roleName is not None:
+       oprot.writeFieldBegin('roleName', TType.STRING, 1)
+       oprot.writeString(self.roleName)
+       oprot.writeFieldEnd()
+     if self.principalName is not None:
+       oprot.writeFieldBegin('principalName', TType.STRING, 2)
+       oprot.writeString(self.principalName)
+       oprot.writeFieldEnd()
+     if self.principalType is not None:
+       oprot.writeFieldBegin('principalType', TType.I32, 3)
+       oprot.writeI32(self.principalType)
+       oprot.writeFieldEnd()
+     if self.grantOption is not None:
+       oprot.writeFieldBegin('grantOption', TType.BOOL, 4)
+       oprot.writeBool(self.grantOption)
+       oprot.writeFieldEnd()
+     if self.grantTime is not None:
+       oprot.writeFieldBegin('grantTime', TType.I32, 5)
+       oprot.writeI32(self.grantTime)
+       oprot.writeFieldEnd()
+     if self.grantorName is not None:
+       oprot.writeFieldBegin('grantorName', TType.STRING, 6)
+       oprot.writeString(self.grantorName)
+       oprot.writeFieldEnd()
+     if self.grantorPrincipalType is not None:
+       oprot.writeFieldBegin('grantorPrincipalType', TType.I32, 7)
+       oprot.writeI32(self.grantorPrincipalType)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.roleName)
+     value = (value * 31) ^ hash(self.principalName)
+     value = (value * 31) ^ hash(self.principalType)
+     value = (value * 31) ^ hash(self.grantOption)
+     value = (value * 31) ^ hash(self.grantTime)
+     value = (value * 31) ^ hash(self.grantorName)
+     value = (value * 31) ^ hash(self.grantorPrincipalType)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class GetRoleGrantsForPrincipalRequest:
+   """
+   Attributes:
+    - principal_name
+    - principal_type
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'principal_name', None, None, ), # 1
+     (2, TType.I32, 'principal_type', None, None, ), # 2
+   )
+ 
+   def __init__(self, principal_name=None, principal_type=None,):
+     self.principal_name = principal_name
+     self.principal_type = principal_type
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.principal_name = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.I32:
+           self.principal_type = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('GetRoleGrantsForPrincipalRequest')
+     if self.principal_name is not None:
+       oprot.writeFieldBegin('principal_name', TType.STRING, 1)
+       oprot.writeString(self.principal_name)
+       oprot.writeFieldEnd()
+     if self.principal_type is not None:
+       oprot.writeFieldBegin('principal_type', TType.I32, 2)
+       oprot.writeI32(self.principal_type)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     if self.principal_name is None:
+       raise TProtocol.TProtocolException(message='Required field principal_name is unset!')
+     if self.principal_type is None:
+       raise TProtocol.TProtocolException(message='Required field principal_type is unset!')
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.principal_name)
+     value = (value * 31) ^ hash(self.principal_type)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class GetRoleGrantsForPrincipalResponse:
+   """
+   Attributes:
+    - principalGrants
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.LIST, 'principalGrants', (TType.STRUCT,(RolePrincipalGrant, RolePrincipalGrant.thrift_spec)), None, ), # 1
+   )
+ 
+   def __init__(self, principalGrants=None,):
+     self.principalGrants = principalGrants
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.LIST:
+           self.principalGrants = []
+           (_etype72, _size69) = iprot.readListBegin()
+           for _i73 in xrange(_size69):
+             _elem74 = RolePrincipalGrant()
+             _elem74.read(iprot)
+             self.principalGrants.append(_elem74)
+           iprot.readListEnd()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('GetRoleGrantsForPrincipalResponse')
+     if self.principalGrants is not None:
+       oprot.writeFieldBegin('principalGrants', TType.LIST, 1)
+       oprot.writeListBegin(TType.STRUCT, len(self.principalGrants))
+       for iter75 in self.principalGrants:
+         iter75.write(oprot)
+       oprot.writeListEnd()
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     if self.principalGrants is None:
+       raise TProtocol.TProtocolException(message='Required field principalGrants is unset!')
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.principalGrants)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class GetPrincipalsInRoleRequest:
+   """
+   Attributes:
+    - roleName
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.STRING, 'roleName', None, None, ), # 1
+   )
+ 
+   def __init__(self, roleName=None,):
+     self.roleName = roleName
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.STRING:
+           self.roleName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('GetPrincipalsInRoleRequest')
+     if self.roleName is not None:
+       oprot.writeFieldBegin('roleName', TType.STRING, 1)
+       oprot.writeString(self.roleName)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     if self.roleName is None:
+       raise TProtocol.TProtocolException(message='Required field roleName is unset!')
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.roleName)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class GetPrincipalsInRoleResponse:
+   """
+   Attributes:
+    - principalGrants
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.LIST, 'principalGrants', (TType.STRUCT,(RolePrincipalGrant, RolePrincipalGrant.thrift_spec)), None, ), # 1
+   )
+ 
+   def __init__(self, principalGrants=None,):
+     self.principalGrants = principalGrants
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.LIST:
+           self.principalGrants = []
+           (_etype79, _size76) = iprot.readListBegin()
+           for _i80 in xrange(_size76):
+             _elem81 = RolePrincipalGrant()
+             _elem81.read(iprot)
+             self.principalGrants.append(_elem81)
+           iprot.readListEnd()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('GetPrincipalsInRoleResponse')
+     if self.principalGrants is not None:
+       oprot.writeFieldBegin('principalGrants', TType.LIST, 1)
+       oprot.writeListBegin(TType.STRUCT, len(self.principalGrants))
+       for iter82 in self.principalGrants:
+         iter82.write(oprot)
+       oprot.writeListEnd()
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     if self.principalGrants is None:
+       raise TProtocol.TProtocolException(message='Required field principalGrants is unset!')
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.principalGrants)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class GrantRevokeRoleRequest:
+   """
+   Attributes:
+    - requestType
+    - roleName
+    - principalName
+    - principalType
+    - grantor
+    - grantorType
+    - grantOption
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.I32, 'requestType', None, None, ), # 1
+     (2, TType.STRING, 'roleName', None, None, ), # 2
+     (3, TType.STRING, 'principalName', None, None, ), # 3
+     (4, TType.I32, 'principalType', None, None, ), # 4
+     (5, TType.STRING, 'grantor', None, None, ), # 5
+     (6, TType.I32, 'grantorType', None, None, ), # 6
+     (7, TType.BOOL, 'grantOption', None, None, ), # 7
+   )
+ 
+   def __init__(self, requestType=None, roleName=None, principalName=None, principalType=None, grantor=None, grantorType=None, grantOption=None,):
+     self.requestType = requestType
+     self.roleName = roleName
+     self.principalName = principalName
+     self.principalType = principalType
+     self.grantor = grantor
+     self.grantorType = grantorType
+     self.grantOption = grantOption
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.I32:
+           self.requestType = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 2:
+         if ftype == TType.STRING:
+           self.roleName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 3:
+         if ftype == TType.STRING:
+           self.principalName = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 4:
+         if ftype == TType.I32:
+           self.principalType = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 5:
+         if ftype == TType.STRING:
+           self.grantor = iprot.readString()
+         else:
+           iprot.skip(ftype)
+       elif fid == 6:
+         if ftype == TType.I32:
+           self.grantorType = iprot.readI32()
+         else:
+           iprot.skip(ftype)
+       elif fid == 7:
+         if ftype == TType.BOOL:
+           self.grantOption = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('GrantRevokeRoleRequest')
+     if self.requestType is not None:
+       oprot.writeFieldBegin('requestType', TType.I32, 1)
+       oprot.writeI32(self.requestType)
+       oprot.writeFieldEnd()
+     if self.roleName is not None:
+       oprot.writeFieldBegin('roleName', TType.STRING, 2)
+       oprot.writeString(self.roleName)
+       oprot.writeFieldEnd()
+     if self.principalName is not None:
+       oprot.writeFieldBegin('principalName', TType.STRING, 3)
+       oprot.writeString(self.principalName)
+       oprot.writeFieldEnd()
+     if self.principalType is not None:
+       oprot.writeFieldBegin('principalType', TType.I32, 4)
+       oprot.writeI32(self.principalType)
+       oprot.writeFieldEnd()
+     if self.grantor is not None:
+       oprot.writeFieldBegin('grantor', TType.STRING, 5)
+       oprot.writeString(self.grantor)
+       oprot.writeFieldEnd()
+     if self.grantorType is not None:
+       oprot.writeFieldBegin('grantorType', TType.I32, 6)
+       oprot.writeI32(self.grantorType)
+       oprot.writeFieldEnd()
+     if self.grantOption is not None:
+       oprot.writeFieldBegin('grantOption', TType.BOOL, 7)
+       oprot.writeBool(self.grantOption)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.requestType)
+     value = (value * 31) ^ hash(self.roleName)
+     value = (value * 31) ^ hash(self.principalName)
+     value = (value * 31) ^ hash(self.principalType)
+     value = (value * 31) ^ hash(self.grantor)
+     value = (value * 31) ^ hash(self.grantorType)
+     value = (value * 31) ^ hash(self.grantOption)
+     return value
+ 
+   def __repr__(self):
+     L = ['%s=%r' % (key, value)
+       for key, value in self.__dict__.iteritems()]
+     return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
+ 
+   def __eq__(self, other):
+     return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
+ 
+   def __ne__(self, other):
+     return not (self == other)
+ 
+ class GrantRevokeRoleResponse:
+   """
+   Attributes:
+    - success
+   """
+ 
+   thrift_spec = (
+     None, # 0
+     (1, TType.BOOL, 'success', None, None, ), # 1
+   )
+ 
+   def __init__(self, success=None,):
+     self.success = success
+ 
+   def read(self, iprot):
+     if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
+       fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
+       return
+     iprot.readStructBegin()
+     while True:
+       (fname, ftype, fid) = iprot.readFieldBegin()
+       if ftype == TType.STOP:
+         break
+       if fid == 1:
+         if ftype == TType.BOOL:
+           self.success = iprot.readBool()
+         else:
+           iprot.skip(ftype)
+       else:
+         iprot.skip(ftype)
+       iprot.readFieldEnd()
+     iprot.readStructEnd()
+ 
+   def write(self, oprot):
+     if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
+       oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
+       return
+     oprot.writeStructBegin('GrantRevokeRoleResponse')
+     if self.success is not None:
+       oprot.writeFieldBegin('success', TType.BOOL, 1)
+       oprot.writeBool(self.success)
+       oprot.writeFieldEnd()
+     oprot.writeFieldStop()
+     oprot.writeStructEnd()
+ 
+   def validate(self):
+     return
+ 
+ 
+   def __hash__(self):
+     value = 17
+     value = (value * 31) ^ hash(self.

<TRUNCATED>

[17/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
index 0000000,a1187c4..6354f77
mode 000000,100644..100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
@@@ -1,0 -1,243 +1,245 @@@
+ /*
+   Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+ 
+       http://www.apache.org/licenses/LICENSE-2.0
+ 
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+  */
+ package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table;
+ 
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+ import org.apache.hadoop.hive.metastore.TableType;
+ import org.apache.hadoop.hive.metastore.api.Database;
+ import org.apache.hadoop.hive.metastore.api.MetaException;
+ import org.apache.hadoop.hive.ql.ErrorMsg;
+ import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
+ import org.apache.hadoop.hive.ql.exec.Task;
+ import org.apache.hadoop.hive.ql.exec.TaskFactory;
+ import org.apache.hadoop.hive.ql.exec.Utilities;
+ import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadTask;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.events.TableEvent;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.TaskTracker;
+ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.util.Context;
+ import org.apache.hadoop.hive.ql.metadata.Table;
+ import org.apache.hadoop.hive.ql.parse.EximUtil;
+ import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer;
+ import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+ import org.apache.hadoop.hive.ql.parse.SemanticException;
+ import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+ import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
+ import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+ import org.apache.hadoop.hive.ql.plan.MoveWork;
++import org.apache.hadoop.hive.ql.session.SessionState;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ 
+ import java.io.IOException;
+ import java.io.Serializable;
+ import java.util.ArrayList;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.TreeMap;
+ 
+ import static org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer.isPartitioned;
+ 
+ public class LoadTable {
+   private final static Logger LOG = LoggerFactory.getLogger(LoadTable.class);
+   //  private final Helper helper;
+   private final Context context;
+   private final ReplLogger replLogger;
+   private final TableContext tableContext;
+   private final TaskTracker tracker;
+   private final TableEvent event;
+ 
+   public LoadTable(TableEvent event, Context context, ReplLogger replLogger,
+                    TableContext tableContext, TaskTracker limiter)
+       throws SemanticException, IOException {
+     this.event = event;
+     this.context = context;
+     this.replLogger = replLogger;
+     this.tableContext = tableContext;
+     this.tracker = new TaskTracker(limiter);
+   }
+ 
+   private void createTableReplLogTask(String tableName, TableType tableType) throws SemanticException {
+     ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger,tableName, tableType);
+     Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf);
+     ReplLoadTask.dependency(tracker.tasks(), replLogTask);
+ 
+     if (tracker.tasks().isEmpty()) {
+       tracker.addTask(replLogTask);
+     } else {
+       ReplLoadTask.dependency(tracker.tasks(), replLogTask);
+ 
+       List<Task<? extends Serializable>> visited = new ArrayList<>();
+       tracker.updateTaskCount(replLogTask, visited);
+     }
+   }
+ 
+   public TaskTracker tasks() throws SemanticException {
+     // Path being passed to us is a table dump location. We go ahead and load it in as needed.
+     // If tblName is null, then we default to the table name specified in _metadata, which is good.
+     // or are both specified, in which case, that's what we are intended to create the new table as.
+     try {
+       if (event.shouldNotReplicate()) {
+         return tracker;
+       }
+       String dbName = tableContext.dbNameToLoadIn; //this can never be null or empty;
+       // Create table associated with the import
+       // Executed if relevant, and used to contain all the other details about the table if not.
+       ImportTableDesc tableDesc = tableContext.overrideProperties(event.tableDesc(dbName));
+       Table table = ImportSemanticAnalyzer.tableIfExists(tableDesc, context.hiveDb);
+       ReplicationSpec replicationSpec = event.replicationSpec();
+ 
+       // Normally, on import, trying to create a table or a partition in a db that does not yet exist
+       // is a error condition. However, in the case of a REPL LOAD, it is possible that we are trying
+       // to create tasks to create a table inside a db that as-of-now does not exist, but there is
+       // a precursor Task waiting that will create it before this is encountered. Thus, we instantiate
+       // defaults and do not error out in that case.
+       // the above will change now since we are going to split replication load in multiple execution
+       // tasks and hence we could have created the database earlier in which case the waitOnPrecursor will
+       // be false and hence if db Not found we should error out.
+       Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName());
+       if (parentDb == null) {
+         if (!tableContext.waitOnPrecursor()) {
+           throw new SemanticException(
+               ErrorMsg.DATABASE_NOT_EXISTS.getMsg(tableDesc.getDatabaseName()));
+         }
+       }
+ 
+       if (table == null) {
+         // If table doesn't exist, allow creating a new one only if the database state is older than the update.
+         if ((parentDb != null) && (!replicationSpec
+             .allowReplacementInto(parentDb.getParameters()))) {
+           // If the target table exists and is newer or same as current update based on repl.last.id, then just noop it.
+           return tracker;
+         }
+       } else {
+         if (!replicationSpec.allowReplacementInto(table.getParameters())) {
+           // If the target table exists and is newer or same as current update based on repl.last.id, then just noop it.
+           return tracker;
+         }
+       }
+ 
+       if (tableDesc.getLocation() == null) {
+         tableDesc.setLocation(location(tableDesc, parentDb));
+       }
+ 
+ 
+   /* Note: In the following section, Metadata-only import handling logic is
+      interleaved with regular repl-import logic. The rule of thumb being
+      followed here is that MD-only imports are essentially ALTERs. They do
+      not load data, and should not be "creating" any metadata - they should
+      be replacing instead. The only place it makes sense for a MD-only import
+      to create is in the case of a table that's been dropped and recreated,
+      or in the case of an unpartitioned table. In all other cases, it should
+      behave like a noop or a pure MD alter.
+   */
+       if (table == null) {
+         newTableTasks(tableDesc);
+       } else {
+         existingTableTasks(tableDesc, table, replicationSpec);
+       }
+ 
+       if (!isPartitioned(tableDesc)) {
+         createTableReplLogTask(tableDesc.getTableName(), tableDesc.tableType());
+       }
+       return tracker;
+     } catch (Exception e) {
+       throw new SemanticException(e);
+     }
+   }
+ 
+   private void existingTableTasks(ImportTableDesc tblDesc, Table table,
+       ReplicationSpec replicationSpec) {
+     if (!table.isPartitioned()) {
+ 
+       LOG.debug("table non-partitioned");
+       if (!replicationSpec.allowReplacementInto(table.getParameters())) {
+         return; // silently return, table is newer than our replacement.
+       }
+ 
+       Task<? extends Serializable> alterTableTask = alterTableTask(tblDesc, replicationSpec);
+       if (replicationSpec.isMetadataOnly()) {
+         tracker.addTask(alterTableTask);
+       } else {
+         Task<?> loadTableTask =
+             loadTableTask(table, replicationSpec, event.metadataPath(), event.metadataPath());
+         alterTableTask.addDependentTask(loadTableTask);
+         tracker.addTask(alterTableTask);
+       }
+     }
+   }
+ 
+   private void newTableTasks(ImportTableDesc tblDesc) throws SemanticException {
+     Table table;
+     table = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName());
+     // Either we're dropping and re-creating, or the table didn't exist, and we're creating.
+     Task<?> createTableTask =
+         tblDesc.getCreateTableTask(new HashSet<>(), new HashSet<>(), context.hiveConf);
+     if (event.replicationSpec().isMetadataOnly()) {
+       tracker.addTask(createTableTask);
+       return;
+     }
+     if (!isPartitioned(tblDesc)) {
+       LOG.debug("adding dependent CopyWork/MoveWork for table");
+       Task<?> loadTableTask =
+           loadTableTask(table, event.replicationSpec(), new Path(tblDesc.getLocation()),
+               event.metadataPath());
+       createTableTask.addDependentTask(loadTableTask);
+     }
+     tracker.addTask(createTableTask);
+   }
+ 
+   private String location(ImportTableDesc tblDesc, Database parentDb)
+       throws MetaException, SemanticException {
+     if (!tableContext.waitOnPrecursor()) {
+       return context.warehouse.getDefaultTablePath(parentDb, tblDesc.getTableName()).toString();
+     } else {
+       Path tablePath = new Path(
+           context.warehouse.getDefaultDatabasePath(tblDesc.getDatabaseName()),
+           MetaStoreUtils.encodeTableName(tblDesc.getTableName().toLowerCase())
+       );
+       return context.warehouse.getDnsPath(tablePath).toString();
+     }
+   }
+ 
+   private Task<?> loadTableTask(Table table, ReplicationSpec replicationSpec, Path tgtPath,
+       Path fromURI) {
+     Path dataPath = new Path(fromURI, EximUtil.DATA_PATH_NAME);
+     Path tmpPath = context.utils.getExternalTmpPath(tgtPath);
+     Task<?> copyTask =
+         ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, tmpPath, context.hiveConf);
+ 
+     LoadTableDesc loadTableWork = new LoadTableDesc(
 -        tmpPath, Utilities.getTableDesc(table), new TreeMap<>(), replicationSpec.isReplace()
++        tmpPath, Utilities.getTableDesc(table), new TreeMap<>(), replicationSpec.isReplace(),
++        SessionState.get().getTxnMgr().getCurrentTxnId()
+     );
+     MoveWork moveWork =
+         new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false);
+     Task<?> loadTableTask = TaskFactory.get(moveWork, context.hiveConf);
+     copyTask.addDependentTask(loadTableTask);
+     return copyTask;
+   }
+ 
+   private Task<? extends Serializable> alterTableTask(ImportTableDesc tableDesc,
+       ReplicationSpec replicationSpec) {
+     tableDesc.setReplaceMode(true);
+     if ((replicationSpec != null) && (replicationSpec.isInReplicationScope())) {
+       tableDesc.setReplicationSpec(replicationSpec);
+     }
+     return tableDesc.getCreateTableTask(new HashSet<>(), new HashSet<>(), context.hiveConf);
+   }
+ }

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
index cfa4e2c,90d1372..b5733ec
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
@@@ -145,12 -162,18 +162,18 @@@ public class VectorizedRowBatchCtx 
  
    public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx, Configuration hiveConf,
        FileSplit split, Object[] partitionValues) throws IOException {
+     // TODO: this is invalid for SMB. Keep this for now for legacy reasons. See the other overload.
+     MapWork mapWork = Utilities.getMapWork(hiveConf);
+     getPartitionValues(vrbCtx, mapWork, split, partitionValues);
+   }
  
-     Map<Path, PartitionDesc> pathToPartitionInfo = Utilities
-         .getMapWork(hiveConf).getPathToPartitionInfo();
+   public static void getPartitionValues(VectorizedRowBatchCtx vrbCtx,
+       MapWork mapWork, FileSplit split, Object[] partitionValues)
+       throws IOException {
+     Map<Path, PartitionDesc> pathToPartitionInfo = mapWork.getPathToPartitionInfo();
  
      PartitionDesc partDesc = HiveFileFormatUtils
 -        .getPartitionDescFromPathRecursively(pathToPartitionInfo,
 +        .getFromPathRecursively(pathToPartitionInfo,
              split.getPath(), IOPrepareCache.get().getPartitionDescMap());
  
      getPartitionValues(vrbCtx, partDesc, partitionValues);

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index f325c0e,feacdd8..9c9a2d3
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@@ -36,9 -37,9 +37,10 @@@ import org.apache.hadoop.hive.conf.Hive
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
  import org.apache.hadoop.hive.metastore.api.DataOperationType;
  import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 +import org.apache.hadoop.hive.metastore.MetaStoreUtils;
  import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
  import org.apache.hadoop.hive.ql.ErrorMsg;
+ import org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater;
  import org.apache.hadoop.hive.ql.metadata.Table;
  import org.apache.hadoop.hive.shims.HadoopShims;
  import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
@@@ -345,11 -384,7 +385,10 @@@ public class AcidUtils 
      public static final String SPLIT_UPDATE_STRING = "split_update";
      public static final int HASH_BASED_MERGE_BIT = 0x02;
      public static final String HASH_BASED_MERGE_STRING = "hash_merge";
 +    public static final int INSERT_ONLY_BIT = 0x04;
 +    public static final String INSERT_ONLY_STRING = "insert_only";
      public static final String DEFAULT_VALUE_STRING = TransactionalValidationListener.DEFAULT_TRANSACTIONAL_PROPERTY;
-     public static final String LEGACY_VALUE_STRING = TransactionalValidationListener.LEGACY_TRANSACTIONAL_PROPERTY;
 +    public static final String INSERTONLY_VALUE_STRING = TransactionalValidationListener.INSERTONLY_TRANSACTIONAL_PROPERTY;
  
      private AcidOperationalProperties() {
      }
@@@ -401,12 -414,6 +430,9 @@@
        if (propertiesStr.equalsIgnoreCase(DEFAULT_VALUE_STRING)) {
          return AcidOperationalProperties.getDefault();
        }
-       if (propertiesStr.equalsIgnoreCase(LEGACY_VALUE_STRING)) {
-         return AcidOperationalProperties.getLegacy();
-       }
 +      if (propertiesStr.equalsIgnoreCase(INSERTONLY_VALUE_STRING)) {
 +        return AcidOperationalProperties.getInsertOnly();
 +      }
        AcidOperationalProperties obj = new AcidOperationalProperties();
        String[] options = propertiesStr.split("\\|");
        for (String option : options) {

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
index a70fde6,38aaeed..6178de2
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
@@@ -14,16 -14,12 +14,20 @@@
  package org.apache.hadoop.hive.ql.io.parquet;
  
  import java.io.IOException;
 +import java.util.Map;
  
+ import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.hive.common.io.DataCache;
+ import org.apache.hadoop.hive.common.io.FileMetadataCache;
  import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
 +import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
+ import org.apache.hadoop.hive.ql.io.LlapCacheOnlyInputFormatInterface;
 +import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetTableUtils;
 +import org.apache.hadoop.hive.ql.plan.MapWork;
 +import org.apache.hadoop.hive.ql.plan.PartitionDesc;
 +import org.apache.hadoop.mapred.FileSplit;
 +import org.apache.hadoop.mapred.JobConf;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.apache.hadoop.hive.ql.exec.Utilities;

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ProjectionPusher.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 01e8a48,5dec791..82804b2
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@@ -423,11 -423,19 +423,19 @@@ public final class DbTxnManager extend
            compBuilder.setExclusive();
            compBuilder.setOperationType(DataOperationType.NO_TXN);
            break;
- 
+         case INSERT_OVERWRITE:
+           t = getTable(output);
+           if (AcidUtils.isAcidTable(t)) {
+             compBuilder.setSemiShared();
+             compBuilder.setOperationType(DataOperationType.UPDATE);
+           } else {
+             compBuilder.setExclusive();
+             compBuilder.setOperationType(DataOperationType.NO_TXN);
+           }
+           break;
          case INSERT:
            assert t != null;
 -          if(AcidUtils.isAcidTable(t)) {
 +          if(AcidUtils.isFullAcidTable(t)) {
              compBuilder.setShared();
            }
            else {

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index ae451e2,d661f10..a0b735c
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@@ -65,9 -62,9 +65,10 @@@ import org.apache.hadoop.fs.FileStatus
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.fs.PathFilter;
+ import org.apache.hadoop.hive.common.BlobStorageUtils;
  import org.apache.hadoop.hive.common.FileUtils;
  import org.apache.hadoop.hive.common.HiveStatsUtils;
 +import org.apache.hadoop.hive.common.JavaUtils;
  import org.apache.hadoop.hive.common.ObjectPair;
  import org.apache.hadoop.hive.common.StatsSetupConst;
  import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
@@@ -3338,8 -3250,9 +3389,9 @@@ private void constructOneLBLocationMap(
      // (1) Do not delete the dest dir before doing the move operation.
      // (2) It is assumed that subdir and dir are in same encryption zone.
      // (3) Move individual files from scr dir to dest dir.
 -    boolean destIsSubDir = isSubDir(srcf, destf, srcFs, destFs, isSrcLocal);
 +    boolean srcIsSubDirOfDest = isSubDir(srcf, destf, srcFs, destFs, isSrcLocal),
 +        destIsSubDirOfSrc = isSubDir(destf, srcf, destFs, srcFs, false);
+     final String msg = "Unable to move source " + srcf + " to destination " + destf;
 -
      try {
        if (replace) {
          try{

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 7fd8f04,a054abb..021507f
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@@ -1507,9 -1517,14 +1530,15 @@@ public class DDLSemanticAnalyzer extend
      }
  
      Table tab = getTable(tableName, true);
+     // cascade only occurs with partitioned table
+     if (isCascade && !tab.isPartitioned()) {
+       throw new SemanticException(
+           ErrorMsg.ALTER_TABLE_NON_PARTITIONED_TABLE_CASCADE_NOT_SUPPORTED);
+     }
+ 
      // Determine the lock type to acquire
 -    WriteEntity.WriteType writeType = WriteEntity.determineAlterTableWriteType(op);
 +    WriteEntity.WriteType writeType = doForceExclusive
 +        ? WriteType.DDL_EXCLUSIVE : WriteEntity.determineAlterTableWriteType(op);
  
      if (!alterPartitions) {
        inputs.add(new ReadEntity(tab));

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
index 2401036,40c34bf..4109de4
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
@@@ -140,15 -141,15 +141,16 @@@ public class EximUtil 
     * Initialize the URI where the exported data collection is
     * to created for export, or is present for import
     */
-   static URI getValidatedURI(HiveConf conf, String dcPath) throws SemanticException {
+   public static URI getValidatedURI(HiveConf conf, String dcPath) throws SemanticException {
      try {
 -      boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE);
 +      boolean testMode = conf.getBoolVar(HiveConf.ConfVars.HIVETESTMODE)
 +          || conf.getBoolVar(HiveConf.ConfVars.HIVEEXIMTESTMODE);
        URI uri = new Path(dcPath).toUri();
-       String scheme = uri.getScheme();
+       FileSystem fs = FileSystem.get(uri, conf);
+       // Get scheme from FileSystem
+       String scheme = fs.getScheme();
        String authority = uri.getAuthority();
        String path = uri.getPath();
-       FileSystem fs = FileSystem.get(uri, conf);
  
        LOG.info("Path before norm :" + path);
        // generate absolute path relative to home directory

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
index e534272,b8c6ea9..54ee7ae
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
@@@ -18,42 -18,12 +18,46 @@@
  
  package org.apache.hadoop.hive.ql.parse;
  
 +
 +import org.apache.hadoop.hive.ql.metadata.HiveException;
 +
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.io.Serializable;
 +import java.net.URI;
 +import java.util.HashSet;
 +import java.util.List;
 +
 +import org.antlr.runtime.tree.Tree;
 +import org.apache.hadoop.conf.Configuration;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.hive.common.FileUtils;
 +import org.apache.hadoop.hive.common.ValidReadTxnList;
 +import org.apache.hadoop.hive.common.ValidTxnList;
 +import org.apache.hadoop.hive.conf.HiveConf;
 +import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 +import org.apache.hadoop.hive.ql.Context;
 +import org.apache.hadoop.hive.ql.ErrorMsg;
 +import org.apache.hadoop.hive.ql.QueryState;
 +import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
 +import org.apache.hadoop.hive.ql.exec.Task;
 +import org.apache.hadoop.hive.ql.exec.TaskFactory;
 +import org.apache.hadoop.hive.ql.exec.Utilities;
 +import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 +import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 +import org.apache.hadoop.hive.ql.metadata.Hive;
+ import org.antlr.runtime.tree.Tree;
+ import org.apache.hadoop.hive.ql.ErrorMsg;
+ import org.apache.hadoop.hive.ql.QueryState;
  import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
 +import org.apache.hadoop.hive.ql.metadata.Partition;
 +import org.apache.hadoop.hive.ql.metadata.PartitionIterable;
  import org.apache.hadoop.hive.ql.metadata.Table;
 +import org.apache.hadoop.hive.ql.plan.CopyWork;
 +import org.slf4j.Logger;
+ import org.apache.hadoop.hive.ql.parse.repl.dump.TableExport;
  
  /**
   * ExportSemanticAnalyzer.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 5ac15af,7f3460f..387582b
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@@ -62,12 -59,10 +62,12 @@@ import org.apache.hadoop.hive.ql.metada
  import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
  import org.apache.hadoop.hive.ql.metadata.Table;
  import org.apache.hadoop.hive.ql.parse.repl.load.MetaData;
+ import org.apache.hadoop.hive.ql.parse.repl.load.UpdatedMetaDataTracker;
  import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
 +import org.apache.hadoop.hive.ql.plan.CopyWork;
 +import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
  import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
  import org.apache.hadoop.hive.ql.plan.DDLWork;
- import org.apache.hadoop.hive.ql.plan.DropTableDesc;
  import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
  import org.apache.hadoop.hive.ql.plan.MoveWork;
  import org.apache.hadoop.hive.ql.session.SessionState;
@@@ -223,9 -228,8 +234,9 @@@ public class ImportSemanticAnalyzer ext
      } catch (Exception e) {
        throw new HiveException(e);
      }
 +    boolean isSourceMm = MetaStoreUtils.isInsertOnlyTable(tblDesc.getTblProps());
  
-     if ((replicationSpec!= null) && replicationSpec.isInReplicationScope()){
+     if ((replicationSpec != null) && replicationSpec.isInReplicationScope()){
        tblDesc.setReplicationSpec(replicationSpec);
      }
  
@@@ -318,8 -306,8 +324,8 @@@
      } else {
        createReplImportTasks(
            tblDesc, partitionDescs,
-           isPartSpecSet, replicationSpec, waitOnPrecursor, table,
-           fromURI, fs, wh, x, txnId, stmtId, isSourceMm);
+           replicationSpec, waitOnPrecursor, table,
 -          fromURI, fs, wh, x, updatedMetadata);
++          fromURI, fs, wh, x, txnId, stmtId, isSourceMm, updatedMetadata);
      }
      return tableExists;
    }
@@@ -800,8 -734,8 +799,8 @@@
          // ensure if destination is not empty only for regular import
          Path tgtPath = new Path(table.getDataLocation().toString());
          FileSystem tgtFs = FileSystem.get(tgtPath.toUri(), x.getConf());
-         checkTargetLocationEmpty(tgtFs, tgtPath, replicationSpec, x);
+         checkTargetLocationEmpty(tgtFs, tgtPath, replicationSpec, x.getLOG());
 -        loadTable(fromURI, table, false, tgtPath, replicationSpec,x);
 +        loadTable(fromURI, table, false, tgtPath, replicationSpec, x, txnId, stmtId, isSourceMm);
        }
        // Set this to read because we can't overwrite any existing partitions
        x.getOutputs().add(new WriteEntity(table, WriteEntity.WriteType.DDL_NO_LOCK));
@@@ -838,14 -769,8 +837,14 @@@
              tablePath = wh.getDefaultTablePath(parentDb, tblDesc.getTableName());
            }
            FileSystem tgtFs = FileSystem.get(tablePath.toUri(), x.getConf());
-           checkTargetLocationEmpty(tgtFs, tablePath, replicationSpec,x);
+           checkTargetLocationEmpty(tgtFs, tablePath, replicationSpec,x.getLOG());
 -          t.addDependentTask(loadTable(fromURI, table, false, tablePath, replicationSpec, x));
 +          if (isSourceMm) { // since target table doesn't exist, it should inherit soruce table's properties
 +            Map<String, String> tblproperties = table.getParameters();
 +            tblproperties.put("transactional", "true");
 +            tblproperties.put("transactional_properties", "insert_only");
 +            table.setParameters(tblproperties);
 +          }
 +          t.addDependentTask(loadTable(fromURI, table, false, tablePath, replicationSpec, x, txnId, stmtId, isSourceMm));
          }
        }
        x.getTasks().add(t);
@@@ -866,29 -783,14 +865,15 @@@
    private static void createReplImportTasks(
        ImportTableDesc tblDesc,
        List<AddPartitionDesc> partitionDescs,
-       boolean isPartSpecSet, ReplicationSpec replicationSpec, boolean waitOnPrecursor,
+       ReplicationSpec replicationSpec, boolean waitOnPrecursor,
        Table table, URI fromURI, FileSystem fs, Warehouse wh,
-       EximUtil.SemanticAnalyzerWrapperContext x, Long txnId, int stmtId, boolean isSourceMm)
 -      EximUtil.SemanticAnalyzerWrapperContext x,
++      EximUtil.SemanticAnalyzerWrapperContext x, Long txnId, int stmtId, boolean isSourceMm,
+       UpdatedMetaDataTracker updatedMetadata)
        throws HiveException, URISyntaxException, IOException, MetaException {
  
 +    Task<?> dr = null;
      WriteEntity.WriteType lockType = WriteEntity.WriteType.DDL_NO_LOCK;
  
-     if ((table != null) && (isPartitioned(tblDesc) != table.isPartitioned())){
-       // If destination table exists, but is partitioned, and we think we're writing to an unpartitioned
-       // or if destination table exists, but is unpartitioned and we think we're writing to a partitioned
-       // table, then this can only happen because there are drops in the queue that are yet to be processed.
-       // So, we check the repl.last.id of the destination, and if it's newer, we no-op. If it's older, we
-       // drop and re-create.
-       if (replicationSpec.allowReplacementInto(table)){
-         dr = dropTableTask(table, x);
-         lockType = WriteEntity.WriteType.DDL_EXCLUSIVE;
-         table = null; // null it out so we go into the table re-create flow.
-       } else {
-         return; // noop out of here.
-       }
-     }
- 
      // Normally, on import, trying to create a table or a partition in a db that does not yet exist
      // is a error condition. However, in the case of a REPL LOAD, it is possible that we are trying
      // to create tasks to create a table inside a db that as-of-now does not exist, but there is
@@@ -942,21 -863,18 +949,18 @@@
            for (AddPartitionDesc addPartitionDesc : partitionDescs) {
              addPartitionDesc.setReplicationSpec(replicationSpec);
              t.addDependentTask(
 -                addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x));
 +                addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x, txnId, stmtId, isSourceMm, ict));
+             if (updatedMetadata != null) {
+               updatedMetadata.addPartition(addPartitionDesc.getPartition(0).getPartSpec());
+             }
            }
          } else {
            x.getLOG().debug("adding dependent CopyWork/MoveWork for table");
 -          t.addDependentTask(loadTable(fromURI, table, true, new Path(tblDesc.getLocation()),replicationSpec, x));
 +          t.addDependentTask(loadTable(fromURI, table, true, new Path(tblDesc.getLocation()), replicationSpec, x, txnId, stmtId, isSourceMm));
          }
        }
-       if (dr == null){
-         // Simply create
-         x.getTasks().add(t);
-       } else {
-         // Drop and recreate
-         dr.addDependentTask(t);
-         x.getTasks().add(dr);
-       }
+       // Simply create
+       x.getTasks().add(t);
      } else {
        // Table existed, and is okay to replicate into, not dropping and re-creating.
        if (table.isPartitioned()) {
@@@ -971,15 -887,18 +975,18 @@@
            if ((ptn = x.getHive().getPartition(table, partSpec, false)) == null) {
              if (!replicationSpec.isMetadataOnly()){
                x.getTasks().add(addSinglePartition(
 -                  fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x));
 +                  fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x, txnId, stmtId, isSourceMm, ict));
+               if (updatedMetadata != null) {
+                 updatedMetadata.addPartition(addPartitionDesc.getPartition(0).getPartSpec());
+               }
              }
            } else {
              // If replicating, then the partition already existing means we need to replace, maybe, if
              // the destination ptn's repl.last.id is older than the replacement's.
-             if (replicationSpec.allowReplacementInto(ptn)){
+             if (replicationSpec.allowReplacementInto(ptn.getParameters())){
                if (!replicationSpec.isMetadataOnly()){
                  x.getTasks().add(addSinglePartition(
 -                    fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x));
 +                    fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, x, txnId, stmtId, isSourceMm, ict));
                } else {
                  x.getTasks().add(alterSinglePartition(
                      fromURI, fs, tblDesc, table, wh, addPartitionDesc, replicationSpec, ptn, x));
@@@ -1002,13 -921,9 +1009,10 @@@
          }
        } else {
          x.getLOG().debug("table non-partitioned");
-         if (!replicationSpec.allowReplacementInto(table)){
-           return; // silently return, table is newer than our replacement.
-         }
          if (!replicationSpec.isMetadataOnly()) {
            // repl-imports are replace-into unless the event is insert-into
 -          loadTable(fromURI, table, replicationSpec.isReplace(), new Path(fromURI), replicationSpec, x);
 +          loadTable(fromURI, table, replicationSpec.isReplace(), new Path(fromURI),
 +            replicationSpec, x, txnId, stmtId, isSourceMm);
          } else {
            x.getTasks().add(alterTableTask(tblDesc, x, replicationSpec));
          }

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 9b39cd0,fa79700..5e70863
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@@ -227,6 -228,9 +229,9 @@@ public class LoadSemanticAnalyzer exten
            + " and use 'insert... select' to allow Hive to enforce bucketing. " + error);
      }
  
 -    if(AcidUtils.isAcidTable(ts.tableHandle)) {
++    if(AcidUtils.isAcidTable(ts.tableHandle) && !MetaStoreUtils.isInsertOnlyTable(ts.tableHandle.getParameters())) {
+       throw new SemanticException(ErrorMsg.LOAD_DATA_ON_ACID_TABLE, ts.tableHandle.getCompleteName());
+     }
      // make sure the arguments make sense
      List<FileStatus> files = applyConstraintsAndGetFiles(fromURI, fromTree, isLocal);
  

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 19882eb,6f379da..ab3cbe1
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@@ -6851,17 -6896,13 +6855,21 @@@ public class SemanticAnalyzer extends B
          AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID;
          if (destTableIsAcid) {
            acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
-           checkAcidConstraints(qb, table_desc, dest_tab, acidOp);
+           checkAcidConstraints(qb, table_desc, dest_tab);
          }
 -        ltd = new LoadTableDesc(queryTmpdir, table_desc, dpCtx, acidOp);
 +        if (MetaStoreUtils.isInsertOnlyTable(table_desc.getProperties())) {
 +          acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
 +        }
 +        if (isMmTable) {
 +          txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
 +        }
 +        boolean isReplace = !qb.getParseInfo().isInsertIntoTable(
 +            dest_tab.getDbName(), dest_tab.getTableName());
 +        ltd = new LoadTableDesc(queryTmpdir, table_desc, dpCtx, acidOp, isReplace, txnId);
+         // For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old
+         // deltas and base and leave them up to the cleaner to clean up
+         ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
+             dest_tab.getTableName()) && !destTableIsAcid);
          ltd.setLbCtx(lbCtx);
          loadTableWork.add(ltd);
        } else {
@@@ -6915,17 -7011,13 +6923,19 @@@
        AcidUtils.Operation acidOp = AcidUtils.Operation.NOT_ACID;
        if (destTableIsAcid) {
          acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
-         checkAcidConstraints(qb, table_desc, dest_tab, acidOp);
+         checkAcidConstraints(qb, table_desc, dest_tab);
        }
 -      ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp);
 +      if (MetaStoreUtils.isInsertOnlyTable(dest_part.getTable().getParameters())) {
 +        acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
 +      }
 +      if (isMmTable) {
 +        txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
 +      }
 +      ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp, txnId);
+       // For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old
+       // deltas and base and leave them up to the cleaner to clean up
        ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
-           dest_tab.getTableName()));
+           dest_tab.getTableName()) && !destTableIsAcid);
        ltd.setLbCtx(lbCtx);
  
        loadTableWork.add(ltd);
@@@ -7091,122 -7226,18 +7101,122 @@@
        genPartnCols(dest, input, qb, table_desc, dest_tab, rsCtx);
      }
  
 -    FileSinkDesc fileSinkDesc = new FileSinkDesc(
 -      queryTmpdir,
 -      table_desc,
 -      conf.getBoolVar(HiveConf.ConfVars.COMPRESSRESULT),
 -      currentTableId,
 -      rsCtx.isMultiFileSpray(),
 -      canBeMerged,
 -      rsCtx.getNumFiles(),
 -      rsCtx.getTotalFiles(),
 -      rsCtx.getPartnCols(),
 -      dpCtx,
 -      dest_path);
 +    FileSinkDesc fileSinkDesc = createFileSinkDesc(dest, table_desc, dest_part,
 +        dest_path, currentTableId, destTableIsAcid, destTableIsTemporary,
 +        destTableIsMaterialization, queryTmpdir, rsCtx, dpCtx, lbCtx, fsRS,
-         canBeMerged, txnId, isMmCtas);
++        canBeMerged, dest_tab, txnId, isMmCtas);
 +    if (isMmCtas) {
 +      // Add FSD so that the LoadTask compilation could fix up its path to avoid the move.
 +      tableDesc.setWriter(fileSinkDesc);
 +    }
 +
 +    if (SessionState.get().isHiveServerQuery() &&
 +      null != table_desc &&
 +      table_desc.getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()) &&
 +      HiveConf.getBoolVar(conf,HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) {
 +        fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(true);
 +    } else {
 +        fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(false);
 +    }
 +
 +    Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(
 +        fileSinkDesc, fsRS, input), inputRR);
 +
 +    handleLineage(ltd, output);
 +
 +    if (LOG.isDebugEnabled()) {
 +      LOG.debug("Created FileSink Plan for clause: " + dest + "dest_path: "
 +          + dest_path + " row schema: " + inputRR.toString());
 +    }
 +
 +    FileSinkOperator fso = (FileSinkOperator) output;
 +    fso.getConf().setTable(dest_tab);
 +    // the following code is used to collect column stats when
 +    // hive.stats.autogather=true
 +    // and it is an insert overwrite or insert into table
 +    if (dest_tab != null && conf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER)
 +        && conf.getBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER)
 +        && ColumnStatsAutoGatherContext.canRunAutogatherStats(fso)) {
 +      if (dest_type.intValue() == QBMetaData.DEST_TABLE) {
 +        genAutoColumnStatsGatheringPipeline(qb, table_desc, partSpec, input, qb.getParseInfo()
 +            .isInsertIntoTable(dest_tab.getDbName(), dest_tab.getTableName()));
 +      } else if (dest_type.intValue() == QBMetaData.DEST_PARTITION) {
 +        genAutoColumnStatsGatheringPipeline(qb, table_desc, dest_part.getSpec(), input, qb
 +            .getParseInfo().isInsertIntoTable(dest_tab.getDbName(), dest_tab.getTableName()));
 +
 +      }
 +    }
 +    return output;
 +  }
 +
 +  private ColsAndTypes deriveFileSinkColTypes(
 +      RowResolver inputRR, List<FieldSchema> field_schemas) throws SemanticException {
 +    ColsAndTypes result = new ColsAndTypes("", "");
 +    ArrayList<ColumnInfo> colInfos = inputRR.getColumnInfos();
 +    boolean first = true;
 +    for (ColumnInfo colInfo : colInfos) {
 +      String[] nm = inputRR.reverseLookup(colInfo.getInternalName());
 +
 +      if (nm[1] != null) { // non-null column alias
 +        colInfo.setAlias(nm[1]);
 +      }
 +
 +      String colName = colInfo.getInternalName();  //default column name
 +      if (field_schemas != null) {
 +        FieldSchema col = new FieldSchema();
 +        if (!("".equals(nm[0])) && nm[1] != null) {
 +          colName = unescapeIdentifier(colInfo.getAlias()).toLowerCase(); // remove ``
 +        }
 +        colName = fixCtasColumnName(colName);
 +        col.setName(colName);
 +        String typeName = colInfo.getType().getTypeName();
 +        // CTAS should NOT create a VOID type
 +        if (typeName.equals(serdeConstants.VOID_TYPE_NAME)) {
 +            throw new SemanticException(ErrorMsg.CTAS_CREATES_VOID_TYPE.getMsg(colName));
 +        }
 +        col.setType(typeName);
 +        field_schemas.add(col);
 +      }
 +
 +      if (!first) {
 +        result.cols = result.cols.concat(",");
 +        result.colTypes = result.colTypes.concat(":");
 +      }
 +
 +      first = false;
 +      result.cols = result.cols.concat(colName);
 +
 +      // Replace VOID type with string when the output is a temp table or
 +      // local files.
 +      // A VOID type can be generated under the query:
 +      //
 +      // select NULL from tt;
 +      // or
 +      // insert overwrite local directory "abc" select NULL from tt;
 +      //
 +      // where there is no column type to which the NULL value should be
 +      // converted.
 +      //
 +      String tName = colInfo.getType().getTypeName();
 +      if (tName.equals(serdeConstants.VOID_TYPE_NAME)) {
 +        result.colTypes = result.colTypes.concat(serdeConstants.STRING_TYPE_NAME);
 +      } else {
 +        result.colTypes = result.colTypes.concat(tName);
 +      }
 +    }
 +    return result;
 +  }
 +
 +  private FileSinkDesc createFileSinkDesc(String dest, TableDesc table_desc,
 +      Partition dest_part, Path dest_path, int currentTableId,
 +      boolean destTableIsAcid, boolean destTableIsTemporary,
 +      boolean destTableIsMaterialization, Path queryTmpdir,
 +      SortBucketRSCtx rsCtx, DynamicPartitionCtx dpCtx, ListBucketingCtx lbCtx,
-       RowSchema fsRS, boolean canBeMerged, Long mmWriteId, boolean isMmCtas) throws SemanticException {
++      RowSchema fsRS, boolean canBeMerged, Table dest_tab, Long mmWriteId, boolean isMmCtas) throws SemanticException {
 +    FileSinkDesc fileSinkDesc = new FileSinkDesc(queryTmpdir, table_desc,
 +        conf.getBoolVar(HiveConf.ConfVars.COMPRESSRESULT), currentTableId, rsCtx.isMultiFileSpray(),
 +        canBeMerged, rsCtx.getNumFiles(), rsCtx.getTotalFiles(), rsCtx.getPartnCols(), dpCtx,
 +        dest_path, mmWriteId, isMmCtas);
  
      boolean isHiveServerQuery = SessionState.get().isHiveServerQuery();
      fileSinkDesc.setHiveServerQuery(isHiveServerQuery);
@@@ -7448,14 -7395,12 +7456,9 @@@
      */
      conf.set(AcidUtils.CONF_ACID_KEY, "true");
  
-     if (!Operation.NOT_ACID.equals(acidOp)) {
-       if (table.getNumBuckets() < 1) {
-         throw new SemanticException(ErrorMsg.ACID_OP_ON_NONACID_TABLE, table.getTableName());
-       }
-       if (table.getSortCols() != null && table.getSortCols().size() > 0) {
-         throw new SemanticException(ErrorMsg.ACID_NO_SORTED_BUCKETS, table.getTableName());
-       }
+     if (table.getSortCols() != null && table.getSortCols().size() > 0) {
+       throw new SemanticException(ErrorMsg.ACID_NO_SORTED_BUCKETS, table.getTableName());
      }
 -
 -
 -
    }
  
    /**

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
index d0a82af,4f8ba6f..ffc2daf
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
@@@ -355,6 -396,52 +396,52 @@@ public class GenSparkUtils 
    }
  
    /**
+    * Create and add any dependent move tasks.
+    *
+    * This is forked from {@link GenMapRedUtils}. The difference is that it doesn't check
+    * 'isLinkedFileSink' and does not set parent dir for the linked file sinks.
+    */
+   public static Path createMoveTask(Task<? extends Serializable> currTask, boolean chDir,
+       FileSinkOperator fsOp, ParseContext parseCtx, List<Task<MoveWork>> mvTasks,
+       HiveConf hconf, DependencyCollectionTask dependencyTask) {
+ 
+     Path dest = null;
++    FileSinkDesc fileSinkDesc = fsOp.getConf();
+ 
+     if (chDir) {
+       dest = fsOp.getConf().getFinalDirName();
+ 
+       // generate the temporary file
+       // it must be on the same file system as the current destination
+       Context baseCtx = parseCtx.getContext();
+ 
+       Path tmpDir = baseCtx.getExternalTmpPath(dest);
+ 
 -      FileSinkDesc fileSinkDesc = fsOp.getConf();
+       // Change all the linked file sink descriptors
+       if (fileSinkDesc.getLinkedFileSinkDesc() != null) {
+         for (FileSinkDesc fsConf : fileSinkDesc.getLinkedFileSinkDesc()) {
+           fsConf.setDirName(tmpDir);
+         }
+       } else {
+         fileSinkDesc.setDirName(tmpDir);
+       }
+     }
+ 
+     Task<MoveWork> mvTask = null;
+ 
+     if (!chDir) {
 -      mvTask = GenMapRedUtils.findMoveTask(mvTasks, fsOp);
++      mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(mvTasks, fileSinkDesc.getFinalDirName(), false);
+     }
+ 
+     // Set the move task to be dependent on the current task
+     if (mvTask != null) {
+       GenMapRedUtils.addDependentMoveTasks(mvTask, hconf, currTask, dependencyTask);
+     }
+ 
+     return dest;
+   }
+ 
+   /**
     * Populate partition pruning information from the pruning sink operator to the
     * target MapWork (the MapWork for the big table side). The information include the source table
     * name, column name, and partition key expression. It also set up the temporary path used to

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index 955e0e5,a3df166..4732f0a
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@@ -20,9 -20,9 +20,10 @@@ package org.apache.hadoop.hive.ql.plan
  
  import java.util.ArrayList;
  import java.util.List;
+ import java.util.Objects;
  
  import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.hive.metastore.MetaStoreUtils;
  import org.apache.hadoop.hive.ql.io.AcidUtils;
  import org.apache.hadoop.hive.ql.metadata.Table;
  import org.apache.hadoop.hive.ql.plan.Explain.Level;

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 3600a6b,0f129fc..5a8ac31
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@@ -487,79 -439,7 +439,6 @@@ public class TestTxnCommands extends Te
      }
    }
  
-   /**
-    * takes raw data and turns it into a string as if from Driver.getResults()
-    * sorts rows in dictionary order
-    */
-   private List<String> stringifyValues(int[][] rowsIn) {
-     assert rowsIn.length > 0;
-     int[][] rows = rowsIn.clone();
-     Arrays.sort(rows, new RowComp());
-     List<String> rs = new ArrayList<String>();
-     for(int[] row : rows) {
-       assert row.length > 0;
-       StringBuilder sb = new StringBuilder();
-       for(int value : row) {
-         sb.append(value).append("\t");
-       }
-       sb.setLength(sb.length() - 1);
-       rs.add(sb.toString());
-     }
-     return rs;
-   }
-   private static final class RowComp implements Comparator<int[]> {
-     @Override
-     public int compare(int[] row1, int[] row2) {
-       assert row1 != null && row2 != null && row1.length == row2.length;
-       for(int i = 0; i < row1.length; i++) {
-         int comp = Integer.compare(row1[i], row2[i]);
-         if(comp != 0) {
-           return comp;
-         }
-       }
-       return 0;
-     }
-   }
-   private String makeValuesClause(int[][] rows) {
-     assert rows.length > 0;
-     StringBuilder sb = new StringBuilder("values");
-     for(int[] row : rows) {
-       assert row.length > 0;
-       if(row.length > 1) {
-         sb.append("(");
-       }
-       for(int value : row) {
-         sb.append(value).append(",");
-       }
-       sb.setLength(sb.length() - 1);//remove trailing comma
-       if(row.length > 1) {
-         sb.append(")");
-       }
-       sb.append(",");
-     }
-     sb.setLength(sb.length() - 1);//remove trailing comma
-     return sb.toString();
-   }
- 
-   private List<String> runStatementOnDriver(String stmt) throws Exception {
-     LOG.info("Running " + stmt);
-     CommandProcessorResponse cpr = d.run(stmt);
-     if(cpr.getResponseCode() != 0) {
-       throw new RuntimeException(stmt + " failed: " + cpr);
-     }
-     List<String> rs = new ArrayList<String>();
-     d.getResults(rs);
-     return rs;
-   }
-   private CommandProcessorResponse runStatementOnDriverNegative(String stmt) throws Exception {
-     CommandProcessorResponse cpr = d.run(stmt);
-     if(cpr.getResponseCode() != 0) {
-       return cpr;
-     }
-     throw new RuntimeException("Didn't get expected failure!");
-   }
--
- //  @Ignore
    @Test
    public void exchangePartition() throws Exception {
      runStatementOnDriver("create database ex1");

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 7a73a17,21b4a2c..bd6e6a0
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@@ -85,16 -84,16 +84,17 @@@ public class TestTxnCommands2 
  
    protected HiveConf hiveConf;
    protected Driver d;
-   protected static enum Table {
+   protected enum Table {
      ACIDTBL("acidTbl"),
-     ACIDTBLPART("acidTblPart"),
+     ACIDTBLPART("acidTblPart", "p"),
      NONACIDORCTBL("nonAcidOrcTbl"),
-     NONACIDPART("nonAcidPart"),
-     NONACIDPART2("nonAcidPart2"),
-     ACIDNESTEDPART("acidNestedPart"),
+     NONACIDPART("nonAcidPart", "p"),
+     NONACIDPART2("nonAcidPart2", "p2"),
 -    ACIDNESTEDPART("acidNestedPart", "p,q");
++    ACIDNESTEDPART("acidNestedPart", "p,q"),
 +    MMTBL("mmTbl");
  
      private final String name;
+     private final String partitionColumns;
      @Override
      public String toString() {
        return name;
@@@ -738,14 -841,13 +843,13 @@@
      // There should be only 1 directory left: base_0000001.
      // Original bucket files, delta directories and previous base directory should have been cleaned up.
      status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
-         (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+       (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
      Assert.assertEquals(1, status.length);
 -    Assert.assertEquals("base_0000023", status[0].getPath().getName());
 +    Assert.assertEquals("base_0000025", status[0].getPath().getName());
      FileStatus[] buckets = fs.listStatus(status[0].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
      Arrays.sort(buckets);
-     Assert.assertEquals(BUCKET_COUNT, buckets.length);
-     Assert.assertEquals("bucket_00000", buckets[0].getPath().getName());
-     Assert.assertEquals("bucket_00001", buckets[1].getPath().getName());
+     Assert.assertEquals(1, buckets.length);
+     Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
      rs = runStatementOnDriver("select a,b from " + Table.NONACIDORCTBL);
      resultData = new int[][] {{1, 3}, {3, 4}};
      Assert.assertEquals(stringifyValues(resultData), rs);
@@@ -1643,65 -1755,241 +1757,299 @@@
      Assert.assertEquals(stringifyValues(rExpected), r);
    }
  
+   @Test
+   public void testBucketCodec() throws Exception {
+     d.destroy();
+     //insert data in "legacy" format
+     hiveConf.setIntVar(HiveConf.ConfVars.TESTMODE_BUCKET_CODEC_VERSION, 0);
+     d = new Driver(hiveConf);
+ 
+     int[][] targetVals = {{2,1},{4,3},{5,6},{7,8}};
+     runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(targetVals));
+ 
+     d.destroy();
+     hiveConf.setIntVar(HiveConf.ConfVars.TESTMODE_BUCKET_CODEC_VERSION, 1);
+     d = new Driver(hiveConf);
+     //do some operations with new format
+     runStatementOnDriver("update " + Table.ACIDTBL + " set b=11 where a in (5,7)");
+     runStatementOnDriver("insert into " + Table.ACIDTBL + " values(11,11)");
+     runStatementOnDriver("delete from " + Table.ACIDTBL + " where a = 7");
+ 
+     //make sure we get the right data back before/after compactions
+     List<String> r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
+     int[][] rExpected = {{2,1},{4,3},{5,11},{11,11}};
+     Assert.assertEquals(stringifyValues(rExpected), r);
+ 
+     runStatementOnDriver("ALTER TABLE " + Table.ACIDTBL + " COMPACT 'MINOR'");
+     runWorker(hiveConf);
+ 
+     r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
+     Assert.assertEquals(stringifyValues(rExpected), r);
+ 
+     runStatementOnDriver("ALTER TABLE " + Table.ACIDTBL + " COMPACT 'MAJOR'");
+     runWorker(hiveConf);
+ 
+     r = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
+     Assert.assertEquals(stringifyValues(rExpected), r);
+   }
+   /**
+    * Test the scenario when IOW comes in before a MAJOR compaction happens
+    * @throws Exception
+    */
+   @Test
+   public void testInsertOverwrite1() throws Exception {
+     FileSystem fs = FileSystem.get(hiveConf);
+     FileStatus[] status;
+ 
+     // 1. Insert two rows to an ACID table
+     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)");
+     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(3,4)");
+     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+         (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+     // There should be 2 delta dirs in the location
+     Assert.assertEquals(2, status.length);
+     for (int i = 0; i < status.length; i++) {
+       Assert.assertTrue(status[i].getPath().getName().matches("delta_.*"));
+     }
+ 
+     // 2. INSERT OVERWRITE
+     // Prepare data for the source table
+     runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(5,6),(7,8)");
+     // Insert overwrite ACID table from source table
+     runStatementOnDriver("insert overwrite table " + Table.ACIDTBL + " select a,b from " + Table.NONACIDORCTBL);
+     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+         (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+     // There should be 2 delta dirs, plus a base dir in the location
+     Assert.assertEquals(3, status.length);
+     boolean sawBase = false;
+     String baseDir = "";
+     int deltaCount = 0;
+     for (int i = 0; i < status.length; i++) {
+       String dirName = status[i].getPath().getName();
+       if (dirName.matches("delta_.*")) {
+         deltaCount++;
+       } else {
+         sawBase = true;
+         baseDir = dirName;
+         Assert.assertTrue(baseDir.matches("base_.*"));
+       }
+     }
+     Assert.assertEquals(2, deltaCount);
+     Assert.assertTrue(sawBase);
+     // Verify query result
+     List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+     int [][] resultData = new int[][] {{5,6},{7,8}};
+     Assert.assertEquals(stringifyValues(resultData), rs);
+ 
+     // 3. Perform a major compaction. Nothing should change. Both deltas and base dirs should have the same name.
+     // Re-verify directory layout and query result by using the same logic as above
+     runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'");
+     runWorker(hiveConf);
+     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+         (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+     // There should be 2 delta dirs, plus a base dir in the location
+     Assert.assertEquals(3, status.length);
+     sawBase = false;
+     deltaCount = 0;
+     for (int i = 0; i < status.length; i++) {
+       String dirName = status[i].getPath().getName();
+       if (dirName.matches("delta_.*")) {
+         deltaCount++;
+       } else {
+         sawBase = true;
+         Assert.assertTrue(dirName.matches("base_.*"));
+         Assert.assertEquals(baseDir, dirName);
+       }
+     }
+     Assert.assertEquals(2, deltaCount);
+     Assert.assertTrue(sawBase);
+     // Verify query result
+     rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+     Assert.assertEquals(stringifyValues(resultData), rs);
+ 
+     // 4. Run Cleaner. It should remove the 2 delta dirs.
+     runCleaner(hiveConf);
+     // There should be only 1 directory left: base_xxxxxxx.
+     // The delta dirs should have been cleaned up.
+     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+         (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+     Assert.assertEquals(1, status.length);
+     Assert.assertTrue(status[0].getPath().getName().matches("base_.*"));
+     Assert.assertEquals(baseDir, status[0].getPath().getName());
+     // Verify query result
+     rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+     Assert.assertEquals(stringifyValues(resultData), rs);
+   }
+ 
+   /**
+    * Test the scenario when IOW comes in after a MAJOR compaction happens
+    * @throws Exception
+    */
+   @Test
+   public void testInsertOverwrite2() throws Exception {
+     FileSystem fs = FileSystem.get(hiveConf);
+     FileStatus[] status;
+ 
+     // 1. Insert two rows to an ACID table
+     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)");
+     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(3,4)");
+     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+         (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+     // There should be 2 delta dirs in the location
+     Assert.assertEquals(2, status.length);
+     for (int i = 0; i < status.length; i++) {
+       Assert.assertTrue(status[i].getPath().getName().matches("delta_.*"));
+     }
+ 
+     // 2. Perform a major compaction. There should be an extra base dir now.
+     runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'");
+     runWorker(hiveConf);
+     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+         (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+     // There should be 2 delta dirs, plus a base dir in the location
+     Assert.assertEquals(3, status.length);
+     boolean sawBase = false;
+     int deltaCount = 0;
+     for (int i = 0; i < status.length; i++) {
+       String dirName = status[i].getPath().getName();
+       if (dirName.matches("delta_.*")) {
+         deltaCount++;
+       } else {
+         sawBase = true;
+         Assert.assertTrue(dirName.matches("base_.*"));
+       }
+     }
+     Assert.assertEquals(2, deltaCount);
+     Assert.assertTrue(sawBase);
+     // Verify query result
+     int [][] resultData = new int[][] {{1,2},{3,4}};
+     List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+     Assert.assertEquals(stringifyValues(resultData), rs);
+ 
+     // 3. INSERT OVERWRITE
+     // Prepare data for the source table
+     runStatementOnDriver("insert into " + Table.NONACIDORCTBL + "(a,b) values(5,6),(7,8)");
+     // Insert overwrite ACID table from source table
+     runStatementOnDriver("insert overwrite table " + Table.ACIDTBL + " select a,b from " + Table.NONACIDORCTBL);
+     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+         (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+     // There should be 2 delta dirs, plus 2 base dirs in the location
+     Assert.assertEquals(4, status.length);
+     int baseCount = 0;
+     deltaCount = 0;
+     for (int i = 0; i < status.length; i++) {
+       String dirName = status[i].getPath().getName();
+       if (dirName.matches("delta_.*")) {
+         deltaCount++;
+       } else {
+         baseCount++;
+       }
+     }
+     Assert.assertEquals(2, deltaCount);
+     Assert.assertEquals(2, baseCount);
+     // Verify query result
+     resultData = new int[][] {{5,6},{7,8}};
+     rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+     Assert.assertEquals(stringifyValues(resultData), rs);
+ 
+     // 4. Perform another major compaction. Nothing should change. Both deltas and  both base dirs
+     // should have the same name.
+     // Re-verify directory layout and query result by using the same logic as above
+     runStatementOnDriver("alter table "+ Table.ACIDTBL + " compact 'MAJOR'");
+     runWorker(hiveConf);
+     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+         (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+     // There should be 2 delta dirs, plus 2 base dirs in the location
+     Assert.assertEquals(4, status.length);
+     baseCount = 0;
+     deltaCount = 0;
+     for (int i = 0; i < status.length; i++) {
+       String dirName = status[i].getPath().getName();
+       if (dirName.matches("delta_.*")) {
+         deltaCount++;
+       } else {
+         Assert.assertTrue(dirName.matches("base_.*"));
+         baseCount++;
+       }
+     }
+     Assert.assertEquals(2, deltaCount);
+     Assert.assertEquals(2, baseCount);
+     // Verify query result
+     rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+     Assert.assertEquals(stringifyValues(resultData), rs);
+ 
+     // 5. Run Cleaner. It should remove the 2 delta dirs and 1 old base dir.
+     runCleaner(hiveConf);
+     // There should be only 1 directory left: base_xxxxxxx.
+     // The delta dirs should have been cleaned up.
+     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+         (Table.ACIDTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+     Assert.assertEquals(1, status.length);
+     Assert.assertTrue(status[0].getPath().getName().matches("base_.*"));
+     // Verify query result
+     rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a");
+     Assert.assertEquals(stringifyValues(resultData), rs);
+   }
+ 
    /**
 +   * Test compaction for Micro-managed table
 +   * 1. Regular compaction shouldn't impact any valid subdirectories of MM tables
 +   * 2. Compactions will only remove subdirectories for aborted transactions of MM tables, if any
 +   * @throws Exception
 +   */
 +  @Test
 +  public void testMmTableCompaction() throws Exception {
 +    // 1. Insert some rows into MM table
 +    runStatementOnDriver("insert into " + Table.MMTBL + "(a,b) values(1,2)");
 +    runStatementOnDriver("insert into " + Table.MMTBL + "(a,b) values(3,4)");
 +    // There should be 2 delta directories
 +    verifyDirAndResult(2);
 +
 +    // 2. Perform a MINOR compaction. Since nothing was aborted, subdirs should stay.
 +    runStatementOnDriver("alter table "+ Table.MMTBL + " compact 'MINOR'");
 +    runWorker(hiveConf);
 +    verifyDirAndResult(2);
 +
 +    // 3. Let a transaction be aborted
 +    hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true);
 +    runStatementOnDriver("insert into " + Table.MMTBL + "(a,b) values(5,6)");
 +    hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false);
 +    // There should be 3 delta directories. The new one is the aborted one.
 +    verifyDirAndResult(3);
 +
 +    // 4. Perform a MINOR compaction again. This time it will remove the subdir for aborted transaction.
 +    runStatementOnDriver("alter table "+ Table.MMTBL + " compact 'MINOR'");
 +    runWorker(hiveConf);
 +    // The worker should remove the subdir for aborted transaction
 +    verifyDirAndResult(2);
 +
 +    // 5. Run Cleaner. Shouldn't impact anything.
 +    runCleaner(hiveConf);
 +    verifyDirAndResult(2);
 +  }
 +
 +  private void verifyDirAndResult(int expectedDeltas) throws Exception {
 +    FileSystem fs = FileSystem.get(hiveConf);
 +    // Verify the content of subdirs
 +    FileStatus[] status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
 +        (Table.MMTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
 +    int sawDeltaTimes = 0;
 +    for (int i = 0; i < status.length; i++) {
 +      Assert.assertTrue(status[i].getPath().getName().matches("delta_.*"));
 +      sawDeltaTimes++;
 +      FileStatus[] files = fs.listStatus(status[i].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
 +      Assert.assertEquals(1, files.length);
 +      Assert.assertTrue(files[0].getPath().getName().equals("000000_0"));
 +    }
 +    Assert.assertEquals(expectedDeltas, sawDeltaTimes);
 +
 +    // Verify query result
 +    int [][] resultData = new int[][] {{1,2}, {3,4}};
 +    List<String> rs = runStatementOnDriver("select a,b from " + Table.MMTBL);
 +    Assert.assertEquals(stringifyValues(resultData), rs);
 +  }
 +
 +  /**
     * takes raw data and turns it into a string as if from Driver.getResults()
     * sorts rows in dictionary order
     */

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index 6b19eb1,4c30732..ccd7d8e
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@@ -40,10 -46,15 +46,7 @@@ import java.util.TreeSet
  
  import org.apache.commons.codec.binary.Base64;
  import org.apache.hadoop.conf.Configuration;
 -import org.apache.hadoop.fs.BlockLocation;
 -import org.apache.hadoop.fs.FSDataInputStream;
--import org.apache.hadoop.fs.FSDataOutputStream;
 -import org.apache.hadoop.fs.FSInputStream;
--import org.apache.hadoop.fs.FileStatus;
--import org.apache.hadoop.fs.FileSystem;
 -import org.apache.hadoop.fs.LocatedFileStatus;
--import org.apache.hadoop.fs.Path;
 -import org.apache.hadoop.fs.RemoteIterator;
++import org.apache.hadoop.fs.*;
  import org.apache.hadoop.fs.permission.FsPermission;
  import org.apache.hadoop.hive.common.ValidTxnList;
  import org.apache.hadoop.hive.common.type.HiveDecimal;

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/queries/clientpositive/mm_all.q
----------------------------------------------------------------------
diff --cc ql/src/test/queries/clientpositive/mm_all.q
index 6001e9f,0000000..e23260f
mode 100644,000000..100644
--- a/ql/src/test/queries/clientpositive/mm_all.q
+++ b/ql/src/test/queries/clientpositive/mm_all.q
@@@ -1,370 -1,0 +1,325 @@@
 +set hive.mapred.mode=nonstrict;
 +set hive.explain.user=false;
 +set hive.fetch.task.conversion=none;
 +set tez.grouping.min-size=1;
 +set tez.grouping.max-size=2;
 +set hive.exec.dynamic.partition.mode=nonstrict;
 +set hive.support.concurrency=true;
 +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
 +
 +
 +-- Force multiple writers when reading
 +drop table intermediate;
 +create table intermediate(key int) partitioned by (p int) stored as orc;
 +insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2;
 +insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2;
 +insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2;
 +
 +
 +drop table part_mm;
 +create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only");
 +explain insert into table part_mm partition(key_mm=455) select key from intermediate;
 +insert into table part_mm partition(key_mm=455) select key from intermediate;
 +insert into table part_mm partition(key_mm=456) select key from intermediate;
 +insert into table part_mm partition(key_mm=455) select key from intermediate;
 +select * from part_mm order by key, key_mm;
 +
 +-- TODO: doesn't work truncate table part_mm partition(key_mm=455);
 +select * from part_mm order by key, key_mm;
 +truncate table part_mm;
 +select * from part_mm order by key, key_mm;
 +drop table part_mm;
 +
 +drop table simple_mm;
 +create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only");
 +insert into table simple_mm select key from intermediate;
 +select * from simple_mm order by key;
 +insert into table simple_mm select key from intermediate;
 +select * from simple_mm order by key;
 +truncate table simple_mm;
 +select * from simple_mm;
 +drop table simple_mm;
 +
 +
 +-- simple DP (no bucketing)
 +drop table dp_mm;
 +
 +set hive.exec.dynamic.partition.mode=nonstrict;
 +
 +set hive.merge.mapredfiles=false;
 +set hive.merge.sparkfiles=false;
 +set hive.merge.tezfiles=false;
 +
 +create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
 +  tblproperties ("transactional"="true", "transactional_properties"="insert_only");
 +
 +insert into table dp_mm partition (key1='123', key2) select key, key from intermediate;
 +
 +select * from dp_mm order by key;
 +
 +drop table dp_mm;
 +
 +
 +-- union
 +
 +create table union_mm(id int)  tblproperties ("transactional"="true", "transactional_properties"="insert_only");
 +insert into table union_mm 
 +select temps.p from (
 +select key as p from intermediate 
 +union all 
 +select key + 1 as p from intermediate ) temps;
 +
 +select * from union_mm order by id;
 +
 +insert into table union_mm 
 +select p from
 +(
 +select key + 1 as p from intermediate
 +union all
 +select key from intermediate
 +) tab group by p
 +union all
 +select key + 2 as p from intermediate;
 +
 +select * from union_mm order by id;
 +
 +insert into table union_mm
 +SELECT p FROM
 +(
 +  SELECT key + 1 as p FROM intermediate
 +  UNION ALL
 +  SELECT key as p FROM ( 
 +    SELECT distinct key FROM (
 +      SELECT key FROM (
 +        SELECT key + 2 as key FROM intermediate
 +        UNION ALL
 +        SELECT key FROM intermediate
 +      )t1 
 +    group by key)t2
 +  )t3
 +)t4
 +group by p;
 +
 +
 +select * from union_mm order by id;
 +drop table union_mm;
 +
 +
 +create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only");
 +insert into table partunion_mm partition(key)
 +select temps.* from (
 +select key as p, key from intermediate 
 +union all 
 +select key + 1 as p, key + 1 from intermediate ) temps;
 +
 +select * from partunion_mm order by id;
 +drop table partunion_mm;
 +
 +
 +
 +create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
 + stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only");
 +
 +insert into table skew_mm 
 +select key, key, key from intermediate;
 +
 +select * from skew_mm order by k2, k1, k4;
 +drop table skew_mm;
 +
 +
 +create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) 
 +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only");
 +
 +insert into table skew_dp_union_mm partition (k3)
 +select key as i, key as j, key as k, key as l from intermediate
 +union all 
 +select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate;
 +
 +
 +select * from skew_dp_union_mm order by k2, k1, k4;
 +drop table skew_dp_union_mm;
 +
 +
 +
 +set hive.merge.orcfile.stripe.level=true;
 +set hive.merge.tezfiles=true;
 +set hive.merge.mapfiles=true;
 +set hive.merge.mapredfiles=true;
 +
 +
 +create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only");
 +
 +insert into table merge0_mm select key from intermediate;
 +select * from merge0_mm;
 +
 +set tez.grouping.split-count=1;
 +insert into table merge0_mm select key from intermediate;
 +set tez.grouping.split-count=0;
 +select * from merge0_mm;
 +
 +drop table merge0_mm;
 +
 +
 +create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
 +
 +insert into table merge2_mm select key from intermediate;
 +select * from merge2_mm;
 +
 +set tez.grouping.split-count=1;
 +insert into table merge2_mm select key from intermediate;
 +set tez.grouping.split-count=0;
 +select * from merge2_mm;
 +
 +drop table merge2_mm;
 +
 +
 +create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only");
 +
 +insert into table merge1_mm partition (key) select key, key from intermediate;
 +select * from merge1_mm order by id, key;
 +
 +set tez.grouping.split-count=1;
 +insert into table merge1_mm partition (key) select key, key from intermediate;
 +set tez.grouping.split-count=0;
 +select * from merge1_mm order by id, key;
 +
 +drop table merge1_mm;
 +
 +set hive.merge.tezfiles=false;
 +set hive.merge.mapfiles=false;
 +set hive.merge.mapredfiles=false;
 +
 +-- TODO: need to include merge+union+DP, but it's broken for now
 +
 +
 +drop table ctas0_mm;
 +create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate;
 +select * from ctas0_mm;
 +drop table ctas0_mm;
 +
 +drop table ctas1_mm;
 +create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
 +  select * from intermediate union all select * from intermediate;
 +select * from ctas1_mm;
 +drop table ctas1_mm;
 +
 +
- drop table load0_mm;
- create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
- load data local inpath '../../data/files/kv1.txt' into table load0_mm;
- select count(1) from load0_mm;
- load data local inpath '../../data/files/kv2.txt' into table load0_mm;
- select count(1) from load0_mm;
- load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm;
- select count(1) from load0_mm;
- drop table load0_mm;
- 
- 
- drop table intermediate2;
- create table intermediate2 (key string, value string) stored as textfile
- location 'file:${system:test.tmp.dir}/intermediate2';
- load data local inpath '../../data/files/kv1.txt' into table intermediate2;
- load data local inpath '../../data/files/kv2.txt' into table intermediate2;
- load data local inpath '../../data/files/kv3.txt' into table intermediate2;
- 
- drop table load1_mm;
- create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
- load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' into table load1_mm;
- load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv1.txt' into table load1_mm;
- select count(1) from load1_mm;
- load data local inpath '../../data/files/kv1.txt' into table intermediate2;
- load data local inpath '../../data/files/kv2.txt' into table intermediate2;
- load data local inpath '../../data/files/kv3.txt' into table intermediate2;
- load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' overwrite into table load1_mm;
- select count(1) from load1_mm;
- load data local inpath '../../data/files/kv2.txt' into table intermediate2;
- load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' overwrite into table load1_mm;
- select count(1) from load1_mm;
- drop table load1_mm;
- 
- drop table load2_mm;
- create table load2_mm (key string, value string)
-   partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
- load data local inpath '../../data/files/kv1.txt' into table intermediate2;
- load data local inpath '../../data/files/kv2.txt' into table intermediate2;
- load data local inpath '../../data/files/kv3.txt' into table intermediate2;
- load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' into table load2_mm partition(k=5, l=5);
- select count(1) from load2_mm;
- drop table load2_mm;
- drop table intermediate2;
- 
- 
 +drop table multi0_1_mm;
 +drop table multi0_2_mm;
 +create table multi0_1_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only");
 +create table multi0_2_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only");
 +
 +--from intermediate
 +--insert overwrite table multi0_1_mm select key, p
 +--insert overwrite table multi0_2_mm select p, key;
 +insert into table multi0_1_mm select key, p from intermediate;
 +insert into table multi0_2_mm select p, key from intermediate;
 +
 +select * from multi0_1_mm order by key, key2;
 +select * from multi0_2_mm order by key, key2;
 +
 +set hive.merge.mapredfiles=true;
 +set hive.merge.sparkfiles=true;
 +set hive.merge.tezfiles=true;
 +
 +--from intermediate
 +--insert into table multi0_1_mm select p, key
 +--insert overwrite table multi0_2_mm select key, p;
 +insert into table multi0_1_mm select p, key from intermediate;
 +insert into table multi0_2_mm select key, p from intermediate;
 +select * from multi0_1_mm order by key, key2;
 +select * from multi0_2_mm order by key, key2;
 +
 +set hive.merge.mapredfiles=false;
 +set hive.merge.sparkfiles=false;
 +set hive.merge.tezfiles=false;
 +
 +drop table multi0_1_mm;
 +drop table multi0_2_mm;
 +
 +
 +drop table multi1_mm;
 +create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only");
 +from intermediate
 +insert into table multi1_mm partition(p=1) select p, key
 +insert into table multi1_mm partition(p=2) select key, p;
 +select * from multi1_mm order by key, key2, p;
 +--from intermediate
 +--insert into table multi1_mm partition(p=2) select p, key
 +--insert overwrite table multi1_mm partition(p=1) select key, p;
 +insert into table multi1_mm partition(p=2) select p, key from intermediate;
 +insert into table multi1_mm partition(p=1) select key, p from intermediate;
 +select * from multi1_mm order by key, key2, p;
 +
 +from intermediate
 +insert into table multi1_mm partition(p) select p, key, p
 +insert into table multi1_mm partition(p=1) select key, p;
 +select key, key2, p from multi1_mm order by key, key2, p;
 +
 +from intermediate
 +insert into table multi1_mm partition(p) select p, key, 1
 +insert into table multi1_mm partition(p=1) select key, p;
 +select key, key2, p from multi1_mm order by key, key2, p;
 +drop table multi1_mm;
 +
 +
 +
 +
 +set datanucleus.cache.collections=false;
 +set hive.stats.autogather=true;
 +
 +drop table stats_mm;
 +create table stats_mm(key int)  tblproperties("transactional"="true", "transactional_properties"="insert_only");
 +--insert overwrite table stats_mm  select key from intermediate;
 +insert into table stats_mm  select key from intermediate;
 +desc formatted stats_mm;
 +
 +insert into table stats_mm  select key from intermediate;
 +desc formatted stats_mm;
 +drop table stats_mm;
 +
 +drop table stats2_mm;
 +create table stats2_mm tblproperties("transactional"="true", "transactional_properties"="insert_only") as select array(key, value) from src;
 +desc formatted stats2_mm;
 +drop table stats2_mm;
 +
 +
 +set hive.optimize.skewjoin=true;
 +set hive.skewjoin.key=2;
 +set hive.optimize.metadataonly=false;
 +
 +CREATE TABLE skewjoin_mm(key INT, value STRING) STORED AS TEXTFILE tblproperties ("transactional"="true", "transactional_properties"="insert_only");
 +FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT into TABLE skewjoin_mm SELECT src1.key, src2.value;
 +select count(distinct key) from skewjoin_mm;
 +drop table skewjoin_mm;
 +
 +set hive.optimize.skewjoin=false;
 +
 +set hive.optimize.index.filter=true;
 +set hive.auto.convert.join=false;
 +CREATE TABLE parquet1_mm(id INT) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only");
 +INSERT INTO parquet1_mm VALUES(1), (2);
 +CREATE TABLE parquet2_mm(id INT, value STRING) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only");
 +INSERT INTO parquet2_mm VALUES(1, 'value1');
 +INSERT INTO parquet2_mm VALUES(1, 'value2');
 +select parquet1_mm.id, t1.value, t2.value FROM parquet1_mm
 +  JOIN parquet2_mm t1 ON parquet1_mm.id=t1.id
 +  JOIN parquet2_mm t2 ON parquet1_mm.id=t2.id
 +where t1.value = 'value1' and t2.value = 'value2';
 +drop table parquet1_mm;
 +drop table parquet2_mm;
 +
 +set hive.auto.convert.join=true;
 +
 +
 +DROP TABLE IF EXISTS temp1;
 +CREATE TEMPORARY TABLE temp1 (a int) TBLPROPERTIES ("transactional"="true", "transactional_properties"="insert_only");
 +INSERT INTO temp1 SELECT key FROM intermediate;
 +DESC EXTENDED temp1;
 +SELECT * FROM temp1;
 +
 +
 +drop table intermediate;
 +
 +
 +

[28/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7a40541f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7a40541f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7a40541f

Branch: refs/heads/master
Commit: 7a40541f5f2d302e4577c5c57b4b0b7e1ae13fa8
Parents: e3fb116 a51ae9c
Author: sergey <se...@apache.org>
Authored: Sun Sep 17 17:15:13 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Sun Sep 17 17:15:13 2017 -0700

----------------------------------------------------------------------
 .../hive/hcatalog/common/HCatContext.java       |   2 +-
 .../hive/hcatalog/common/HCatException.java     |   5 +
 .../apache/hive/hcatalog/data/HCatRecord.java   |   2 +-
 .../hcatalog/data/schema/HCatFieldSchema.java   |   2 +-
 .../hive/hcatalog/data/schema/HCatSchema.java   |   2 +-
 .../hcatalog/mapreduce/HCatInputFormat.java     |   2 +-
 .../hcatalog/mapreduce/HCatOutputFormat.java    |   2 +-
 .../hive/hcatalog/mapreduce/OutputJobInfo.java  |   4 +
 .../apache/hive/hcatalog/pig/HCatLoader.java    |   2 +-
 .../apache/hive/hcatalog/pig/HCatStorer.java    |   2 +-
 .../hcatalog/messaging/AddPartitionMessage.java |   5 +
 .../hcatalog/messaging/AlterIndexMessage.java   |   7 +-
 .../messaging/AlterPartitionMessage.java        |   5 +
 .../hcatalog/messaging/AlterTableMessage.java   |   5 +
 .../messaging/CreateDatabaseMessage.java        |   5 +
 .../messaging/CreateFunctionMessage.java        |   5 +
 .../hcatalog/messaging/CreateIndexMessage.java  |   7 +-
 .../hcatalog/messaging/CreateTableMessage.java  |   5 +
 .../hcatalog/messaging/DropDatabaseMessage.java |   5 +
 .../hcatalog/messaging/DropFunctionMessage.java |   7 +-
 .../hcatalog/messaging/DropIndexMessage.java    |   7 +-
 .../messaging/DropPartitionMessage.java         |   5 +
 .../hcatalog/messaging/DropTableMessage.java    |   5 +
 .../hcatalog/messaging/HCatEventMessage.java    |   4 +
 .../hive/hcatalog/messaging/InsertMessage.java  |   5 +
 .../hcatalog/messaging/MessageDeserializer.java |   4 +
 .../hive/hcatalog/messaging/MessageFactory.java |   4 +
 .../hive/hcatalog/api/HCatAddPartitionDesc.java |   4 +
 .../apache/hive/hcatalog/api/HCatClient.java    |   2 +
 .../hive/hcatalog/api/HCatCreateDBDesc.java     |   4 +
 .../hive/hcatalog/api/HCatCreateTableDesc.java  |   4 +
 .../apache/hive/hcatalog/api/HCatDatabase.java  |   4 +
 .../hcatalog/api/HCatNotificationEvent.java     |   4 +
 .../apache/hive/hcatalog/api/HCatPartition.java |   4 +
 .../org/apache/hive/hcatalog/api/HCatTable.java |   4 +
 .../hive/hcatalog/api/MetadataSerializer.java   |   2 +
 .../apache/hive/hcatalog/api/repl/Command.java  |   4 +
 .../hive/hcatalog/api/repl/ReplicationTask.java |   4 +-
 .../java/org/apache/hadoop/hive/ql/Context.java |  12 +-
 .../stats/annotation/StatsRulesProcFactory.java |  36 ++++-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |   2 +-
 .../queries/clientpositive/acid_view_delete.q   |  35 +++++
 .../clientpositive/acid_view_delete.q.out       | 141 +++++++++++++++++++
 43 files changed, 363 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/7a40541f/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/7a40541f/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/7a40541f/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------

[25/50] [abbrv] hive git commit: Merge branch 'master' into hive-14535

Posted by se...@apache.org.

Merge branch 'master' into hive-14535


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9a9f2540
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9a9f2540
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9a9f2540

Branch: refs/heads/master
Commit: 9a9f25409f4db9b3798aa529bb5531264cecd282
Parents: bb1c0f2 50fb6f3
Author: sergey <se...@apache.org>
Authored: Fri Sep 15 12:47:20 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Sep 15 12:47:20 2017 -0700

----------------------------------------------------------------------
 .gitignore                                      |    1 +
 .../org/apache/hive/beeline/HiveSchemaTool.java |    5 +-
 .../apache/hive/beeline/TestHiveSchemaTool.java |    9 +-
 .../common/classification/RetrySemantics.java   |   54 -
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   34 +-
 .../listener/DbNotificationListener.java        |   14 +-
 .../hive/hcatalog/streaming/TestStreaming.java  |    2 +-
 .../src/test/queries/clientpositive/load_data.q |   20 +
 .../test/results/clientpositive/load_data.q.out |   59 +
 .../hadoop/hive/metastore/TestFilterHooks.java  |    3 +-
 .../hive/ql/parse/TestReplicationScenarios.java |  124 +-
 ...TestReplicationScenariosAcrossInstances.java |    9 +-
 .../hive/ql/txn/compactor/TestCompactor.java    |   40 +-
 .../test/resources/testconfiguration.properties |    1 +
 .../impl/LlapZookeeperRegistryImpl.java         |   17 +-
 .../hive/registry/impl/TezAmInstance.java       |   76 +
 .../hive/registry/impl/TezAmRegistryImpl.java   |  118 ++
 .../hive/registry/impl/ZkRegistryBase.java      |   37 +-
 .../hive/llap/cache/EvictionDispatcher.java     |   27 +-
 .../hive/llap/io/api/impl/LlapIoImpl.java       |   42 +-
 .../llap/io/decode/OrcColumnVectorProducer.java |    6 +-
 .../llap/io/encoded/OrcEncodedDataReader.java   |  345 +++--
 .../hive/llap/io/metadata/MetadataCache.java    |  537 +++++++
 .../llap/io/metadata/OrcFileEstimateErrors.java |    5 +-
 .../hive/llap/io/metadata/OrcFileMetadata.java  |  114 +-
 .../hive/llap/io/metadata/OrcMetadataCache.java |  163 ---
 .../llap/io/metadata/OrcStripeMetadata.java     |  123 +-
 .../io/metadata/ParquetMetadataCacheImpl.java   |  353 -----
 .../TestIncrementalObjectSizeEstimator.java     |    4 +-
 .../hive/llap/cache/TestOrcMetadataCache.java   |  159 ++-
 llap-tez/pom.xml                                |   34 +
 .../tezplugins/LlapTaskSchedulerService.java    |   83 +-
 .../endpoint/LlapPluginServerImpl.java          |   11 +-
 .../TestLlapTaskSchedulerService.java           |    1 +
 metastore/pom.xml                               |   39 -
 .../upgrade/derby/045-HIVE-16886.derby.sql      |    1 +
 .../upgrade/derby/hive-schema-3.0.0.derby.sql   |    6 +
 .../derby/upgrade-2.3.0-to-3.0.0.derby.sql      |    1 +
 .../upgrade/mssql/030-HIVE-16886.mssql.sql      |    1 +
 .../upgrade/mssql/hive-schema-3.0.0.mssql.sql   |    2 +
 .../mssql/upgrade-2.3.0-to-3.0.0.mssql.sql      |    1 +
 .../upgrade/mysql/045-HIVE-16886.mysql.sql      |    1 +
 .../upgrade/mysql/hive-schema-3.0.0.mysql.sql   |    2 +
 .../mysql/upgrade-2.3.0-to-3.0.0.mysql.sql      |    1 +
 .../upgrade/oracle/045-HIVE-16886.oracle.sql    |    1 +
 .../upgrade/oracle/hive-schema-3.0.0.oracle.sql |    1 +
 .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql    |    1 +
 .../postgres/044-HIVE-16886.postgres.sql        |    1 +
 .../postgres/hive-schema-3.0.0.postgres.sql     |    2 +
 .../upgrade-2.3.0-to-3.0.0.postgres.sql         |    1 +
 .../apache/hadoop/hive/metastore/Metastore.java | 1331 ------------------
 .../hive/metastore/AggregateStatsCache.java     |  573 --------
 .../hadoop/hive/metastore/DatabaseProduct.java  |   75 -
 .../hive/metastore/DefaultHiveMetaHook.java     |   51 -
 .../DefaultMetaStoreFilterHookImpl.java         |  102 --
 .../hadoop/hive/metastore/FileFormatProxy.java  |   64 -
 .../hive/metastore/HiveMetaException.java       |   42 -
 .../hadoop/hive/metastore/HiveMetaHook.java     |   96 --
 .../hive/metastore/HiveMetaHookLoader.java      |   39 -
 .../hadoop/hive/metastore/HiveMetaStore.java    |   11 +-
 .../hive/metastore/HiveMetaStoreClient.java     |    5 +-
 .../hive/metastore/HiveMetaStoreFsImpl.java     |   52 -
 .../hive/metastore/IExtrapolatePartStatus.java  |   85 --
 .../hadoop/hive/metastore/IHMSHandler.java      |   28 -
 .../hadoop/hive/metastore/IMetaStoreClient.java |    2 +-
 .../hive/metastore/IMetaStoreSchemaInfo.java    |  107 --
 .../metastore/LinearExtrapolatePartStatus.java  |  106 --
 .../hive/metastore/LockComponentBuilder.java    |  121 --
 .../hive/metastore/LockRequestBuilder.java      |  168 ---
 .../hive/metastore/MetaStoreDirectSql.java      |   21 +-
 .../metastore/MetaStoreEndFunctionContext.java  |   59 -
 .../metastore/MetaStoreEndFunctionListener.java |   58 -
 .../hadoop/hive/metastore/MetaStoreFS.java      |   43 -
 .../hive/metastore/MetaStoreFilterHook.java     |  132 --
 .../hadoop/hive/metastore/MetaStoreInit.java    |  112 --
 .../hive/metastore/MetaStoreSchemaInfo.java     |  234 ---
 .../metastore/MetaStoreSchemaInfoFactory.java   |   65 -
 .../hadoop/hive/metastore/MetaStoreThread.java  |   63 -
 .../hadoop/hive/metastore/ObjectStore.java      |  102 +-
 .../hive/metastore/PartitionDropOptions.java    |   54 -
 .../metastore/PartitionExpressionProxy.java     |   73 -
 .../hive/metastore/TServerSocketKeepAlive.java  |   47 -
 .../apache/hadoop/hive/metastore/TableType.java |   26 -
 .../hive/metastore/annotation/NoReconnect.java  |   29 -
 .../metastore/hooks/JDOConnectionURLHook.java   |   53 -
 .../spec/CompositePartitionSpecProxy.java       |  228 ---
 .../spec/PartitionListComposingSpecProxy.java   |  171 ---
 .../partition/spec/PartitionSpecProxy.java      |  199 ---
 .../spec/PartitionSpecWithSharedSDProxy.java    |  172 ---
 .../hive/metastore/repl/DumpDirCleanerTask.java |   45 +
 .../hive/metastore/tools/HiveSchemaHelper.java  |  640 ---------
 .../hive/metastore/tools/SQLGenerator.java      |  172 +++
 .../hadoop/hive/metastore/txn/TxnHandler.java   |  134 +-
 .../hadoop/hive/metastore/metastore.proto       |   29 -
 .../hive/metastore/TestAggregateStatsCache.java |  266 ----
 .../hive/metastore/TestLockRequestBuilder.java  |  584 --------
 .../metastore/TestMetaStoreSchemaFactory.java   |   67 -
 .../hive/metastore/TestMetaStoreSchemaInfo.java |   52 -
 .../hadoop/hive/metastore/TestObjectStore.java  |  163 ++-
 .../hadoop/hive/metastore/txn/TestTxnUtils.java |    9 +-
 .../org/apache/hadoop/hive/ql/ErrorMsg.java     |    9 +-
 .../hive/ql/exec/AbstractFileMergeOperator.java |   60 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   24 +-
 .../apache/hadoop/hive/ql/exec/Utilities.java   |    5 +
 .../hadoop/hive/ql/exec/repl/ReplDumpTask.java  |   32 +
 .../ql/exec/repl/bootstrap/ReplLoadTask.java    |   32 +-
 .../ql/exec/repl/bootstrap/ReplLoadWork.java    |    9 +-
 .../repl/bootstrap/events/BootstrapEvent.java   |    2 +-
 .../repl/bootstrap/events/ConstraintEvent.java  |   24 +
 .../filesystem/ConstraintEventsIterator.java    |   90 ++
 .../events/filesystem/FSConstraintEvent.java    |   39 +
 .../repl/bootstrap/load/LoadConstraint.java     |  119 ++
 .../bootstrap/load/table/LoadPartitions.java    |  110 +-
 .../repl/bootstrap/load/table/LoadTable.java    |    3 +-
 .../exec/repl/bootstrap/load/util/Context.java  |    4 +-
 .../exec/repl/bootstrap/load/util/PathInfo.java |   90 ++
 .../repl/bootstrap/load/util/PathUtils.java     |   68 +-
 .../ql/exec/tez/SessionExpirationTracker.java   |    3 +-
 .../hadoop/hive/ql/exec/tez/TezSessionPool.java |   93 +-
 .../hive/ql/exec/tez/TezSessionPoolManager.java |   34 +-
 .../hive/ql/exec/tez/TezSessionPoolSession.java |    8 +-
 .../hive/ql/exec/tez/TezSessionState.java       |   40 +-
 .../apache/hadoop/hive/ql/exec/tez/TezTask.java |    5 +-
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |    4 +-
 .../hadoop/hive/ql/io/orc/LocalCache.java       |    8 +-
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |   11 +
 .../hive/ql/io/orc/encoded/EncodedReader.java   |   14 +
 .../ql/io/orc/encoded/EncodedReaderImpl.java    |  534 +++++--
 .../hadoop/hive/ql/io/orc/encoded/Reader.java   |    4 +
 .../hive/ql/io/orc/encoded/ReaderImpl.java      |   11 +-
 .../apache/hadoop/hive/ql/metadata/Hive.java    |   43 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java      |   25 +-
 .../apache/hadoop/hive/ql/parse/EximUtil.java   |    5 +-
 .../hadoop/hive/ql/parse/GenTezUtils.java       |    3 +-
 .../ql/parse/ReplicationSemanticAnalyzer.java   |    1 +
 .../hive/ql/parse/repl/dump/TableExport.java    |    9 +-
 .../repl/dump/io/ConstraintsSerializer.java     |   71 +
 .../repl/load/message/AddForeignKeyHandler.java |   16 +-
 .../message/AddNotNullConstraintHandler.java    |    7 +-
 .../repl/load/message/AddPrimaryKeyHandler.java |    7 +-
 .../message/AddUniqueConstraintHandler.java     |    6 +-
 .../hadoop/hive/ql/plan/DropTableDesc.java      |    7 +
 .../AuthorizationMetaStoreFilterHook.java       |    4 +-
 .../hadoop/hive/ql/session/SessionState.java    |   10 +-
 .../hive/ql/txn/compactor/CompactorThread.java  |   23 +-
 .../apache/hadoop/hive/ql/TestTxnCommands2.java |   18 +-
 .../hive/ql/exec/tez/SampleTezSessionState.java |   12 +-
 .../hive/ql/exec/tez/TestTezSessionPool.java    |   14 +-
 .../hadoop/hive/ql/exec/tez/TestTezTask.java    |    3 +-
 .../hive/ql/txn/compactor/CompactorTest.java    |    2 +-
 .../queries/clientnegative/merge_negative_4.q   |    6 +
 .../queries/clientnegative/merge_negative_5.q   |   14 +
 .../clientpositive/dynamic_semijoin_reduction.q |    1 +
 .../test/queries/clientpositive/mapjoin_hint.q  |    1 +
 .../test/queries/clientpositive/orc_merge13.q   |   44 +
 .../tez_union_dynamic_partition_2.q             |   27 +
 .../clientnegative/merge_negative_3.q.out       |    2 +-
 .../clientnegative/merge_negative_4.q.out       |   19 +
 .../clientnegative/merge_negative_5.q.out       |   67 +
 .../llap/dynamic_semijoin_reduction.q.out       |   48 +-
 .../llap/dynamic_semijoin_reduction_2.q.out     |   24 +-
 .../llap/dynamic_semijoin_reduction_3.q.out     |   14 +-
 .../llap/dynamic_semijoin_reduction_sw.q.out    |   16 +-
 .../llap/dynamic_semijoin_user_level.q.out      |   32 +-
 .../clientpositive/llap/mapjoin_hint.q.out      |    4 +-
 .../results/clientpositive/llap/mergejoin.q.out |   12 +-
 .../clientpositive/llap/orc_llap_counters.q.out |   98 +-
 .../llap/orc_llap_counters1.q.out               |   11 +-
 .../clientpositive/llap/orc_ppd_basic.q.out     |  147 +-
 .../llap/orc_ppd_schema_evol_3a.q.out           |  156 +-
 .../clientpositive/llap/semijoin_hint.q.out     |   12 +-
 .../llap/tez_union_dynamic_partition_2.q.out    |  219 +++
 .../vectorized_dynamic_semijoin_reduction.q.out |   36 +-
 ...vectorized_dynamic_semijoin_reduction2.q.out |   32 +-
 .../results/clientpositive/orc_merge13.q.out    |  248 ++++
 .../hive/serde2/avro/AvroDeserializer.java      |   22 +-
 .../lazy/fast/LazySimpleSerializeWrite.java     |    4 +-
 .../ObjectInspectorConverters.java              |    8 +-
 standalone-metastore/pom.xml                    |   50 +
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp  |   32 +-
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.h    |   12 +-
 .../hive/metastore/api/ThriftHiveMetastore.java |  140 +-
 .../gen-php/metastore/ThriftHiveMetastore.php   |   31 +-
 .../hive_metastore/ThriftHiveMetastore.py       |   25 +-
 .../gen/thrift/gen-rb/thrift_hive_metastore.rb  |    7 +-
 .../common/classification/RetrySemantics.java   |   57 +
 .../hive/metastore/AggregateStatsCache.java     |  567 ++++++++
 .../hadoop/hive/metastore/DatabaseProduct.java  |   75 +
 .../hive/metastore/DefaultHiveMetaHook.java     |   51 +
 .../DefaultMetaStoreFilterHookImpl.java         |  102 ++
 .../hadoop/hive/metastore/FileFormatProxy.java  |   64 +
 .../hive/metastore/HiveMetaException.java       |   42 +
 .../hadoop/hive/metastore/HiveMetaHook.java     |   95 ++
 .../hive/metastore/HiveMetaHookLoader.java      |   39 +
 .../hive/metastore/HiveMetaStoreFsImpl.java     |   53 +
 .../hive/metastore/IExtrapolatePartStatus.java  |   85 ++
 .../hadoop/hive/metastore/IHMSHandler.java      |   28 +
 .../hive/metastore/IMetaStoreSchemaInfo.java    |  108 ++
 .../metastore/LinearExtrapolatePartStatus.java  |  106 ++
 .../hive/metastore/LockComponentBuilder.java    |  121 ++
 .../hive/metastore/LockRequestBuilder.java      |  168 +++
 .../metastore/MetaStoreEndFunctionContext.java  |   59 +
 .../metastore/MetaStoreEndFunctionListener.java |   58 +
 .../hadoop/hive/metastore/MetaStoreFS.java      |   43 +
 .../hive/metastore/MetaStoreFilterHook.java     |  132 ++
 .../hadoop/hive/metastore/MetaStoreInit.java    |  109 ++
 .../hive/metastore/MetaStoreSchemaInfo.java     |  234 +++
 .../metastore/MetaStoreSchemaInfoFactory.java   |   64 +
 .../hadoop/hive/metastore/MetaStoreThread.java  |   58 +
 .../hive/metastore/PartitionDropOptions.java    |   54 +
 .../metastore/PartitionExpressionProxy.java     |   73 +
 .../hive/metastore/TServerSocketKeepAlive.java  |   47 +
 .../apache/hadoop/hive/metastore/TableType.java |   26 +
 .../annotation/MetastoreVersionAnnotation.java  |   85 ++
 .../hive/metastore/annotation/NoReconnect.java  |   29 +
 .../metastore/hooks/JDOConnectionURLHook.java   |   53 +
 .../spec/CompositePartitionSpecProxy.java       |  226 +++
 .../spec/PartitionListComposingSpecProxy.java   |  171 +++
 .../partition/spec/PartitionSpecProxy.java      |  199 +++
 .../spec/PartitionSpecWithSharedSDProxy.java    |  172 +++
 .../hive/metastore/tools/HiveSchemaHelper.java  |  641 +++++++++
 .../hadoop/hive/metastore/utils/FileUtils.java  |   66 +
 .../hadoop/hive/metastore/utils/JavaUtils.java  |   37 +
 .../hive/metastore/utils/MetaStoreUtils.java    |   41 +
 .../metastore/utils/MetastoreVersionInfo.java   |  133 ++
 .../hadoop/hive/metastore/metastore.proto       |   29 +
 .../src/main/resources/saveVersion.sh           |   91 ++
 .../src/main/thrift/hive_metastore.thrift       |    2 +-
 .../hive/metastore/TestAggregateStatsCache.java |  267 ++++
 .../hive/metastore/TestLockRequestBuilder.java  |  584 ++++++++
 .../metastore/TestMetaStoreSchemaFactory.java   |   69 +
 .../hive/metastore/TestMetaStoreSchemaInfo.java |   52 +
 .../hive/common/io/FileMetadataCache.java       |    8 +-
 233 files changed, 10377 insertions(+), 8544 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index d2038c5,3053dcb..c65baa4
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@@ -241,8 -242,8 +244,9 @@@ public class ObjectStore implements Raw
  
    private boolean isInitialized = false;
    private PersistenceManager pm = null;
+   private SQLGenerator sqlGenerator = null;
    private MetaStoreDirectSql directSql = null;
 +  private DatabaseProduct dbType = null;
    private PartitionExpressionProxy expressionProxy = null;
    private Configuration hiveConf;
    private volatile int openTrasactionCalls = 0;
@@@ -430,9 -431,17 +434,18 @@@
      LOG.info("ObjectStore, initialize called");
      prop = dsProps;
      pm = getPersistenceManager();
+     try {
+       String productName = MetaStoreDirectSql.getProductName(pm);
+       sqlGenerator = new SQLGenerator(
+           DatabaseProduct.determineDatabaseProduct(productName),
+           new HiveConf(hiveConf, ObjectStore.class));
+     } catch (SQLException e) {
+       LOG.error("error trying to figure out the database product", e);
+       throw new RuntimeException(e);
+     }
      isInitialized = pm != null;
      if (isInitialized) {
 +      dbType = determineDatabaseProduct();
        expressionProxy = createExpressionProxy(hiveConf);
        if (HiveConf.getBoolVar(getConf(), ConfVars.METASTORE_TRY_DIRECT_SQL)) {
          String schema = prop.getProperty("javax.jdo.mapping.Schema");

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 75eba79,7882dad..176fc4c
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@@ -59,9 -57,21 +57,23 @@@ import org.mockito.Mockito
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
 +import com.google.common.base.Supplier;
 +import com.google.common.collect.ImmutableList;
  import javax.jdo.Query;
+ import java.util.Arrays;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.Set;
+ import java.util.concurrent.BrokenBarrierException;
+ import java.util.concurrent.CyclicBarrier;
+ import java.util.concurrent.ExecutionException;
+ import java.util.concurrent.ExecutorService;
+ import java.util.concurrent.Executors;
+ import java.util.concurrent.TimeUnit;
+ 
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertTrue;
  
  public class TestObjectStore {
    private ObjectStore objectStore = null;
@@@ -287,8 -282,8 +299,8 @@@
    public void testPartitionOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
      Database db1 = new Database(DB1, "description", "locationurl", null);
      objectStore.createDatabase(db1);
 -    StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
 +    StorageDescriptor sd = createFakeSd("location");
-     HashMap<String,String> tableParams = new HashMap<String,String>();
+     HashMap<String, String> tableParams = new HashMap<>();
      tableParams.put("EXTERNAL", "false");
      FieldSchema partitionKey1 = new FieldSchema("Country", ColumnType.STRING_TYPE_NAME, "");
      FieldSchema partitionKey2 = new FieldSchema("State", ColumnType.STRING_TYPE_NAME, "");

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
index 70e764e,6bba057..6b98ecd
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractFileMergeOperator.java
@@@ -224,50 -216,63 +232,78 @@@ public abstract class AbstractFileMerge
    @Override
    public void closeOp(boolean abort) throws HiveException {
      try {
 -      if (!abort) {
 -        // if outPath does not exist, then it means all paths within combine split are skipped as
 -        // they are incompatible for merge (for example: files without stripe stats).
 -        // Those files will be added to incompatFileSet
 -        if (fs.exists(outPath)) {
 -          FileStatus fss = fs.getFileStatus(outPath);
 +      if (abort) {
 +        if (!autoDelete || isMmTable) {
 +          fs.delete(outPath, true);
 +        }
 +        return;
 +      }
 +      // if outPath does not exist, then it means all paths within combine split are skipped as
 +      // they are incompatible for merge (for example: files without stripe stats).
 +      // Those files will be added to incompatFileSet
 +      if (fs.exists(outPath)) {
 +        FileStatus fss = fs.getFileStatus(outPath);
 +        if (!isMmTable) {
            if (!fs.rename(outPath, finalPath)) {
 -            throw new IOException(
 -                "Unable to rename " + outPath + " to " + finalPath);
 +            throw new IOException("Unable to rename " + outPath + " to " + finalPath);
            }
 -          LOG.info("renamed path " + outPath + " to " + finalPath + " . File" +
 -              " size is "
 -              + fss.getLen());
 +          LOG.info("Renamed path " + outPath + " to " + finalPath
 +              + "(" + fss.getLen() + " bytes).");
 +        } else {
 +          assert finalPath.equals(outPath);
 +          // There's always just one file that we have merged.
 +          // The union/DP/etc. should already be account for in the path.
 +          Utilities.writeMmCommitManifest(Lists.newArrayList(outPath),
 +              tmpPath.getParent(), fs, taskId, conf.getTxnId(), conf.getStmtId(), null);
 +          LOG.info("Merged into " + finalPath + "(" + fss.getLen() + " bytes).");
          }
 +      }
  
 +      // move any incompatible files to final path
 +      if (incompatFileSet != null && !incompatFileSet.isEmpty()) {
 +        if (isMmTable) {
 +          // We only support query-time merge for MM tables, so don't handle this.
 +          throw new HiveException("Incompatible files should not happen in MM tables.");
 +        }
-         for (Path incompatFile : incompatFileSet) {
-           Path destDir = finalPath.getParent();
-           try {
-             Utilities.renameOrMoveFiles(fs, incompatFile, destDir);
-             LOG.info("Moved incompatible file " + incompatFile + " to " +
-                 destDir);
-           } catch (HiveException e) {
-             LOG.error("Unable to move " + incompatFile + " to " + destDir);
-             throw new IOException(e);
+         Path destDir = finalPath.getParent();
+         Path destPath = destDir;
+         // move any incompatible files to final path
+         if (incompatFileSet != null && !incompatFileSet.isEmpty()) {
+           for (Path incompatFile : incompatFileSet) {
+             // check if path conforms to Hive's file name convention. Hive expects filenames to be in specific format
+             // like 000000_0, but "LOAD DATA" commands can let you add any files to any partitions/tables without
+             // renaming. This can cause MoveTask to remove files in some cases where MoveTask assumes the files are
+             // are generated by speculatively executed tasks.
+             // Example: MoveTask thinks the following files are same
+             // part-m-00000_1417075294718
+             // part-m-00001_1417075294718
+             // Assumes 1417075294718 as taskId and retains only large file supposedly generated by speculative execution.
+             // This can result in data loss in case of CONCATENATE/merging. Filter out files that does not match Hive's
+             // filename convention.
+             if (!Utilities.isHiveManagedFile(incompatFile)) {
+               // rename un-managed files to conform to Hive's naming standard
+               // Example:
+               // /warehouse/table/part-m-00000_1417075294718 will get renamed to /warehouse/table/.hive-staging/000000_0
+               // If staging directory already contains the file, taskId_copy_N naming will be used.
+               final String taskId = Utilities.getTaskId(jc);
+               Path destFilePath = new Path(destDir, new Path(taskId));
+               for (int counter = 1; fs.exists(destFilePath); counter++) {
+                 destFilePath = new Path(destDir, taskId + (Utilities.COPY_KEYWORD + counter));
+               }
+               LOG.warn("Path doesn't conform to Hive's expectation. Renaming {} to {}", incompatFile, destFilePath);
+               destPath = destFilePath;
+             }
+ 
+             try {
+               Utilities.renameOrMoveFiles(fs, incompatFile, destPath);
+               LOG.info("Moved incompatible file " + incompatFile + " to " + destPath);
+             } catch (HiveException e) {
+               LOG.error("Unable to move " + incompatFile + " to " + destPath);
+               throw new IOException(e);
+             }
            }
          }
 -      } else {
 -        if (!autoDelete) {
 -          fs.delete(outPath, true);
 -        }
++
        }
      } catch (IOException e) {
        throw new HiveException("Failed to close AbstractFileMergeOperator", e);

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
index 2ae18cf,acf9746..522444e
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
@@@ -44,8 -45,7 +45,9 @@@ import org.apache.hadoop.hive.ql.plan.D
  import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
  import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
  import org.apache.hadoop.hive.ql.plan.MoveWork;
 +import org.apache.hadoop.hive.ql.session.SessionState;
 +import org.mortbay.jetty.servlet.AbstractSessionManager;
+ import org.datanucleus.util.StringUtils;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index 8f94679,40d4fad..cf1d14c
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@@ -314,9 -315,9 +315,9 @@@ public class GenTezUtils 
          linked = context.linkedFileSinks.get(path);
          linked.add(desc);
  
-         desc.setDirName(new Path(path, "" + linked.size()));
+         desc.setDirName(new Path(path, AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + linked.size()));
 +        Utilities.LOG14535.info("removing union - new desc with " + desc.getDirName() + "; parent " + path);
          desc.setLinkedFileSink(true);
 -        desc.setParentDir(path);
          desc.setLinkedFileSinkDesc(linked);
        }
  

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
----------------------------------------------------------------------
diff --cc standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
index 0000000,10f7732..6ef2e35
mode 000000,100644..100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java
@@@ -1,0 -1,57 +1,58 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.hadoop.hive.metastore;
+ 
+ import org.apache.hadoop.conf.Configurable;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.hive.metastore.api.MetaException;
+ 
+ import java.util.concurrent.atomic.AtomicBoolean;
+ 
+ /**
+  * A thread that runs in the metastore, separate from the threads in the thrift service.
+  */
+ public interface MetaStoreThread extends Configurable {
+ 
+   /**
+    * Set the id for this thread.
+    * @param threadId id of the thread
+    */
+   void setThreadId(int threadId);
+ 
+   /**
+    * Initialize the thread.  This must not be called until after
+    * {@link #setConf(Configuration)}  and  {@link #setThreadId(int)}
+    * have been called.
+    * @param stop a flag to watch for when to stop.  If this value is set to true,
+    *             the thread will terminate the next time through its main loop.
+    * @param looped a flag that is set to true everytime a thread goes through it's main loop.
+    *               This is purely for testing so that tests can assure themselves that the thread
+    *               has run through it's loop once.  The test can set this value to false.  The
+    *               thread should then assure that the loop has been gone completely through at
+    *               least once.
+    */
++  // TODO: move these test parameters to more specific places... there's no need to have them here
+   void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException;
+ 
+   /**
+    * Run the thread in the background.  This must not be called until
+    * {@link MetaStoreThread#init(java.util.concurrent.atomic.AtomicBoolean,java.util.concurrent.atomic.AtomicBoolean)} has
+    * been called.
+    */
+   void start();
+ }

http://git-wip-us.apache.org/repos/asf/hive/blob/9a9f2540/standalone-metastore/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------

[21/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/ql/src/test/results/clientpositive/mm_all.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/mm_all.q.out
index ea60414,0000000..b69ac00
mode 100644,000000..100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@@ -1,2147 -1,0 +1,2116 @@@
 +PREHOOK: query: drop table intermediate
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table intermediate
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@intermediate
 +POSTHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@intermediate
 +PREHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=455
 +POSTHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=455
 +POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=456
 +POSTHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=456
 +POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=457
 +POSTHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=457
 +POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: drop table part_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table part_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@part_mm
 +POSTHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@part_mm
 +PREHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
 +PREHOOK: type: QUERY
 +POSTHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
 +POSTHOOK: type: QUERY
 +STAGE DEPENDENCIES:
 +  Stage-1 is a root stage
 +  Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
 +  Stage-4
 +  Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
 +  Stage-2 depends on stages: Stage-0
 +  Stage-3
 +  Stage-5
 +  Stage-6 depends on stages: Stage-5
 +
 +STAGE PLANS:
 +  Stage: Stage-1
 +    Map Reduce
 +      Map Operator Tree:
 +          TableScan
 +            alias: intermediate
 +            Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
 +            Select Operator
 +              expressions: key (type: int)
 +              outputColumnNames: _col0
 +              Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
 +              File Output Operator
 +                compressed: false
 +                Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
 +                table:
 +                    input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +                    output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
 +                    serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
 +                    name: default.part_mm
 +                Write Type: INSERT
 +
 +  Stage: Stage-7
 +    Conditional Operator
 +
 +  Stage: Stage-4
 +    Move Operator
 +      files:
 +          hdfs directory: true
 +#### A masked pattern was here ####
 +
 +  Stage: Stage-0
 +    Move Operator
 +      tables:
 +          partition:
 +            key_mm 455
 +          replace: false
 +          table:
 +              input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +              output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
 +              serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
 +              name: default.part_mm
 +          micromanaged table: true
 +
 +  Stage: Stage-2
 +    Stats-Aggr Operator
 +
 +  Stage: Stage-3
 +    Merge File Operator
 +      Map Operator Tree:
 +          ORC File Merge Operator
 +      merge level: stripe
 +      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +
 +  Stage: Stage-5
 +    Merge File Operator
 +      Map Operator Tree:
 +          ORC File Merge Operator
 +      merge level: stripe
 +      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +
 +  Stage: Stage-6
 +    Move Operator
 +      files:
 +          hdfs directory: true
 +#### A masked pattern was here ####
 +
 +PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@part_mm@key_mm=456
 +POSTHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@part_mm@key_mm=456
 +POSTHOOK: Lineage: part_mm PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from part_mm order by key, key_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Input: default@part_mm@key_mm=455
 +PREHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from part_mm order by key, key_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Input: default@part_mm@key_mm=455
 +POSTHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +0	455
 +0	455
 +0	456
 +10	455
 +10	455
 +10	456
 +97	455
 +97	455
 +97	456
 +98	455
 +98	455
 +98	456
 +100	455
 +100	455
 +100	456
 +103	455
 +103	455
 +103	456
 +PREHOOK: query: select * from part_mm order by key, key_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Input: default@part_mm@key_mm=455
 +PREHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from part_mm order by key, key_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Input: default@part_mm@key_mm=455
 +POSTHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +0	455
 +0	455
 +0	456
 +10	455
 +10	455
 +10	456
 +97	455
 +97	455
 +97	456
 +98	455
 +98	455
 +98	456
 +100	455
 +100	455
 +100	456
 +103	455
 +103	455
 +103	456
 +PREHOOK: query: truncate table part_mm
 +PREHOOK: type: TRUNCATETABLE
 +PREHOOK: Output: default@part_mm@key_mm=455
 +PREHOOK: Output: default@part_mm@key_mm=456
 +POSTHOOK: query: truncate table part_mm
 +POSTHOOK: type: TRUNCATETABLE
 +POSTHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: Output: default@part_mm@key_mm=456
 +PREHOOK: query: select * from part_mm order by key, key_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Input: default@part_mm@key_mm=455
 +PREHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from part_mm order by key, key_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Input: default@part_mm@key_mm=455
 +POSTHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +PREHOOK: query: drop table part_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Output: default@part_mm
 +POSTHOOK: query: drop table part_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Output: default@part_mm
 +PREHOOK: query: drop table simple_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table simple_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@simple_mm
 +PREHOOK: query: insert into table simple_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: insert into table simple_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@simple_mm
 +POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from simple_mm order by key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from simple_mm order by key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +0
 +10
 +97
 +98
 +100
 +103
 +PREHOOK: query: insert into table simple_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: insert into table simple_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@simple_mm
 +POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from simple_mm order by key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from simple_mm order by key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +0
 +0
 +10
 +10
 +97
 +97
 +98
 +98
 +100
 +100
 +103
 +103
 +PREHOOK: query: truncate table simple_mm
 +PREHOOK: type: TRUNCATETABLE
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: truncate table simple_mm
 +POSTHOOK: type: TRUNCATETABLE
 +POSTHOOK: Output: default@simple_mm
 +PREHOOK: query: select * from simple_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from simple_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +PREHOOK: query: drop table simple_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@simple_mm
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: drop table simple_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@simple_mm
 +POSTHOOK: Output: default@simple_mm
 +PREHOOK: query: drop table dp_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table dp_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
 +  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@dp_mm
 +POSTHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
 +  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@dp_mm
 +PREHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@dp_mm@key1=123
 +POSTHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=0
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=10
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=100
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=103
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=97
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=98
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=0).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=100).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=103).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=10).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=97).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=98).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from dp_mm order by key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@dp_mm
 +PREHOOK: Input: default@dp_mm@key1=123/key2=0
 +PREHOOK: Input: default@dp_mm@key1=123/key2=10
 +PREHOOK: Input: default@dp_mm@key1=123/key2=100
 +PREHOOK: Input: default@dp_mm@key1=123/key2=103
 +PREHOOK: Input: default@dp_mm@key1=123/key2=97
 +PREHOOK: Input: default@dp_mm@key1=123/key2=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from dp_mm order by key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@dp_mm
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=0
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=10
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=100
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=103
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=97
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=98
 +#### A masked pattern was here ####
 +0	123	0
 +10	123	10
 +97	123	97
 +98	123	98
 +100	123	100
 +103	123	103
 +PREHOOK: query: drop table dp_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@dp_mm
 +PREHOOK: Output: default@dp_mm
 +POSTHOOK: query: drop table dp_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@dp_mm
 +POSTHOOK: Output: default@dp_mm
 +PREHOOK: query: create table union_mm(id int)  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: create table union_mm(id int)  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@union_mm
 +PREHOOK: query: insert into table union_mm 
 +select temps.p from (
 +select key as p from intermediate 
 +union all 
 +select key + 1 as p from intermediate ) temps
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: insert into table union_mm 
 +select temps.p from (
 +select key as p from intermediate 
 +union all 
 +select key + 1 as p from intermediate ) temps
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@union_mm
 +POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from union_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from union_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +0
 +1
 +10
 +11
 +97
 +98
 +98
 +99
 +100
 +101
 +103
 +104
 +PREHOOK: query: insert into table union_mm 
 +select p from
 +(
 +select key + 1 as p from intermediate
 +union all
 +select key from intermediate
 +) tab group by p
 +union all
 +select key + 2 as p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: insert into table union_mm 
 +select p from
 +(
 +select key + 1 as p from intermediate
 +union all
 +select key from intermediate
 +) tab group by p
 +union all
 +select key + 2 as p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@union_mm
 +POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from union_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from union_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +0
 +0
 +1
 +1
 +2
 +10
 +10
 +11
 +11
 +12
 +97
 +97
 +98
 +98
 +98
 +99
 +99
 +99
 +100
 +100
 +100
 +101
 +101
 +102
 +103
 +103
 +104
 +104
 +105
 +PREHOOK: query: insert into table union_mm
 +SELECT p FROM
 +(
 +  SELECT key + 1 as p FROM intermediate
 +  UNION ALL
 +  SELECT key as p FROM ( 
 +    SELECT distinct key FROM (
 +      SELECT key FROM (
 +        SELECT key + 2 as key FROM intermediate
 +        UNION ALL
 +        SELECT key FROM intermediate
 +      )t1 
 +    group by key)t2
 +  )t3
 +)t4
 +group by p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: insert into table union_mm
 +SELECT p FROM
 +(
 +  SELECT key + 1 as p FROM intermediate
 +  UNION ALL
 +  SELECT key as p FROM ( 
 +    SELECT distinct key FROM (
 +      SELECT key FROM (
 +        SELECT key + 2 as key FROM intermediate
 +        UNION ALL
 +        SELECT key FROM intermediate
 +      )t1 
 +    group by key)t2
 +  )t3
 +)t4
 +group by p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@union_mm
 +POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from union_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from union_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +0
 +0
 +0
 +1
 +1
 +1
 +2
 +2
 +10
 +10
 +10
 +11
 +11
 +11
 +12
 +12
 +97
 +97
 +97
 +98
 +98
 +98
 +98
 +99
 +99
 +99
 +99
 +100
 +100
 +100
 +100
 +101
 +101
 +101
 +102
 +102
 +103
 +103
 +103
 +104
 +104
 +104
 +105
 +105
 +PREHOOK: query: drop table union_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@union_mm
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: drop table union_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@union_mm
 +POSTHOOK: Output: default@union_mm
 +PREHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@partunion_mm
 +POSTHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@partunion_mm
 +PREHOOK: query: insert into table partunion_mm partition(key)
 +select temps.* from (
 +select key as p, key from intermediate 
 +union all 
 +select key + 1 as p, key + 1 from intermediate ) temps
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@partunion_mm
 +POSTHOOK: query: insert into table partunion_mm partition(key)
 +select temps.* from (
 +select key as p, key from intermediate 
 +union all 
 +select key + 1 as p, key + 1 from intermediate ) temps
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@partunion_mm@key=0
 +POSTHOOK: Output: default@partunion_mm@key=1
 +POSTHOOK: Output: default@partunion_mm@key=10
 +POSTHOOK: Output: default@partunion_mm@key=100
 +POSTHOOK: Output: default@partunion_mm@key=101
 +POSTHOOK: Output: default@partunion_mm@key=103
 +POSTHOOK: Output: default@partunion_mm@key=104
 +POSTHOOK: Output: default@partunion_mm@key=11
 +POSTHOOK: Output: default@partunion_mm@key=97
 +POSTHOOK: Output: default@partunion_mm@key=98
 +POSTHOOK: Output: default@partunion_mm@key=99
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=0).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=100).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=101).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=103).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=104).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=10).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=11).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=1).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=97).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=98).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=99).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from partunion_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@partunion_mm
 +PREHOOK: Input: default@partunion_mm@key=0
 +PREHOOK: Input: default@partunion_mm@key=1
 +PREHOOK: Input: default@partunion_mm@key=10
 +PREHOOK: Input: default@partunion_mm@key=100
 +PREHOOK: Input: default@partunion_mm@key=101
 +PREHOOK: Input: default@partunion_mm@key=103
 +PREHOOK: Input: default@partunion_mm@key=104
 +PREHOOK: Input: default@partunion_mm@key=11
 +PREHOOK: Input: default@partunion_mm@key=97
 +PREHOOK: Input: default@partunion_mm@key=98
 +PREHOOK: Input: default@partunion_mm@key=99
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from partunion_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@partunion_mm
 +POSTHOOK: Input: default@partunion_mm@key=0
 +POSTHOOK: Input: default@partunion_mm@key=1
 +POSTHOOK: Input: default@partunion_mm@key=10
 +POSTHOOK: Input: default@partunion_mm@key=100
 +POSTHOOK: Input: default@partunion_mm@key=101
 +POSTHOOK: Input: default@partunion_mm@key=103
 +POSTHOOK: Input: default@partunion_mm@key=104
 +POSTHOOK: Input: default@partunion_mm@key=11
 +POSTHOOK: Input: default@partunion_mm@key=97
 +POSTHOOK: Input: default@partunion_mm@key=98
 +POSTHOOK: Input: default@partunion_mm@key=99
 +#### A masked pattern was here ####
 +0	0
 +1	1
 +10	10
 +11	11
 +97	97
 +98	98
 +98	98
 +99	99
 +100	100
 +101	101
 +103	103
 +104	104
 +PREHOOK: query: drop table partunion_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@partunion_mm
 +PREHOOK: Output: default@partunion_mm
 +POSTHOOK: query: drop table partunion_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@partunion_mm
 +POSTHOOK: Output: default@partunion_mm
 +PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
 + stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@skew_mm
 +POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
 + stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@skew_mm
 +PREHOOK: query: insert into table skew_mm 
 +select key, key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@skew_mm
 +POSTHOOK: query: insert into table skew_mm 
 +select key, key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@skew_mm
 +POSTHOOK: Lineage: skew_mm.k1 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_mm.k2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_mm.k4 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from skew_mm order by k2, k1, k4
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@skew_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from skew_mm order by k2, k1, k4
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@skew_mm
 +#### A masked pattern was here ####
 +0	0	0
 +10	10	10
 +97	97	97
 +98	98	98
 +100	100	100
 +103	103	103
 +PREHOOK: query: drop table skew_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@skew_mm
 +PREHOOK: Output: default@skew_mm
 +POSTHOOK: query: drop table skew_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@skew_mm
 +POSTHOOK: Output: default@skew_mm
 +PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) 
 +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@skew_dp_union_mm
 +POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) 
 +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@skew_dp_union_mm
 +PREHOOK: query: insert into table skew_dp_union_mm partition (k3)
 +select key as i, key as j, key as k, key as l from intermediate
 +union all 
 +select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@skew_dp_union_mm
 +POSTHOOK: query: insert into table skew_dp_union_mm partition (k3)
 +select key as i, key as j, key as k, key as l from intermediate
 +union all 
 +select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=0
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=10
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=100
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=101
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=102
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=103
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=104
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=107
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=14
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=4
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=97
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=98
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@skew_dp_union_mm
 +PREHOOK: Input: default@skew_dp_union_mm@k3=0
 +PREHOOK: Input: default@skew_dp_union_mm@k3=10
 +PREHOOK: Input: default@skew_dp_union_mm@k3=100
 +PREHOOK: Input: default@skew_dp_union_mm@k3=101
 +PREHOOK: Input: default@skew_dp_union_mm@k3=102
 +PREHOOK: Input: default@skew_dp_union_mm@k3=103
 +PREHOOK: Input: default@skew_dp_union_mm@k3=104
 +PREHOOK: Input: default@skew_dp_union_mm@k3=107
 +PREHOOK: Input: default@skew_dp_union_mm@k3=14
 +PREHOOK: Input: default@skew_dp_union_mm@k3=4
 +PREHOOK: Input: default@skew_dp_union_mm@k3=97
 +PREHOOK: Input: default@skew_dp_union_mm@k3=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@skew_dp_union_mm
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=0
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=10
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=100
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=101
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=102
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=103
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=104
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=107
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=14
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=4
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=97
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=98
 +#### A masked pattern was here ####
 +0	0	0	0
 +1	2	3	4
 +10	10	10	10
 +11	12	13	14
 +97	97	97	97
 +98	98	98	98
 +98	99	100	101
 +99	100	101	102
 +100	100	100	100
 +101	102	103	104
 +103	103	103	103
 +104	105	106	107
 +PREHOOK: query: drop table skew_dp_union_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@skew_dp_union_mm
 +PREHOOK: Output: default@skew_dp_union_mm
 +POSTHOOK: query: drop table skew_dp_union_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@skew_dp_union_mm
 +POSTHOOK: Output: default@skew_dp_union_mm
 +PREHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@merge0_mm
 +PREHOOK: query: insert into table merge0_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: insert into table merge0_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge0_mm
 +POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: insert into table merge0_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: insert into table merge0_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge0_mm
 +POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: drop table merge0_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@merge0_mm
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: drop table merge0_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@merge0_mm
 +POSTHOOK: Output: default@merge0_mm
 +PREHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@merge2_mm
 +PREHOOK: query: insert into table merge2_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: insert into table merge2_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge2_mm
 +POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge2_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge2_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: insert into table merge2_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: insert into table merge2_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge2_mm
 +POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge2_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge2_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: drop table merge2_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@merge2_mm
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: drop table merge2_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@merge2_mm
 +POSTHOOK: Output: default@merge2_mm
 +PREHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@merge1_mm
 +PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge1_mm@key=0
 +POSTHOOK: Output: default@merge1_mm@key=10
 +POSTHOOK: Output: default@merge1_mm@key=100
 +POSTHOOK: Output: default@merge1_mm@key=103
 +POSTHOOK: Output: default@merge1_mm@key=97
 +POSTHOOK: Output: default@merge1_mm@key=98
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge1_mm order by id, key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge1_mm
 +PREHOOK: Input: default@merge1_mm@key=0
 +PREHOOK: Input: default@merge1_mm@key=10
 +PREHOOK: Input: default@merge1_mm@key=100
 +PREHOOK: Input: default@merge1_mm@key=103
 +PREHOOK: Input: default@merge1_mm@key=97
 +PREHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge1_mm order by id, key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge1_mm
 +POSTHOOK: Input: default@merge1_mm@key=0
 +POSTHOOK: Input: default@merge1_mm@key=10
 +POSTHOOK: Input: default@merge1_mm@key=100
 +POSTHOOK: Input: default@merge1_mm@key=103
 +POSTHOOK: Input: default@merge1_mm@key=97
 +POSTHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +0	0
 +10	10
 +97	97
 +98	98
 +100	100
 +103	103
 +PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge1_mm@key=0
 +POSTHOOK: Output: default@merge1_mm@key=10
 +POSTHOOK: Output: default@merge1_mm@key=100
 +POSTHOOK: Output: default@merge1_mm@key=103
 +POSTHOOK: Output: default@merge1_mm@key=97
 +POSTHOOK: Output: default@merge1_mm@key=98
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge1_mm order by id, key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge1_mm
 +PREHOOK: Input: default@merge1_mm@key=0
 +PREHOOK: Input: default@merge1_mm@key=10
 +PREHOOK: Input: default@merge1_mm@key=100
 +PREHOOK: Input: default@merge1_mm@key=103
 +PREHOOK: Input: default@merge1_mm@key=97
 +PREHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge1_mm order by id, key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge1_mm
 +POSTHOOK: Input: default@merge1_mm@key=0
 +POSTHOOK: Input: default@merge1_mm@key=10
 +POSTHOOK: Input: default@merge1_mm@key=100
 +POSTHOOK: Input: default@merge1_mm@key=103
 +POSTHOOK: Input: default@merge1_mm@key=97
 +POSTHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +0	0
 +0	0
 +10	10
 +10	10
 +97	97
 +97	97
 +98	98
 +98	98
 +100	100
 +100	100
 +103	103
 +103	103
 +PREHOOK: query: drop table merge1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@merge1_mm
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: drop table merge1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@merge1_mm
 +POSTHOOK: Output: default@merge1_mm
 +PREHOOK: query: drop table ctas0_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table ctas0_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate
 +PREHOOK: type: CREATETABLE_AS_SELECT
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@ctas0_mm
 +POSTHOOK: query: create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate
 +POSTHOOK: type: CREATETABLE_AS_SELECT
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@ctas0_mm
 +POSTHOOK: Lineage: ctas0_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: ctas0_mm.p SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from ctas0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@ctas0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from ctas0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@ctas0_mm
 +#### A masked pattern was here ####
 +98	455
 +97	455
 +0	456
 +10	456
 +100	457
 +103	457
 +PREHOOK: query: drop table ctas0_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@ctas0_mm
 +PREHOOK: Output: default@ctas0_mm
 +POSTHOOK: query: drop table ctas0_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@ctas0_mm
 +POSTHOOK: Output: default@ctas0_mm
 +PREHOOK: query: drop table ctas1_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table ctas1_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
 +  select * from intermediate union all select * from intermediate
 +PREHOOK: type: CREATETABLE_AS_SELECT
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@ctas1_mm
 +POSTHOOK: query: create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
 +  select * from intermediate union all select * from intermediate
 +POSTHOOK: type: CREATETABLE_AS_SELECT
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@ctas1_mm
 +POSTHOOK: Lineage: ctas1_mm.key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: ctas1_mm.p EXPRESSION [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from ctas1_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@ctas1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from ctas1_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@ctas1_mm
 +#### A masked pattern was here ####
 +98	455
 +98	455
 +97	455
 +97	455
 +0	456
 +0	456
 +10	456
 +10	456
 +100	457
 +100	457
 +103	457
 +103	457
 +PREHOOK: query: drop table ctas1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@ctas1_mm
 +PREHOOK: Output: default@ctas1_mm
 +POSTHOOK: query: drop table ctas1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@ctas1_mm
 +POSTHOOK: Output: default@ctas1_mm
 +PREHOOK: query: drop table multi0_1_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table multi0_1_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: drop table multi0_2_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table multi0_2_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table multi0_1_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: create table multi0_1_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@multi0_1_mm
 +PREHOOK: query: create table multi0_2_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: create table multi0_2_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@multi0_2_mm
 +PREHOOK: query: insert into table multi0_1_mm select key, p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: insert into table multi0_1_mm select key, p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_1_mm
 +POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: insert into table multi0_2_mm select p, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: insert into table multi0_2_mm select p, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_2_mm
 +POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from multi0_1_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_1_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +0	456
 +10	456
 +97	455
 +98	455
 +100	457
 +103	457
 +PREHOOK: query: select * from multi0_2_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_2_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +455	97
 +455	98
 +456	0
 +456	10
 +457	100
 +457	103
 +PREHOOK: query: insert into table multi0_1_mm select p, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: insert into table multi0_1_mm select p, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_1_mm
 +POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table multi0_2_mm select key, p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: insert into table multi0_2_mm select key, p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_2_mm
 +POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from multi0_1_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_1_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +0	456
 +10	456
 +97	455
 +98	455
 +100	457
 +103	457
 +455	97
 +455	98
 +456	0
 +456	10
 +457	100
 +457	103
 +PREHOOK: query: select * from multi0_2_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_2_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +0	456
 +10	456
 +97	455
 +98	455
 +100	457
 +103	457
 +455	97
 +455	98
 +456	0
 +456	10
 +457	100
 +457	103
 +PREHOOK: query: drop table multi0_1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@multi0_1_mm
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: drop table multi0_1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@multi0_1_mm
 +POSTHOOK: Output: default@multi0_1_mm
 +PREHOOK: query: drop table multi0_2_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@multi0_2_mm
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: drop table multi0_2_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@multi0_2_mm
 +POSTHOOK: Output: default@multi0_2_mm
 +PREHOOK: query: drop table multi1_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table multi1_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@multi1_mm
 +POSTHOOK: query: create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@multi1_mm
 +PREHOOK: query: from intermediate
 +insert into table multi1_mm partition(p=1) select p, key
 +insert into table multi1_mm partition(p=2) select key, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm@p=1
 +PREHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: query: from intermediate
 +insert into table multi1_mm partition(p=1) select p, key
 +insert into table multi1_mm partition(p=2) select key, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +0	456	2
 +10	456	2
 +97	455	2
 +98	455	2
 +100	457	2
 +103	457	2
 +455	97	1
 +455	98	1
 +456	0	1
 +456	10	1
 +457	100	1
 +457	103	1
 +PREHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +0	456	1
 +0	456	2
 +10	456	1
 +10	456	2
 +97	455	1
 +97	455	2
 +98	455	1
 +98	455	2
 +100	457	1
 +100	457	2
 +103	457	1
 +103	457	2
 +455	97	1
 +455	97	2
 +455	98	1
 +455	98	2
 +456	0	1
 +456	0	2
 +456	10	1
 +456	10	2
 +457	100	1
 +457	100	2
 +457	103	1
 +457	103	2
 +PREHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, p
 +insert into table multi1_mm partition(p=1) select key, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm
 +PREHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, p
 +insert into table multi1_mm partition(p=1) select key, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
- POSTHOOK: Output: default@multi1_mm@p=455
- POSTHOOK: Output: default@multi1_mm@p=456
- POSTHOOK: Output: default@multi1_mm@p=457
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
- POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
- POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
- POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
- POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
- POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
- POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
++POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
- PREHOOK: Input: default@multi1_mm@p=455
- PREHOOK: Input: default@multi1_mm@p=456
- PREHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
- POSTHOOK: Input: default@multi1_mm@p=455
- POSTHOOK: Input: default@multi1_mm@p=456
- POSTHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +0	456	1
 +0	456	1
 +0	456	2
 +10	456	1
 +10	456	1
 +10	456	2
 +97	455	1
 +97	455	1
 +97	455	2
 +98	455	1
 +98	455	1
 +98	455	2
 +100	457	1
 +100	457	1
 +100	457	2
 +103	457	1
 +103	457	1
 +103	457	2
 +455	97	1
 +455	97	2
- 455	97	455
 +455	98	1
 +455	98	2
- 455	98	455
 +456	0	1
 +456	0	2
- 456	0	456
 +456	10	1
 +456	10	2
- 456	10	456
 +457	100	1
 +457	100	2
- 457	100	457
 +457	103	1
 +457	103	2
- 457	103	457
 +PREHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, 1
 +insert into table multi1_mm partition(p=1) select key, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm
 +PREHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, 1
 +insert into table multi1_mm partition(p=1) select key, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
- PREHOOK: Input: default@multi1_mm@p=455
- PREHOOK: Input: default@multi1_mm@p=456
- PREHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
- POSTHOOK: Input: default@multi1_mm@p=455
- POSTHOOK: Input: default@multi1_mm@p=456
- POSTHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +0	456	1
 +0	456	1
 +0	456	1
 +0	456	2
 +10	456	1
 +10	456	1
 +10	456	1
 +10	456	2
 +97	455	1
 +97	455	1
 +97	455	1
 +97	455	2
 +98	455	1
 +98	455	1
 +98	455	1
 +98	455	2
 +100	457	1
 +100	457	1
 +100	457	1
 +100	457	2
 +103	457	1
 +103	457	1
 +103	457	1
 +103	457	2
 +455	97	1
 +455	97	1
 +455	97	2
- 455	97	455
 +455	98	1
 +455	98	1
 +455	98	2
- 455	98	455
 +456	0	1
 +456	0	1
 +456	0	2
- 456	0	456
 +456	10	1
 +456	10	1
 +456	10	2
- 456	10	456
 +457	100	1
 +457	100	1
 +457	100	2
- 457	100	457
 +457	103	1
 +457	103	1
 +457	103	2
- 457	103	457
 +PREHOOK: query: drop table multi1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Output: default@multi1_mm
 +POSTHOOK: query: drop table multi1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Output: default@multi1_mm
 +PREHOOK: query: drop table stats_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table stats_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table stats_mm(key int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@stats_mm
 +POSTHOOK: query: create table stats_mm(key int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@stats_mm
 +PREHOOK: query: insert into table stats_mm  select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@stats_mm
 +POSTHOOK: query: insert into table stats_mm  select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@stats_mm
 +POSTHOOK: Lineage: stats_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: desc formatted stats_mm
 +PREHOOK: type: DESCTABLE
 +PREHOOK: Input: default@stats_mm
 +POSTHOOK: query: desc formatted stats_mm
 +POSTHOOK: type: DESCTABLE
 +POSTHOOK: Input: default@stats_mm
 +# col_name            	data_type           	comment             
 +	 	 
 +key                 	int                 	                    
 +	 	 
 +# Detailed Table Information	 	 
 +Database:           	default             	 
 +#### A masked pattern was here ####
 +Retention:          	0                   	 
 +#### A masked pattern was here ####
 +Table Type:         	MANAGED_TABLE       	 
 +Table Parameters:	 	 
 +	COLUMN_STATS_ACCURATE	{\"BASIC_STATS\":\"true\"}
 +	numFiles            	1                   
 +	numRows             	6                   
 +	rawDataSize         	13                  
 +	totalSize           	19                  
 +	transactional       	true                
 +	transactional_properties	insert_only         
 +#### A masked pattern was here ####
 +	 	 
 +# Storage Information	 	 
 +SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
 +InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
 +OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
 +Compressed:         	No                  	 
 +Num Buckets:        	-1                  	 
 +Bucket Columns:     	[]                  	 
 +Sort Columns:       	[]                  	 
 +Storage Desc Params:	 	 
 +	serialization.format	1                   
 +PREHOOK: query: insert into table stats_mm  select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@stats_mm
 +POSTHOOK: query: insert into table stats_mm  select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@stats_mm
 +POSTHOOK: Lineage: stats_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: desc formatted stats_mm
 +PREHOOK: type: DESCTABLE
 +PREHOOK: Input: default@stats_mm
 +POSTHOOK: query: desc formatted stats_mm
 +POSTHOOK: type: DESCTABLE
 +POSTHOOK: Input: default@stats_mm
 +# col_name            	data_type           	comment             
 +	 	 
 +key                 	int                 	                    
 +	 	 
 +# Detailed Table Information	 	 
 +Database:           	default             	 
 +#### A masked pattern was here ####
 +Retention:          	0                   	 
 +#### A masked pattern was here ####
 +Table Type:         	MANAGED_TABLE       	 
 +Table Parameters:	 	 
 +	COLUMN_STATS_ACCURATE	{\"BASIC_STATS\":\"true\"}
 +	numFiles            	2                   
 +	numRows             	12                  
 +	rawDataSize         	26                  
 +	totalSize           	38                  
 +	transactional       	true                
 +	transactional_properties	insert_only         
 +#### A masked pattern was here ####
 +	 	 
 +# Storage Information	 	 
 +SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
 +InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
 +OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
 +Compressed:         	No                  	 
 +Num Buckets:        	-1                  	 
 +Bucket Columns:     	[]                  	 
 +Sort Columns:       	[]                  	 
 +Storage Desc Params:	 	 
 +	serialization.format	1                   
 +PREHOOK: query: drop table stats_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@stats_mm
 +PREHOOK: Output: default@stats_mm
 +POSTHOOK: query: drop table stats_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@stats_mm
 +POSTHOOK: Output: default@stats_mm
 +PREHOOK: query: drop table stats2_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table stats2_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table stats2_mm tblproperties("transactional"="true", "transactional_properties"="insert_only") as select array(key, value) from src
 +PREHOOK: type: CREATETABLE_AS_SELECT
 +PREHOOK: Input: default@src
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@stats2_mm
 +POSTHOOK: query: create table stats2_mm tblproperties("transactional"="true", "transactional_properties"="insert_only") as select array(key, value) from src
 +POSTHOOK: type: CREATETABLE_AS_SELECT
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@stats2_mm
 +POSTHOOK: Lineage: stats2_mm._c0 EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
 +PREHOOK: query: desc formatted stats2_mm
 +PREHOOK: type: DESCTABLE
 +PREHOOK: Input: default@stats2_mm
 +POSTHOOK: query: desc formatted stats2_mm
 +POSTHOOK: type: DESCTABLE
 +POSTHOOK: Input: default@stats2_mm
 +# col_name            	data_type           	comment             
 +	 	 
 +_c0                 	array<string>       	                    
 +	 	 
 +# Detailed Table Information	 	 
 +Database:           	default             	 
 +#### A masked pattern was here ####
 +Retention:          	0                   	 
 +#### A masked pattern was here ####
 +Table Type:         	MANAGED_TABLE       	 
 +Table Parameters:	 	 
 +	COLUMN_STATS_ACCURATE	{\"BASIC_STATS\":\"true\"}
 +	numFiles            	1                   
 +	numRows             	500                 
 +	rawDataSize         	5312                
 +	totalSize           	5812                
 +	transactional       	true                
 +	transactional_properties	insert_only         
 +#### A masked pattern was here ####
 +	 	 
 +# Storage Information	 	 
 +SerDe Library:      	org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe	 
 +InputFormat:        	org.apache.hadoop.mapred.TextInputFormat	 
 +OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat	 
 +Compressed:         	No                  	 
 +Num Buckets:        	-1                  	 
 +Bucket Columns:     	[]                  	 
 +Sort Columns:       	[]                  	 
 +Storage Desc Params:	 	 
 +	serialization.format	1                   
 +PREHOOK: query: drop table stats2_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@stats2_mm
 +PREHOOK: Output: default@stats2_mm
 +POSTHOOK: query: drop table stats2_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@stats2_mm
 +POSTHOOK: Output: default@stats2_mm
 +PREHOOK: query: CREATE TABLE skewjoin_mm(key INT, value STRING) STORED AS TEXTFILE tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@skewjoin_mm
 +POSTHOOK: query: CREATE TABLE skewjoin_mm(key INT, value STRING) STORED AS TEXTFILE tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@skewjoin_mm
 +PREHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT into TABLE skewjoin_mm SELECT src1.key, src2.value
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@skewjoin_mm
 +POSTHOOK: query: FROM src src1 JOIN src src2 ON (src1.key = src2.key) INSERT into TABLE skewjoin_mm SELECT src1.key, src2.value
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@skewjoin_mm
 +POSTHOOK: Lineage: skewjoin_mm.key EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
 +POSTHOOK: Lineage: skewjoin_mm.value SIMPLE [(src)src2.FieldSchema(name:value, type:string, comment:default), ]
 +PREHOOK: query: select count(distinct key) from skewjoin_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@skewjoin_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select count(distinct key) from skewjoin_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@skewjoin_mm
 +#### A masked pattern was here ####
 +309
 +PREHOOK: query: drop table skewjoin_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@skewjoin_mm
 +PREHOOK: Output: default@skewjoin_mm
 +POSTHOOK: query: drop table skewjoin_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@skewjoin_mm
 +POSTHOOK: Output: default@skewjoin_mm
 +PREHOOK: query: CREATE TABLE parquet1_mm(id INT) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@parquet1_mm
 +POSTHOOK: query: CREATE TABLE parquet1_mm(id INT) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@parquet1_mm
 +PREHOOK: query: INSERT INTO parquet1_mm VALUES(1), (2)
 +PREHOOK: type: QUERY
 +PREHOOK: Output: default@parquet1_mm
 +POSTHOOK: query: INSERT INTO parquet1_mm VALUES(1), (2)
 +POSTHOOK: type: QUERY
 +POSTHOOK: Output: default@parquet1_mm
 +POSTHOOK: Lineage: parquet1_mm.id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
 +PREHOOK: query: CREATE TABLE parquet2_mm(id INT, value STRING) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@parquet2_mm
 +POSTHOOK: query: CREATE TABLE parquet2_mm(id INT, value STRING) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@parquet2_mm
 +PREHOOK: query: INSERT INTO parquet2_mm VALUES(1, 'value1')
 +PREHOOK: type: QUERY
 +PREHOOK: Output: default@parquet2_mm
 +POSTHOOK: query: INSERT INTO parquet2_mm VALUES(1, 'value1')
 +POSTHOOK: type: QUERY
 +POSTHOOK: Output: default@parquet2_mm
 +POSTHOOK: Lineage: parquet2_mm.id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
 +POSTHOOK: Lineage: parquet2_mm.value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
 +PREHOOK: query: INSERT INTO parquet2_mm VALUES(1, 'value2')
 +PREHOOK: type: QUERY
 +PREHOOK: Output: default@parquet2_mm
 +POSTHOOK: query: INSERT INTO parquet2_mm VALUES(1, 'value2')
 +POSTHOOK: type: QUERY
 +POSTHOOK: Output: default@parquet2_mm
 +POSTHOOK: Lineage: parquet2_mm.id EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
 +POSTHOOK: Lineage: parquet2_mm.value SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
 +PREHOOK: query: select parquet1_mm.id, t1.value, t2.value FROM parquet1_mm
 +  JOIN parquet2_mm t1 ON parquet1_mm.id=t1.id
 +  JOIN parquet2_mm t2 ON parquet1_mm.id=t2.id
 +where t1.value = 'value1' and t2.value = 'value2'
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@parquet1_mm
 +PREHOOK: Input: default@parquet2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select parquet1_mm.id, t1.value, t2.value FROM parquet1_mm
 +  JOIN parquet2_mm t1 ON parquet1_mm.id=t1.id
 +  JOIN parquet2_mm t2 ON parquet1_mm.id=t2.id
 +where t1.value = 'value1' and t2.value = 'value2'
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@parquet1_mm
 +POSTHOOK: Input: default@parquet2_mm
 +#### A masked pattern was here ####
 +1	value1	value2
 +PREHOOK: query: drop table parquet1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@parquet1_mm
 +PREHOOK: Output: default@parquet1_mm
 +POSTHOOK: query: drop table parquet1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@parquet1_mm
 +POSTHOOK: Output: default@parquet1_mm
 +PREHOOK: query: drop table parquet2_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@parquet2_mm
 +PREHOOK: Output: default@parquet2_mm
 +POSTHOOK: query: drop table parquet2_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@parquet2_mm
 +POSTHOOK: Output: default@parquet2_mm
 +PREHOOK: query: DROP TABLE IF EXISTS temp1
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: DROP TABLE IF EXISTS temp1
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: CREATE TEMPORARY TABLE temp1 (a int) TBLPROPERTIES 

<TRUNCATED>

[33/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index b000745,c8ad795..ed527b8
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@@ -392,12 -435,110 +390,12 @@@ public class MoveTask extends Task<Move
            // deal with dynamic partitions
            DynamicPartitionCtx dpCtx = tbd.getDPCtx();
            if (dpCtx != null && dpCtx.getNumDPCols() > 0) { // dynamic partitions
 -
 -            List<LinkedHashMap<String, String>> dps = Utilities.getFullDPSpecs(conf, dpCtx);
 -
 -            console.printInfo(System.getProperty("line.separator"));
 -            long startTime = System.currentTimeMillis();
 -            // load the list of DP partitions and return the list of partition specs
 -            // TODO: In a follow-up to HIVE-1361, we should refactor loadDynamicPartitions
 -            // to use Utilities.getFullDPSpecs() to get the list of full partSpecs.
 -            // After that check the number of DPs created to not exceed the limit and
 -            // iterate over it and call loadPartition() here.
 -            // The reason we don't do inside HIVE-1361 is the latter is large and we
 -            // want to isolate any potential issue it may introduce.
 -            Map<Map<String, String>, Partition> dp =
 -              db.loadDynamicPartitions(
 -                tbd.getSourcePath(),
 -                tbd.getTable().getTableName(),
 -                tbd.getPartitionSpec(),
 -                tbd.getReplace(),
 -                dpCtx.getNumDPCols(),
 -                isSkewedStoredAsDirs(tbd),
 -                work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID,
 -                work.getLoadTableWork().getCurrentTransactionId(), hasFollowingStatsTask(),
 -                work.getLoadTableWork().getWriteType());
 -
 -            // publish DP columns to its subscribers
 -            if (dps != null && dps.size() > 0) {
 -              pushFeed(FeedType.DYNAMIC_PARTITIONS, dp.values());
 -            }
 -
 -            String loadTime = "\t Time taken to load dynamic partitions: "  +
 -                (System.currentTimeMillis() - startTime)/1000.0 + " seconds";
 -            console.printInfo(loadTime);
 -            LOG.info(loadTime);
 -
 -            if (dp.size() == 0 && conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) {
 -              throw new HiveException("This query creates no partitions." +
 -                  " To turn off this error, set hive.error.on.empty.partition=false.");
 -            }
 -
 -            startTime = System.currentTimeMillis();
 -            // for each partition spec, get the partition
 -            // and put it to WriteEntity for post-exec hook
 -            for(Map.Entry<Map<String, String>, Partition> entry : dp.entrySet()) {
 -              Partition partn = entry.getValue();
 -
 -              if (bucketCols != null || sortCols != null) {
 -                updatePartitionBucketSortColumns(
 -                    db, table, partn, bucketCols, numBuckets, sortCols);
 -              }
 -
 -              WriteEntity enty = new WriteEntity(partn,
 -                getWriteType(tbd, work.getLoadTableWork().getWriteType()));
 -              if (work.getOutputs() != null) {
 -                DDLTask.addIfAbsentByName(enty, work.getOutputs());
 -              }
 -              // Need to update the queryPlan's output as well so that post-exec hook get executed.
 -              // This is only needed for dynamic partitioning since for SP the the WriteEntity is
 -              // constructed at compile time and the queryPlan already contains that.
 -              // For DP, WriteEntity creation is deferred at this stage so we need to update
 -              // queryPlan here.
 -              if (queryPlan.getOutputs() == null) {
 -                queryPlan.setOutputs(new LinkedHashSet<WriteEntity>());
 -              }
 -              queryPlan.getOutputs().add(enty);
 -
 -              // update columnar lineage for each partition
 -              dc = new DataContainer(table.getTTable(), partn.getTPartition());
 -
 -              // Don't set lineage on delete as we don't have all the columns
 -              if (work.getLineagState() != null &&
 -                  work.getLoadTableWork().getWriteType() != AcidUtils.Operation.DELETE &&
 -                  work.getLoadTableWork().getWriteType() != AcidUtils.Operation.UPDATE) {
 -                work.getLineagState().setLineage(tbd.getSourcePath(), dc,
 -                    table.getCols());
 -              }
 -              LOG.info("\tLoading partition " + entry.getKey());
 -            }
 -            console.printInfo("\t Time taken for adding to write entity : " +
 -                (System.currentTimeMillis() - startTime)/1000.0 + " seconds");
 -            dc = null; // reset data container to prevent it being added again.
 +            dc = handleDynParts(db, table, tbd, ti, dpCtx);
            } else { // static partitions
 -            List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(),
 -                tbd.getPartitionSpec());
 -            db.validatePartitionNameCharacters(partVals);
 -            db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(),
 -                tbd.getPartitionSpec(), tbd.getReplace(),
 -                tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(),
 -                work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID, hasFollowingStatsTask());
 -            Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);
 -
 -            if (bucketCols != null || sortCols != null) {
 -              updatePartitionBucketSortColumns(db, table, partn, bucketCols,
 -                  numBuckets, sortCols);
 -            }
 -
 -            dc = new DataContainer(table.getTTable(), partn.getTPartition());
 -            // add this partition to post-execution hook
 -            if (work.getOutputs() != null) {
 -              DDLTask.addIfAbsentByName(new WriteEntity(partn,
 -                getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs());
 -            }
 -         }
 +            dc = handleStaticParts(db, table, tbd, ti);
 +          }
          }
-         if (SessionState.get() != null && dc != null) {
+         if (work.getLineagState() != null && dc != null) {
            // If we are doing an update or a delete the number of columns in the table will not
            // match the number of columns in the file sink.  For update there will be one too many
            // (because of the ROW__ID), and in the case of the delete there will be just the
@@@ -445,231 -586,6 +443,231 @@@
        return (1);
      }
    }
 +
 +  private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd,
 +      TaskInformation ti) throws HiveException, IOException, InvalidOperationException {
 +    List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(),  tbd.getPartitionSpec());
 +    db.validatePartitionNameCharacters(partVals);
 +    Utilities.LOG14535.info("loadPartition called from " + tbd.getSourcePath()
 +        + " into " + tbd.getTable().getTableName());
 +    db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(),
 +        tbd.getPartitionSpec(), tbd.getReplace(),
 +        tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(),
 +        work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID &&
 +            !tbd.isMmTable(),
 +        hasFollowingStatsTask(), tbd.getTxnId(), tbd.getStmtId());
 +    Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);
 +
 +    // See the comment inside updatePartitionBucketSortColumns.
 +    if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) {
 +      updatePartitionBucketSortColumns(db, table, partn, ti.bucketCols,
 +          ti.numBuckets, ti.sortCols);
 +    }
 +
 +    DataContainer dc = new DataContainer(table.getTTable(), partn.getTPartition());
 +    // add this partition to post-execution hook
 +    if (work.getOutputs() != null) {
 +      DDLTask.addIfAbsentByName(new WriteEntity(partn,
 +        getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs());
 +    }
 +    return dc;
 +  }
 +
 +  private DataContainer handleDynParts(Hive db, Table table, LoadTableDesc tbd,
 +      TaskInformation ti, DynamicPartitionCtx dpCtx) throws HiveException,
 +      IOException, InvalidOperationException {
 +    DataContainer dc;
 +    List<LinkedHashMap<String, String>> dps = Utilities.getFullDPSpecs(conf, dpCtx);
 +
 +    console.printInfo(System.getProperty("line.separator"));
 +    long startTime = System.currentTimeMillis();
 +    // load the list of DP partitions and return the list of partition specs
 +    // TODO: In a follow-up to HIVE-1361, we should refactor loadDynamicPartitions
 +    // to use Utilities.getFullDPSpecs() to get the list of full partSpecs.
 +    // After that check the number of DPs created to not exceed the limit and
 +    // iterate over it and call loadPartition() here.
 +    // The reason we don't do inside HIVE-1361 is the latter is large and we
 +    // want to isolate any potential issue it may introduce.
 +    if (tbd.isMmTable() && !tbd.isCommitMmWrite()) {
 +      throw new HiveException("Only single-partition LoadTableDesc can skip commiting write ID");
 +    }
 +    Map<Map<String, String>, Partition> dp =
 +      db.loadDynamicPartitions(
 +        tbd.getSourcePath(),
 +        tbd.getTable().getTableName(),
 +        tbd.getPartitionSpec(),
 +        tbd.getReplace(),
 +        dpCtx.getNumDPCols(),
 +        (tbd.getLbCtx() == null) ? 0 : tbd.getLbCtx().calculateListBucketingLevel(),
 +        work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID &&
 +            !tbd.isMmTable(),
-         SessionState.get().getTxnMgr().getCurrentTxnId(), tbd.getStmtId(), hasFollowingStatsTask(),
++        work.getLoadTableWork().getTxnId(), tbd.getStmtId(), hasFollowingStatsTask(),
 +        work.getLoadTableWork().getWriteType());
 +
 +    // publish DP columns to its subscribers
 +    if (dps != null && dps.size() > 0) {
 +      pushFeed(FeedType.DYNAMIC_PARTITIONS, dp.values());
 +    }
 +
 +    String loadTime = "\t Time taken to load dynamic partitions: "  +
 +        (System.currentTimeMillis() - startTime)/1000.0 + " seconds";
 +    console.printInfo(loadTime);
 +    LOG.info(loadTime);
 +
 +    if (dp.size() == 0 && conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) {
 +      throw new HiveException("This query creates no partitions." +
 +          " To turn off this error, set hive.error.on.empty.partition=false.");
 +    }
 +
 +    startTime = System.currentTimeMillis();
 +    // for each partition spec, get the partition
 +    // and put it to WriteEntity for post-exec hook
 +    for(Map.Entry<Map<String, String>, Partition> entry : dp.entrySet()) {
 +      Partition partn = entry.getValue();
 +
 +      // See the comment inside updatePartitionBucketSortColumns.
 +      if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) {
 +        updatePartitionBucketSortColumns(
 +            db, table, partn, ti.bucketCols, ti.numBuckets, ti.sortCols);
 +      }
 +
 +      WriteEntity enty = new WriteEntity(partn,
 +        getWriteType(tbd, work.getLoadTableWork().getWriteType()));
 +      if (work.getOutputs() != null) {
 +        DDLTask.addIfAbsentByName(enty, work.getOutputs());
 +      }
 +      // Need to update the queryPlan's output as well so that post-exec hook get executed.
 +      // This is only needed for dynamic partitioning since for SP the the WriteEntity is
 +      // constructed at compile time and the queryPlan already contains that.
 +      // For DP, WriteEntity creation is deferred at this stage so we need to update
 +      // queryPlan here.
 +      if (queryPlan.getOutputs() == null) {
 +        queryPlan.setOutputs(new LinkedHashSet<WriteEntity>());
 +      }
 +      queryPlan.getOutputs().add(enty);
 +
 +      // update columnar lineage for each partition
 +      dc = new DataContainer(table.getTTable(), partn.getTPartition());
 +
 +      // Don't set lineage on delete as we don't have all the columns
-       if (SessionState.get() != null &&
++      if (work.getLineagState() != null &&
 +          work.getLoadTableWork().getWriteType() != AcidUtils.Operation.DELETE &&
 +          work.getLoadTableWork().getWriteType() != AcidUtils.Operation.UPDATE) {
-         SessionState.get().getLineageState().setLineage(tbd.getSourcePath(), dc,
++        work.getLineagState().setLineage(tbd.getSourcePath(), dc,
 +            table.getCols());
 +      }
 +      LOG.info("\tLoading partition " + entry.getKey());
 +    }
 +    console.printInfo("\t Time taken for adding to write entity : " +
 +        (System.currentTimeMillis() - startTime)/1000.0 + " seconds");
 +    dc = null; // reset data container to prevent it being added again.
 +    return dc;
 +  }
 +
 +  private void inferTaskInformation(TaskInformation ti) {
 +    // Find the first ancestor of this MoveTask which is some form of map reduce task
 +    // (Either standard, local, or a merge)
 +    while (ti.task.getParentTasks() != null && ti.task.getParentTasks().size() == 1) {
 +      ti.task = (Task)ti.task.getParentTasks().get(0);
 +      // If it was a merge task or a local map reduce task, nothing can be inferred
 +      if (ti.task instanceof MergeFileTask || ti.task instanceof MapredLocalTask) {
 +        break;
 +      }
 +
 +      // If it's a standard map reduce task, check what, if anything, it inferred about
 +      // the directory this move task is moving
 +      if (ti.task instanceof MapRedTask) {
 +        MapredWork work = (MapredWork)ti.task.getWork();
 +        MapWork mapWork = work.getMapWork();
 +        ti.bucketCols = mapWork.getBucketedColsByDirectory().get(ti.path);
 +        ti.sortCols = mapWork.getSortedColsByDirectory().get(ti.path);
 +        if (work.getReduceWork() != null) {
 +          ti.numBuckets = work.getReduceWork().getNumReduceTasks();
 +        }
 +
 +        if (ti.bucketCols != null || ti.sortCols != null) {
 +          // This must be a final map reduce task (the task containing the file sink
 +          // operator that writes the final output)
 +          assert work.isFinalMapRed();
 +        }
 +        break;
 +      }
 +
 +      // If it's a move task, get the path the files were moved from, this is what any
 +      // preceding map reduce task inferred information about, and moving does not invalidate
 +      // those assumptions
 +      // This can happen when a conditional merge is added before the final MoveTask, but the
 +      // condition for merging is not met, see GenMRFileSink1.
 +      if (ti.task instanceof MoveTask) {
 +        MoveTask mt = (MoveTask)ti.task;
 +        if (mt.getWork().getLoadFileWork() != null) {
 +          ti.path = mt.getWork().getLoadFileWork().getSourcePath().toUri().toString();
 +        }
 +      }
 +    }
 +  }
 +
 +  private void checkFileFormats(Hive db, LoadTableDesc tbd, Table table)
 +      throws HiveException {
 +    if (work.getCheckFileFormat()) {
 +      // Get all files from the src directory
 +      FileStatus[] dirs;
 +      ArrayList<FileStatus> files;
 +      FileSystem srcFs; // source filesystem
 +      try {
 +        srcFs = tbd.getSourcePath().getFileSystem(conf);
 +        dirs = srcFs.globStatus(tbd.getSourcePath());
 +        files = new ArrayList<FileStatus>();
 +        for (int i = 0; (dirs != null && i < dirs.length); i++) {
 +          files.addAll(Arrays.asList(srcFs.listStatus(dirs[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER)));
 +          // We only check one file, so exit the loop when we have at least
 +          // one.
 +          if (files.size() > 0) {
 +            break;
 +          }
 +        }
 +      } catch (IOException e) {
 +        throw new HiveException(
 +            "addFiles: filesystem error in check phase", e);
 +      }
 +
 +      // handle file format check for table level
 +      if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVECHECKFILEFORMAT)) {
 +        boolean flag = true;
 +        // work.checkFileFormat is set to true only for Load Task, so assumption here is
 +        // dynamic partition context is null
 +        if (tbd.getDPCtx() == null) {
 +          if (tbd.getPartitionSpec() == null || tbd.getPartitionSpec().isEmpty()) {
 +            // Check if the file format of the file matches that of the table.
 +            flag = HiveFileFormatUtils.checkInputFormat(
 +                srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
 +          } else {
 +            // Check if the file format of the file matches that of the partition
 +            Partition oldPart = db.getPartition(table, tbd.getPartitionSpec(), false);
 +            if (oldPart == null) {
 +              // this means we have just created a table and are specifying partition in the
 +              // load statement (without pre-creating the partition), in which case lets use
 +              // table input format class. inheritTableSpecs defaults to true so when a new
 +              // partition is created later it will automatically inherit input format
 +              // from table object
 +              flag = HiveFileFormatUtils.checkInputFormat(
 +                  srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
 +            } else {
 +              flag = HiveFileFormatUtils.checkInputFormat(
 +                  srcFs, conf, oldPart.getInputFormatClass(), files);
 +            }
 +          }
 +          if (!flag) {
 +            throw new HiveException(ErrorMsg.WRONG_FILE_FORMAT);
 +          }
 +        } else {
 +          LOG.warn("Skipping file format check as dpCtx is not null");
 +        }
 +      }
 +    }
 +  }
 +
 +
    /**
     * so to make sure we crate WriteEntity with the right WriteType.  This is (at this point) only
     * for consistency since LockManager (which is the only thing that pays attention to WriteType)

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
index 05698d1,5c6ef9f..51c4090
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
@@@ -240,10 -238,10 +240,11 @@@ public class LoadPartitions 
        Path tmpPath) {
      LoadTableDesc loadTableWork = new LoadTableDesc(
          tmpPath, Utilities.getTableDesc(table), partSpec.getPartSpec(),
 -        event.replicationSpec().isReplace());
 +        event.replicationSpec().isReplace(), SessionState.get().getTxnMgr().getCurrentTxnId()
 +    );
      loadTableWork.setInheritTableSpecs(false);
-     MoveWork work = new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false);
+     MoveWork work = new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false,
+         context.sessionStateLineageState);
      return TaskFactory.get(work, context.hiveConf);
    }
  

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
index afc04a3,a9a9162..65a3a59
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
@@@ -225,11 -224,10 +225,12 @@@ public class LoadTable 
          ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, tmpPath, context.hiveConf);
  
      LoadTableDesc loadTableWork = new LoadTableDesc(
 -        tmpPath, Utilities.getTableDesc(table), new TreeMap<>(), replicationSpec.isReplace());
 +        tmpPath, Utilities.getTableDesc(table), new TreeMap<>(), replicationSpec.isReplace(),
 +        SessionState.get().getTxnMgr().getCurrentTxnId()
 +    );
      MoveWork moveWork =
-         new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false);
+         new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false,
+             context.sessionStateLineageState);
      Task<?> loadTableTask = TaskFactory.get(moveWork, context.hiveConf);
      copyTask.addDependentTask(loadTableTask);
      return copyTask;

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index 6dfa607,25cca7b..6a56407
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@@ -1364,29 -1353,9 +1364,31 @@@ public final class GenMapRedUtils 
      cplan.setInputformat("org.apache.hadoop.hive.ql.io.CombineHiveInputFormat");
      // NOTE: we should gather stats in MR1 rather than MR2 at merge job since we don't
      // know if merge MR2 will be triggered at execution time
 -    Task<MoveWork> mvTask = GenMapRedUtils.findMoveTask(mvTasks, fsOutput);
 +    MoveWork dummyMv = null;
 +    if (srcMmWriteId == null) {
 +      // Only create the movework for non-MM table. No action needed for a MM table.
 +      Utilities.LOG14535.info("creating dummy movetask for merge (with lfd)");
 +      dummyMv = new MoveWork(null, null, null,
-          new LoadFileDesc(inputDirName, finalName, true, null, null, false), false);
++          new LoadFileDesc(inputDirName, finalName, true, null, null, false), false,
++          SessionState.get().getLineageState());
 +    } else {
 +      // TODO# create the noop MoveWork to avoid q file changes for now. Should be removed w/the flag just before merge
 +      dummyMv = new MoveWork(null, null, null,
-           new LoadFileDesc(inputDirName, finalName, true, null, null, false), false);
++          new LoadFileDesc(inputDirName, finalName, true, null, null, false), false,
++          SessionState.get().getLineageState());
 +      dummyMv.setNoop(true);
 +    }
 +    // Use the original fsOp path here in case of MM - while the new FSOP merges files inside the
 +    // MM directory, the original MoveTask still commits based on the parent. Note that this path
 +    // can only be triggered for a merge that's part of insert for now; MM tables do not support
 +    // concatenate. Keeping the old logic for non-MM tables with temp directories and stuff.
 +    Path fsopPath = srcMmWriteId != null ? fsInputDesc.getFinalDirName() : finalName;
 +    Utilities.LOG14535.info("Looking for MoveTask to make it dependant on the conditional tasks");
 +
 +    Task<MoveWork> mvTask = GenMapRedUtils.findMoveTaskForFsopOutput(
 +        mvTasks, fsopPath, fsInputDesc.isMmTable());
      ConditionalTask cndTsk = GenMapRedUtils.createCondTask(conf, currTask, dummyMv, work,
 -        fsInputDesc.getFinalDirName(), finalName, mvTask, dependencyTask);
 +        fsInputDesc.getMergeInputDirName(), finalName, mvTask, dependencyTask);
  
      // keep the dynamic partition context in conditional task resolver context
      ConditionalResolverMergeFilesCtx mrCtx =

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index fdeeacf,e70a72c..36bb89f
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@@ -1090,10 -1088,11 +1095,11 @@@ public class DDLSemanticAnalyzer extend
          Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc);
          truncateTblDesc.setOutputDir(queryTmpdir);
          LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc,
-             partSpec == null ? new HashMap<String, String>() : partSpec, null);
 -            partSpec == null ? new HashMap<>() : partSpec);
++            partSpec == null ? new HashMap<>() : partSpec, null);
          ltd.setLbCtx(lbCtx);
 -        Task<MoveWork> moveTsk = TaskFactory
 -            .get(new MoveWork(null, null, ltd, null, false, SessionState.get().getLineageState()),
 -            conf);
 +        @SuppressWarnings("unchecked")
-         Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false), conf);
++        Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(
++          null, null, ltd, null, false, SessionState.get().getLineageState()), conf);
          truncateTask.addDependentTask(moveTsk);
  
          // Recalculate the HDFS stats if auto gather stats is set
@@@ -1736,11 -1722,12 +1742,12 @@@
        TableDesc tblDesc = Utilities.getTableDesc(tblObj);
        Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc);
        mergeDesc.setOutputDir(queryTmpdir);
 +      // No need to handle MM tables - unsupported path.
        LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc,
 -          partSpec == null ? new HashMap<>() : partSpec);
 +          partSpec == null ? new HashMap<>() : partSpec, null);
        ltd.setLbCtx(lbCtx);
-       Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false), conf);
 -      Task<MoveWork> moveTsk = TaskFactory
 -          .get(new MoveWork(null, null, ltd, null, false, SessionState.get().getLineageState()),
 -          conf);
++      Task<MoveWork> moveTsk = TaskFactory.get(
++        new MoveWork(null, null, ltd, null, false, SessionState.get().getLineageState()), conf);
        mergeTask.addDependentTask(moveTsk);
  
        if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 36c0c98,751bda0..b9f28eb
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@@ -360,36 -343,17 +360,36 @@@ public class ImportSemanticAnalyzer ext
      return tblDesc;
    }
  
 +
    private static Task<?> loadTable(URI fromURI, Table table, boolean replace, Path tgtPath,
 -                            ReplicationSpec replicationSpec, EximUtil.SemanticAnalyzerWrapperContext x) {
 +      ReplicationSpec replicationSpec, EximUtil.SemanticAnalyzerWrapperContext x,
 +      Long txnId, int stmtId, boolean isSourceMm) {
      Path dataPath = new Path(fromURI.toString(), EximUtil.DATA_PATH_NAME);
 -    Path tmpPath = x.getCtx().getExternalTmpPath(tgtPath);
 -    Task<?> copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, tmpPath, x.getConf());
 -    LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
 -        Utilities.getTableDesc(table), new TreeMap<>(),
 -        replace);
 -    Task<?> loadTableTask = TaskFactory.get(new MoveWork(x.getInputs(),
 -            x.getOutputs(), loadTableWork, null, false, SessionState.get().getLineageState()),
 -        x.getConf());
 +    Path destPath = !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtPath)
 +        : new Path(tgtPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
 +    Utilities.LOG14535.info("adding import work for table with source location: "
 +        + dataPath + "; table: " + tgtPath + "; copy destination " + destPath + "; mm "
 +        + txnId + " (src " + isSourceMm + ") for " + (table == null ? "a new table" : table.getTableName()));
 +
 +    Task<?> copyTask = null;
 +    if (replicationSpec.isInReplicationScope()) {
 +      if (isSourceMm || isAcid(txnId)) {
 +        // TODO: ReplCopyTask is completely screwed. Need to support when it's not as screwed.
 +        throw new RuntimeException("Replicating MM and ACID tables is not supported");
 +      }
 +      copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, destPath, x.getConf());
 +    } else {
 +      CopyWork cw = new CopyWork(dataPath, destPath, false);
 +      cw.setSkipSourceMmDirs(isSourceMm);
 +      copyTask = TaskFactory.get(cw, x.getConf());
 +    }
 +
 +    LoadTableDesc loadTableWork = new LoadTableDesc(destPath,
-         Utilities.getTableDesc(table), new TreeMap<String, String>(), replace, txnId);
++        Utilities.getTableDesc(table), new TreeMap<>(), replace, txnId);
 +    loadTableWork.setTxnId(txnId);
 +    loadTableWork.setStmtId(stmtId);
-     MoveWork mv = new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, null, false);
++    MoveWork mv = new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, null, false, SessionState.get().getLineageState());
 +    Task<?> loadTableTask = TaskFactory.get(mv, x.getConf());
      copyTask.addDependentTask(loadTableTask);
      x.getTasks().add(copyTask);
      return loadTableTask;
@@@ -450,40 -413,19 +455,40 @@@
            + partSpecToString(partSpec.getPartSpec())
            + " with source location: " + srcLocation);
        Path tgtLocation = new Path(partSpec.getLocation());
 -      Path tmpPath = x.getCtx().getExternalTmpPath(tgtLocation);
 -      Task<?> copyTask = ReplCopyTask.getLoadCopyTask(
 -          replicationSpec, new Path(srcLocation), tmpPath, x.getConf());
 +      Path destPath = !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtLocation)
 +          : new Path(tgtLocation, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
 +      Path moveTaskSrc =  !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? destPath : tgtLocation;
 +      Utilities.LOG14535.info("adding import work for partition with source location: "
 +          + srcLocation + "; target: " + tgtLocation + "; copy dest " + destPath + "; mm "
 +          + txnId + " (src " + isSourceMm + ") for " + partSpecToString(partSpec.getPartSpec()));
 +
 +
 +      Task<?> copyTask = null;
 +      if (replicationSpec.isInReplicationScope()) {
 +        if (isSourceMm || isAcid(txnId)) {
 +          // TODO: ReplCopyTask is completely screwed. Need to support when it's not as screwed.
 +          throw new RuntimeException("Replicating MM and ACID tables is not supported");
 +        }
 +        copyTask = ReplCopyTask.getLoadCopyTask(
 +            replicationSpec, new Path(srcLocation), destPath, x.getConf());
 +      } else {
 +        CopyWork cw = new CopyWork(new Path(srcLocation), destPath, false);
 +        cw.setSkipSourceMmDirs(isSourceMm);
 +        copyTask = TaskFactory.get(cw, x.getConf());
 +      }
 +
        Task<?> addPartTask = TaskFactory.get(new DDLWork(x.getInputs(),
            x.getOutputs(), addPartitionDesc), x.getConf());
 -      LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
 -          Utilities.getTableDesc(table),
 -          partSpec.getPartSpec(), replicationSpec.isReplace());
 +      LoadTableDesc loadTableWork = new LoadTableDesc(moveTaskSrc, Utilities.getTableDesc(table),
 +          partSpec.getPartSpec(), replicationSpec.isReplace(), txnId);
 +      loadTableWork.setTxnId(txnId);
 +      loadTableWork.setStmtId(stmtId);
        loadTableWork.setInheritTableSpecs(false);
 +      // Do not commit the write ID from each task; need to commit once.
 +      // TODO: we should just change the import to use a single MoveTask, like dynparts.
 +      loadTableWork.setIntermediateInMmWrite(isAcid(txnId));
        Task<?> loadPartTask = TaskFactory.get(new MoveWork(
-           x.getInputs(), x.getOutputs(), loadTableWork, null, false), x.getConf());
 -              x.getInputs(), x.getOutputs(), loadTableWork, null, false,
 -              SessionState.get().getLineageState()),
 -          x.getConf());
++          x.getInputs(), x.getOutputs(), loadTableWork, null, false, SessionState.get().getLineageState()), x.getConf());
        copyTask.addDependentTask(loadPartTask);
        addPartTask.addDependentTask(loadPartTask);
        x.getTasks().add(copyTask);

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 3475c7c,4814fcd..df50aab
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@@ -6858,15 -6902,10 +6861,18 @@@ public class SemanticAnalyzer extends B
            acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
            checkAcidConstraints(qb, table_desc, dest_tab);
          }
 -        Long currentTransactionId = acidOp == Operation.NOT_ACID ? null :
 +        if (MetaStoreUtils.isInsertOnlyTable(table_desc.getProperties())) {
 +          acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
 +        }
 +        if (isMmTable) {
 +          txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
++        } else {
++          txnId = acidOp == Operation.NOT_ACID ? null :
+             SessionState.get().getTxnMgr().getCurrentTxnId();
 -        ltd = new LoadTableDesc(queryTmpdir, table_desc, dpCtx, acidOp,
 -            currentTransactionId);
 +        }
 +        boolean isReplace = !qb.getParseInfo().isInsertIntoTable(
 +            dest_tab.getDbName(), dest_tab.getTableName());
 +        ltd = new LoadTableDesc(queryTmpdir, table_desc, dpCtx, acidOp, isReplace, txnId);
          // For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old
          // deltas and base and leave them up to the cleaner to clean up
          ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
@@@ -6926,13 -7020,10 +6932,16 @@@
          acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
          checkAcidConstraints(qb, table_desc, dest_tab);
        }
 -      Long currentTransactionId = (acidOp == Operation.NOT_ACID) ? null :
 +      if (MetaStoreUtils.isInsertOnlyTable(dest_part.getTable().getParameters())) {
 +        acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
 +      }
 +      if (isMmTable) {
 +        txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
++      } else {
++        txnId = (acidOp == Operation.NOT_ACID) ? null :
+           SessionState.get().getTxnMgr().getCurrentTxnId();
 -      ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp,
 -          currentTransactionId);
 +      }
 +      ltd = new LoadTableDesc(queryTmpdir, table_desc, dest_part.getSpec(), acidOp, txnId);
        // For Acid table, Insert Overwrite shouldn't replace the table content. We keep the old
        // deltas and base and leave them up to the cleaner to clean up
        ltd.setReplace(!qb.getParseInfo().isInsertIntoTable(dest_tab.getDbName(),
@@@ -7015,9 -7150,8 +7024,9 @@@
        }
  
        boolean isDfsDir = (dest_type.intValue() == QBMetaData.DEST_DFS_FILE);
 +      // Create LFD even for MM CTAS - it's a no-op move, but it still seems to be used for stats.
        loadFileWork.add(new LoadFileDesc(tblDesc, viewDesc, queryTmpdir, dest_path, isDfsDir, cols,
-         colTypes, destTableIsAcid ? Operation.INSERT : Operation.NOT_ACID, isMmCtas));
 -          colTypes, destTableIsAcid ? Operation.INSERT : Operation.NOT_ACID));
++          colTypes, destTableIsAcid ? Operation.INSERT : Operation.NOT_ACID, isMmCtas));
        if (tblDesc == null) {
          if (viewDesc != null) {
            table_desc = PlanUtils.getTableDesc(viewDesc, cols, colTypes);

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 356ab6f,f0089fc..752a934
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@@ -18,17 -18,8 +18,19 @@@
  
  package org.apache.hadoop.hive.ql.parse;
  
 +import java.io.Serializable;
 +import java.util.ArrayList;
 +import java.util.HashSet;
 +import java.util.Iterator;
 +import java.util.LinkedHashSet;
 +import java.util.List;
 +import java.util.Set;
 +
 +import org.apache.hadoop.hive.ql.io.AcidUtils;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
+ import com.google.common.collect.Interner;
+ import com.google.common.collect.Interners;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.hive.common.HiveStatsUtils;
  import org.apache.hadoop.hive.conf.HiveConf;
@@@ -228,17 -227,47 +239,19 @@@ public abstract class TaskCompiler 
          }
        }
  
 -      boolean oneLoadFile = true;
 +      boolean oneLoadFileForCtas = true;
        for (LoadFileDesc lfd : loadFileWork) {
          if (pCtx.getQueryProperties().isCTAS() || pCtx.getQueryProperties().isMaterializedView()) {
 -          assert (oneLoadFile); // should not have more than 1 load file for
 -          // CTAS
 -          // make the movetask's destination directory the table's destination.
 -          Path location;
 -          String loc = pCtx.getQueryProperties().isCTAS() ?
 -                  pCtx.getCreateTable().getLocation() : pCtx.getCreateViewDesc().getLocation();
 -          if (loc == null) {
 -            // get the default location
 -            Path targetPath;
 -            try {
 -              String protoName = null;
 -              if (pCtx.getQueryProperties().isCTAS()) {
 -                protoName = pCtx.getCreateTable().getTableName();
 -              } else if (pCtx.getQueryProperties().isMaterializedView()) {
 -                protoName = pCtx.getCreateViewDesc().getViewName();
 -              }
 -              String[] names = Utilities.getDbTableName(protoName);
 -              if (!db.databaseExists(names[0])) {
 -                throw new SemanticException("ERROR: The database " + names[0]
 -                    + " does not exist.");
 -              }
 -              Warehouse wh = new Warehouse(conf);
 -              targetPath = wh.getDefaultTablePath(db.getDatabase(names[0]), names[1]);
 -            } catch (HiveException | MetaException e) {
 -              throw new SemanticException(e);
 -            }
 -
 -            location = targetPath;
 -          } else {
 -            location = new Path(loc);
 +          if (!oneLoadFileForCtas) { // should not have more than 1 load file for CTAS.
 +            throw new SemanticException(
 +                "One query is not expected to contain multiple CTAS loads statements");
            }
 -          lfd.setTargetDir(location);
 -
 -          oneLoadFile = false;
 +          setLoadFileLocation(pCtx, lfd);
 +          oneLoadFileForCtas = false;
          }
-         mvTask.add(TaskFactory.get(new MoveWork(null, null, null, lfd, false), conf));
+         mvTask.add(TaskFactory
+             .get(new MoveWork(null, null, null, lfd, false, SessionState.get().getLineageState()),
+                 conf));
        }
      }
  

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java
index 45d4fb0,023d247..9477df6
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java
@@@ -35,8 -35,8 +35,9 @@@ public class LoadDesc implements Serial
     * Need to remember whether this is an acid compliant operation, and if so whether it is an
     * insert, update, or delete.
     */
-   private final AcidUtils.Operation writeType;
+   final AcidUtils.Operation writeType;
+ 
 +
    public LoadDesc(final Path sourcePath, AcidUtils.Operation writeType) {
      this.sourcePath = sourcePath;
      this.writeType = writeType;

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
index 0032648,0292af5..d90acce
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
@@@ -51,8 -48,8 +51,8 @@@ public class LoadFileDesc extends LoadD
  
    public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateViewDesc  createViewDesc,
                        final Path sourcePath, final Path targetDir, final boolean isDfsDir,
 -                      final String columns, final String columnTypes, AcidUtils.Operation writeType) {
 -    this(sourcePath, targetDir, isDfsDir, columns, columnTypes, writeType);
 +                      final String columns, final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) {
-    this(sourcePath, targetDir, isDfsDir, columns, columnTypes, writeType, isMmCtas);
++    this(sourcePath, targetDir, isDfsDir, columns, columnTypes, writeType, isMmCtas);
      if (createTableDesc != null && createTableDesc.getDatabaseName() != null
          && createTableDesc.getTableName() != null) {
        destinationCreateTable = (createTableDesc.getTableName().contains(".") ? "" : createTableDesc

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
index e893ab5,90a970c..85d1324
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
@@@ -18,13 -18,7 +18,9 @@@
  
  package org.apache.hadoop.hive.ql.plan;
  
- import java.io.Serializable;
- import java.util.LinkedHashMap;
- import java.util.Map;
- 
  import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 +import org.apache.hadoop.hive.ql.exec.Utilities;
  import org.apache.hadoop.hive.ql.io.AcidUtils;
  import org.apache.hadoop.hive.ql.plan.Explain.Level;
  
@@@ -39,8 -37,10 +39,8 @@@ public class LoadTableDesc extends Load
    private ListBucketingCtx lbCtx;
    private boolean inheritTableSpecs = true; //For partitions, flag controlling whether the current
                                              //table specs are to be used
-   private Long txnId;
 -  /*
 -  if the writeType above is NOT_ACID then the currentTransactionId will be null
 -   */
 -  private final Long currentTransactionId;
 +  private int stmtId;
++  private Long currentTransactionId;
  
    // TODO: the below seems like they should just be combined into partitionDesc
    private org.apache.hadoop.hive.ql.plan.TableDesc table;
@@@ -59,15 -59,13 +60,14 @@@
    }
  
    public LoadTableDesc(final Path sourcePath,
-       final org.apache.hadoop.hive.ql.plan.TableDesc table,
+       final TableDesc table,
        final Map<String, String> partitionSpec,
        final boolean replace,
-       final AcidUtils.Operation writeType,
-       Long txnId) {
+       final AcidUtils.Operation writeType, Long currentTransactionId) {
      super(sourcePath, writeType);
 -    this.currentTransactionId = currentTransactionId;
 -    init(table, partitionSpec, replace);
 +    Utilities.LOG14535.info("creating part LTD from " + sourcePath + " to "
 +        + ((table.getProperties() == null) ? "null" : table.getTableName()));
-     init(table, partitionSpec, replace, txnId);
++    init(table, partitionSpec, replace, currentTransactionId);
    }
  
    /**
@@@ -105,17 -103,16 +105,17 @@@
    }
  
    public LoadTableDesc(final Path sourcePath,
-       final org.apache.hadoop.hive.ql.plan.TableDesc table,
+       final TableDesc table,
        final DynamicPartitionCtx dpCtx,
 -      final AcidUtils.Operation writeType, Long currentTransactionId) {
 +      final AcidUtils.Operation writeType,
 +      boolean isReplace, Long txnId) {
      super(sourcePath, writeType);
 +    Utilities.LOG14535.info("creating LTD from " + sourcePath + " to " + table.getTableName()/*, new Exception()*/);
      this.dpCtx = dpCtx;
 -    this.currentTransactionId = currentTransactionId;
      if (dpCtx != null && dpCtx.getPartSpec() != null && partitionSpec == null) {
 -      init(table, dpCtx.getPartSpec(), true);
 +      init(table, dpCtx.getPartSpec(), isReplace, txnId);
      } else {
-       init(table, new LinkedHashMap<String, String>(), isReplace, txnId);
 -      init(table, new LinkedHashMap<>(), true);
++      init(table, new LinkedHashMap<>(), isReplace, txnId);
      }
    }
  
@@@ -127,7 -123,6 +127,7 @@@
      this.table = table;
      this.partitionSpec = partitionSpec;
      this.replace = replace;
-     this.txnId = txnId;
++    this.currentTransactionId = txnId;
    }
  
    @Explain(displayName = "table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
@@@ -196,27 -182,7 +196,27 @@@
      this.lbCtx = lbCtx;
    }
  
-   public Long getTxnId() {
-     return txnId;
 -  public long getCurrentTransactionId() {
 -    return writeType == AcidUtils.Operation.NOT_ACID ? 0L : currentTransactionId;
++  public long getTxnId() {
++    return currentTransactionId == null ? 0 : currentTransactionId;
 +  }
 +
 +  public void setTxnId(Long txnId) {
-     this.txnId = txnId;
++    this.currentTransactionId = txnId;
 +  }
 +
 +  public int getStmtId() {
 +    return stmtId;
 +  }
 +
 +  public void setStmtId(int stmtId) {
 +    this.stmtId = stmtId;
 +  }
 +
 +  public void setIntermediateInMmWrite(boolean b) {
 +    this.commitMmWriteId = !b;
 +  }
 +
 +  public boolean isCommitMmWrite() {
 +    return commitMmWriteId;
    }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
index 8594edf,00c0ce3..f6303ba
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
@@@ -56,22 -61,22 +62,26 @@@ public class MoveWork implements Serial
     * List of inserted partitions
     */
    protected List<Partition> movedParts;
 +  private boolean isNoop;
  
    public MoveWork() {
+     sessionStateLineageState = null;
    }
  
-   private MoveWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs) {
++
+   private MoveWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
+       LineageState lineageState) {
      this.inputs = inputs;
      this.outputs = outputs;
+     sessionStateLineageState = lineageState;
    }
  
    public MoveWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
        final LoadTableDesc loadTableWork, final LoadFileDesc loadFileWork,
-       boolean checkFileFormat, boolean srcLocal) {
-     this(inputs, outputs);
+       boolean checkFileFormat, boolean srcLocal, LineageState lineageState) {
+     this(inputs, outputs, lineageState);
 +    Utilities.LOG14535.info("Creating MoveWork " + System.identityHashCode(this)
 +        + " with " + loadTableWork + "; " + loadFileWork);
      this.loadTableWork = loadTableWork;
      this.loadFileWork = loadFileWork;
      this.checkFileFormat = checkFileFormat;
@@@ -80,8 -85,11 +90,8 @@@
  
    public MoveWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
        final LoadTableDesc loadTableWork, final LoadFileDesc loadFileWork,
-       boolean checkFileFormat) {
-     this(inputs, outputs, loadTableWork, loadFileWork, checkFileFormat, false);
+       boolean checkFileFormat, LineageState lineageState) {
 -    this(inputs, outputs, lineageState);
 -    this.loadTableWork = loadTableWork;
 -    this.loadFileWork = loadFileWork;
 -    this.checkFileFormat = checkFileFormat;
++    this(inputs, outputs, loadTableWork, loadFileWork, checkFileFormat, false, lineageState);
    }
  
    public MoveWork(final MoveWork o) {
@@@ -153,11 -162,7 +164,15 @@@
      this.srcLocal = srcLocal;
    }
  
 +  public void setNoop(boolean b) {
 +    this.isNoop = true;
 +  }
 +
 +  public boolean isNoop() {
 +    return this.isNoop;
 +  }
++  
+   public LineageState getLineagState() {
+     return sessionStateLineageState;
+   }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/tez/explainuser_3.q.out
index 9f764b8,a331bf7..7e89478
--- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
@@@ -509,13 -509,13 +509,13 @@@ Stage-
                    Conditional Operator
                      Stage-1
                        Map 1 vectorized
 -                      File Output Operator [FS_10]
 +                      File Output Operator [FS_8]
                          table:{"name:":"default.orc_merge5"}
-                         Select Operator [SEL_9] (rows=306 width=335)
+                         Select Operator [SEL_9] (rows=1 width=352)
                            Output:["_col0","_col1","_col2","_col3","_col4"]
-                           Filter Operator [FIL_8] (rows=306 width=335)
+                           Filter Operator [FIL_8] (rows=1 width=352)
                              predicate:(userid <= 13)
-                             TableScan [TS_0] (rows=919 width=335)
+                             TableScan [TS_0] (rows=1 width=352)
                                default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"]
              Stage-4(CONDITIONAL)
                File Merge

[34/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/379d9bab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/379d9bab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/379d9bab

Branch: refs/heads/master
Commit: 379d9babdd43d459b0acfb7663527850320fdbb6
Parents: 5b3c797 fb07a11
Author: sergey <se...@apache.org>
Authored: Wed Sep 27 13:22:07 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed Sep 27 13:22:07 2017 -0700

----------------------------------------------------------------------
 .../hive/common/ValidCompactorTxnList.java      |     8 +-
 .../hadoop/hive/common/ValidReadTxnList.java    |     7 +-
 .../apache/hadoop/hive/common/ValidTxnList.java |     4 +
 .../org/apache/hadoop/hive/conf/HiveConf.java   |    62 +-
 .../druid/io/DruidQueryBasedInputFormat.java    |     7 +-
 .../results/positive/hbase_custom_key2.q.out    |     2 +-
 .../results/positive/hbase_custom_key3.q.out    |     2 +-
 .../results/positive/hbase_ppd_key_range.q.out  |     2 +-
 .../test/results/positive/hbase_pushdown.q.out  |     2 +-
 .../test/results/positive/hbase_queries.q.out   |     2 +-
 .../hbase_single_sourced_multi_insert.q.out     |     4 +-
 hcatalog/core/pom.xml                           |     7 +
 .../apache/hive/hcatalog/cli/TestPermsGrp.java  |     5 +-
 .../mapreduce/TestHCatMultiOutputFormat.java    |     4 +-
 .../mapreduce/TestHCatPartitionPublish.java     |     5 +-
 hcatalog/webhcat/java-client/pom.xml            |     7 +
 .../hive/hcatalog/api/TestHCatClient.java       |     6 +-
 hcatalog/webhcat/svr/pom.xml                    |     7 +
 .../hive/hcatalog/templeton/TestWebHCatE2e.java |     4 +-
 .../antlr4/org/apache/hive/hplsql/Hplsql.g4     |     7 +
 .../main/java/org/apache/hive/hplsql/Copy.java  |    10 +-
 .../main/java/org/apache/hive/hplsql/Exec.java  |     8 +
 .../main/java/org/apache/hive/hplsql/Meta.java  |    71 +-
 .../main/java/org/apache/hive/hplsql/Row.java   |     7 +
 .../main/java/org/apache/hive/hplsql/Stmt.java  |   281 +
 .../apache/hive/hplsql/functions/Function.java  |     3 +
 hplsql/src/main/resources/hplsql-site.xml       |     2 -
 .../org/apache/hive/hplsql/TestHplsqlLocal.java |    10 +
 hplsql/src/test/queries/db/summary.sql          |    15 +
 .../src/test/queries/local/create_function5.sql |    11 +
 .../test/queries/local/create_procedure4.sql    |    10 +
 hplsql/src/test/results/db/summary.out.txt      |    47 +
 .../test/results/local/create_function3.out.txt |     2 +
 .../test/results/local/create_function4.out.txt |     1 +
 .../test/results/local/create_function5.out.txt |     8 +
 .../results/local/create_procedure4.out.txt     |     8 +
 .../mapjoin/MapJoinMultiKeyBench.java           |    94 +-
 .../mapjoin/MapJoinOneLongKeyBench.java         |    94 +-
 .../mapjoin/MapJoinOneStringKeyBench.java       |    94 +-
 .../security/TestHadoopAuthBridge23.java        |     4 +-
 itests/hive-unit/pom.xml                        |     8 +-
 .../AbstractTestAuthorizationApiAuthorizer.java |     4 +-
 .../hadoop/hive/metastore/TestFilterHooks.java  |     4 +-
 ...TestHiveMetaStoreWithEnvironmentContext.java |     4 +-
 .../hive/metastore/TestMarkPartitionRemote.java |     2 +-
 .../metastore/TestMetaStoreAuthorization.java   |     2 +-
 .../TestMetaStoreEndFunctionListener.java       |     4 +-
 .../metastore/TestMetaStoreEventListener.java   |     4 +-
 .../TestMetaStoreEventListenerOnlyOnCommit.java |     4 +-
 .../metastore/TestMetaStoreInitListener.java    |     4 +-
 .../metastore/TestMetaStoreListenersError.java  |     4 +-
 .../hive/metastore/TestMetaStoreMetrics.java    |     4 +-
 .../hive/metastore/TestRemoteHiveMetaStore.java |     4 +-
 .../TestRemoteHiveMetaStoreIpAddress.java       |     4 +-
 .../hive/metastore/TestRetryingHMSHandler.java  |     4 +-
 ...estDDLWithRemoteMetastoreSecondNamenode.java |     6 +-
 .../hadoop/hive/ql/parse/TestCopyUtils.java     |   137 +
 .../hadoop/hive/ql/parse/TestExportImport.java  |    10 +-
 .../hive/ql/parse/TestReplicationScenarios.java |   163 +-
 ...TestReplicationScenariosAcrossInstances.java |    29 +
 .../hadoop/hive/ql/parse/WarehouseInstance.java |    97 +-
 .../security/StorageBasedMetastoreTestBase.java |     6 +-
 .../TestAuthorizationPreEventListener.java      |     6 +-
 .../TestClientSideAuthorizationProvider.java    |     6 +-
 .../TestMetastoreAuthorizationProvider.java     |     6 +-
 .../TestMultiAuthorizationPreEventListener.java |     6 +-
 .../hive/ql/txn/compactor/TestCompactor.java    |     4 +-
 .../apache/hive/jdbc/TestJdbcWithMiniHS2.java   |    82 +
 .../apache/hive/jdbc/TestJdbcWithMiniLlap.java  |    74 +
 .../jdbc/TestServiceDiscoveryWithMiniHS2.java   |    22 +
 .../cli/session/TestHiveSessionImpl.java        |     3 +-
 .../service/cli/session/TestQueryDisplay.java   |     3 +-
 .../thrift/ThriftCliServiceMessageSizeTest.java |     4 +-
 .../test/resources/testconfiguration.properties |     6 +-
 itests/util/pom.xml                             |     6 +
 .../org/apache/hadoop/hive/ql/QTestUtil.java    |     2 +-
 .../org/apache/hive/jdbc/miniHS2/MiniHS2.java   |    12 +-
 .../hive/cli/control/TestDanglingQOuts.java     |   120 +
 .../org/apache/hive/jdbc/HiveConnection.java    |    15 +
 .../org/apache/hive/jdbc/HiveStatement.java     |    11 +
 jdbc/src/java/org/apache/hive/jdbc/Utils.java   |    15 +
 .../hive/jdbc/ZooKeeperHiveClientHelper.java    |   118 +-
 .../hive/llap/tez/LlapProtocolClientProxy.java  |    37 +-
 .../hive/registry/impl/TezAmInstance.java       |     9 +-
 .../hadoop/hive/llap/TestAsyncPbRpcProxy.java   |     8 +-
 .../plugin/rpc/LlapPluginProtocolProtos.java    |   260 +-
 .../hadoop/hive/llap/AsyncPbRpcProxy.java       |    83 +-
 .../llap/impl/LlapPluginProtocolClientImpl.java |     5 +-
 .../hadoop/hive/llap/impl/ProtobufProxy.java    |    31 +-
 .../src/protobuf/LlapPluginProtocol.proto       |     5 +-
 llap-server/pom.xml                             |     8 +
 .../services/impl/TestLlapWebServices.java      |     4 +-
 .../tezplugins/LlapTaskSchedulerService.java    |     4 +-
 .../endpoint/LlapPluginServerImpl.java          |    23 +-
 metastore/pom.xml                               |    11 +
 .../hadoop/hive/metastore/HiveAlterHandler.java |    31 +-
 .../hadoop/hive/metastore/HiveMetaStore.java    |    42 +-
 .../hive/metastore/MetaStoreDirectSql.java      |    10 +-
 .../hadoop/hive/metastore/MetaStoreUtils.java   |   165 +-
 .../hadoop/hive/metastore/ObjectStore.java      |     3 +-
 .../hadoop/hive/metastore/txn/TxnUtils.java     |     8 +-
 .../hive/metastore/MetaStoreTestUtils.java      |   147 +
 .../metastore/TestHiveMetaStoreGetMetaConf.java |     2 +-
 .../TestHiveMetaStorePartitionSpecs.java        |     2 +-
 .../TestRetriesInRetryingHMSHandler.java        |     2 +-
 .../txn/TestValidCompactorTxnList.java          |     6 +-
 pom.xml                                         |     8 +-
 ql/pom.xml                                      |    21 +-
 .../metastore/SynchronizedMetaStoreClient.java  |    24 +
 .../java/org/apache/hadoop/hive/ql/Driver.java  |    41 +-
 .../apache/hadoop/hive/ql/DriverContext.java    |     2 +-
 .../hadoop/hive/ql/exec/ColumnStatsTask.java    |     3 +-
 .../hive/ql/exec/ColumnStatsUpdateTask.java     |     3 +-
 .../hadoop/hive/ql/exec/ConditionalTask.java    |     7 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |    34 +-
 .../hive/ql/exec/ExplainSQRewriteTask.java      |     5 +
 .../apache/hadoop/hive/ql/exec/ExplainTask.java |    68 +-
 .../apache/hadoop/hive/ql/exec/FetchTask.java   |     5 +
 .../hadoop/hive/ql/exec/FunctionTask.java       |     8 +
 .../apache/hadoop/hive/ql/exec/MoveTask.java    |    32 +-
 .../apache/hadoop/hive/ql/exec/NodeUtils.java   |     7 +-
 .../hadoop/hive/ql/exec/OperatorUtils.java      |     4 +-
 .../org/apache/hadoop/hive/ql/exec/Task.java    |    61 +-
 .../apache/hadoop/hive/ql/exec/Utilities.java   |   106 +-
 .../hadoop/hive/ql/exec/mr/ExecDriver.java      |     7 +-
 .../hadoop/hive/ql/exec/mr/MapRedTask.java      |     2 -
 .../hadoop/hive/ql/exec/repl/ReplDumpTask.java  |     5 +
 .../ql/exec/repl/bootstrap/ReplLoadTask.java    |     8 +-
 .../ql/exec/repl/bootstrap/ReplLoadWork.java    |    20 +-
 .../bootstrap/load/table/LoadPartitions.java    |     3 +-
 .../repl/bootstrap/load/table/LoadTable.java    |     3 +-
 .../exec/repl/bootstrap/load/util/Context.java  |    15 +-
 .../ql/exec/spark/HiveSparkClientFactory.java   |    16 +-
 .../ql/exec/spark/RemoteHiveSparkClient.java    |     6 +-
 .../hadoop/hive/ql/exec/tez/AmPluginNode.java   |    43 +
 .../ql/exec/tez/GuaranteedTasksAllocator.java   |   171 +
 .../ql/exec/tez/LlapPluginEndpointClient.java   |    30 +
 .../exec/tez/LlapPluginEndpointClientImpl.java  |   123 +
 .../ql/exec/tez/QueryAllocationManager.java     |    32 +
 .../hadoop/hive/ql/exec/tez/TezSessionPool.java |    85 +-
 .../hive/ql/exec/tez/TezSessionPoolManager.java |    59 +-
 .../hive/ql/exec/tez/TezSessionPoolSession.java |    55 +-
 .../hive/ql/exec/tez/TezSessionState.java       |    29 +-
 .../apache/hadoop/hive/ql/exec/tez/TezTask.java |    23 +-
 .../hadoop/hive/ql/exec/tez/WmTezSession.java   |   121 +
 .../hive/ql/exec/tez/WorkloadManager.java       |   372 +
 .../ql/exec/tez/monitoring/TezJobMonitor.java   |     2 +
 .../org/apache/hadoop/hive/ql/hooks/Entity.java |    28 +-
 .../hadoop/hive/ql/hooks/LineageInfo.java       |     2 +-
 .../hadoop/hive/ql/hooks/WriteEntity.java       |     5 +
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |     5 +-
 .../apache/hadoop/hive/ql/io/BucketCodec.java   |    23 +-
 .../hadoop/hive/ql/io/merge/MergeFileTask.java  |     1 -
 .../hadoop/hive/ql/log/LogDivertAppender.java   |    25 +-
 .../hive/ql/log/LogDivertAppenderForTest.java   |    18 +-
 .../apache/hadoop/hive/ql/metadata/Hive.java    |    20 +-
 .../hive/ql/optimizer/GenMapRedUtils.java       |     8 +-
 .../HiveJoinPushTransitivePredicatesRule.java   |    21 +-
 .../HiveReduceExpressionsWithStatsRule.java     |    60 +-
 .../hive/ql/optimizer/lineage/LineageCtx.java   |     3 +-
 .../physical/LlapClusterStateForCompile.java    |   105 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |    19 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java      |    45 +-
 .../apache/hadoop/hive/ql/parse/EximUtil.java   |    15 +
 .../org/apache/hadoop/hive/ql/parse/HiveLexer.g |     2 +
 .../apache/hadoop/hive/ql/parse/HiveParser.g    |    13 +-
 .../hadoop/hive/ql/parse/IdentifiersParser.g    |     4 +-
 .../hive/ql/parse/ImportSemanticAnalyzer.java   |    47 +-
 .../hive/ql/parse/LoadSemanticAnalyzer.java     |     7 +-
 .../ql/parse/ReplicationSemanticAnalyzer.java   |    10 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |    16 +-
 .../hive/ql/parse/SemanticAnalyzerFactory.java  |     2 +
 .../hadoop/hive/ql/parse/TaskCompiler.java      |    40 +-
 .../hadoop/hive/ql/parse/repl/CopyUtils.java    |    11 +-
 .../repl/dump/events/AddForeignKeyHandler.java  |     8 +
 .../events/AddNotNullConstraintHandler.java     |     8 +
 .../repl/dump/events/AddPrimaryKeyHandler.java  |     8 +
 .../dump/events/AddUniqueConstraintHandler.java |     8 +
 .../ql/parse/repl/dump/io/FileOperations.java   |     5 +-
 .../repl/load/message/DropTableHandler.java     |     2 +-
 .../hive/ql/parse/spark/SparkCompiler.java      |     6 +-
 .../hive/ql/plan/ColumnStatsUpdateWork.java     |    12 +-
 .../hadoop/hive/ql/plan/ColumnStatsWork.java    |    11 +-
 .../org/apache/hadoop/hive/ql/plan/DDLWork.java |    16 +
 .../hadoop/hive/ql/plan/DropTableDesc.java      |    28 +-
 .../org/apache/hadoop/hive/ql/plan/Explain.java |     3 +
 .../hadoop/hive/ql/plan/ExprNodeDesc.java       |     5 +
 .../hive/ql/plan/ExprNodeGenericFuncDesc.java   |    23 +
 .../apache/hadoop/hive/ql/plan/FilterDesc.java  |    12 +-
 .../hadoop/hive/ql/plan/HiveOperation.java      |     3 +-
 .../hadoop/hive/ql/plan/KillQueryDesc.java      |    45 +
 .../apache/hadoop/hive/ql/plan/LoadDesc.java    |     3 +-
 .../hadoop/hive/ql/plan/LoadFileDesc.java       |     2 +-
 .../hadoop/hive/ql/plan/LoadTableDesc.java      |    36 +-
 .../apache/hadoop/hive/ql/plan/MoveWork.java    |    27 +-
 .../apache/hadoop/hive/ql/plan/PlanUtils.java   |    17 +-
 .../hive/ql/plan/ShowCompactionsDesc.java       |     5 +-
 .../authorization/AuthorizationUtils.java       |     2 +
 .../authorization/plugin/HiveOperationType.java |     1 +
 .../plugin/HivePrivilegeObject.java             |     8 +-
 .../plugin/sqlstd/Operation2Privilege.java      |     2 +
 .../hadoop/hive/ql/session/KillQuery.java       |    25 +
 .../hadoop/hive/ql/session/LineageState.java    |    26 +-
 .../hadoop/hive/ql/session/NullKillQuery.java   |    28 +
 .../hadoop/hive/ql/session/SessionState.java    |    20 +
 .../apache/hadoop/hive/ql/stats/StatsUtils.java |    50 +-
 .../hive/ql/txn/compactor/CompactorMR.java      |     9 +-
 .../org/apache/hadoop/hive/ql/udf/UDFType.java  |     8 +
 .../hive/ql/udf/generic/GenericUDFOPAnd.java    |     2 +
 .../hive/ql/udf/generic/GenericUDFOPOr.java     |     2 +
 .../hadoop/hive/ql/exec/TestExplainTask.java    |     8 +-
 .../hadoop/hive/ql/exec/TestUtilities.java      |    22 +
 .../hive/ql/exec/tez/SampleTezSessionState.java |    27 +-
 .../exec/tez/TestGuaranteedTaskAllocator.java   |   151 +
 .../TestHostAffinitySplitLocationProvider.java  |     8 +-
 .../hive/ql/exec/tez/TestTezSessionPool.java    |    38 +-
 .../hadoop/hive/ql/exec/tez/TestTezTask.java    |    16 +-
 .../hive/ql/exec/tez/TestWorkloadManager.java   |   173 +
 .../hadoop/hive/ql/metadata/TestHiveRemote.java |     4 +-
 ...TestGenMapRedUtilsCreateConditionalTask.java |    62 +-
 .../TestHiveReduceExpressionsWithStatsRule.java |   203 +
 .../TestSQL11ReservedKeyWordsNegative.java      |    13 +
 .../TestSQLStdHiveAccessControllerHS2.java      |     2 +-
 .../clientnegative/authorization_kill_query.q   |    15 +
 .../clientnegative/subquery_missing_from.q      |     1 -
 .../clientnegative/subquery_select_no_source.q  |     2 -
 .../clientpositive/bucket_map_join_tez1.q       |   149 +-
 .../clientpositive/bucket_map_join_tez2.q       |    57 +-
 .../clientpositive/drop_deleted_partitions.q    |    18 +
 ql/src/test/queries/clientpositive/kill_query.q |     7 +
 .../queries/clientpositive/optimize_join_ptp.q  |    16 +
 .../queries/clientpositive/remove_exprs_stats.q |    20 +
 .../spark_combine_equivalent_work_2.q           |    41 +
 .../authorization_kill_query.q.out              |    34 +
 .../authorization_uri_import.q.out              |    40 -
 .../results/clientnegative/orc_merge1.q.out     |    75 -
 .../results/clientnegative/orc_merge2.q.out     |    75 -
 .../results/clientnegative/orc_merge3.q.out     |    75 -
 .../results/clientnegative/orc_merge4.q.out     |    87 -
 .../results/clientnegative/orc_merge5.q.out     |    75 -
 .../clientnegative/subquery_missing_from.q.out  |     3 -
 .../subquery_select_no_source.q.out             |     1 -
 .../clientnegative/subquery_shared_alias.q.out  |     1 -
 .../clientpositive/alter_merge_orc.q.out        |   263 -
 .../clientpositive/alter_merge_stats_orc.q.out  |   382 -
 .../annotate_stats_deep_filters.q.out           |     4 +-
 .../clientpositive/annotate_stats_filter.q.out  |     2 +-
 .../clientpositive/annotate_stats_groupby.q.out |    22 +-
 .../annotate_stats_groupby2.q.out               |    12 +-
 .../annotate_stats_join_pkfk.q.out              |    14 +-
 .../clientpositive/annotate_stats_select.q.out  |     8 +-
 .../clientpositive/annotate_stats_table.q.out   |     4 +-
 .../clientpositive/authorization_2.q.out        |   667 -
 .../clientpositive/autoColumnStats_1.q.out      |  1379 -
 .../clientpositive/autoColumnStats_2.q.out      |  1500 -
 .../results/clientpositive/auto_join20.q.out    |     2 +-
 .../results/clientpositive/auto_join28.q.out    |     2 +-
 .../results/clientpositive/auto_join29.q.out    |    14 +-
 .../results/clientpositive/auto_join30.q.out    |  2183 --
 .../results/clientpositive/auto_join4.q.out     |     4 +-
 .../results/clientpositive/auto_join5.q.out     |     2 +-
 .../results/clientpositive/auto_join6.q.out     |     4 +-
 .../results/clientpositive/auto_join7.q.out     |     6 +-
 .../results/clientpositive/auto_join8.q.out     |     4 +-
 .../clientpositive/auto_join_filters.q.out      |   540 -
 .../clientpositive/auto_join_nulls.q.out        |   217 -
 .../auto_join_reordering_values.q.out           |     2 +-
 .../clientpositive/auto_smb_mapjoin_14.q.out    |  1792 -
 .../clientpositive/auto_sortmerge_join_16.q.out |   256 -
 .../clientpositive/auto_sortmerge_join_5.q.out  |    42 +-
 .../clientpositive/auto_sortmerge_join_6.q.out  |  1353 -
 .../clientpositive/auto_sortmerge_join_8.q.out  |  1459 -
 .../clientpositive/auto_sortmerge_join_9.q.out  |  4820 ---
 .../clientpositive/beeline/mapjoin2.q.out       |     4 +
 .../materialized_view_create_rewrite.q.out      |     4 +-
 .../beeline/select_dummy_source.q.out           |    22 +-
 .../clientpositive/beeline/smb_mapjoin_1.q.out  |    10 +-
 .../clientpositive/beeline/smb_mapjoin_10.q.out |     2 +-
 .../clientpositive/beeline/smb_mapjoin_2.q.out  |    20 +-
 .../clientpositive/beeline/smb_mapjoin_3.q.out  |    10 +-
 .../test/results/clientpositive/bucket4.q.out   |   467 -
 .../test/results/clientpositive/bucket5.q.out   |   610 -
 .../test/results/clientpositive/bucket6.q.out   |   189 -
 .../results/clientpositive/bucket_groupby.q.out |  1635 -
 .../results/clientpositive/bucket_many.q.out    |   222 -
 .../clientpositive/bucketcontext_5.q.out        |    14 +-
 .../clientpositive/bucketcontext_6.q.out        |     6 +-
 .../bucketizedhiveinputformat.q.out             |   116 -
 .../results/clientpositive/bucketmapjoin1.q.out |  1341 -
 .../results/clientpositive/bucketmapjoin2.q.out |  1738 -
 .../results/clientpositive/bucketmapjoin3.q.out |  1168 -
 .../results/clientpositive/bucketmapjoin4.q.out |  1050 -
 .../results/clientpositive/bucketmapjoin5.q.out |     8 +-
 .../results/clientpositive/bucketmapjoin6.q.out |   146 -
 .../results/clientpositive/bucketmapjoin7.q.out |   298 -
 .../clientpositive/bucketmapjoin_negative.q.out |     4 +-
 .../bucketmapjoin_negative2.q.out               |     4 +-
 .../bucketmapjoin_negative3.q.out               |   126 +-
 .../results/clientpositive/bucketpruning1.q.out |  2282 --
 .../bucketsortoptimize_insert_2.q.out           |  1244 -
 .../bucketsortoptimize_insert_6.q.out           |  1323 -
 .../bucketsortoptimize_insert_7.q.out           |   468 -
 .../test/results/clientpositive/cbo_const.q.out |     2 +-
 .../test/results/clientpositive/cbo_gby.q.out   |   124 -
 .../test/results/clientpositive/cbo_join.q.out  | 15028 --------
 .../test/results/clientpositive/cbo_limit.q.out |    90 -
 .../cbo_rp_annotate_stats_groupby.q.out         |    22 +-
 .../results/clientpositive/cbo_rp_gby.q.out     |   124 -
 .../results/clientpositive/cbo_rp_join.q.out    | 15028 --------
 .../results/clientpositive/cbo_rp_join1.q.out   |    64 +-
 .../results/clientpositive/cbo_rp_limit.q.out   |    90 -
 .../clientpositive/cbo_rp_lineage2.q.out        |   675 -
 .../clientpositive/cbo_rp_outer_join_ppr.q.out  |     4 +-
 .../clientpositive/cbo_rp_semijoin.q.out        |   440 -
 .../clientpositive/cbo_rp_subq_exists.q.out     |   283 -
 .../results/clientpositive/cbo_rp_subq_in.q.out |   151 -
 .../clientpositive/cbo_rp_subq_not_in.q.out     |   365 -
 .../cbo_rp_udaf_percentile_approx_23.q.out      |     8 +-
 .../clientpositive/cbo_rp_udf_udaf.q.out        |   123 -
 .../cbo_rp_udf_udaf_stats_opt.q.out             |   124 -
 .../clientpositive/cbo_rp_unionDistinct_2.q.out |   551 -
 .../results/clientpositive/cbo_rp_views.q.out   |   237 -
 .../clientpositive/cbo_rp_windowing_2.q.out     |  2339 --
 .../results/clientpositive/cbo_semijoin.q.out   |   440 -
 .../results/clientpositive/cbo_subq_in.q.out    |   151 -
 .../clientpositive/cbo_subq_not_in.q.out        |   365 -
 .../results/clientpositive/cbo_udf_udaf.q.out   |   125 -
 .../results/clientpositive/cbo_union_view.q.out |     6 +-
 .../test/results/clientpositive/cbo_views.q.out |   237 -
 .../test/results/clientpositive/cluster.q.out   |  1449 -
 .../columnStatsUpdateForStatsOptimizer_1.q.out  |   961 -
 .../clientpositive/column_access_stats.q.out    |   869 -
 .../columnstats_part_coltype.q.out              |   441 -
 .../clientpositive/columnstats_tbllvl.q.out     |    24 +-
 .../clientpositive/compute_stats_date.q.out     |     4 +-
 .../clientpositive/constant_prop_1.q.out        |     2 +-
 .../clientpositive/constant_prop_3.q.out        |     2 +-
 .../clientpositive/constprog_semijoin.q.out     |   812 -
 .../clientpositive/correlated_join_keys.q.out   |     8 +-
 .../clientpositive/correlationoptimizer1.q.out  |  3091 --
 .../clientpositive/correlationoptimizer10.q.out |     4 +-
 .../clientpositive/correlationoptimizer13.q.out |     2 +-
 .../clientpositive/correlationoptimizer2.q.out  |  2257 --
 .../clientpositive/correlationoptimizer3.q.out  |  1422 -
 .../clientpositive/correlationoptimizer4.q.out  |  1862 -
 .../clientpositive/correlationoptimizer5.q.out  |   350 +-
 .../clientpositive/correlationoptimizer6.q.out  |  3918 ---
 .../clientpositive/correlationoptimizer9.q.out  |     8 +-
 ql/src/test/results/clientpositive/count.q.out  |   752 -
 .../test/results/clientpositive/database.q.out  |  1484 -
 .../clientpositive/dbtxnmgr_showlocks.q.out     |     4 +-
 .../clientpositive/decimal_precision.q.out      |     4 +-
 .../clientpositive/decimal_precision2.q.out     |    10 +-
 .../results/clientpositive/decimal_udf.q.out    |   178 +-
 .../results/clientpositive/decimal_udf2.q.out   |    16 +-
 .../disable_merge_for_bucketing.q.out           |   474 -
 .../display_colstats_tbllvl.q.out               |     8 +-
 .../clientpositive/distinct_windowing.q.out     |    60 +-
 .../distinct_windowing_no_cbo.q.out             |   104 +-
 .../drop_deleted_partitions.q.out               |    74 +
 .../drop_partition_with_stats.q.out             |   496 -
 .../clientpositive/drop_table_with_index.q.out  |    12 +-
 .../dynpart_sort_opt_vectorization.q.out        |  2430 --
 .../dynpart_sort_optimization.q.out             |  3041 --
 .../dynpart_sort_optimization2.q.out            |  1750 -
 .../clientpositive/empty_dir_in_table.q.out     |    46 -
 .../encryption_join_unencrypted_tbl.q.out       |     8 +-
 .../test/results/clientpositive/escape1.q.out   |   Bin 54831 -> 0 bytes
 .../test/results/clientpositive/escape2.q.out   |   Bin 78516 -> 0 bytes
 .../results/clientpositive/except_all.q.out     |     6 +-
 .../clientpositive/exchgpartition2lel.q.out     |   203 -
 ...rnal_table_with_space_in_location_path.q.out |    88 -
 .../extrapolate_part_stats_partial_ndv.q.out    |  1307 -
 .../file_with_header_footer.q.out               |   184 -
 .../clientpositive/filter_cond_pushdown.q.out   |     6 +-
 .../clientpositive/filter_cond_pushdown2.q.out  |    42 +-
 .../filter_cond_pushdown_HIVE_15647.q.out       |    10 +-
 .../clientpositive/filter_join_breaktask.q.out  |    60 +-
 .../results/clientpositive/flatten_and_or.q.out |     2 +-
 .../clientpositive/fold_eq_with_case_when.q.out |     2 +-
 .../clientpositive/folder_predicate.q.out       |    14 +-
 .../clientpositive/fouter_join_ppr.q.out        |     8 +-
 .../clientpositive/gen_udf_example_add10.q.out  |    12 +-
 .../results/clientpositive/global_limit.q.out   |  1677 -
 .../test/results/clientpositive/groupby10.q.out |    80 +-
 .../test/results/clientpositive/groupby2.q.out  |   130 -
 .../clientpositive/groupby_grouping_id2.q.out   |   234 -
 .../clientpositive/groupby_grouping_id3.q.out   |    32 +-
 .../groupby_grouping_sets_grouping.q.out        |   226 +-
 .../groupby_multi_single_reducer3.q.out         |    20 +-
 .../clientpositive/groupby_position.q.out       |     2 +-
 .../clientpositive/groupby_resolution.q.out     |   792 -
 .../test/results/clientpositive/having2.q.out   |     2 +-
 .../identity_project_remove_skip.q.out          |     2 +-
 .../clientpositive/import_exported_table.q.out  |    28 -
 .../results/clientpositive/index_auto.q.out     |     6 +-
 .../clientpositive/index_auto_mult_tables.q.out |     6 +-
 .../index_auto_mult_tables_compact.q.out        |     6 +-
 .../clientpositive/index_auto_self_join.q.out   |    12 +-
 .../clientpositive/index_auto_unused.q.out      |     6 +-
 .../index_bitmap_compression.q.out              |     4 +-
 .../clientpositive/index_compression.q.out      |     4 +-
 .../results/clientpositive/index_serde.q.out    |     8 +-
 .../infer_bucket_sort_bucketed_table.q.out      |   121 -
 .../infer_bucket_sort_map_operators.q.out       |     8 +-
 .../clientpositive/infer_const_type.q.out       |     8 +-
 .../test/results/clientpositive/input12.q.out   |     2 +-
 .../test/results/clientpositive/input13.q.out   |     4 +-
 .../results/clientpositive/input16_cc.q.out     |   534 -
 .../test/results/clientpositive/input22.q.out   |    12 +-
 .../results/clientpositive/input3_limit.q.out   |    16 +-
 ql/src/test/results/clientpositive/input4.q.out |     2 +-
 .../results/clientpositive/input_part2.q.out    |     4 +-
 .../results/clientpositive/input_part4.q.out    |     2 +-
 .../clientpositive/input_testxpath4.q.out       |     4 +-
 .../test/results/clientpositive/insert1.q.out   |     2 +-
 .../clientpositive/insert_dir_distcp.q.out      |    14 -
 .../insert_into_with_schema.q.out               |   348 -
 ql/src/test/results/clientpositive/join1.q.out  |  1143 -
 ql/src/test/results/clientpositive/join19.q.out |     6 +-
 ql/src/test/results/clientpositive/join20.q.out |     2 +-
 .../clientpositive/join32_lessSize.q.out        |  2651 --
 ql/src/test/results/clientpositive/join4.q.out  |     4 +-
 ql/src/test/results/clientpositive/join40.q.out |     2 +-
 ql/src/test/results/clientpositive/join45.q.out |     2 +-
 ql/src/test/results/clientpositive/join5.q.out  |     2 +-
 ql/src/test/results/clientpositive/join6.q.out  |     4 +-
 ql/src/test/results/clientpositive/join7.q.out  |     6 +-
 ql/src/test/results/clientpositive/join8.q.out  |     4 +-
 .../clientpositive/join_acid_non_acid.q.out     |    54 -
 .../results/clientpositive/join_filters.q.out   |  1484 -
 .../results/clientpositive/join_hive_626.q.out  |     2 +-
 .../results/clientpositive/join_nulls.q.out     |   652 -
 .../results/clientpositive/join_nullsafe.q.out  |  1589 -
 .../results/clientpositive/join_parse.q.out     |     2 +-
 .../results/clientpositive/join_reorder2.q.out  |     4 +-
 .../results/clientpositive/join_reorder3.q.out  |     4 +-
 .../test/results/clientpositive/join_star.q.out |   170 +-
 .../results/clientpositive/kill_query.q.out     |    36 +
 .../results/clientpositive/lateral_view.q.out   |   900 -
 .../clientpositive/leftsemijoin_mr.q.out        |    98 -
 .../clientpositive/limit_join_transpose.q.out   |  1813 -
 .../results/clientpositive/limit_pushdown.q.out |  1432 -
 .../clientpositive/limit_pushdown3.q.out        |  1407 -
 .../test/results/clientpositive/lineage2.q.out  |   705 -
 .../test/results/clientpositive/lineage3.q.out  |   388 -
 .../clientpositive/list_bucket_dml_10.q.out     |   357 -
 .../list_bucket_query_multiskew_2.q.out         |     2 +-
 .../llap/acid_bucket_pruning.q.out              |     8 +-
 .../llap/acid_vectorization.q.out               |   120 -
 .../llap/acid_vectorization_partition.q.out     |    60 -
 .../llap/acid_vectorization_project.q.out       |    73 -
 .../alter_table_invalidate_column_stats.q.out   |   966 -
 .../clientpositive/llap/auto_join29.q.out       |    14 +-
 .../llap/auto_sortmerge_join_5.q.out            |    46 +-
 .../llap/bucket_map_join_tez1.q.out             |  4401 ++-
 .../llap/bucket_map_join_tez2.q.out             |  1340 +-
 .../clientpositive/llap/bucketmapjoin1.q.out    |    16 +-
 .../clientpositive/llap/bucketmapjoin4.q.out    |    44 +-
 .../clientpositive/llap/bucketpruning1.q.out    |    26 +-
 .../llap/bucketsortoptimize_insert_7.q.out      |     4 +-
 .../clientpositive/llap/cbo_rp_subq_in.q.out    |   123 -
 .../llap/cbo_rp_subq_not_in.q.out               |   345 -
 .../llap/column_table_stats.q.out               |     4 +-
 .../llap/constprog_semijoin.q.out               |     2 +-
 .../llap/correlationoptimizer4.q.out            |   324 +-
 .../results/clientpositive/llap/count.q.out     |   130 +-
 .../clientpositive/llap/delete_orig_table.q.out |    61 -
 .../llap/dynamic_partition_pruning.q.out        |    10 +-
 .../llap/dynamic_semijoin_reduction.q.out       |    22 +-
 .../llap/dynamic_semijoin_reduction_2.q.out     |     8 +-
 .../llap/dynamic_semijoin_reduction_3.q.out     |     8 +-
 .../llap/dynamic_semijoin_reduction_sw.q.out    |     8 +-
 .../llap/dynamic_semijoin_user_level.q.out      |    14 +-
 .../llap/dynpart_sort_opt_vectorization.q.out   |    30 +-
 .../llap/dynpart_sort_optimization.q.out        |   322 +-
 .../llap/dynpart_sort_optimization2.q.out       |    64 +-
 .../clientpositive/llap/except_distinct.q.out   |    10 +-
 .../clientpositive/llap/explainuser_1.q.out     |   142 +-
 .../clientpositive/llap/explainuser_2.q.out     |    70 +-
 .../clientpositive/llap/explainuser_3.q.out     |   522 -
 .../clientpositive/llap/explainuser_4.q.out     |    12 +-
 .../llap/filter_join_breaktask.q.out            |    76 +-
 .../clientpositive/llap/groupby_sort_1_23.q.out |  4310 ---
 .../llap/groupby_sort_skew_1_23.q.out           |  4482 ---
 .../llap/hybridgrace_hashjoin_2.q.out           |     8 +-
 .../llap/identity_project_remove_skip.q.out     |     2 +-
 .../results/clientpositive/llap/insert1.q.out   |     2 +-
 .../llap/insert_acid_dynamic_partition.q.out    |    48 -
 .../llap/insert_acid_not_bucketed.q.out         |    36 -
 .../llap/insert_values_acid_not_bucketed.q.out  |    26 -
 .../llap/insert_values_orig_table.q.out         |    80 -
 .../clientpositive/llap/join32_lessSize.q.out   |     2 +-
 .../results/clientpositive/llap/join43.q.out    |   616 -
 .../llap/join_is_not_distinct_from.q.out        |   102 +-
 .../clientpositive/llap/join_nullsafe.q.out     |   102 +-
 .../clientpositive/llap/kill_query.q.out        |    36 +
 .../clientpositive/llap/llap_reader.q.out       |   167 -
 .../results/clientpositive/llap/mapjoin2.q.out  |     4 +
 .../clientpositive/llap/mapjoin_hint.q.out      |    79 +-
 .../clientpositive/llap/mapjoin_mapjoin.q.out   |     4 +-
 .../results/clientpositive/llap/mergejoin.q.out |     6 +-
 .../clientpositive/llap/multiMapJoin1.q.out     |     8 +-
 .../clientpositive/llap/multi_insert.q.out      |    32 +-
 ...i_insert_move_tasks_share_dependencies.q.out |  3765 --
 .../clientpositive/llap/optimize_join_ptp.q.out |   130 +
 .../results/clientpositive/llap/orc_llap.q.out  |  1205 -
 .../clientpositive/llap/orc_merge12.q.out       |   822 -
 .../clientpositive/llap/orc_merge5.q.out        |    16 +-
 .../clientpositive/llap/orc_merge6.q.out        |    16 +-
 .../clientpositive/llap/orc_merge7.q.out        |    12 +-
 .../llap/orc_merge_incompat1.q.out              |     8 +-
 .../llap/orc_merge_incompat2.q.out              |     6 +-
 .../llap/orc_predicate_pushdown.q.out           |    20 +-
 .../llap/orc_vectorization_ppd.q.out            |   288 -
 .../llap/parquet_ppd_decimal.q.out              |   856 -
 .../llap/parquet_predicate_pushdown.q.out       |    22 +-
 .../test/results/clientpositive/llap/pcs.q.out  |  1780 -
 .../clientpositive/llap/ptf_matchpath.q.out     |    42 +-
 .../clientpositive/llap/script_env_var1.q.out   |    14 -
 .../clientpositive/llap/script_env_var2.q.out   |    14 -
 .../clientpositive/llap/script_pipe.q.out       |   125 -
 .../clientpositive/llap/scriptfile1.q.out       |    59 -
 .../llap/select_dummy_source.q.out              |    10 +-
 .../clientpositive/llap/semijoin_hint.q.out     |    32 +-
 .../llap/show_create_database.q.out             |    19 -
 .../results/clientpositive/llap/skewjoin.q.out  |     6 +-
 .../clientpositive/llap/smb_mapjoin_15.q.out    |     4 +-
 .../clientpositive/llap/smb_mapjoin_4.q.out     |   224 +-
 .../clientpositive/llap/smb_mapjoin_5.q.out     |   224 +-
 .../results/clientpositive/llap/sqlmerge.q.out  |     4 +-
 .../results/clientpositive/llap/stats11.q.out   |    16 +-
 .../clientpositive/llap/subquery_in.q.out       |     4 +-
 .../clientpositive/llap/subquery_multi.q.out    |   348 +-
 .../clientpositive/llap/subquery_notin.q.out    |    12 +-
 .../clientpositive/llap/subquery_null_agg.q.out |     2 +-
 .../clientpositive/llap/subquery_scalar.q.out   |   122 +-
 .../llap/tez_dynpart_hashjoin_1.q.out           |    12 +-
 .../llap/tez_join_result_complex.q.out          |   212 +-
 .../clientpositive/llap/tez_nway_join.q.out     |    90 +-
 .../llap/tez_union_group_by.q.out               |     2 +-
 .../llap/tez_vector_dynpart_hashjoin_1.q.out    |    12 +-
 .../clientpositive/llap/transform1.q.out        |   138 -
 .../clientpositive/llap/transform2.q.out        |     9 -
 .../clientpositive/llap/transform_ppr1.q.out    |   493 -
 .../clientpositive/llap/transform_ppr2.q.out    |   399 -
 .../clientpositive/llap/union_view.q.out        |  1209 -
 .../clientpositive/llap/update_orig_table.q.out |    62 -
 .../clientpositive/llap/vector_const.q.out      |    64 -
 .../clientpositive/llap/vector_date_1.q.out     |     4 +-
 .../llap/vector_decimal_cast.q.out              |     2 +-
 .../llap/vector_decimal_expressions.q.out       |     2 +-
 .../llap/vector_empty_where.q.out               |   676 -
 .../vector_groupby_grouping_sets_grouping.q.out |     4 +-
 .../clientpositive/llap/vector_interval_2.q.out |    12 +-
 .../llap/vector_interval_mapjoin.q.out          |     4 +-
 .../clientpositive/llap/vector_join.q.out       |   104 -
 .../llap/vector_join_part_col_char.q.out        |   173 -
 .../llap/vector_mapjoin_reduce.q.out            |     2 +-
 .../llap/vector_mr_diff_schema_alias.q.out      |     2 +-
 .../llap/vector_multi_insert.q.out              |     2 +-
 .../llap/vector_non_constant_in_expr.q.out      |    51 -
 .../llap/vector_non_string_partition.q.out      |   276 -
 .../vector_orc_string_reader_empty_dict.q.out   |    62 -
 .../clientpositive/llap/vector_order_null.q.out |  1382 -
 .../llap/vector_outer_reference_windowed.q.out  |  1435 -
 .../llap/vector_string_decimal.q.out            |   135 -
 .../llap/vector_tablesample_rows.q.out          |   403 -
 .../clientpositive/llap/vector_udf2.q.out       |   188 -
 .../clientpositive/llap/vector_windowing.q.out  |  9768 ------
 .../llap/vector_windowing_expressions.q.out     |  2022 --
 .../llap/vector_windowing_gby.q.out             |   314 -
 .../llap/vector_windowing_gby2.q.out            |  1158 -
 .../vector_windowing_multipartitioning.q.out    | 11587 ------
 .../llap/vector_windowing_navfn.q.out           |    84 +-
 .../llap/vector_windowing_order_null.q.out      |  1232 -
 .../vector_windowing_range_multiorder.q.out     | 12584 -------
 .../llap/vector_windowing_rank.q.out            |  1860 -
 .../llap/vector_windowing_streaming.q.out       |  1036 -
 .../llap/vector_windowing_windowspec.q.out      |  2374 --
 .../llap/vector_windowing_windowspec4.q.out     |   223 -
 .../clientpositive/llap/vectorization_0.q.out   |     8 +-
 .../clientpositive/llap/vectorization_1.q.out   |     2 +-
 .../clientpositive/llap/vectorization_10.q.out  |     2 +-
 .../clientpositive/llap/vectorization_12.q.out  |     2 +-
 .../clientpositive/llap/vectorization_13.q.out  |     4 +-
 .../clientpositive/llap/vectorization_14.q.out  |     2 +-
 .../clientpositive/llap/vectorization_15.q.out  |     2 +-
 .../clientpositive/llap/vectorization_16.q.out  |     2 +-
 .../clientpositive/llap/vectorization_17.q.out  |     2 +-
 .../clientpositive/llap/vectorization_2.q.out   |     2 +-
 .../clientpositive/llap/vectorization_3.q.out   |     2 +-
 .../clientpositive/llap/vectorization_4.q.out   |     2 +-
 .../clientpositive/llap/vectorization_5.q.out   |     2 +-
 .../clientpositive/llap/vectorization_6.q.out   |     2 +-
 .../clientpositive/llap/vectorization_7.q.out   |     4 +-
 .../clientpositive/llap/vectorization_8.q.out   |     4 +-
 .../clientpositive/llap/vectorization_9.q.out   |     2 +-
 .../llap/vectorization_decimal_date.q.out       |     2 +-
 .../llap/vectorization_div0.q.out               |   615 -
 .../llap/vectorization_limit.q.out              |   942 -
 .../llap/vectorization_offset_limit.q.out       |   187 -
 .../llap/vectorization_short_regress.q.out      |    20 +-
 .../clientpositive/llap/vectorized_case.q.out   |     4 +-
 .../llap/vectorized_context.q.out               |     2 +-
 .../vectorized_dynamic_partition_pruning.q.out  |    10 +-
 .../vectorized_dynamic_semijoin_reduction.q.out |    16 +-
 ...vectorized_dynamic_semijoin_reduction2.q.out |    16 +-
 .../llap/vectorized_mapjoin2.q.out              |   213 -
 .../llap/vectorized_nested_mapjoin.q.out        |     2 +-
 .../llap/vectorized_string_funcs.q.out          |     2 +-
 .../llap/windowing_windowspec.q.out             |   955 -
 .../clientpositive/llap_partitioned.q.out       |  2003 --
 .../clientpositive/llap_uncompressed.q.out      |     2 +-
 .../clientpositive/load_dyn_part13.q.out        |     2 +-
 .../results/clientpositive/load_dyn_part5.q.out |  1033 -
 .../test/results/clientpositive/load_fs2.q.out  |   121 -
 .../load_hdfs_file_with_space_in_the_name.q.out |    25 -
 .../clientpositive/louter_join_ppr.q.out        |     8 +-
 .../test/results/clientpositive/mapjoin2.q.out  |     4 +
 .../mapjoin_filter_on_outerjoin.q.out           |     4 +-
 .../clientpositive/mapjoin_mapjoin.q.out        |     4 +-
 .../test/results/clientpositive/masking_1.q.out |     2 +-
 .../test/results/clientpositive/masking_3.q.out |     2 +-
 .../clientpositive/masking_disablecbo_1.q.out   |     2 +-
 .../clientpositive/masking_disablecbo_3.q.out   |    12 +-
 .../materialized_view_create_rewrite.q.out      |     4 +-
 .../results/clientpositive/metadataonly1.q.out  |  1956 --
 .../clientpositive/min_structvalue.q.out        |    45 -
 .../results/clientpositive/multiMapJoin1.q.out  |  3081 --
 .../results/clientpositive/multiMapJoin2.q.out  |  3464 --
 .../clientpositive/multi_column_in.q.out        |   408 -
 .../clientpositive/multi_column_in_single.q.out |   370 -
 .../results/clientpositive/multi_insert.q.out   |  2749 --
 .../clientpositive/multi_insert_gby.q.out       |     2 +-
 .../clientpositive/multi_insert_gby4.q.out      |     2 +-
 .../multi_insert_lateral_view.q.out             |  1586 -
 ...i_insert_move_tasks_share_dependencies.q.out |    56 +-
 .../multi_insert_with_join2.q.out               |    14 +-
 .../clientpositive/nested_column_pruning.q.out  |     2 +-
 .../clientpositive/non_native_window_udf.q.out  |    52 -
 .../results/clientpositive/nonmr_fetch.q.out    |     2 +-
 .../results/clientpositive/nullscript.q.out     |     8 +-
 .../results/clientpositive/offset_limit.q.out   |   276 -
 .../offset_limit_ppd_optimizer.q.out            |  1387 -
 .../clientpositive/optimize_nullscan.q.out      |  2054 --
 .../results/clientpositive/orc_analyze.q.out    |  1544 -
 .../results/clientpositive/orc_create.q.out     |   782 -
 .../test/results/clientpositive/orc_llap.q.out  |  1153 -
 .../clientpositive/orc_llap_nonvector.q.out     |  1302 -
 .../results/clientpositive/orc_merge5.q.out     |    16 +-
 .../results/clientpositive/orc_merge6.q.out     |    16 +-
 .../results/clientpositive/orc_merge7.q.out     |   605 -
 .../clientpositive/orc_merge_incompat1.q.out    |     8 +-
 .../clientpositive/orc_merge_incompat2.q.out    |     6 +-
 .../results/clientpositive/orc_ppd_date.q.out   |   301 -
 .../clientpositive/orc_ppd_decimal.q.out        |   490 -
 .../clientpositive/orc_ppd_schema_evol_3a.q.out |   544 -
 .../clientpositive/orc_ppd_timestamp.q.out      |   292 -
 .../clientpositive/orc_ppd_varchar.q.out        |   220 -
 .../clientpositive/orc_predicate_pushdown.q.out |  1132 -
 .../clientpositive/orc_split_elimination.q.out  |   532 -
 .../results/clientpositive/order_null.q.out     |   210 -
 .../results/clientpositive/outer_join_ppr.q.out |     4 +-
 .../clientpositive/parallel_orderby.q.out       |    10 +-
 .../parquet_int96_timestamp.q.out               |   535 -
 .../parquet_predicate_pushdown.q.out            |  1276 -
 .../results/clientpositive/parquet_types.q.out  |   379 -
 .../clientpositive/partition_multilevels.q.out  |  1624 -
 .../clientpositive/partition_pruning.q.out      |   396 -
 .../partition_wise_fileformat2.q.out            |     4 +-
 ql/src/test/results/clientpositive/pcr.q.out    |     2 +-
 ql/src/test/results/clientpositive/pcs.q.out    |     6 +-
 .../results/clientpositive/perf/query1.q.out    |     2 +-
 .../results/clientpositive/perf/query10.q.out   |     2 +-
 .../results/clientpositive/perf/query11.q.out   |     2 +-
 .../results/clientpositive/perf/query13.q.out   |     8 +-
 .../results/clientpositive/perf/query14.q.out   |    30 +-
 .../results/clientpositive/perf/query15.q.out   |     4 +-
 .../results/clientpositive/perf/query16.q.out   |     2 +-
 .../results/clientpositive/perf/query17.q.out   |     4 +-
 .../results/clientpositive/perf/query18.q.out   |     6 +-
 .../results/clientpositive/perf/query19.q.out   |     4 +-
 .../results/clientpositive/perf/query21.q.out   |     2 +-
 .../results/clientpositive/perf/query22.q.out   |     2 +-
 .../results/clientpositive/perf/query23.q.out   |     8 +-
 .../results/clientpositive/perf/query24.q.out   |     8 +-
 .../results/clientpositive/perf/query25.q.out   |     8 +-
 .../results/clientpositive/perf/query26.q.out   |     4 +-
 .../results/clientpositive/perf/query27.q.out   |     4 +-
 .../results/clientpositive/perf/query28.q.out   |    12 +-
 .../results/clientpositive/perf/query29.q.out   |     6 +-
 .../results/clientpositive/perf/query3.q.out    |     2 +-
 .../results/clientpositive/perf/query31.q.out   |     4 +-
 .../results/clientpositive/perf/query33.q.out   |    10 +-
 .../results/clientpositive/perf/query34.q.out   |     4 +-
 .../results/clientpositive/perf/query35.q.out   |     2 +-
 .../results/clientpositive/perf/query36.q.out   |     2 +-
 .../results/clientpositive/perf/query37.q.out   |     2 +-
 .../results/clientpositive/perf/query38.q.out   |     8 +-
 .../results/clientpositive/perf/query39.q.out   |     6 +-
 .../results/clientpositive/perf/query4.q.out    |     2 +-
 .../results/clientpositive/perf/query40.q.out   |     2 +-
 .../results/clientpositive/perf/query42.q.out   |     2 +-
 .../results/clientpositive/perf/query45.q.out   |     4 +-
 .../results/clientpositive/perf/query46.q.out   |     4 +-
 .../results/clientpositive/perf/query47.q.out   |     6 +-
 .../results/clientpositive/perf/query48.q.out   |     6 +-
 .../results/clientpositive/perf/query49.q.out   |    14 +-
 .../results/clientpositive/perf/query5.q.out    |     8 +-
 .../results/clientpositive/perf/query50.q.out   |     6 +-
 .../results/clientpositive/perf/query51.q.out   |     2 +-
 .../results/clientpositive/perf/query52.q.out   |     2 +-
 .../results/clientpositive/perf/query53.q.out   |     2 +-
 .../results/clientpositive/perf/query54.q.out   |     8 +-
 .../results/clientpositive/perf/query55.q.out   |     2 +-
 .../results/clientpositive/perf/query56.q.out   |     8 +-
 .../results/clientpositive/perf/query57.q.out   |     6 +-
 .../results/clientpositive/perf/query58.q.out   |    10 +-
 .../results/clientpositive/perf/query59.q.out   |     4 +-
 .../results/clientpositive/perf/query6.q.out    |     6 +-
 .../results/clientpositive/perf/query60.q.out   |     8 +-
 .../results/clientpositive/perf/query61.q.out   |     8 +-
 .../results/clientpositive/perf/query62.q.out   |   164 -
 .../results/clientpositive/perf/query63.q.out   |     2 +-
 .../results/clientpositive/perf/query64.q.out   |     6 +-
 .../results/clientpositive/perf/query65.q.out   |     4 +-
 .../results/clientpositive/perf/query66.q.out   |     4 +-
 .../results/clientpositive/perf/query67.q.out   |     4 +-
 .../results/clientpositive/perf/query68.q.out   |     6 +-
 .../results/clientpositive/perf/query69.q.out   |     2 +-
 .../results/clientpositive/perf/query7.q.out    |     4 +-
 .../results/clientpositive/perf/query70.q.out   |     4 +-
 .../results/clientpositive/perf/query71.q.out   |     6 +-
 .../results/clientpositive/perf/query72.q.out   |     4 +-
 .../results/clientpositive/perf/query73.q.out   |     4 +-
 .../results/clientpositive/perf/query74.q.out   |     8 +-
 .../results/clientpositive/perf/query75.q.out   |     2 +-
 .../results/clientpositive/perf/query76.q.out   |     4 +-
 .../results/clientpositive/perf/query79.q.out   |     4 +-
 .../results/clientpositive/perf/query80.q.out   |     6 +-
 .../results/clientpositive/perf/query82.q.out   |     2 +-
 .../results/clientpositive/perf/query83.q.out   |     4 +-
 .../results/clientpositive/perf/query85.q.out   |    10 +-
 .../results/clientpositive/perf/query86.q.out   |     4 +-
 .../results/clientpositive/perf/query87.q.out   |    12 +-
 .../results/clientpositive/perf/query89.q.out   |     2 +-
 .../results/clientpositive/perf/query91.q.out   |     6 +-
 .../results/clientpositive/perf/query93.q.out   |     2 +-
 .../results/clientpositive/perf/query94.q.out   |     2 +-
 .../results/clientpositive/perf/query95.q.out   |     2 +-
 .../results/clientpositive/perf/query97.q.out   |     2 +-
 .../results/clientpositive/perf/query99.q.out   |     4 +-
 .../results/clientpositive/pointlookup.q.out    |     2 +-
 .../results/clientpositive/ppd_clusterby.q.out  |     4 +-
 .../test/results/clientpositive/ppd_gby.q.out   |     4 +-
 .../test/results/clientpositive/ppd_gby2.q.out  |     4 +-
 .../results/clientpositive/ppd_gby_join.q.out   |     8 +-
 .../test/results/clientpositive/ppd_join.q.out  |     8 +-
 .../test/results/clientpositive/ppd_join2.q.out |     8 +-
 .../test/results/clientpositive/ppd_join3.q.out |    16 +-
 .../test/results/clientpositive/ppd_join4.q.out |     2 +-
 .../test/results/clientpositive/ppd_join5.q.out |     2 +-
 .../clientpositive/ppd_multi_insert.q.out       |     8 +-
 .../clientpositive/ppd_outer_join1.q.out        |     4 +-
 .../clientpositive/ppd_outer_join2.q.out        |     4 +-
 .../clientpositive/ppd_outer_join3.q.out        |     4 +-
 .../clientpositive/ppd_outer_join4.q.out        |     8 +-
 .../clientpositive/ppd_repeated_alias.q.out     |    52 +-
 .../results/clientpositive/ppd_udf_case.q.out   |     2 +-
 .../results/clientpositive/ppd_udf_col.q.out    |     4 +-
 .../results/clientpositive/ppd_union_view.q.out |   744 -
 .../clientpositive/ppr_allchildsarenull.q.out   |     2 +-
 .../results/clientpositive/ppr_pushdown.q.out   |   328 -
 ql/src/test/results/clientpositive/ptf.q.out    |  5014 ---
 .../results/clientpositive/ptf_matchpath.q.out  |    42 +-
 .../results/clientpositive/ptf_streaming.q.out  |  2724 --
 .../clientpositive/ql_rewrite_gbtoidx.q.out     |  2531 --
 .../ql_rewrite_gbtoidx_cbo_1.q.out              |  2527 --
 .../ql_rewrite_gbtoidx_cbo_2.q.out              |  3863 --
 ql/src/test/results/clientpositive/quote1.q.out |     2 +-
 .../results/clientpositive/quotedid_smb.q.out   |    81 -
 .../clientpositive/rand_partitionpruner3.q.out  |     2 +-
 .../clientpositive/rcfile_createas1.q.out       |   205 -
 .../results/clientpositive/rcfile_merge1.q.out  |  1270 -
 .../results/clientpositive/rcfile_merge2.q.out  |   255 -
 .../results/clientpositive/rcfile_merge3.q.out  |   194 -
 .../results/clientpositive/rcfile_merge4.q.out  |   204 -
 .../clientpositive/rcfile_null_value.q.out      |     2 +-
 .../clientpositive/reduce_deduplicate.q.out     |   360 -
 .../reduce_deduplicate_extended.q.out           |  4976 ---
 .../results/clientpositive/remote_script.q.out  |    49 -
 .../clientpositive/remove_exprs_stats.q.out     |   224 +
 .../clientpositive/router_join_ppr.q.out        |     8 +-
 .../test/results/clientpositive/sample10.q.out  |   473 -
 .../test/results/clientpositive/sample8.q.out   |     4 +-
 .../clientpositive/schema_evol_stats.q.out      |   384 -
 .../clientpositive/schemeAuthority.q.out        |    84 -
 .../clientpositive/schemeAuthority2.q.out       |    53 -
 .../clientpositive/selectDistinctStar.q.out     |  4856 ---
 .../clientpositive/select_dummy_source.q.out    |    22 +-
 .../clientpositive/select_unquote_and.q.out     |     2 +-
 .../clientpositive/select_unquote_or.q.out      |     2 +-
 .../test/results/clientpositive/semijoin.q.out  |  2601 --
 .../test/results/clientpositive/semijoin2.q.out |     4 +-
 .../test/results/clientpositive/semijoin4.q.out |     2 +-
 .../test/results/clientpositive/semijoin5.q.out |     4 +-
 .../test/results/clientpositive/skewjoin.q.out  |     6 +-
 .../clientpositive/skewjoin_mapjoin1.q.out      |    16 +-
 .../clientpositive/skewjoin_mapjoin10.q.out     |    16 +-
 .../clientpositive/skewjoin_mapjoin11.q.out     |     8 +-
 .../clientpositive/skewjoin_mapjoin2.q.out      |     8 +-
 .../clientpositive/skewjoin_mapjoin3.q.out      |     8 +-
 .../clientpositive/skewjoin_mapjoin4.q.out      |    12 +-
 .../clientpositive/skewjoin_mapjoin6.q.out      |     8 +-
 .../clientpositive/skewjoin_mapjoin7.q.out      |    16 +-
 .../clientpositive/skewjoin_mapjoin9.q.out      |     8 +-
 .../skewjoin_union_remove_1.q.out               |    16 +-
 .../skewjoin_union_remove_2.q.out               |    12 +-
 .../results/clientpositive/skewjoinopt1.q.out   |    16 +-
 .../results/clientpositive/skewjoinopt10.q.out  |     8 +-
 .../results/clientpositive/skewjoinopt11.q.out  |    16 +-
 .../results/clientpositive/skewjoinopt12.q.out  |     8 +-
 .../results/clientpositive/skewjoinopt14.q.out  |     8 +-
 .../results/clientpositive/skewjoinopt15.q.out  |   740 -
 .../results/clientpositive/skewjoinopt16.q.out  |     8 +-
 .../results/clientpositive/skewjoinopt17.q.out  |    16 +-
 .../results/clientpositive/skewjoinopt19.q.out  |     8 +-
 .../results/clientpositive/skewjoinopt2.q.out   |    16 +-
 .../results/clientpositive/skewjoinopt20.q.out  |     8 +-
 .../results/clientpositive/skewjoinopt21.q.out  |     8 +-
 .../results/clientpositive/skewjoinopt3.q.out   |     8 +-
 .../results/clientpositive/skewjoinopt4.q.out   |    16 +-
 .../results/clientpositive/skewjoinopt5.q.out   |     8 +-
 .../results/clientpositive/skewjoinopt6.q.out   |     8 +-
 .../results/clientpositive/skewjoinopt7.q.out   |    12 +-
 .../results/clientpositive/skewjoinopt8.q.out   |    12 +-
 .../results/clientpositive/skiphf_aggr.q.out    |   255 -
 .../results/clientpositive/smb_mapjoin9.q.out   |     4 +-
 .../results/clientpositive/smb_mapjoin_1.q.out  |    10 +-
 .../results/clientpositive/smb_mapjoin_10.q.out |     2 +-
 .../results/clientpositive/smb_mapjoin_14.q.out |  1233 -
 .../results/clientpositive/smb_mapjoin_15.q.out |   766 -
 .../results/clientpositive/smb_mapjoin_17.q.out |   674 -
 .../results/clientpositive/smb_mapjoin_18.q.out |   448 -
 .../results/clientpositive/smb_mapjoin_19.q.out |   250 -
 .../results/clientpositive/smb_mapjoin_2.q.out  |    20 +-
 .../results/clientpositive/smb_mapjoin_25.q.out |    96 +-
 .../results/clientpositive/smb_mapjoin_3.q.out  |    10 +-
 .../results/clientpositive/smb_mapjoin_4.q.out  |   829 -
 .../results/clientpositive/smb_mapjoin_5.q.out  |   829 -
 .../results/clientpositive/smb_mapjoin_6.q.out  |  2719 --
 .../results/clientpositive/smb_mapjoin_8.q.out  |   468 -
 .../clientpositive/sort_merge_join_desc_2.q.out |     2 +-
 .../clientpositive/sort_merge_join_desc_3.q.out |     2 +-
 .../clientpositive/sort_merge_join_desc_4.q.out |     4 +-
 .../clientpositive/sort_merge_join_desc_8.q.out |     4 +-
 .../clientpositive/spark/auto_join20.q.out      |     2 +-
 .../clientpositive/spark/auto_join25.q.out      |   116 -
 .../clientpositive/spark/auto_join28.q.out      |     2 +-
 .../clientpositive/spark/auto_join29.q.out      |    14 +-
 .../clientpositive/spark/auto_join32.q.out      |   580 -
 .../clientpositive/spark/auto_join4.q.out       |     4 +-
 .../clientpositive/spark/auto_join5.q.out       |     2 +-
 .../clientpositive/spark/auto_join6.q.out       |     4 +-
 .../clientpositive/spark/auto_join7.q.out       |     6 +-
 .../clientpositive/spark/auto_join8.q.out       |     4 +-
 .../spark/auto_join_reordering_values.q.out     |     2 +-
 .../spark/auto_sortmerge_join_11.q.out          |  1548 -
 .../spark/auto_sortmerge_join_2.q.out           |   644 -
 .../spark/auto_sortmerge_join_5.q.out           |    30 +-
 .../spark/bucket_map_join_tez1.q.out            |  4265 ++-
 .../spark/bucket_map_join_tez2.q.out            |  1330 +-
 .../clientpositive/spark/bucketmapjoin1.q.out   |    16 +-
 .../clientpositive/spark/bucketmapjoin4.q.out   |    44 +-
 .../clientpositive/spark/bucketmapjoin5.q.out   |     8 +-
 .../spark/bucketmapjoin_negative.q.out          |     4 +-
 .../spark/bucketmapjoin_negative2.q.out         |     4 +-
 .../spark/bucketmapjoin_negative3.q.out         |   126 +-
 .../spark/bucketsortoptimize_insert_7.q.out     |     4 +-
 .../clientpositive/spark/char_join1.q.out       |   153 -
 .../spark/constprog_semijoin.q.out              |     2 +-
 .../results/clientpositive/spark/count.q.out    |   130 +-
 .../spark/dynamic_rdd_cache.q.out               |     8 +-
 .../spark/filter_join_breaktask.q.out           |    48 +-
 .../spark/gen_udf_example_add10.q.out           |    12 +-
 .../clientpositive/spark/groupby10.q.out        |    72 +-
 .../spark/groupby_multi_single_reducer3.q.out   |    20 +-
 .../clientpositive/spark/groupby_position.q.out |     2 +-
 .../spark/identity_project_remove_skip.q.out    |     2 +-
 .../spark/index_auto_self_join.q.out            |     8 +-
 .../spark/infer_bucket_sort_convert_join.q.out  |   128 -
 .../results/clientpositive/spark/input12.q.out  |     2 +-
 .../results/clientpositive/spark/input13.q.out  |     4 +-
 .../clientpositive/spark/input_part2.q.out      |     4 +-
 .../results/clientpositive/spark/insert1.q.out  |   420 -
 .../results/clientpositive/spark/join19.q.out   |     6 +-
 .../results/clientpositive/spark/join20.q.out   |     2 +-
 .../clientpositive/spark/join32_lessSize.q.out  |     2 +-
 .../results/clientpositive/spark/join4.q.out    |     4 +-
 .../results/clientpositive/spark/join40.q.out   |  3846 --
 .../results/clientpositive/spark/join5.q.out    |     2 +-
 .../results/clientpositive/spark/join6.q.out    |     4 +-
 .../results/clientpositive/spark/join7.q.out    |     6 +-
 .../results/clientpositive/spark/join8.q.out    |     4 +-
 .../clientpositive/spark/join_empty.q.out       |    56 -
 .../clientpositive/spark/join_filters.q.out     |  1364 -
 .../clientpositive/spark/join_hive_626.q.out    |     2 +-
 .../clientpositive/spark/join_nulls.q.out       |   652 -
 .../clientpositive/spark/join_nullsafe.q.out    |   116 +-
 .../clientpositive/spark/join_reorder2.q.out    |     4 +-
 .../clientpositive/spark/join_reorder3.q.out    |     4 +-
 .../clientpositive/spark/join_star.q.out        |   140 +-
 .../clientpositive/spark/load_dyn_part13.q.out  |     2 +-
 .../clientpositive/spark/louter_join_ppr.q.out  |     8 +-
 .../spark/mapjoin_filter_on_outerjoin.q.out     |     4 +-
 .../clientpositive/spark/mapjoin_hook.q.out     |    61 -
 .../clientpositive/spark/mapjoin_mapjoin.q.out  |     4 +-
 .../clientpositive/spark/multi_insert.q.out     |    32 +-
 .../clientpositive/spark/multi_insert_gby.q.out |     2 +-
 ...i_insert_move_tasks_share_dependencies.q.out |    56 +-
 .../spark/multi_insert_union_src.q.out          |   372 -
 .../spark/multi_insert_with_join2.q.out         |  1006 -
 .../clientpositive/spark/orc_analyze.q.out      |  1038 -
 .../clientpositive/spark/orc_merge5.q.out       |    16 +-
 .../clientpositive/spark/orc_merge6.q.out       |    16 +-
 .../clientpositive/spark/orc_merge7.q.out       |    12 +-
 .../spark/orc_merge_incompat1.q.out             |     8 +-
 .../spark/orc_merge_incompat2.q.out             |     6 +-
 .../clientpositive/spark/outer_join_ppr.q.out   |     4 +-
 .../clientpositive/spark/parallel_orderby.q.out |    10 +-
 .../test/results/clientpositive/spark/pcr.q.out |     2 +-
 .../clientpositive/spark/ppd_gby_join.q.out     |     8 +-
 .../results/clientpositive/spark/ppd_join.q.out |     8 +-
 .../clientpositive/spark/ppd_join2.q.out        |     8 +-
 .../clientpositive/spark/ppd_join3.q.out        |    16 +-
 .../clientpositive/spark/ppd_join4.q.out        |   137 -
 .../clientpositive/spark/ppd_join5.q.out        |     2 +-
 .../clientpositive/spark/ppd_multi_insert.q.out |     8 +-
 .../clientpositive/spark/ppd_outer_join1.q.out  |     4 +-
 .../clientpositive/spark/ppd_outer_join2.q.out  |     4 +-
 .../clientpositive/spark/ppd_outer_join3.q.out  |     4 +-
 .../clientpositive/spark/ppd_outer_join4.q.out  |     8 +-
 .../clientpositive/spark/ptf_matchpath.q.out    |    42 +-
 .../spark/ql_rewrite_gbtoidx.q.out              |  2645 --
 .../spark/ql_rewrite_gbtoidx_cbo_1.q.out        |  2641 --
 .../clientpositive/spark/router_join_ppr.q.out  |     8 +-
 .../results/clientpositive/spark/sample8.q.out  |     4 +-
 .../results/clientpositive/spark/skewjoin.q.out |     6 +-
 .../spark/skewjoin_union_remove_1.q.out         |    16 +-
 .../spark/skewjoin_union_remove_2.q.out         |    12 +-
 .../clientpositive/spark/skewjoinopt1.q.out     |    16 +-
 .../clientpositive/spark/skewjoinopt10.q.out    |     4 +-
 .../clientpositive/spark/skewjoinopt11.q.out    |     4 +-
 .../clientpositive/spark/skewjoinopt12.q.out    |     8 +-
 .../clientpositive/spark/skewjoinopt14.q.out    |     6 +-
 .../clientpositive/spark/skewjoinopt15.q.out    |    16 +-
 .../clientpositive/spark/skewjoinopt16.q.out    |     8 +-
 .../clientpositive/spark/skewjoinopt17.q.out    |    12 +-
 .../clientpositive/spark/skewjoinopt19.q.out    |     4 +-
 .../clientpositive/spark/skewjoinopt2.q.out     |    16 +-
 .../clientpositive/spark/skewjoinopt20.q.out    |     4 +-
 .../clientpositive/spark/skewjoinopt3.q.out     |     8 +-
 .../clientpositive/spark/skewjoinopt4.q.out     |     8 +-
 .../clientpositive/spark/skewjoinopt5.q.out     |     8 +-
 .../clientpositive/spark/skewjoinopt6.q.out     |     8 +-
 .../clientpositive/spark/skewjoinopt7.q.out     |    12 +-
 .../clientpositive/spark/skewjoinopt8.q.out     |    12 +-
 .../clientpositive/spark/smb_mapjoin9.q.out     |   402 -
 .../clientpositive/spark/smb_mapjoin_1.q.out    |    58 +-
 .../clientpositive/spark/smb_mapjoin_10.q.out   |     2 +-
 .../clientpositive/spark/smb_mapjoin_15.q.out   |     4 +-
 .../clientpositive/spark/smb_mapjoin_2.q.out    |    68 +-
 .../clientpositive/spark/smb_mapjoin_25.q.out   |    80 +-
 .../clientpositive/spark/smb_mapjoin_3.q.out    |    58 +-
 .../clientpositive/spark/smb_mapjoin_4.q.out    |   224 +-
 .../clientpositive/spark/smb_mapjoin_5.q.out    |   224 +-
 .../spark/sort_merge_join_desc_1.q.out          |   127 -
 .../spark/sort_merge_join_desc_2.q.out          |   133 -
 .../spark/sort_merge_join_desc_3.q.out          |   133 -
 .../spark/sort_merge_join_desc_4.q.out          |   155 -
 .../spark/sort_merge_join_desc_5.q.out          |   262 -
 .../spark/sort_merge_join_desc_6.q.out          |   355 -
 .../spark/sort_merge_join_desc_7.q.out          |   479 -
 .../spark/sort_merge_join_desc_8.q.out          |   283 -
 .../spark/spark_combine_equivalent_work_2.q.out |   232 +
 .../spark/spark_dynamic_partition_pruning.q.out |    14 +-
 .../spark_dynamic_partition_pruning_3.q.out     |     2 +-
 .../spark/spark_explainuser_1.q.out             |   138 +-
 .../spark/spark_use_file_size_for_mapjoin.q.out |   257 -
 ...k_vectorized_dynamic_partition_pruning.q.out |    14 +-
 .../clientpositive/spark/subquery_in.q.out      |     4 +-
 .../clientpositive/spark/transform1.q.out       |   138 -
 .../clientpositive/spark/transform_ppr1.q.out   |     2 +-
 .../clientpositive/spark/union_view.q.out       |    22 +-
 .../spark/vector_mapjoin_reduce.q.out           |     2 +-
 .../spark/vector_tablesample_rows.q.out         |   424 -
 .../clientpositive/spark/vectorization_0.q.out  |     8 +-
 .../clientpositive/spark/vectorization_1.q.out  |     2 +-
 .../clientpositive/spark/vectorization_10.q.out |     2 +-
 .../clientpositive/spark/vectorization_12.q.out |     2 +-
 .../clientpositive/spark/vectorization_13.q.out |     4 +-
 .../clientpositive/spark/vectorization_14.q.out |     2 +-
 .../clientpositive/spark/vectorization_15.q.out |     2 +-
 .../clientpositive/spark/vectorization_16.q.out |     2 +-
 .../clientpositive/spark/vectorization_17.q.out |     2 +-
 .../clientpositive/spark/vectorization_2.q.out  |     2 +-
 .../clientpositive/spark/vectorization_3.q.out  |     2 +-
 .../clientpositive/spark/vectorization_4.q.out  |     2 +-
 .../clientpositive/spark/vectorization_5.q.out  |     2 +-
 .../clientpositive/spark/vectorization_6.q.out  |     2 +-
 .../clientpositive/spark/vectorization_7.q.out  |   480 -
 .../clientpositive/spark/vectorization_8.q.out  |   454 -
 .../clientpositive/spark/vectorization_9.q.out  |     2 +-
 .../spark/vectorization_decimal_date.q.out      |     2 +-
 .../spark/vectorization_div0.q.out              |     4 +-
 .../spark/vectorization_short_regress.q.out     |    20 +-
 .../spark/vectorized_bucketmapjoin1.q.out       |   375 -
 .../clientpositive/spark/vectorized_case.q.out  |     4 +-
 .../spark/vectorized_nested_mapjoin.q.out       |     2 +-
 .../spark/vectorized_string_funcs.q.out         |     2 +-
 .../special_character_in_tabnames_1.q.out       | 19607 -----------
 .../special_character_in_tabnames_2.q.out       |     6 +-
 .../test/results/clientpositive/stats11.q.out   |  1291 -
 .../results/clientpositive/subquery_in.q.out    |   956 -
 .../clientpositive/subquery_in_having.q.out     |   312 +-
 .../results/clientpositive/subquery_notin.q.out |  1620 -
 .../results/clientpositive/subquery_views.q.out |   595 -
 .../table_access_keys_stats.q.out               |   606 -
 .../clientpositive/table_nonprintable.q.out     |    72 -
 .../results/clientpositive/temp_table.q.out     |   634 -
 .../clientpositive/temp_table_external.q.out    |    34 -
 .../clientpositive/tez/explainanalyze_2.q.out   |    58 +-
 .../clientpositive/tez/explainanalyze_3.q.out   |     6 +-
 .../clientpositive/tez/explainanalyze_4.q.out   |    12 +-
 .../clientpositive/tez/explainanalyze_5.q.out   |     8 +-
 .../clientpositive/tez/explainuser_3.q.out      |     6 +-
 .../tez/hybridgrace_hashjoin_2.q.out            |     8 +-
 .../clientpositive/tez/stats_filemetadata.q.out |    54 -
 .../clientpositive/tez/vector_acid3.q.out       |    29 -
 .../tez/vector_adaptor_usage_mode.q.out         |   706 -
 .../clientpositive/tez/vector_aggregate_9.q.out |   201 -
 .../tez/vector_aggregate_without_gby.q.out      |    70 -
 .../tez/vector_auto_smb_mapjoin_14.q.out        |  2314 --
 .../tez/vector_between_columns.q.out            |   365 -
 .../clientpositive/tez/vector_between_in.q.out  |  1691 -
 .../tez/vector_binary_join_groupby.q.out        |   586 -
 .../clientpositive/tez/vector_bround.q.out      |    67 -
 .../clientpositive/tez/vector_bucket.q.out      |   125 -
 .../tez/vector_cast_constant.q.out              |   262 -
 .../clientpositive/tez/vector_char_2.q.out      |   430 -
 .../clientpositive/tez/vector_char_4.q.out      |   198 -
 .../clientpositive/tez/vector_char_cast.q.out   |    33 -
 .../tez/vector_char_mapjoin1.q.out              |   678 -
 .../clientpositive/tez/vector_char_simple.q.out |   311 -
 .../clientpositive/tez/vector_coalesce.q.out    |   601 -
 .../clientpositive/tez/vector_coalesce_2.q.out  |   393 -
 .../clientpositive/tez/vectorization_div0.q.out |     4 +-
 .../tez_join_result_complex.q.out               |  1337 -
 .../results/clientpositive/timestamptz_1.q.out  |     2 +-
 .../results/clientpositive/transform_ppr1.q.out |     2 +-
 .../results/clientpositive/uber_reduce.q.out    |    31 -
 .../clientpositive/udaf_collect_set_2.q.out     |   742 -
 .../udaf_percentile_approx_23.q.out             |     8 +-
 .../results/clientpositive/udf_mask_hash.q.out  |     6 +-
 .../clientpositive/udf_trunc_number.q.out       |    20 +-
 .../results/clientpositive/udtf_stack.q.out     |    18 +-
 .../clientpositive/unionDistinct_1.q.out        | 16654 ---------
 .../clientpositive/unionDistinct_2.q.out        |   551 -
 .../clientpositive/union_fast_stats.q.out       |   646 -
 .../clientpositive/union_remove_26.q.out        |   823 -
 .../clientpositive/union_top_level.q.out        |  1396 -
 .../results/clientpositive/union_view.q.out     |    16 +-
 .../results/clientpositive/varchar_udf1.q.out   |   453 -
 .../vector_adaptor_usage_mode.q.out             |   931 -
 .../vector_auto_smb_mapjoin_14.q.out            |  1805 -
 .../clientpositive/vector_between_in.q.out      |   943 -
 .../clientpositive/vector_complex_all.q.out     |   229 -
 .../clientpositive/vector_count_distinct.q.out  |  1356 -
 .../results/clientpositive/vector_date_1.q.out  |     4 +-
 .../clientpositive/vector_decimal_2.q.out       |  1458 -
 .../clientpositive/vector_decimal_cast.q.out    |     2 +-
 .../vector_decimal_expressions.q.out            |     2 +-
 .../clientpositive/vector_decimal_udf.q.out     |  2638 --
 .../clientpositive/vector_inner_join.q.out      |   851 -
 .../clientpositive/vector_interval_2.q.out      |  1550 -
 .../vector_interval_mapjoin.q.out               |     4 +-
 .../results/clientpositive/vector_join30.q.out  |  2220 --
 .../clientpositive/vector_join_filters.q.out    |   224 -
 .../clientpositive/vector_join_nulls.q.out      |   197 -
 .../vector_leftsemi_mapjoin.q.out               | 13610 --------
 .../clientpositive/vector_mapjoin_reduce.q.out  |     2 +-
 .../vector_mr_diff_schema_alias.q.out           |     2 +-
 .../clientpositive/vector_multi_insert.q.out    |     2 +-
 .../clientpositive/vector_nullsafe_join.q.out   |  1210 -
 .../vector_number_compare_projection.q.out      |   716 -
 .../clientpositive/vector_outer_join5.q.out     |  1308 -
 .../vector_partition_diff_num_cols.q.out        |   579 -
 .../vector_partitioned_date_time.q.out          |  2048 --
 .../clientpositive/vector_ptf_part_simple.q.out |  2895 --
 .../vector_tablesample_rows.q.out               |     2 +-
 .../results/clientpositive/vector_udf1.q.out    |  1641 -
 .../vector_windowing_expressions.q.out          |    74 +-
 .../vector_windowing_multipartitioning.q.out    |   132 +-
 .../vector_windowing_order_null.q.out           |   112 +-
 .../vector_windowing_range_multiorder.q.out     |   152 +-
 .../clientpositive/vector_windowing_rank.q.out  |   152 +-
 .../vector_windowing_streaming.q.out            |    16 +-
 .../vector_windowing_windowspec.q.out           |   154 +-
 .../clientpositive/vectorization_0.q.out        | 30981 -----------------
 .../clientpositive/vectorization_1.q.out        |     2 +-
 .../clientpositive/vectorization_10.q.out       |     2 +-
 .../clientpositive/vectorization_12.q.out       |     2 +-
 .../clientpositive/vectorization_13.q.out       |     4 +-
 .../clientpositive/vectorization_14.q.out       |     2 +-
 .../clientpositive/vectorization_15.q.out       |     2 +-
 .../clientpositive/vectorization_16.q.out       |     2 +-
 .../clientpositive/vectorization_17.q.out       |     2 +-
 .../clientpositive/vectorization_2.q.out        |     2 +-
 .../clientpositive/vectorization_3.q.out        |     2 +-
 .../clientpositive/vectorization_4.q.out        |     2 +-
 .../clientpositive/vectorization_5.q.out        |     2 +-
 .../clientpositive/vectorization_6.q.out        |     2 +-
 .../clientpositive/vectorization_7.q.out        |     4 +-
 .../clientpositive/vectorization_8.q.out        |     4 +-
 .../clientpositive/vectorization_9.q.out        |     2 +-
 .../vectorization_decimal_date.q.out            |     2 +-
 .../clientpositive/vectorization_div0.q.out     |     4 +-
 .../vectorization_short_regress.q.out           |  3336 --
 .../clientpositive/vectorized_case.q.out        |     4 +-
 .../clientpositive/vectorized_context.q.out     |     2 +-
 .../clientpositive/vectorized_mapjoin2.q.out    |    14 +-
 .../vectorized_nested_mapjoin.q.out             |   129 -
 .../clientpositive/vectorized_parquet.q.out     |   333 -
 .../results/clientpositive/vectorized_ptf.q.out |  8440 -----
 .../vectorized_string_funcs.q.out               |     2 +-
 .../test/results/clientpositive/view_cbo.q.out  |     2 +-
 .../test/results/clientpositive/windowing.q.out |  2439 --
 .../clientpositive/windowing_windowspec2.q.out  |  3084 --
 .../ConstantObjectInspector.java                |     5 +
 .../objectinspector/ListObjectInspector.java    |     5 +
 .../objectinspector/MapObjectInspector.java     |     5 +
 .../serde2/objectinspector/ObjectInspector.java |     5 +
 .../serde2/objectinspector/StructField.java     |     5 +
 .../objectinspector/StructObjectInspector.java  |     5 +
 .../serde2/objectinspector/UnionObject.java     |     5 +
 .../objectinspector/UnionObjectInspector.java   |     5 +
 .../primitive/BinaryObjectInspector.java        |     5 +
 .../primitive/BooleanObjectInspector.java       |     4 +
 .../primitive/ByteObjectInspector.java          |     4 +
 .../primitive/DateObjectInspector.java          |     4 +
 .../primitive/DoubleObjectInspector.java        |     4 +
 .../primitive/FloatObjectInspector.java         |     4 +
 .../primitive/HiveCharObjectInspector.java      |     4 +
 .../primitive/HiveDecimalObjectInspector.java   |     4 +
 .../HiveIntervalDayTimeObjectInspector.java     |     4 +
 .../HiveIntervalYearMonthObjectInspector.java   |     4 +
 .../primitive/HiveVarcharObjectInspector.java   |     4 +
 .../primitive/IntObjectInspector.java           |     4 +
 .../primitive/LongObjectInspector.java          |     4 +
 .../PrimitiveObjectInspectorUtils.java          |     4 +
 .../primitive/ShortObjectInspector.java         |     4 +
 .../primitive/StringObjectInspector.java        |     4 +
 .../primitive/TimestampObjectInspector.java     |     4 +
 .../primitive/VoidObjectInspector.java          |     4 +
 .../hive/serde2/typeinfo/BaseCharTypeInfo.java  |     5 +
 .../hive/serde2/typeinfo/ListTypeInfo.java      |     4 +
 .../hive/serde2/typeinfo/MapTypeInfo.java       |     4 +
 .../hive/serde2/typeinfo/PrimitiveTypeInfo.java |     4 +
 .../hive/serde2/typeinfo/StructTypeInfo.java    |     4 +
 .../hadoop/hive/serde2/typeinfo/TypeInfo.java   |     4 +
 .../hive/serde2/typeinfo/UnionTypeInfo.java     |     4 +
 service-rpc/if/TCLIService.thrift               |    10 +
 .../src/gen/thrift/gen-cpp/TCLIService.cpp      |   383 +
 .../src/gen/thrift/gen-cpp/TCLIService.h        |   126 +
 .../gen-cpp/TCLIService_server.skeleton.cpp     |     5 +
 .../gen/thrift/gen-cpp/TCLIService_types.cpp    |   172 +
 .../src/gen/thrift/gen-cpp/TCLIService_types.h  |    84 +
 .../hive/service/rpc/thrift/TCLIService.java    |   858 +
 .../hive/service/rpc/thrift/TGetQueryIdReq.java |   394 +
 .../service/rpc/thrift/TGetQueryIdResp.java     |   389 +
 .../src/gen/thrift/gen-php/TCLIService.php      |   216 +
 service-rpc/src/gen/thrift/gen-php/Types.php    |   155 +
 .../gen-py/TCLIService/TCLIService-remote       |     7 +
 .../thrift/gen-py/TCLIService/TCLIService.py    |   189 +
 .../src/gen/thrift/gen-py/TCLIService/ttypes.py |   135 +
 .../src/gen/thrift/gen-rb/t_c_l_i_service.rb    |    54 +
 .../gen/thrift/gen-rb/t_c_l_i_service_types.rb  |    34 +
 service/pom.xml                                 |     7 +
 .../org/apache/hive/service/cli/CLIService.java |    10 +
 .../service/cli/EmbeddedCLIServiceClient.java   |     6 +
 .../apache/hive/service/cli/ICLIService.java    |     3 +
 .../service/cli/operation/OperationManager.java |    25 +-
 .../service/cli/session/HiveSessionImpl.java    |     9 +
 .../service/cli/session/SessionManager.java     |     7 +
 .../thrift/RetryingThriftCLIServiceClient.java  |     6 +
 .../service/cli/thrift/ThriftCLIService.java    |    11 +
 .../cli/thrift/ThriftCLIServiceClient.java      |    11 +
 .../apache/hive/service/server/HiveServer2.java |    24 +-
 .../hive/service/server/KillQueryImpl.java      |    53 +
 .../cli/session/TestSessionManagerMetrics.java  |     3 +-
 .../cli/thrift/ThriftCLIServiceTest.java        |     4 +-
 .../thrift/ThriftCliServiceTestWithCookie.java  |     4 +-
 .../hive/service/server/TestHS2HttpServer.java  |     4 +-
 storage-api/pom.xml                             |     2 +-
 1205 files changed, 24074 insertions(+), 443973 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 1672222,b51446d..0442e21
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@@ -1948,61 -1814,6 +1817,59 @@@ public class MetaStoreUtils 
      return cols;
    }
  
 +  // TODO The following two utility methods can be moved to AcidUtils once no class in metastore is relying on them,
 +  // right now ObjectStore.getAllMmTablesForCleanup is calling these method
 +  /**
 +   * Checks if a table is an ACID table that only supports INSERT, but not UPDATE/DELETE
 +   * @param params table properties
 +   * @return true if table is an INSERT_ONLY table, false otherwise
 +   */
 +  public static boolean isInsertOnlyTable(Map<String, String> params) {
 +    return isInsertOnlyTable(params, false);
 +  }
 +
++  // TODO: CTAS for MM may currently be broken. It used to work. See the old code and why isCtas isn't used?
 +  public static boolean isInsertOnlyTable(Map<String, String> params, boolean isCtas) {
 +    String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
 +    return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
 +  }
 +
 +   public static boolean isInsertOnlyTable(Properties params) {
-     // TODO#  redirect for now - fix before merge
-     HashMap<String, String> testMap = new HashMap<String, String>();
-     for (String n  : params.stringPropertyNames()) {
-       testMap.put(n, params.getProperty(n));
-     }
-     return isInsertOnlyTable(testMap);
++     String transactionalProp = params.getProperty(
++         hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
++     return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
 +  }
 +
 +   /** The method for altering table props; may set the table to MM, non-MM, or not affect MM. */
 +  public static Boolean isToInsertOnlyTable(Map<String, String> props) {
 +    // TODO: Setting these separately is a very hairy issue in certain combinations, since we
 +    //       cannot decide what type of table this becomes without taking both into account, and
 +    //       in many cases the conversion might be illegal.
 +    //       The only thing we allow is tx = true w/o tx-props, for backward compat.
 +    String transactional = props.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
 +    String transactionalProp = props.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
 +    if (transactional == null && transactionalProp == null) return null; // Not affected.
 +    boolean isSetToTxn = "true".equalsIgnoreCase(transactional);
 +    if (transactionalProp == null) {
 +      if (isSetToTxn) return false; // Assume the full ACID table.
 +      throw new RuntimeException("Cannot change '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL
 +          + "' without '" + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "'");
 +    }
 +    if (!"insert_only".equalsIgnoreCase(transactionalProp)) return false; // Not MM.
 +    if (!isSetToTxn) {
 +      throw new RuntimeException("Cannot set '"
 +          + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "' to 'insert_only' without "
 +          + "setting '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL + "' to 'true'");
 +    }
 +    return true;
 +  }
 +
 +  public static boolean isRemovedInsertOnlyTable(Set<String> removedSet) {
 +    boolean hasTxn = removedSet.contains(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL),
 +        hasProps = removedSet.contains(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
 +    return hasTxn || hasProps;
 +  }
 +
    // given a list of partStats, this function will give you an aggr stats
    public static List<ColumnStatisticsObj> aggrPartitionStats(List<ColumnStatistics> partStats,
        String dbName, String tableName, List<String> partNames, List<String> colNames,

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 5dc0cea,2cf5bfd..8414a78
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@@ -182,11 -179,10 +182,12 @@@ import org.apache.hadoop.hive.ql.plan.F
  import org.apache.hadoop.hive.ql.plan.GrantDesc;
  import org.apache.hadoop.hive.ql.plan.GrantRevokeRoleDDL;
  import org.apache.hadoop.hive.ql.plan.InsertTableDesc;
+ import org.apache.hadoop.hive.ql.plan.KillQueryDesc;
  import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
 +import org.apache.hadoop.hive.ql.plan.LoadMultiFilesDesc;
  import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
  import org.apache.hadoop.hive.ql.plan.LockTableDesc;
 +import org.apache.hadoop.hive.ql.plan.MoveWork;
  import org.apache.hadoop.hive.ql.plan.MsckDesc;
  import org.apache.hadoop.hive.ql.plan.OperatorDesc;
  import org.apache.hadoop.hive.ql.plan.OrcFileMergeDesc;
@@@ -4067,215 -4089,7 +4086,215 @@@ public class DDLTask extends Task<DDLWo
        throw new HiveException(ErrorMsg.UNSUPPORTED_ALTER_TBL_OP, alterTbl.getOp().toString());
      }
  
 -    return 0;
 +    return null;
 +  }
 +
 +  private List<Task<?>> alterTableDropProps(AlterTableDesc alterTbl, Table tbl,
 +      Partition part, EnvironmentContext environmentContext) throws HiveException {
 +    if (StatsSetupConst.USER.equals(environmentContext.getProperties()
 +        .get(StatsSetupConst.STATS_GENERATED))) {
 +      // drop a stats parameter, which triggers recompute stats update automatically
 +      environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
 +    }
 +
 +    List<Task<?>> result = null;
 +    if (part == null) {
 +      Set<String> removedSet = alterTbl.getProps().keySet();
 +      boolean isFromMmTable = MetaStoreUtils.isInsertOnlyTable(tbl.getParameters()),
 +          isRemoved = MetaStoreUtils.isRemovedInsertOnlyTable(removedSet);
 +      if (isFromMmTable && isRemoved) {
 +        result = generateRemoveMmTasks(tbl);
 +      }
 +    }
 +    Iterator<String> keyItr = alterTbl.getProps().keySet().iterator();
 +    while (keyItr.hasNext()) {
 +      if (part != null) {
 +        part.getTPartition().getParameters().remove(keyItr.next());
 +      } else {
 +        tbl.getTTable().getParameters().remove(keyItr.next());
 +      }
 +    }
 +    return result;
 +  }
 +
 +  private List<Task<?>> generateRemoveMmTasks(Table tbl) throws HiveException {
 +    // To avoid confusion from nested MM directories when table is converted back and forth, we
 +    // want to rename mm_ dirs to remove the prefix; however, given the unpredictable nested
 +    // directory handling in Hive/MR, we will instead move all the files into the root directory.
 +    // We will also delete any directories that are not committed. 
 +    // Note that this relies on locks. Note also that we only do the renames AFTER the metastore
 +    // operation commits. Deleting uncommitted things is safe, but moving stuff before we convert
 +    // could cause data loss.
 +    List<Path> allMmDirs = new ArrayList<>();
 +    if (tbl.isStoredAsSubDirectories()) {
 +      // TODO: support this? we only bail because it's a PITA and hardly anyone seems to care.
 +      throw new HiveException("Converting list bucketed tables stored as subdirectories "
 +          + " to and from MM is not supported");
 +    }
 +    List<String> bucketCols = tbl.getBucketCols();
 +    if (bucketCols != null && !bucketCols.isEmpty()
 +        && HiveConf.getBoolVar(conf, ConfVars.HIVE_STRICT_CHECKS_BUCKETING)) {
 +      throw new HiveException("Converting bucketed tables from MM is not supported by default; "
 +          + "copying files from multiple MM directories may potentially break the buckets. You "
 +          + "can set " + ConfVars.HIVE_STRICT_CHECKS_BUCKETING.varname
 +          + " to false for this query if you want to force the conversion.");
 +    }
 +    Hive db = getHive();
 +    String value = conf.get(ValidTxnList.VALID_TXNS_KEY);
 +    ValidTxnList validTxnList = value == null ? new ValidReadTxnList() : new ValidReadTxnList(value);
 +    if (tbl.getPartitionKeys().size() > 0) {
 +      PartitionIterable parts = new PartitionIterable(db, tbl, null,
 +          HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
 +      Iterator<Partition> partIter = parts.iterator();
 +      while (partIter.hasNext()) {
 +        Partition part = partIter.next();
 +        checkMmLb(part);
 +        handleRemoveMm(part.getDataLocation(), validTxnList, allMmDirs);
 +      }
 +    } else {
 +      checkMmLb(tbl);
 +      handleRemoveMm(tbl.getDataLocation(), validTxnList, allMmDirs);
 +    }
 +    List<Path> targetPaths = new ArrayList<>(allMmDirs.size());
 +    List<String> targetPrefix = new ArrayList<>(allMmDirs.size());
 +    int prefixLen = JavaUtils.DELTA_PREFIX.length();
 +    for (int i = 0; i < allMmDirs.size(); ++i) {
 +      Path src = allMmDirs.get(i);
 +      Path tgt = src.getParent();
 +      String prefix = src.getName().substring(prefixLen + 1) + "_";
 +      Utilities.LOG14535.info("Will move " + src + " to " + tgt + " (prefix " + prefix + ")");
 +      targetPaths.add(tgt);
 +      targetPrefix.add(prefix);
 +    }
 +    // Don't set inputs and outputs - the locks have already been taken so it's pointless.
-     MoveWork mw = new MoveWork(null, null, null, null, false);
++    MoveWork mw = new MoveWork(null, null, null, null, false, SessionState.get().getLineageState());
 +    mw.setMultiFilesDesc(new LoadMultiFilesDesc(
 +        allMmDirs, targetPaths, targetPrefix, true, null, null));
 +    return Lists.<Task<?>>newArrayList(TaskFactory.get(mw, conf));
 +  }
 +
 +  private void checkMmLb(Table tbl) throws HiveException {
 +    if (!tbl.isStoredAsSubDirectories()) return;
 +    // TODO: support this?
 +    throw new HiveException("Converting list bucketed tables stored as subdirectories "
 +        + " to and from MM is not supported");
 +  }
 +
 +  private void checkMmLb(Partition part) throws HiveException {
 +    if (!part.isStoredAsSubDirectories()) return;
 +    // TODO: support this?
 +    throw new HiveException("Converting list bucketed tables stored as subdirectories "
 +        + " to and from MM is not supported. Please create a table in the desired format.");
 +  }
 +
 +  private void handleRemoveMm(
 +      Path path, ValidTxnList validTxnList, List<Path> result) throws HiveException {
 +    // Note: doesn't take LB into account; that is not presently supported here (throws above).
 +    try {
 +      FileSystem fs = path.getFileSystem(conf);
 +      for (FileStatus file : fs.listStatus(path)) {
 +        Path childPath = file.getPath();
 +        if (!file.isDirectory()) {
 +          ensureDelete(fs, childPath, "a non-directory file");
 +          continue;
 +        }
 +        Long writeId = JavaUtils.extractTxnId(childPath);
 +        if (writeId == null) {
 +          ensureDelete(fs, childPath, "an unknown directory");
 +        } else if (!validTxnList.isTxnValid(writeId)) {
 +          // Assume no concurrent active writes - we rely on locks here. We could check and fail.
 +          ensureDelete(fs, childPath, "an uncommitted directory");
 +        } else {
 +          result.add(childPath);
 +        }
 +      }
 +    } catch (IOException ex) {
 +      throw new HiveException(ex);
 +    }
 +  }
 +
 +  private static void ensureDelete(FileSystem fs, Path path, String what) throws IOException {
 +    Utilities.LOG14535.info("Deleting " + what + " " + path);
 +    try {
 +      if (!fs.delete(path, true)) throw new IOException("delete returned false");
 +    } catch (Exception ex) {
 +      String error = "Couldn't delete " + path + "; cannot remove MM setting from the table";
 +      LOG.error(error, ex);
 +      throw (ex instanceof IOException) ? (IOException)ex : new IOException(ex);
 +    }
 +  }
 +
 +  private List<Task<?>> generateAddMmTasks(Table tbl) throws HiveException {
 +    // We will move all the files in the table/partition directories into the first MM
 +    // directory, then commit the first write ID.
 +    List<Path> srcs = new ArrayList<>(), tgts = new ArrayList<>();
 +    long mmWriteId = 0;
 +    try {
 +      HiveTxnManager txnManager = SessionState.get().getTxnMgr();
 +      if (txnManager.isTxnOpen()) {
 +        mmWriteId = txnManager.getCurrentTxnId();
 +      } else {
 +        mmWriteId = txnManager.openTxn(new Context(conf), conf.getUser());
 +        txnManager.commitTxn();
 +      }
 +    } catch (Exception e) {
 +      String errorMessage = "FAILED: Error in acquiring locks: " + e.getMessage();
 +      console.printError(errorMessage, "\n"
 +          + org.apache.hadoop.util.StringUtils.stringifyException(e));
 +    }
 +    int stmtId = 0;
 +    String mmDir = AcidUtils.deltaSubdir(mmWriteId, mmWriteId, stmtId);
 +    Hive db = getHive();
 +    if (tbl.getPartitionKeys().size() > 0) {
 +      PartitionIterable parts = new PartitionIterable(db, tbl, null,
 +          HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
 +      Iterator<Partition> partIter = parts.iterator();
 +      while (partIter.hasNext()) {
 +        Partition part = partIter.next();
 +        checkMmLb(part);
 +        Path src = part.getDataLocation(), tgt = new Path(src, mmDir);
 +        srcs.add(src);
 +        tgts.add(tgt);
 +        Utilities.LOG14535.info("Will move " + src + " to " + tgt);
 +      }
 +    } else {
 +      checkMmLb(tbl);
 +      Path src = tbl.getDataLocation(), tgt = new Path(src, mmDir);
 +      srcs.add(src);
 +      tgts.add(tgt);
 +      Utilities.LOG14535.info("Will move " + src + " to " + tgt);
 +    }
 +    // Don't set inputs and outputs - the locks have already been taken so it's pointless.
-     MoveWork mw = new MoveWork(null, null, null, null, false);
++    MoveWork mw = new MoveWork(null, null, null, null, false, SessionState.get().getLineageState());
 +    mw.setMultiFilesDesc(new LoadMultiFilesDesc(srcs, tgts, true, null, null));
 +    ImportCommitWork icw = new ImportCommitWork(tbl.getDbName(), tbl.getTableName(), mmWriteId, stmtId);
 +    Task<?> mv = TaskFactory.get(mw, conf), ic = TaskFactory.get(icw, conf);
 +    mv.addDependentTask(ic);
 +    return Lists.<Task<?>>newArrayList(mv);
 +  }
 +
 +  private List<Task<?>> alterTableAddProps(AlterTableDesc alterTbl, Table tbl,
 +      Partition part, EnvironmentContext environmentContext) throws HiveException {
 +    if (StatsSetupConst.USER.equals(environmentContext.getProperties()
 +        .get(StatsSetupConst.STATS_GENERATED))) {
 +      environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
 +    }
 +    List<Task<?>> result = null;
 +    if (part != null) {
 +      part.getTPartition().getParameters().putAll(alterTbl.getProps());
 +    } else {
 +      boolean isFromMmTable = MetaStoreUtils.isInsertOnlyTable(tbl.getParameters());
 +      Boolean isToMmTable = MetaStoreUtils.isToInsertOnlyTable(alterTbl.getProps());
 +      if (isToMmTable != null) {
 +        if (!isFromMmTable && isToMmTable) {
 +          result = generateAddMmTasks(tbl);
 +        } else if (isFromMmTable && !isToMmTable) {
 +          result = generateRemoveMmTasks(tbl);
 +        }
 +      }
 +      tbl.getTTable().getParameters().putAll(alterTbl.getProps());
 +    }
 +    return result;
    }
  
    private int dropConstraint(Hive db, AlterTableDesc alterTbl)

http://git-wip-us.apache.org/repos/asf/hive/blob/379d9bab/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
----------------------------------------------------------------------

[48/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d7b6272a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d7b6272a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d7b6272a

Branch: refs/heads/master
Commit: d7b6272ad4b59619f5bfa7698c2823efdc938273
Parents: 98c5b63 857347f
Author: sergey <se...@apache.org>
Authored: Tue Oct 10 15:23:31 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Tue Oct 10 15:23:31 2017 -0700

----------------------------------------------------------------------
 .../hadoop/hive/cli/control/CliConfigs.java     |  1 +
 .../spark/spark_explainuser_1.q.out             | 72 ++++++++++----------
 2 files changed, 38 insertions(+), 35 deletions(-)
----------------------------------------------------------------------

[43/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/14c94820
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/14c94820
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/14c94820

Branch: refs/heads/master
Commit: 14c9482017f5bb7263e0893023cc47a8d37ec3de
Parents: 6401597 d376157
Author: sergey <se...@apache.org>
Authored: Tue Oct 3 20:33:47 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Tue Oct 3 20:33:47 2017 -0700

----------------------------------------------------------------------
 .../hadoop/hive/druid/DruidStorageHandler.java  |    8 +-
 .../ExpressionTemplates/IfExprColumnScalar.txt  |    4 +
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   49 +-
 .../apache/hadoop/hive/ql/metadata/Hive.java    |   95 +-
 .../metadata/HiveMaterializedViewsRegistry.java |    8 +-
 .../apache/hadoop/hive/ql/metadata/Table.java   |    5 +-
 ...tedDynPartitionTimeGranularityOptimizer.java |   12 +-
 .../ql/optimizer/calcite/HiveCalciteUtil.java   |   12 +-
 .../ql/optimizer/calcite/RelOptHiveTable.java   |   78 +-
 .../rules/views/HiveMaterializedViewRule.java   |   27 +-
 .../hadoop/hive/ql/parse/CalcitePlanner.java    |  168 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |    5 +-
 .../hadoop/hive/ql/plan/CreateTableDesc.java    |   19 +-
 .../hadoop/hive/ql/plan/CreateViewDesc.java     |    1 +
 .../apache/hadoop/hive/ql/plan/PlanUtils.java   |   99 +-
 .../materialized_view_create_rewrite_2.q        |   88 +
 .../materialized_view_rewrite_ssb.q             |  346 ++++
 .../materialized_view_rewrite_ssb_2.q           |  347 ++++
 .../queries/clientpositive/vector_if_expr_2.q   |   19 +
 .../clientpositive/llap/subquery_notin.q.out    |  116 +-
 .../materialized_view_create_rewrite_2.q.out    |  574 ++++++
 .../materialized_view_describe.q.out            |   12 +-
 .../materialized_view_rewrite_ssb.q.out         | 1621 +++++++++++++++++
 .../materialized_view_rewrite_ssb_2.q.out       | 1627 ++++++++++++++++++
 .../outer_reference_windowed.q.out              |   34 +-
 .../clientpositive/vector_if_expr_2.q.out       |  117 ++
 .../vector_outer_reference_windowed.q.out       |   34 +-
 27 files changed, 5303 insertions(+), 222 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/14c94820/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/14c94820/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 7464a29,b0e68b1..37ee6b6
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@@ -51,10 -51,8 +51,11 @@@ import java.util.concurrent.ExecutorSer
  import java.util.concurrent.Executors;
  import java.util.concurrent.Future;
  import java.util.concurrent.atomic.AtomicInteger;
+ import java.util.stream.Collectors;
  
 +import com.google.common.base.Preconditions;
 +import com.google.common.collect.ImmutableMap;
 +
  import javax.jdo.JDODataStoreException;
  
  import org.apache.calcite.plan.RelOptMaterialization;
@@@ -63,13 -61,10 +64,12 @@@ import org.apache.hadoop.conf.Configura
  import org.apache.hadoop.fs.FileChecksum;
  import org.apache.hadoop.fs.FileStatus;
  import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.LocatedFileStatus;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.fs.PathFilter;
- import org.apache.hadoop.hive.common.BlobStorageUtils;
  import org.apache.hadoop.hive.common.FileUtils;
  import org.apache.hadoop.hive.common.HiveStatsUtils;
 +import org.apache.hadoop.hive.common.JavaUtils;
  import org.apache.hadoop.hive.common.ObjectPair;
  import org.apache.hadoop.hive.common.StatsSetupConst;
  import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;

http://git-wip-us.apache.org/repos/asf/hive/blob/14c94820/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/14c94820/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/14c94820/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/14c94820/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------

[50/50] [abbrv] hive git commit: HIVE-15212 : merge branch into master (Sergey Shelukhin, reviewed by Eugene Koifman, Gunther Hagleitner, Wei Zheng, Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-15212 : merge branch into master (Sergey Shelukhin, reviewed by Eugene Koifman, Gunther Hagleitner, Wei Zheng, Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c1a1d596
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c1a1d596
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c1a1d596

Branch: refs/heads/master
Commit: c1a1d5960a639155e5c48b8223b501ad17b37e1f
Parents: 9375cf3 ba9cd79
Author: sergey <se...@apache.org>
Authored: Thu Oct 12 17:11:09 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Oct 12 17:11:09 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/common/FileUtils.java    |   25 +-
 .../hadoop/hive/common/HiveStatsUtils.java      |   47 +-
 .../apache/hadoop/hive/common/JavaUtils.java    |   67 +
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   15 +-
 .../apache/hive/common/util/MockFileSystem.java |  622 ++++++
 .../mapreduce/FileOutputCommitterContainer.java |    2 +-
 .../hcatalog/mapreduce/HCatOutputFormat.java    |    6 +
 .../listener/DummyRawStoreFailEvent.java        |    1 +
 .../hadoop/hive/ql/history/TestHiveHistory.java |    3 +-
 .../test/resources/testconfiguration.properties |    4 +-
 .../io/encoded/VectorDeserializeOrcWriter.java  |    3 +-
 .../upgrade/derby/hive-schema-2.2.0.derby.sql   |    1 +
 .../upgrade/mssql/hive-schema-2.2.0.mssql.sql   |    1 +
 .../upgrade/oracle/hive-schema-2.2.0.oracle.sql |    1 -
 .../postgres/hive-schema-2.2.0.postgres.sql     |    1 +
 .../hadoop/hive/metastore/HiveMetaStore.java    |  150 +-
 .../hive/metastore/HiveMetaStoreClient.java     |    7 +-
 .../hadoop/hive/metastore/MetaStoreUtils.java   |   10 +
 .../hadoop/hive/metastore/ObjectStore.java      |   87 +-
 .../TransactionalValidationListener.java        |   42 +-
 .../hive/metastore/cache/CachedStore.java       |    5 +
 .../DummyRawStoreForJdoConnection.java          |    3 -
 .../hadoop/hive/metastore/TestObjectStore.java  |   22 +-
 .../java/org/apache/hadoop/hive/ql/Context.java |    5 +-
 .../java/org/apache/hadoop/hive/ql/Driver.java  |    4 +-
 .../hive/ql/exec/AbstractFileMergeOperator.java |  168 +-
 .../apache/hadoop/hive/ql/exec/CopyTask.java    |   64 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |  233 +-
 .../hive/ql/exec/DependencyCollectionTask.java  |    1 -
 .../hadoop/hive/ql/exec/FetchOperator.java      |   54 +-
 .../apache/hadoop/hive/ql/exec/FetchTask.java   |    1 +
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |  365 ++-
 .../hive/ql/exec/HashTableSinkOperator.java     |    5 +
 .../hadoop/hive/ql/exec/ImportCommitTask.java   |   62 +
 .../hadoop/hive/ql/exec/ImportCommitWork.java   |   54 +
 .../hadoop/hive/ql/exec/JoinOperator.java       |    4 +-
 .../apache/hadoop/hive/ql/exec/MoveTask.java    |  531 +++--
 .../hive/ql/exec/OrcFileMergeOperator.java      |   11 +-
 .../hive/ql/exec/RCFileMergeOperator.java       |    3 +-
 .../hadoop/hive/ql/exec/ReplCopyTask.java       |    4 +-
 .../hadoop/hive/ql/exec/StatsNoJobTask.java     |    9 +-
 .../apache/hadoop/hive/ql/exec/StatsTask.java   |    9 +-
 .../hadoop/hive/ql/exec/TableScanOperator.java  |    4 +-
 .../apache/hadoop/hive/ql/exec/TaskFactory.java |    2 +
 .../apache/hadoop/hive/ql/exec/Utilities.java   |  800 +++++--
 .../bootstrap/load/table/LoadPartitions.java    |    6 +-
 .../repl/bootstrap/load/table/LoadTable.java    |    5 +-
 .../hadoop/hive/ql/exec/tez/SplitGrouper.java   |    4 +-
 .../ql/exec/vector/VectorizedRowBatchCtx.java   |    2 +-
 .../hadoop/hive/ql/hooks/WriteEntity.java       |    3 +-
 .../hive/ql/index/HiveIndexedInputFormat.java   |    2 +-
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |  124 +-
 .../hive/ql/io/CombineHiveInputFormat.java      |   42 +-
 .../hive/ql/io/CombineHiveRecordReader.java     |    2 +-
 .../ql/io/HiveContextAwareRecordReader.java     |    2 +-
 .../hadoop/hive/ql/io/HiveFileFormatUtils.java  |   42 +-
 .../hadoop/hive/ql/io/HiveInputFormat.java      |   91 +-
 .../hadoop/hive/ql/io/merge/MergeFileWork.java  |    7 +-
 .../hadoop/hive/ql/io/orc/OrcInputFormat.java   |    2 +-
 .../ql/io/parquet/MapredParquetInputFormat.java |    8 +
 .../hive/ql/io/parquet/ProjectionPusher.java    |   24 +-
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java    |    6 +-
 .../apache/hadoop/hive/ql/metadata/Hive.java    |  400 +++-
 .../hadoop/hive/ql/metadata/Partition.java      |    1 +
 .../apache/hadoop/hive/ql/metadata/Table.java   |    1 -
 .../formatting/TextMetaDataFormatter.java       |  119 +-
 .../ql/optimizer/AbstractBucketJoinProc.java    |    7 +
 .../BucketingSortingReduceSinkOptimizer.java    |   15 +-
 .../hive/ql/optimizer/GenMRTableScan1.java      |    1 +
 .../hive/ql/optimizer/GenMapRedUtils.java       |  306 +--
 .../hadoop/hive/ql/optimizer/SamplePruner.java  |    8 +-
 .../physical/BucketingSortingOpProcFactory.java |    7 +
 .../physical/GenMRSkewJoinProcessor.java        |    2 +-
 .../physical/GenSparkSkewJoinProcessor.java     |    4 +-
 .../physical/LocalMapJoinProcFactory.java       |    2 +-
 .../optimizer/physical/SamplingOptimizer.java   |   15 +
 .../ql/optimizer/physical/SkewJoinResolver.java |   27 +-
 .../hive/ql/optimizer/physical/Vectorizer.java  |    1 +
 .../physical/index/IndexWhereProcessor.java     |    2 +-
 .../optimizer/unionproc/UnionProcFactory.java   |    6 +-
 .../ql/parse/AlterTablePartMergeFilesDesc.java  |    9 +
 .../hive/ql/parse/BaseSemanticAnalyzer.java     |    3 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java      |   52 +-
 .../apache/hadoop/hive/ql/parse/EximUtil.java   |    9 +-
 .../hive/ql/parse/ExportSemanticAnalyzer.java   |   34 +
 .../hadoop/hive/ql/parse/GenTezUtils.java       |    5 +-
 .../hive/ql/parse/ImportSemanticAnalyzer.java   |  182 +-
 .../hadoop/hive/ql/parse/IndexUpdater.java      |   19 +-
 .../hive/ql/parse/LoadSemanticAnalyzer.java     |   16 +-
 .../hadoop/hive/ql/parse/ParseContext.java      |    8 -
 .../hive/ql/parse/ProcessAnalyzeTable.java      |    1 +
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  569 ++---
 .../hadoop/hive/ql/parse/TaskCompiler.java      |  155 +-
 .../hadoop/hive/ql/parse/TypeCheckCtx.java      |    2 +-
 .../hive/ql/parse/spark/GenSparkUtils.java      |    4 +-
 .../parse/spark/SparkProcessAnalyzeTable.java   |    2 +-
 .../ql/plan/ConditionalResolverMergeFiles.java  |   38 +-
 .../apache/hadoop/hive/ql/plan/CopyWork.java    |   52 +-
 .../hadoop/hive/ql/plan/CreateTableDesc.java    |   22 +
 .../hadoop/hive/ql/plan/FileMergeDesc.java      |   27 +
 .../hadoop/hive/ql/plan/FileSinkDesc.java       |   56 +-
 .../apache/hadoop/hive/ql/plan/LoadDesc.java    |    1 +
 .../hadoop/hive/ql/plan/LoadFileDesc.java       |   22 +-
 .../hadoop/hive/ql/plan/LoadMultiFilesDesc.java |   12 +
 .../hadoop/hive/ql/plan/LoadTableDesc.java      |   74 +-
 .../org/apache/hadoop/hive/ql/plan/MapWork.java |   12 +
 .../apache/hadoop/hive/ql/plan/MoveWork.java    |   14 +-
 .../hadoop/hive/ql/plan/PartitionDesc.java      |   11 +
 .../apache/hadoop/hive/ql/plan/PlanUtils.java   |   14 -
 .../apache/hadoop/hive/ql/plan/TableDesc.java   |   10 +-
 .../hadoop/hive/ql/plan/TableScanDesc.java      |    2 +-
 .../hive/ql/stats/fs/FSStatsAggregator.java     |    8 +-
 .../hive/ql/stats/fs/FSStatsPublisher.java      |    8 +-
 .../hive/ql/txn/compactor/CompactorMR.java      |   36 +
 .../hive/ql/txn/compactor/CompactorThread.java  |    1 -
 .../hadoop/hive/ql/txn/compactor/Initiator.java |    6 +
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |    1 -
 .../apache/hadoop/hive/ql/TestTxnCommands2.java |   83 +-
 .../hadoop/hive/ql/exec/TestExecDriver.java     |    2 +-
 .../hive/ql/exec/TestFileSinkOperator.java      |    3 +-
 .../apache/hadoop/hive/ql/io/TestAcidUtils.java |    6 +-
 .../hive/ql/io/TestHiveFileFormatUtils.java     |   24 +-
 .../hive/ql/io/orc/TestInputOutputFormat.java   |   11 +-
 .../hadoop/hive/ql/metadata/TestHive.java       |    4 +-
 ...TestGenMapRedUtilsCreateConditionalTask.java |   20 +-
 .../queries/clientnegative/mm_concatenate.q     |    8 +
 ql/src/test/queries/clientnegative/mm_convert.q |   10 +
 .../queries/clientnegative/mm_truncate_cols.q   |    3 +
 ql/src/test/queries/clientpositive/mm_all.q     |  326 +++
 ql/src/test/queries/clientpositive/mm_buckets.q |   66 +
 .../queries/clientpositive/mm_conversions.q     |   42 +
 .../clientpositive/mm_exchangepartition.q       |   68 +
 ql/src/test/queries/clientpositive/mm_exim.q    |   98 +
 .../test/queries/clientpositive/mm_loaddata.q   |   53 +
 .../results/clientnegative/mm_concatenate.q.out |   18 +
 .../results/clientnegative/mm_convert.q.out     |   17 +
 .../clientnegative/mm_truncate_cols.q.out       |    9 +
 .../clientpositive/llap/mm_conversions.q.out    |  309 +++
 ql/src/test/results/clientpositive/mm_all.q.out | 2097 ++++++++++++++++++
 .../results/clientpositive/mm_buckets.q.out     |  489 ++++
 .../results/clientpositive/mm_conversions.q.out |  309 +++
 .../clientpositive/mm_exchangepartition.q.out   |  382 ++++
 .../test/results/clientpositive/mm_exim.q.out   |  557 +++++
 .../results/clientpositive/mm_loaddata.q.out    |  296 +++
 .../clientpositive/tez/explainuser_3.q.out      |    6 +-
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp |    8 +-
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |    3 +-
 .../hive/metastore/api/ClientCapability.java    |    5 +-
 .../src/gen/thrift/gen-php/metastore/Types.php  |    2 +
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |    3 +
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |    5 +-
 .../hadoop/hive/metastore/MetaStoreThread.java  |    1 +
 .../apache/hadoop/hive/metastore/Warehouse.java |   15 +-
 .../hadoop/hive/metastore/model/MTable.java     |    4 +-
 .../src/main/resources/package.jdo              |   12 +-
 .../src/main/thrift/hive_metastore.thrift       |    5 +-
 156 files changed, 9969 insertions(+), 1721 deletions(-)
----------------------------------------------------------------------

[29/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1fb04f36
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1fb04f36
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1fb04f36

Branch: refs/heads/master
Commit: 1fb04f36619483cc5a64e5963ddb8a097a68c9ba
Parents: 7a40541 5608300
Author: sergey <se...@apache.org>
Authored: Wed Sep 20 11:40:50 2017 +1000
Committer: sergey <se...@apache.org>
Committed: Wed Sep 20 11:40:50 2017 +1000

----------------------------------------------------------------------
 .../hadoop/hive/cli/TestCliSessionState.java    |    4 +-
 data/files/filterCard.txt                       |  101 +
 .../apache/hive/hcatalog/cli/HCatDriver.java    |    3 +-
 .../apache/hive/hcatalog/common/HCatUtil.java   |    4 +-
 .../hive/hcatalog/mapreduce/HCatTableInfo.java  |    4 +-
 .../hive/hcatalog/mapreduce/InputJobInfo.java   |    4 +-
 .../hive/hcatalog/mapreduce/OutputJobInfo.java  |    4 +-
 .../apache/hive/hcatalog/cli/TestPermsGrp.java  |    4 +-
 .../hive/hcatalog/cli/TestSemanticAnalysis.java |   24 +-
 .../hcatalog/mapreduce/HCatMapReduceTest.java   |    8 +-
 .../mapreduce/TestHCatPartitionPublish.java     |    3 +-
 .../hcatalog/mapreduce/TestPassProperties.java  |    4 +-
 .../apache/hive/hcatalog/pig/PigHCatUtil.java   |    4 +-
 .../hcatalog/pig/TestHCatLoaderEncryption.java  |    7 +-
 .../hive/hcatalog/api/HCatClientHMSImpl.java    |    4 +-
 .../apache/hive/hcatalog/api/HCatPartition.java |    3 +-
 .../org/apache/hive/hcatalog/api/HCatTable.java |    6 +-
 .../listener/DummyRawStoreFailEvent.java        |    7 +
 .../mapreduce/TestSequenceFileReadWrite.java    |    6 +-
 .../hive/metastore/TestHiveMetaStore.java       |    2 +-
 .../hive/metastore/TestReplChangeManager.java   |   44 +-
 .../hadoop/hive/ql/history/TestHiveHistory.java |    4 +-
 .../TestSemanticAnalyzerHookLoading.java        |    4 +-
 .../hive/ql/parse/TestReplicationScenarios.java |  106 +-
 .../hadoop/hive/hbase/HBaseQTestUtil.java       |    4 +-
 .../org/apache/hadoop/hive/ql/QTestUtil.java    |    6 +-
 .../hadoop/hive/metastore/HiveMetaStore.java    |   15 +-
 .../hive/metastore/HiveMetaStoreClient.java     |    8 +-
 .../hadoop/hive/metastore/IMetaStoreClient.java |    6 +-
 .../hadoop/hive/metastore/MetaStoreUtils.java   |   24 +-
 .../hadoop/hive/metastore/ObjectStore.java      |  250 +-
 .../apache/hadoop/hive/metastore/RawStore.java  |    6 +
 .../hive/metastore/ReplChangeManager.java       |  392 --
 .../apache/hadoop/hive/metastore/Warehouse.java |  644 --
 .../hive/metastore/cache/CachedStore.java       |    9 +
 .../DummyRawStoreControlledCommit.java          |    7 +
 .../DummyRawStoreForJdoConnection.java          |    7 +
 .../java/org/apache/hadoop/hive/ql/Context.java |    3 +
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |    8 +-
 .../apache/hadoop/hive/ql/exec/StatsTask.java   |    3 +-
 .../ql/exec/repl/bootstrap/ReplLoadTask.java    |    5 +-
 .../bootstrap/load/table/LoadPartitions.java    |    2 +-
 .../repl/bootstrap/load/table/LoadTable.java    |    2 +-
 .../exec/repl/bootstrap/load/util/PathInfo.java |    4 +
 .../ql/exec/vector/VectorDeserializeRow.java    |    4 +-
 .../hive/ql/hooks/EnforceReadOnlyTables.java    |    2 +-
 .../hadoop/hive/ql/lockmgr/DummyTxnManager.java |    5 +-
 .../hadoop/hive/ql/lockmgr/HiveLockObject.java  |    5 +-
 .../hive/ql/metadata/HiveMetaStoreChecker.java  |    3 +-
 .../RewriteQueryUsingAggregateIndexCtx.java     |    3 +-
 .../stats/annotation/StatsRulesProcFactory.java |   89 +-
 .../hive/ql/parse/ImportSemanticAnalyzer.java   |    3 +-
 .../hive/ql/parse/MacroSemanticAnalyzer.java    |    4 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |    2 +-
 .../repl/dump/events/AddPartitionHandler.java   |   18 +-
 .../repl/dump/events/AlterPartitionHandler.java |    6 +-
 .../repl/dump/events/AlterTableHandler.java     |   40 +-
 .../parse/repl/dump/events/InsertHandler.java   |    5 +
 .../plugin/sqlstd/SQLAuthorizationUtils.java    |    4 +-
 .../hadoop/hive/ql/session/SessionState.java    |    2 +-
 .../hadoop/hive/ql/exec/TestExecDriver.java     |   22 +-
 .../hadoop/hive/ql/metadata/TestHive.java       |   37 +-
 .../hive/ql/session/TestSessionState.java       |    9 +-
 .../orc_avro_partition_uniontype.q              |    9 +
 .../annotate_stats_deep_filters.q.out           |    4 +-
 .../annotate_stats_join_pkfk.q.out              |   48 +-
 .../clientpositive/llap/explainuser_1.q.out     |   54 +-
 .../llap/vector_decimal_cast.q.out              |    4 +-
 .../clientpositive/llap/vector_if_expr.q.out    |   10 +-
 .../clientpositive/llap/vectorization_0.q.out   |   12 +-
 .../clientpositive/llap/vectorization_10.q.out  |    6 +-
 .../clientpositive/llap/vectorization_17.q.out  |   10 +-
 .../clientpositive/llap/vectorization_7.q.out   |   16 +-
 .../llap/vectorization_short_regress.q.out      |    8 +-
 .../llap/vectorized_nested_mapjoin.q.out        |   12 +-
 .../orc_avro_partition_uniontype.q.out          |   40 +
 .../results/clientpositive/perf/query23.q.out   |    2 +-
 .../clientpositive/spark/vectorization_10.q.out |    6 +-
 .../clientpositive/spark/vectorization_12.q.out |   18 +-
 .../clientpositive/spark/vectorization_17.q.out |   10 +-
 .../clientpositive/vectorization_10.q.out       |    6 +-
 .../clientpositive/vectorization_12.q.out       |   18 +-
 .../clientpositive/vectorization_17.q.out       |   10 +-
 .../clientpositive/vectorization_7.q.out        |   16 +-
 .../ObjectInspectorConverters.java              |    4 +-
 .../SettableUnionObjectInspector.java           |    6 +-
 .../StandardUnionObjectInspector.java           |   16 +-
 .../TestObjectInspectorConverters.java          |   30 +-
 standalone-metastore/pom.xml                    |   11 +
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp  | 2575 ++++----
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.h    |  142 +
 .../ThriftHiveMetastore_server.skeleton.cpp     |    5 +
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 2395 +++++---
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |  181 +
 .../hive/metastore/api/AbortTxnsRequest.java    |   32 +-
 .../metastore/api/AddDynamicPartitions.java     |   32 +-
 .../metastore/api/ClearFileMetadataRequest.java |   32 +-
 .../hive/metastore/api/ClientCapabilities.java  |   32 +-
 .../hive/metastore/api/CompactionRequest.java   |   44 +-
 .../hive/metastore/api/FireEventRequest.java    |   32 +-
 .../hadoop/hive/metastore/api/Function.java     |   36 +-
 .../metastore/api/GetAllFunctionsResponse.java  |   36 +-
 .../api/GetFileMetadataByExprRequest.java       |   32 +-
 .../api/GetFileMetadataByExprResult.java        |   48 +-
 .../metastore/api/GetFileMetadataRequest.java   |   32 +-
 .../metastore/api/GetFileMetadataResult.java    |   44 +-
 .../metastore/api/GetOpenTxnsInfoResponse.java  |   36 +-
 .../hive/metastore/api/GetOpenTxnsResponse.java |   32 +-
 .../hive/metastore/api/GetTablesRequest.java    |   32 +-
 .../hive/metastore/api/GetTablesResult.java     |   36 +-
 .../api/HeartbeatTxnRangeResponse.java          |   64 +-
 .../metastore/api/InsertEventRequestData.java   |   64 +-
 .../hadoop/hive/metastore/api/LockRequest.java  |   36 +-
 .../api/NotificationEventResponse.java          |   36 +-
 .../hive/metastore/api/OpenTxnsResponse.java    |   32 +-
 .../metastore/api/PartitionValuesRequest.java   | 1222 ++++
 .../metastore/api/PartitionValuesResponse.java  |  443 ++
 .../hive/metastore/api/PartitionValuesRow.java  |  438 ++
 .../metastore/api/PutFileMetadataRequest.java   |   64 +-
 .../hive/metastore/api/ShowCompactResponse.java |   36 +-
 .../hive/metastore/api/ShowLocksResponse.java   |   36 +-
 .../hive/metastore/api/ThriftHiveMetastore.java | 5667 +++++++++++-------
 .../gen-php/metastore/ThriftHiveMetastore.php   | 1582 +++--
 .../src/gen/thrift/gen-php/metastore/Types.php  | 1040 +++-
 .../hive_metastore/ThriftHiveMetastore-remote   |    7 +
 .../hive_metastore/ThriftHiveMetastore.py       | 1109 ++--
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |  685 ++-
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |   67 +
 .../gen/thrift/gen-rb/thrift_hive_metastore.rb  |   66 +
 .../hive/metastore/ReplChangeManager.java       |  393 ++
 .../apache/hadoop/hive/metastore/Warehouse.java |  619 ++
 .../hive/metastore/conf/MetastoreConf.java      |    8 +
 .../hadoop/hive/metastore/utils/FileUtils.java  |  304 +-
 .../hadoop/hive/metastore/utils/HdfsUtils.java  |   75 +
 .../hive/metastore/utils/MetaStoreUtils.java    |   15 +
 .../src/main/thrift/hive_metastore.thrift       |   22 +
 136 files changed, 14652 insertions(+), 7762 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
----------------------------------------------------------------------
diff --cc itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
index 39f98e4,bec715d..e519785
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
@@@ -101,10 -101,10 +101,10 @@@ public class TestHiveHistory extends Te
        cols.add("key");
        cols.add("value");
        for (String src : srctables) {
-         db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true);
+         db.dropTable(Warehouse.DEFAULT_DATABASE_NAME, src, true, true);
          db.createTable(src, cols, null, TextInputFormat.class,
              IgnoreKeyTextOutputFormat.class);
 -        db.loadTable(hadoopDataFile[i], src, false, false, false, false, false);
 +        db.loadTable(hadoopDataFile[i], src, false, false, false, false, false, null, 0, false);
          i++;
        }
  

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 814910d,ed004fe..36c0c98
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@@ -64,8 -61,6 +64,7 @@@ import org.apache.hadoop.hive.ql.metada
  import org.apache.hadoop.hive.ql.parse.repl.load.MetaData;
  import org.apache.hadoop.hive.ql.parse.repl.load.UpdatedMetaDataTracker;
  import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
 +import org.apache.hadoop.hive.ql.plan.CopyWork;
- import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
  import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
  import org.apache.hadoop.hive.ql.plan.DDLWork;
  import org.apache.hadoop.hive.ql.plan.LoadTableDesc;

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
index 61eab3d,c0c496f..75ea16f
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
@@@ -137,10 -137,10 +137,10 @@@ public class TestExecDriver extends Tes
        cols.add("key");
        cols.add("value");
        for (String src : srctables) {
-         db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true);
+         db.dropTable(Warehouse.DEFAULT_DATABASE_NAME, src, true, true);
          db.createTable(src, cols, null, TextInputFormat.class,
              HiveIgnoreKeyTextOutputFormat.class);
 -        db.loadTable(hadoopDataFile[i], src, false, true, false, false, false);
 +        db.loadTable(hadoopDataFile[i], src, false, true, false, false, false, null, 0, false);
          i++;
        }
  

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
----------------------------------------------------------------------
diff --cc standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
index 0000000,649437f..df2453b
mode 000000,100755..100755
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
@@@ -1,0 -1,610 +1,619 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ package org.apache.hadoop.hive.metastore;
+ 
+ import java.io.FileNotFoundException;
+ import java.io.IOException;
+ import java.util.AbstractList;
+ import java.util.ArrayList;
+ import java.util.LinkedHashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Map.Entry;
++import java.util.Set;
+ import java.util.regex.Matcher;
+ import java.util.regex.Pattern;
+ 
+ import org.apache.commons.lang.StringUtils;
+ import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+ import org.apache.hadoop.hive.metastore.utils.FileUtils;
+ import org.apache.hadoop.hive.metastore.utils.HdfsUtils;
+ import org.apache.hadoop.hive.metastore.utils.JavaUtils;
+ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.ContentSummary;
+ import org.apache.hadoop.fs.FileStatus;
+ import org.apache.hadoop.fs.FileSystem;
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.fs.permission.FsAction;
+ import org.apache.hadoop.hive.metastore.ReplChangeManager.RecycleType;
+ import org.apache.hadoop.hive.metastore.api.Database;
+ import org.apache.hadoop.hive.metastore.api.FieldSchema;
+ import org.apache.hadoop.hive.metastore.api.MetaException;
+ import org.apache.hadoop.hive.metastore.api.Partition;
+ import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+ import org.apache.hadoop.hive.metastore.api.Table;
+ import org.apache.hadoop.util.ReflectionUtils;
+ 
+ /**
+  * This class represents a warehouse where data of Hive tables is stored
+  */
+ public class Warehouse {
+   public static final String DEFAULT_DATABASE_NAME = "default";
+   public static final String DEFAULT_DATABASE_COMMENT = "Default Hive database";
+   public static final String DEFAULT_SERIALIZATION_FORMAT = "1";
+   public static final String DATABASE_WAREHOUSE_SUFFIX = ".db";
+ 
+   private Path whRoot;
+   private final Configuration conf;
+   private final String whRootString;
+ 
+   public static final Logger LOG = LoggerFactory.getLogger("hive.metastore.warehouse");
+ 
+   private MetaStoreFS fsHandler = null;
+   private boolean storageAuthCheck = false;
+   private ReplChangeManager cm = null;
+ 
+   public Warehouse(Configuration conf) throws MetaException {
+     this.conf = conf;
+     whRootString = MetastoreConf.getVar(conf, ConfVars.WAREHOUSE);
+     if (StringUtils.isBlank(whRootString)) {
+       throw new MetaException(ConfVars.WAREHOUSE.varname
+           + " is not set in the config or blank");
+     }
+     fsHandler = getMetaStoreFsHandler(conf);
+     cm = ReplChangeManager.getInstance(conf);
+     storageAuthCheck = MetastoreConf.getBoolVar(conf, ConfVars.AUTHORIZATION_STORAGE_AUTH_CHECKS);
+   }
+ 
+   private MetaStoreFS getMetaStoreFsHandler(Configuration conf)
+       throws MetaException {
+     String handlerClassStr = MetastoreConf.getVar(conf, ConfVars.FS_HANDLER_CLS);
+     try {
+       Class<? extends MetaStoreFS> handlerClass = (Class<? extends MetaStoreFS>) Class
+           .forName(handlerClassStr, true, JavaUtils.getClassLoader());
+       MetaStoreFS handler = ReflectionUtils.newInstance(handlerClass, conf);
+       return handler;
+     } catch (ClassNotFoundException e) {
+       throw new MetaException("Error in loading MetaStoreFS handler."
+           + e.getMessage());
+     }
+   }
+ 
+ 
+   /**
+    * Helper functions to convert IOException to MetaException
+    */
+   public static FileSystem getFs(Path f, Configuration conf) throws MetaException {
+     try {
+       return f.getFileSystem(conf);
+     } catch (IOException e) {
+       MetaStoreUtils.logAndThrowMetaException(e);
+     }
+     return null;
+   }
+ 
+   public FileSystem getFs(Path f) throws MetaException {
+     return getFs(f, conf);
+   }
+ 
+ 
+   /**
+    * Hadoop File System reverse lookups paths with raw ip addresses The File
+    * System URI always contains the canonical DNS name of the Namenode.
+    * Subsequently, operations on paths with raw ip addresses cause an exception
+    * since they don't match the file system URI.
+    *
+    * This routine solves this problem by replacing the scheme and authority of a
+    * path with the scheme and authority of the FileSystem that it maps to.
+    *
+    * @param path
+    *          Path to be canonicalized
+    * @return Path with canonical scheme and authority
+    */
+   public static Path getDnsPath(Path path, Configuration conf) throws MetaException {
+     FileSystem fs = getFs(path, conf);
+     return (new Path(fs.getUri().getScheme(), fs.getUri().getAuthority(), path
+         .toUri().getPath()));
+   }
+ 
+   public Path getDnsPath(Path path) throws MetaException {
+     return getDnsPath(path, conf);
+   }
+ 
+   /**
+    * Resolve the configured warehouse root dir with respect to the configuration
+    * This involves opening the FileSystem corresponding to the warehouse root
+    * dir (but that should be ok given that this is only called during DDL
+    * statements for non-external tables).
+    */
+   public Path getWhRoot() throws MetaException {
+     if (whRoot != null) {
+       return whRoot;
+     }
+     whRoot = getDnsPath(new Path(whRootString));
+     return whRoot;
+   }
+ 
+   public Path getDatabasePath(Database db) throws MetaException {
+     if (db.getName().equalsIgnoreCase(DEFAULT_DATABASE_NAME)) {
+       return getWhRoot();
+     }
+     return new Path(db.getLocationUri());
+   }
+ 
+   public Path getDefaultDatabasePath(String dbName) throws MetaException {
+     if (dbName.equalsIgnoreCase(DEFAULT_DATABASE_NAME)) {
+       return getWhRoot();
+     }
+     return new Path(getWhRoot(), dbName.toLowerCase() + DATABASE_WAREHOUSE_SUFFIX);
+   }
+ 
+   /**
+    * Returns the default location of the table path using the parent database's location
+    * @param db Database where the table is created
+    * @param tableName table name
+    * @return
+    * @throws MetaException
+    */
+   public Path getDefaultTablePath(Database db, String tableName)
+       throws MetaException {
+     return getDnsPath(new Path(getDatabasePath(db), MetaStoreUtils.encodeTableName(tableName.toLowerCase())));
+   }
+ 
+   public static String getQualifiedName(Table table) {
+     return table.getDbName() + "." + table.getTableName();
+   }
+ 
+   public static String getQualifiedName(Partition partition) {
+     return partition.getDbName() + "." + partition.getTableName() + partition.getValues();
+   }
+ 
+   public boolean mkdirs(Path f) throws MetaException {
+     FileSystem fs;
+     try {
+       fs = getFs(f);
+       return FileUtils.mkdir(fs, f);
+     } catch (IOException e) {
+       MetaStoreUtils.logAndThrowMetaException(e);
+     }
+     return false;
+   }
+ 
+   public boolean renameDir(Path sourcePath, Path destPath, boolean needCmRecycle) throws MetaException {
+     try {
+       if (needCmRecycle) {
+         // Copy the source files to cmroot. As the client will move the source files to another
+         // location, we should make a copy of the files to cmroot instead of moving it.
+         cm.recycle(sourcePath, RecycleType.COPY, true);
+       }
+       FileSystem fs = getFs(sourcePath);
+       return FileUtils.rename(fs, sourcePath, destPath);
+     } catch (Exception ex) {
+       MetaStoreUtils.logAndThrowMetaException(ex);
+     }
+     return false;
+   }
+ 
+   void addToChangeManagement(Path file) throws MetaException {
+     cm.recycle(file, RecycleType.COPY, true);
+   }
+ 
+   public boolean deleteDir(Path f, boolean recursive) throws MetaException {
+     return deleteDir(f, recursive, false);
+   }
+ 
+   public boolean deleteDir(Path f, boolean recursive, boolean ifPurge) throws MetaException {
+     cm.recycle(f, RecycleType.MOVE, ifPurge);
+     FileSystem fs = getFs(f);
+     return fsHandler.deleteDir(fs, f, recursive, ifPurge, conf);
+   }
+ 
+   public void recycleDirToCmPath(Path f, boolean ifPurge) throws MetaException {
+     cm.recycle(f, RecycleType.MOVE, ifPurge);
+     return;
+   }
+ 
+   public boolean isEmpty(Path path) throws IOException, MetaException {
+     ContentSummary contents = getFs(path).getContentSummary(path);
+     if (contents != null && contents.getFileCount() == 0 && contents.getDirectoryCount() == 1) {
+       return true;
+     }
+     return false;
+   }
+ 
+   public boolean isWritable(Path path) throws IOException {
+     if (!storageAuthCheck) {
+       // no checks for non-secure hadoop installations
+       return true;
+     }
+     if (path == null) { //what??!!
+       return false;
+     }
+     final FileStatus stat;
+     final FileSystem fs;
+     try {
+       fs = getFs(path);
+       stat = fs.getFileStatus(path);
+       HdfsUtils.checkFileAccess(fs, stat, FsAction.WRITE);
+       return true;
+     } catch (FileNotFoundException fnfe){
+       // File named by path doesn't exist; nothing to validate.
+       return true;
+     } catch (Exception e) {
+       // all other exceptions are considered as emanating from
+       // unauthorized accesses
+       if (LOG.isDebugEnabled()) {
+         LOG.debug("Exception when checking if path (" + path + ")", e);
+       }
+       return false;
+     }
+   }
+ 
+   private static String escapePathName(String path) {
+     return FileUtils.escapePathName(path);
+   }
+ 
+   private static String unescapePathName(String path) {
+     return FileUtils.unescapePathName(path);
+   }
+ 
+   /**
+    * Given a partition specification, return the path corresponding to the
+    * partition spec. By default, the specification does not include dynamic partitions.
+    * @param spec
+    * @return string representation of the partition specification.
+    * @throws MetaException
+    */
+   public static String makePartPath(Map<String, String> spec)
+       throws MetaException {
+     return makePartName(spec, true);
+   }
+ 
+   /**
+    * Makes a partition name from a specification
+    * @param spec
+    * @param addTrailingSeperator if true, adds a trailing separator e.g. 'ds=1/'
+    * @return partition name
+    * @throws MetaException
+    */
+   public static String makePartName(Map<String, String> spec,
+       boolean addTrailingSeperator)
+       throws MetaException {
+     StringBuilder suffixBuf = new StringBuilder();
+     int i = 0;
+     for (Entry<String, String> e : spec.entrySet()) {
+       if (e.getValue() == null || e.getValue().length() == 0) {
+         throw new MetaException("Partition spec is incorrect. " + spec);
+       }
+       if (i>0) {
+         suffixBuf.append(Path.SEPARATOR);
+       }
+       suffixBuf.append(escapePathName(e.getKey()));
+       suffixBuf.append('=');
+       suffixBuf.append(escapePathName(e.getValue()));
+       i++;
+     }
+     if (addTrailingSeperator) {
+       suffixBuf.append(Path.SEPARATOR);
+     }
+     return suffixBuf.toString();
+   }
+   /**
+    * Given a dynamic partition specification, return the path corresponding to the
+    * static part of partition specification. This is basically a copy of makePartName
+    * but we get rid of MetaException since it is not serializable.
+    * @param spec
+    * @return string representation of the static part of the partition specification.
+    */
+   public static String makeDynamicPartName(Map<String, String> spec) {
+     StringBuilder suffixBuf = new StringBuilder();
+     for (Entry<String, String> e : spec.entrySet()) {
+       if (e.getValue() != null && e.getValue().length() > 0) {
+         suffixBuf.append(escapePathName(e.getKey()));
+         suffixBuf.append('=');
+         suffixBuf.append(escapePathName(e.getValue()));
+         suffixBuf.append(Path.SEPARATOR);
+       } else { // stop once we see a dynamic partition
+         break;
+       }
+     }
+     return suffixBuf.toString();
+   }
+ 
+   static final Pattern pat = Pattern.compile("([^/]+)=([^/]+)");
+ 
+   private static final Pattern slash = Pattern.compile("/");
+ 
+   /**
+    * Extracts values from partition name without the column names.
+    * @param name Partition name.
+    * @param result The result. Must be pre-sized to the expected number of columns.
+    */
+   public static AbstractList<String> makeValsFromName(
+       String name, AbstractList<String> result) throws MetaException {
+     assert name != null;
+     String[] parts = slash.split(name, 0);
+     if (result == null) {
+       result = new ArrayList<>(parts.length);
+       for (int i = 0; i < parts.length; ++i) {
+         result.add(null);
+       }
+     } else if (parts.length != result.size()) {
+       throw new MetaException(
+           "Expected " + result.size() + " components, got " + parts.length + " (" + name + ")");
+     }
+     for (int i = 0; i < parts.length; ++i) {
+       int eq = parts[i].indexOf('=');
+       if (eq <= 0) {
+         throw new MetaException("Unexpected component " + parts[i]);
+       }
+       result.set(i, unescapePathName(parts[i].substring(eq + 1)));
+     }
+     return result;
+   }
+ 
+   public static LinkedHashMap<String, String> makeSpecFromName(String name)
+       throws MetaException {
+     if (name == null || name.isEmpty()) {
+       throw new MetaException("Partition name is invalid. " + name);
+     }
+     LinkedHashMap<String, String> partSpec = new LinkedHashMap<>();
 -    makeSpecFromName(partSpec, new Path(name));
++    makeSpecFromName(partSpec, new Path(name), null);
+     return partSpec;
+   }
+ 
 -  public static void makeSpecFromName(Map<String, String> partSpec, Path currPath) {
++  public static boolean makeSpecFromName(Map<String, String> partSpec, Path currPath,
++      Set<String> requiredKeys) {
+     List<String[]> kvs = new ArrayList<>();
+     do {
+       String component = currPath.getName();
+       Matcher m = pat.matcher(component);
+       if (m.matches()) {
+         String k = unescapePathName(m.group(1));
+         String v = unescapePathName(m.group(2));
+         String[] kv = new String[2];
+         kv[0] = k;
+         kv[1] = v;
+         kvs.add(kv);
+       }
+       currPath = currPath.getParent();
+     } while (currPath != null && !currPath.getName().isEmpty());
+ 
+     // reverse the list since we checked the part from leaf dir to table's base dir
+     for (int i = kvs.size(); i > 0; i--) {
 -      partSpec.put(kvs.get(i - 1)[0], kvs.get(i - 1)[1]);
++      String key = kvs.get(i - 1)[0];
++      if (requiredKeys != null) {
++        requiredKeys.remove(key);
++      }
++      partSpec.put(key, kvs.get(i - 1)[1]);
+     }
++    if (requiredKeys == null || requiredKeys.isEmpty()) return true;
++    LOG.warn("Cannot create partition spec from " + currPath + "; missing keys " + requiredKeys);
++    return false;
+   }
+ 
+   public static Map<String, String> makeEscSpecFromName(String name) throws MetaException {
+ 
+     if (name == null || name.isEmpty()) {
+       throw new MetaException("Partition name is invalid. " + name);
+     }
+     LinkedHashMap<String, String> partSpec = new LinkedHashMap<>();
+ 
+     Path currPath = new Path(name);
+ 
+     List<String[]> kvs = new ArrayList<>();
+     do {
+       String component = currPath.getName();
+       Matcher m = pat.matcher(component);
+       if (m.matches()) {
+         String k = m.group(1);
+         String v = m.group(2);
+         String[] kv = new String[2];
+         kv[0] = k;
+         kv[1] = v;
+         kvs.add(kv);
+       }
+       currPath = currPath.getParent();
+     } while (currPath != null && !currPath.getName().isEmpty());
+ 
+     // reverse the list since we checked the part from leaf dir to table's base dir
+     for (int i = kvs.size(); i > 0; i--) {
+       partSpec.put(kvs.get(i - 1)[0], kvs.get(i - 1)[1]);
+     }
+ 
+     return partSpec;
+   }
+ 
+   /**
+    * Returns the default partition path of a table within a given database and partition key value
+    * pairs. It uses the database location and appends it the table name and the partition key,value
+    * pairs to create the Path for the partition directory
+    *
+    * @param db - parent database which is used to get the base location of the partition directory
+    * @param tableName - table name for the partitions
+    * @param pm - Partition key value pairs
+    * @return
+    * @throws MetaException
+    */
+   public Path getDefaultPartitionPath(Database db, String tableName,
+       Map<String, String> pm) throws MetaException {
+     return getPartitionPath(getDefaultTablePath(db, tableName), pm);
+   }
+ 
+   /**
+    * Returns the path object for the given partition key-value pairs and the base location
+    *
+    * @param tblPath - the base location for the partitions. Typically the table location
+    * @param pm - Partition key value pairs
+    * @return
+    * @throws MetaException
+    */
+   public Path getPartitionPath(Path tblPath, Map<String, String> pm)
+       throws MetaException {
+     return new Path(tblPath, makePartPath(pm));
+   }
+ 
+   /**
+    * Given a database, a table and the partition key value pairs this method returns the Path object
+    * corresponding to the partition key value pairs. It uses the table location if available else
+    * uses the database location for constructing the path corresponding to the partition key-value
+    * pairs
+    *
+    * @param db - Parent database of the given table
+    * @param table - Table for which the partition key-values are given
+    * @param vals - List of values for the partition keys
+    * @return Path corresponding to the partition key-value pairs
+    * @throws MetaException
+    */
+   public Path getPartitionPath(Database db, Table table, List<String> vals)
+       throws MetaException {
+     List<FieldSchema> partKeys = table.getPartitionKeys();
+     if (partKeys == null || (partKeys.size() != vals.size())) {
+       throw new MetaException("Invalid number of partition keys found for " + table.getTableName());
+     }
+     Map<String, String> pm = new LinkedHashMap<>(vals.size());
+     int i = 0;
+     for (FieldSchema key : partKeys) {
+       pm.put(key.getName(), vals.get(i));
+       i++;
+     }
+ 
+     if (table.getSd().getLocation() != null) {
+       return getPartitionPath(getDnsPath(new Path(table.getSd().getLocation())), pm);
+     } else {
+       return getDefaultPartitionPath(db, table.getTableName(), pm);
+     }
+   }
+ 
+   public boolean isDir(Path f) throws MetaException {
+     FileSystem fs;
+     try {
+       fs = getFs(f);
+       FileStatus fstatus = fs.getFileStatus(f);
+       if (!fstatus.isDir()) {
+         return false;
+       }
+     } catch (FileNotFoundException e) {
+       return false;
+     } catch (IOException e) {
+       MetaStoreUtils.logAndThrowMetaException(e);
+     }
+     return true;
+   }
+ 
+   public static String makePartName(List<FieldSchema> partCols,
+       List<String> vals) throws MetaException {
+     return makePartName(partCols, vals, null);
+   }
+ 
+   /**
+    * @param desc
+    * @return array of FileStatus objects corresponding to the files
+    * making up the passed storage description
+    */
+   public FileStatus[] getFileStatusesForSD(StorageDescriptor desc)
+       throws MetaException {
+     return getFileStatusesForLocation(desc.getLocation());
+   }
+ 
+   /**
+    * @param location
+    * @return array of FileStatus objects corresponding to the files
+    * making up the passed storage description
+    */
+   public FileStatus[] getFileStatusesForLocation(String location)
+       throws MetaException {
+     try {
+       Path path = new Path(location);
+       FileSystem fileSys = path.getFileSystem(conf);
+       return FileUtils.getFileStatusRecurse(path, -1, fileSys);
+     } catch (IOException ioe) {
+       MetaStoreUtils.logAndThrowMetaException(ioe);
+     }
+     return null;
+   }
+ 
+   /**
+    * @param db database
+    * @param table table
+    * @return array of FileStatus objects corresponding to the files making up the passed
+    * unpartitioned table
+    */
+   public FileStatus[] getFileStatusesForUnpartitionedTable(Database db, Table table)
+       throws MetaException {
+     Path tablePath = getDnsPath(new Path(table.getSd().getLocation()));
+     try {
+       FileSystem fileSys = tablePath.getFileSystem(conf);
+       return FileUtils.getFileStatusRecurse(tablePath, -1, fileSys);
+     } catch (IOException ioe) {
+       MetaStoreUtils.logAndThrowMetaException(ioe);
+     }
+     return null;
+   }
+ 
+   /**
+    * Makes a valid partition name.
+    * @param partCols The partition columns
+    * @param vals The partition values
+    * @param defaultStr
+    *    The default name given to a partition value if the respective value is empty or null.
+    * @return An escaped, valid partition name.
+    * @throws MetaException
+    */
+   public static String makePartName(List<FieldSchema> partCols,
+       List<String> vals, String defaultStr) throws MetaException {
+     if ((partCols.size() != vals.size()) || (partCols.size() == 0)) {
+       String errorStr = "Invalid partition key & values; keys [";
+       for (FieldSchema fs : partCols) {
+         errorStr += (fs.getName() + ", ");
+       }
+       errorStr += "], values [";
+       for (String val : vals) {
+         errorStr += (val + ", ");
+       }
+       throw new MetaException(errorStr + "]");
+     }
+     List<String> colNames = new ArrayList<>();
+     for (FieldSchema col: partCols) {
+       colNames.add(col.getName());
+     }
+     return FileUtils.makePartName(colNames, vals, defaultStr);
+   }
+ 
+   public static List<String> getPartValuesFromPartName(String partName)
+       throws MetaException {
+     LinkedHashMap<String, String> partSpec = Warehouse.makeSpecFromName(partName);
+     List<String> values = new ArrayList<>();
+     values.addAll(partSpec.values());
+     return values;
+   }
+ 
+   public static Map<String, String> makeSpecFromValues(List<FieldSchema> partCols,
+       List<String> values) {
+     Map<String, String> spec = new LinkedHashMap<>();
+     for (int i = 0; i < values.size(); i++) {
+       spec.put(partCols.get(i).getName(), values.get(i));
+     }
+     return spec;
+   }
+ }

http://git-wip-us.apache.org/repos/asf/hive/blob/1fb04f36/standalone-metastore/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------

[22/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/18d13385
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/18d13385
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/18d13385

Branch: refs/heads/master
Commit: 18d133857b7930242538b49f33b3ca517b2e287d
Parents: 42a3857 8482c5f
Author: Wei Zheng <we...@apache.org>
Authored: Thu Sep 7 23:34:23 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Thu Sep 7 23:34:23 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   23 +-
 .../apache/hadoop/hive/conf/TestHiveConf.java   |    4 +
 .../hive/conf/TestHiveConfRestrictList.java     |    2 +-
 dev-support/checkstyle_YETUS-484.sh             |  406 --
 dev-support/findbugs_YETUS-471.sh               |  488 --
 dev-support/maven_YETUS-506.sh                  |  687 ---
 dev-support/yetus-wrapper.sh                    |   11 +-
 errata.txt                                      |    1 +
 .../mapjoin/MapJoinMultiKeyBench.java           |   94 +-
 .../mapjoin/MapJoinOneLongKeyBench.java         |   94 +-
 .../mapjoin/MapJoinOneStringKeyBench.java       |   94 +-
 .../hive/ql/parse/TestReplicationScenarios.java |  145 +
 .../apache/hive/jdbc/TestJdbcWithMiniLlap.java  |   27 +-
 .../test/resources/testconfiguration.properties |    2 +
 .../apache/hive/beeline/QFileBeeLineClient.java |    6 +-
 .../org/apache/hive/jdbc/HiveStatement.java     |    2 +-
 .../hive/llap/registry/LlapServiceInstance.java |   59 +
 .../llap/registry/LlapServiceInstanceSet.java   |   34 +
 .../hive/llap/registry/ServiceInstance.java     |   86 -
 .../hive/llap/registry/ServiceInstanceSet.java  |   69 -
 .../ServiceInstanceStateChangeListener.java     |   42 -
 .../hive/llap/registry/ServiceRegistry.java     |    8 +-
 .../registry/impl/InactiveServiceInstance.java  |    4 +-
 .../registry/impl/LlapFixedRegistryImpl.java    |   41 +-
 .../llap/registry/impl/LlapRegistryService.java |   18 +-
 .../impl/LlapZookeeperRegistryImpl.java         |  800 +---
 .../hive/llap/security/LlapTokenClient.java     |   18 +-
 .../hive/llap/tez/LlapProtocolClientProxy.java  |   25 +-
 .../hadoop/hive/registry/ServiceInstance.java   |   47 +
 .../hive/registry/ServiceInstanceSet.java       |   61 +
 .../ServiceInstanceStateChangeListener.java     |   42 +
 .../hive/registry/impl/ServiceInstanceBase.java |   93 +
 .../hive/registry/impl/ZkRegistryBase.java      |  549 +++
 .../hive/registry/impl/ZookeeperUtils.java      |  116 +
 llap-common/pom.xml                             |    5 +
 .../plugin/rpc/LlapPluginProtocolProtos.java    | 1273 ++++++
 .../org/apache/hadoop/hive/llap/LlapUtil.java   |   70 +-
 .../llap/impl/LlapPluginProtocolClientImpl.java |   46 +
 .../hadoop/hive/llap/impl/ProtobufProxy.java    |   76 +
 .../llap/protocol/LlapPluginProtocolPB.java     |   27 +
 .../src/protobuf/LlapPluginProtocol.proto       |   36 +
 .../hadoop/hive/llap/LlapBaseInputFormat.java   |  141 +-
 .../hive/llap/cli/LlapStatusServiceDriver.java  |    6 +-
 .../hive/llap/daemon/impl/AMReporter.java       |   75 +-
 .../daemon/impl/LlapProtocolServerImpl.java     |   67 +-
 .../daemon/services/impl/LlapWebServices.java   |    4 +-
 .../daemon/impl/comparator/TestAMReporter.java  |   87 +
 .../llap/tezplugins/LlapTaskCommunicator.java   |   69 +-
 .../tezplugins/LlapTaskSchedulerService.java    |  734 ++-
 .../endpoint/LlapPluginPolicyProvider.java      |   34 +
 .../endpoint/LlapPluginSecurityInfo.java        |   62 +
 .../endpoint/LlapPluginServerImpl.java          |   80 +
 .../org.apache.hadoop.security.SecurityInfo     |   14 +
 .../TestLlapTaskSchedulerService.java           |  567 ++-
 .../TestDataSourceProviderFactory.java          |    6 +
 pom.xml                                         |    8 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   15 +
 .../hadoop/hive/ql/exec/OperatorUtils.java      |   31 +
 .../hadoop/hive/ql/exec/repl/ReplDumpTask.java  |   36 +-
 .../hive/ql/exec/spark/SparkUtilities.java      |    4 +-
 .../apache/hadoop/hive/ql/exec/tez/Utils.java   |    6 +-
 .../hadoop/hive/ql/log/LogDivertAppender.java   |   25 +-
 .../hive/ql/log/LogDivertAppenderForTest.java   |   18 +-
 .../ql/optimizer/calcite/RelOptHiveTable.java   |   93 +-
 .../physical/LlapClusterStateForCompile.java    |    8 +-
 .../SparkDynamicPartitionPruningResolver.java   |  146 +
 .../physical/SparkMapJoinResolver.java          |   35 +-
 .../spark/SparkPartitionPruningSinkDesc.java    |   13 +-
 .../spark/SparkSkewJoinProcFactory.java         |    6 +-
 .../stats/annotation/StatsRulesProcFactory.java |   12 +-
 .../hive/ql/parse/BaseSemanticAnalyzer.java     |   26 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java      |    7 +-
 .../apache/hadoop/hive/ql/parse/EximUtil.java   |   14 +
 .../hive/ql/parse/ImportSemanticAnalyzer.java   |    4 +
 .../hive/ql/parse/SemiJoinBranchInfo.java       |   18 +
 .../hadoop/hive/ql/parse/TezCompiler.java       |  105 +-
 .../dump/BootStrapReplicationSpecFunction.java  |    1 -
 .../hive/ql/parse/repl/dump/HiveWrapper.java    |    5 +
 .../hadoop/hive/ql/parse/repl/dump/Utils.java   |   67 +
 .../hive/ql/parse/spark/GenSparkUtils.java      |    2 +-
 .../hive/ql/parse/spark/SparkCompiler.java      |   18 +-
 .../hadoop/hive/ql/plan/ColStatistics.java      |   12 +
 .../hadoop/hive/ql/session/OperationLog.java    |    4 +-
 .../apache/hadoop/hive/ql/stats/StatsUtils.java |  208 +-
 .../clientpositive/dynamic_semijoin_reduction.q |   10 +
 .../clientpositive/join_reordering_no_stats.q   |   63 +
 ...ynamic_partition_pruning_recursive_mapjoin.q |  147 +
 ...spark_vectorized_dynamic_partition_pruning.q |   43 +-
 .../clientpositive/annotate_stats_filter.q.out  |   12 +-
 .../clientpositive/annotate_stats_groupby.q.out |    4 +-
 .../clientpositive/annotate_stats_part.q.out    |   42 +-
 .../clientpositive/annotate_stats_select.q.out  |    4 +-
 .../clientpositive/annotate_stats_table.q.out   |   12 +-
 .../auto_join_reordering_values.q.out           |    8 +-
 .../clientpositive/auto_join_stats.q.out        |   30 +-
 .../clientpositive/auto_join_stats2.q.out       |   91 +-
 .../clientpositive/auto_sortmerge_join_12.q.out |   12 +-
 .../cbo_rp_annotate_stats_groupby.q.out         |    4 +-
 .../columnStatsUpdateForStatsOptimizer_2.q.out  |   24 +-
 .../clientpositive/explain_rearrange.q.out      |   24 +-
 ql/src/test/results/clientpositive/join19.q.out |  190 +-
 ql/src/test/results/clientpositive/join42.q.out |  154 +-
 ql/src/test/results/clientpositive/join43.q.out |  136 +-
 .../join_cond_pushdown_unqual1.q.out            |  284 +-
 .../join_cond_pushdown_unqual2.q.out            |  162 +-
 .../join_cond_pushdown_unqual3.q.out            |  284 +-
 .../join_cond_pushdown_unqual4.q.out            |  162 +-
 .../results/clientpositive/join_hive_626.q.out  |   60 +-
 .../test/results/clientpositive/join_star.q.out |  124 +-
 .../llap/acid_bucket_pruning.q.out              |    8 +-
 .../llap/auto_smb_mapjoin_14.q.out              |  260 +-
 .../llap/auto_sortmerge_join_1.q.out            |   48 +-
 .../llap/auto_sortmerge_join_10.q.out           |   42 +-
 .../llap/auto_sortmerge_join_11.q.out           |   64 +-
 .../llap/auto_sortmerge_join_12.q.out           |   26 +-
 .../llap/auto_sortmerge_join_13.q.out           |  134 +-
 .../llap/auto_sortmerge_join_14.q.out           |   24 +-
 .../llap/auto_sortmerge_join_15.q.out           |   24 +-
 .../llap/auto_sortmerge_join_2.q.out            |   32 +-
 .../llap/auto_sortmerge_join_3.q.out            |   48 +-
 .../llap/auto_sortmerge_join_4.q.out            |   48 +-
 .../llap/auto_sortmerge_join_5.q.out            |   44 +-
 .../llap/auto_sortmerge_join_6.q.out            |  682 +--
 .../llap/auto_sortmerge_join_7.q.out            |   48 +-
 .../llap/auto_sortmerge_join_8.q.out            |   48 +-
 .../llap/auto_sortmerge_join_9.q.out            |  488 +-
 .../clientpositive/llap/bucket_groupby.q.out    |  198 +-
 .../llap/bucket_map_join_tez1.q.out             | 1121 ++---
 .../llap/bucket_map_join_tez2.q.out             |  322 +-
 .../clientpositive/llap/bucketmapjoin1.q.out    |   88 +-
 .../clientpositive/llap/bucketmapjoin2.q.out    |   66 +-
 .../clientpositive/llap/bucketmapjoin3.q.out    |   44 +-
 .../clientpositive/llap/bucketmapjoin4.q.out    |   44 +-
 .../clientpositive/llap/bucketmapjoin7.q.out    |   24 +-
 .../clientpositive/llap/bucketpruning1.q.out    |  208 +-
 .../llap/bucketsortoptimize_insert_2.q.out      |  144 +-
 .../llap/bucketsortoptimize_insert_6.q.out      |  168 +-
 .../llap/bucketsortoptimize_insert_7.q.out      |   72 +-
 .../columnStatsUpdateForStatsOptimizer_1.q.out  |   24 +-
 .../llap/column_access_stats.q.out              |  138 +-
 .../llap/column_table_stats.q.out               |   24 +-
 .../llap/column_table_stats_orc.q.out           |   20 +-
 .../clientpositive/llap/constprog_dpp.q.out     |   20 +-
 .../llap/constprog_semijoin.q.out               |  150 +-
 .../llap/correlationoptimizer4.q.out            |  324 +-
 .../results/clientpositive/llap/count.q.out     |  144 +-
 .../llap/cross_product_check_1.q.out            |  150 +-
 .../llap/cross_product_check_2.q.out            |  132 +-
 .../results/clientpositive/llap/cte_3.q.out     |    8 +-
 .../results/clientpositive/llap/cte_5.q.out     |   10 +-
 .../results/clientpositive/llap/cte_mat_3.q.out |   18 +-
 .../results/clientpositive/llap/cte_mat_4.q.out |   36 +-
 .../results/clientpositive/llap/cte_mat_5.q.out |   16 +-
 .../llap/disable_merge_for_bucketing.q.out      |   12 +-
 .../llap/dynamic_partition_pruning.q.out        |  655 +--
 .../llap/dynamic_partition_pruning_2.q.out      |  201 +-
 .../llap/dynamic_semijoin_reduction.q.out       |  459 +-
 .../llap/dynamic_semijoin_reduction_2.q.out     |  269 +-
 .../llap/dynamic_semijoin_reduction_3.q.out     |  186 +-
 .../llap/dynamic_semijoin_reduction_sw.q.out    |   34 +-
 .../llap/dynpart_sort_opt_vectorization.q.out   |  208 +-
 .../llap/dynpart_sort_optimization.q.out        |  288 +-
 .../llap/dynpart_sort_optimization2.q.out       |   68 +-
 .../llap/dynpart_sort_optimization_acid.q.out   |  144 +-
 .../clientpositive/llap/empty_join.q.out        |   22 +-
 .../clientpositive/llap/except_distinct.q.out   |   48 +-
 .../clientpositive/llap/explainuser_1.q.out     |  384 +-
 .../clientpositive/llap/explainuser_2.q.out     |  116 +-
 .../llap/filter_join_breaktask.q.out            |   36 +-
 .../llap/hybridgrace_hashjoin_1.q.out           |  197 +-
 .../results/clientpositive/llap/insert1.q.out   |   46 +-
 .../clientpositive/llap/insert_into1.q.out      |    6 +-
 .../clientpositive/llap/intersect_all.q.out     |   44 +-
 .../llap/intersect_distinct.q.out               |   40 +-
 .../clientpositive/llap/intersect_merge.q.out   |  540 +--
 .../clientpositive/llap/jdbc_handler.q.out      |   14 +-
 .../results/clientpositive/llap/join46.q.out    |  326 +-
 .../llap/join_emit_interval.q.out               |   32 +-
 .../llap/join_is_not_distinct_from.q.out        |  102 +-
 .../clientpositive/llap/join_nullsafe.q.out     |  102 +-
 .../llap/join_reordering_no_stats.q.out         |  708 +++
 .../clientpositive/llap/lateral_view.q.out      |   32 +-
 .../clientpositive/llap/llap_nullscan.q.out     |   64 +-
 .../clientpositive/llap/llap_partitioned.q.out  |    8 +-
 .../results/clientpositive/llap/llap_smb.q.out  |   18 +-
 .../clientpositive/llap/llap_stats.q.out        |    4 +-
 .../results/clientpositive/llap/llap_udf.q.out  |   36 +-
 .../clientpositive/llap/llapdecider.q.out       |   32 +-
 .../clientpositive/llap/lvj_mapjoin.q.out       |   60 +-
 .../results/clientpositive/llap/mapjoin3.q.out  |   16 +-
 .../results/clientpositive/llap/mapjoin46.q.out |  280 +-
 .../clientpositive/llap/mapjoin_decimal.q.out   |   22 +-
 .../llap/mapjoin_emit_interval.q.out            |   28 +-
 .../results/clientpositive/llap/merge1.q.out    |   12 +-
 .../results/clientpositive/llap/merge2.q.out    |   12 +-
 .../results/clientpositive/llap/mergejoin.q.out |  370 +-
 .../llap/metadata_only_queries.q.out            |   68 +-
 .../clientpositive/llap/multiMapJoin1.q.out     |  503 ++-
 .../clientpositive/llap/multiMapJoin2.q.out     |    8 +-
 .../llap/multi_count_distinct_null.q.out        |   42 +-
 .../llap/multi_insert_lateral_view.q.out        |  288 +-
 .../clientpositive/llap/optimize_nullscan.q.out |   20 +-
 .../llap/orc_llap_nonvector.q.out               |   16 +-
 .../clientpositive/llap/orc_merge3.q.out        |    6 +-
 .../clientpositive/llap/orc_merge4.q.out        |    6 +-
 .../clientpositive/llap/orc_merge5.q.out        |   16 +-
 .../clientpositive/llap/orc_merge6.q.out        |   16 +-
 .../clientpositive/llap/orc_merge7.q.out        |   12 +-
 .../llap/orc_merge_incompat1.q.out              |    8 +-
 .../llap/orc_merge_incompat2.q.out              |    6 +-
 .../llap/orc_predicate_pushdown.q.out           |  144 +-
 .../llap/parquet_predicate_pushdown.q.out       |  136 +-
 .../llap/parquet_types_vectorization.q.out      |  112 +-
 .../llap/partition_shared_scan.q.out            |  116 +-
 .../clientpositive/llap/ppd_union_view.q.out    |   66 +-
 .../clientpositive/llap/ptf_matchpath.q.out     |   42 +-
 .../clientpositive/llap/rcfile_createas1.q.out  |    6 +-
 .../clientpositive/llap/rcfile_merge3.q.out     |    6 +-
 .../clientpositive/llap/rcfile_merge4.q.out     |    6 +-
 .../llap/reduce_deduplicate.q.out               |   10 +-
 .../llap/reduce_deduplicate_distinct.q.out      |   84 +-
 .../results/clientpositive/llap/sample10.q.out  |    6 +-
 .../llap/schema_evol_orc_nonvec_part.q.out      |   54 +-
 ...chema_evol_orc_nonvec_part_all_complex.q.out |   18 +-
 ...ema_evol_orc_nonvec_part_all_primitive.q.out |   30 +-
 .../llap/schema_evol_orc_nonvec_table.q.out     |   30 +-
 .../llap/schema_evol_orc_vec_part.q.out         |   54 +-
 .../schema_evol_orc_vec_part_all_complex.q.out  |   18 +-
 ...schema_evol_orc_vec_part_all_primitive.q.out |   30 +-
 .../llap/schema_evol_orc_vec_table.q.out        |   30 +-
 .../llap/schema_evol_text_nonvec_part.q.out     |   54 +-
 ...hema_evol_text_nonvec_part_all_complex.q.out |   18 +-
 ...ma_evol_text_nonvec_part_all_primitive.q.out |   30 +-
 .../llap/schema_evol_text_nonvec_table.q.out    |   30 +-
 .../llap/schema_evol_text_vec_part.q.out        |   54 +-
 .../schema_evol_text_vec_part_all_complex.q.out |   18 +-
 ...chema_evol_text_vec_part_all_primitive.q.out |   30 +-
 .../llap/schema_evol_text_vec_table.q.out       |   30 +-
 .../llap/schema_evol_text_vecrow_part.q.out     |   54 +-
 ...hema_evol_text_vecrow_part_all_complex.q.out |   18 +-
 ...ma_evol_text_vecrow_part_all_primitive.q.out |   30 +-
 .../llap/schema_evol_text_vecrow_table.q.out    |   30 +-
 .../results/clientpositive/llap/semijoin.q.out  |  534 +--
 .../results/clientpositive/llap/skewjoin.q.out  |   76 +-
 .../clientpositive/llap/skewjoinopt15.q.out     |   36 +-
 .../results/clientpositive/llap/smb_cache.q.out |   28 +-
 .../clientpositive/llap/smb_mapjoin_14.q.out    |  244 +-
 .../clientpositive/llap/smb_mapjoin_15.q.out    |  104 +-
 .../clientpositive/llap/smb_mapjoin_17.q.out    |  218 +-
 .../clientpositive/llap/smb_mapjoin_18.q.out    |   24 +-
 .../clientpositive/llap/smb_mapjoin_19.q.out    |    6 +-
 .../clientpositive/llap/smb_mapjoin_4.q.out     |  310 +-
 .../clientpositive/llap/smb_mapjoin_5.q.out     |  310 +-
 .../clientpositive/llap/smb_mapjoin_6.q.out     |  108 +-
 .../results/clientpositive/llap/sqlmerge.q.out  |   76 +-
 .../results/clientpositive/llap/stats11.q.out   |   44 +-
 .../clientpositive/llap/stats_only_null.q.out   |   24 +-
 .../clientpositive/llap/subquery_exists.q.out   |   10 +-
 .../clientpositive/llap/subquery_in.q.out       |   36 +-
 .../clientpositive/llap/subquery_multi.q.out    |  416 +-
 .../clientpositive/llap/subquery_notin.q.out    |  366 +-
 .../clientpositive/llap/subquery_scalar.q.out   |  530 +--
 .../clientpositive/llap/subquery_select.q.out   |   48 +-
 .../results/clientpositive/llap/sysdb.q.out     |   12 +-
 .../llap/table_access_keys_stats.q.out          |    8 +-
 .../clientpositive/llap/temp_table.q.out        |   30 +-
 .../llap/tez_bmj_schema_evolution.q.out         |   22 +-
 .../results/clientpositive/llap/tez_dml.q.out   |    6 +-
 .../results/clientpositive/llap/tez_join.q.out  |   24 +-
 .../clientpositive/llap/tez_join_hash.q.out     |    8 +-
 .../llap/tez_join_result_complex.q.out          |   40 +-
 .../clientpositive/llap/tez_nway_join.q.out     |   90 +-
 .../clientpositive/llap/tez_self_join.q.out     |   36 +-
 .../results/clientpositive/llap/tez_smb_1.q.out |  106 +-
 .../clientpositive/llap/tez_smb_empty.q.out     |   82 +-
 .../clientpositive/llap/tez_smb_main.q.out      |  470 +-
 .../llap/tez_union_group_by.q.out               |   66 +-
 .../clientpositive/llap/unionDistinct_1.q.out   |  994 +++--
 .../clientpositive/llap/union_remove_26.q.out   |    8 +-
 .../llap/vector_adaptor_usage_mode.q.out        |  100 +-
 .../llap/vector_aggregate_9.q.out               |   36 +-
 .../llap/vector_aggregate_without_gby.q.out     |   10 +-
 .../llap/vector_auto_smb_mapjoin_14.q.out       |  260 +-
 .../llap/vector_between_columns.q.out           |   34 +-
 .../clientpositive/llap/vector_between_in.q.out |  166 +-
 .../llap/vector_binary_join_groupby.q.out       |   56 +-
 .../clientpositive/llap/vector_bucket.q.out     |   10 +-
 .../llap/vector_cast_constant.q.out             |   18 +-
 .../clientpositive/llap/vector_char_2.q.out     |   36 +-
 .../clientpositive/llap/vector_char_4.q.out     |    6 +-
 .../llap/vector_char_mapjoin1.q.out             |   78 +-
 .../clientpositive/llap/vector_coalesce_2.q.out |   40 +-
 .../llap/vector_complex_all.q.out               |   86 +-
 .../llap/vector_complex_join.q.out              |    8 +-
 .../clientpositive/llap/vector_count.q.out      |   48 +-
 .../llap/vector_count_distinct.q.out            |   18 +-
 .../clientpositive/llap/vector_data_types.q.out |   24 +-
 .../clientpositive/llap/vector_date_1.q.out     |   68 +-
 .../clientpositive/llap/vector_decimal_1.q.out  |   90 +-
 .../llap/vector_decimal_10_0.q.out              |   10 +-
 .../llap/vector_decimal_aggregate.q.out         |   32 +-
 .../llap/vector_decimal_expressions.q.out       |   14 +-
 .../llap/vector_decimal_mapjoin.q.out           |   18 +-
 .../llap/vector_decimal_math_funcs.q.out        |    8 +-
 .../llap/vector_decimal_precision.q.out         |   12 +-
 .../llap/vector_decimal_round.q.out             |   40 +-
 .../llap/vector_decimal_udf.q.out               |  258 +-
 .../llap/vector_decimal_udf2.q.out              |   16 +-
 .../clientpositive/llap/vector_distinct_2.q.out |   14 +-
 .../clientpositive/llap/vector_groupby4.q.out   |   14 +-
 .../clientpositive/llap/vector_groupby6.q.out   |   14 +-
 .../clientpositive/llap/vector_groupby_3.q.out  |   14 +-
 .../llap/vector_groupby_cube1.q.out             |  126 +-
 .../llap/vector_groupby_grouping_id1.q.out      |   84 +-
 .../llap/vector_groupby_grouping_id2.q.out      |  220 +-
 .../llap/vector_groupby_grouping_id3.q.out      |   32 +-
 .../llap/vector_groupby_grouping_sets1.q.out    |   96 +-
 .../llap/vector_groupby_grouping_sets2.q.out    |   72 +-
 .../llap/vector_groupby_grouping_sets3.q.out    |   46 +-
 .../llap/vector_groupby_grouping_sets4.q.out    |   98 +-
 .../llap/vector_groupby_grouping_sets5.q.out    |   64 +-
 .../llap/vector_groupby_grouping_sets6.q.out    |   24 +-
 .../vector_groupby_grouping_sets_grouping.q.out |  182 +-
 .../vector_groupby_grouping_sets_limit.q.out    |  118 +-
 .../llap/vector_groupby_reduce.q.out            |   84 +-
 .../llap/vector_groupby_rollup1.q.out           |   98 +-
 .../llap/vector_grouping_sets.q.out             |   40 +-
 .../llap/vector_include_no_sel.q.out            |   12 +-
 .../clientpositive/llap/vector_inner_join.q.out |  160 +-
 .../clientpositive/llap/vector_interval_1.q.out |   80 +-
 .../clientpositive/llap/vector_interval_2.q.out |  112 +-
 .../llap/vector_interval_arithmetic.q.out       |   60 +-
 .../llap/vector_interval_mapjoin.q.out          |   20 +-
 .../clientpositive/llap/vector_join30.q.out     |  318 +-
 .../llap/vector_left_outer_join2.q.out          |   96 +-
 .../llap/vector_leftsemi_mapjoin.q.out          | 1904 ++++----
 .../llap/vector_mr_diff_schema_alias.q.out      |   55 +-
 .../llap/vector_multi_insert.q.out              |    6 +-
 .../llap/vector_nullsafe_join.q.out             |  176 +-
 .../llap/vector_number_compare_projection.q.out |   24 +-
 .../clientpositive/llap/vector_orderby_5.q.out  |   16 +-
 .../llap/vector_outer_join0.q.out               |   28 +-
 .../llap/vector_partition_diff_num_cols.q.out   |   60 +-
 .../llap/vector_partitioned_date_time.q.out     |  156 +-
 .../llap/vector_ptf_part_simple.q.out           |  312 +-
 .../clientpositive/llap/vector_reduce1.q.out    |   10 +-
 .../clientpositive/llap/vector_reduce2.q.out    |   10 +-
 .../clientpositive/llap/vector_reduce3.q.out    |   10 +-
 .../llap/vector_reduce_groupby_decimal.q.out    |   18 +-
 .../llap/vector_string_concat.q.out             |   26 +-
 .../clientpositive/llap/vector_struct_in.q.out  |   48 +-
 .../clientpositive/llap/vector_udf1.q.out       |  220 +-
 .../llap/vector_udf_character_length.q.out      |   12 +-
 .../llap/vector_udf_octet_length.q.out          |    6 +-
 .../clientpositive/llap/vector_varchar_4.q.out  |    6 +-
 .../llap/vector_varchar_mapjoin1.q.out          |  154 +-
 .../llap/vector_varchar_simple.q.out            |   24 +-
 .../llap/vector_when_case_null.q.out            |   12 +-
 .../llap/vector_windowing_navfn.q.out           |  132 +-
 .../llap/vectorization_decimal_date.q.out       |   10 +-
 .../llap/vectorization_part_project.q.out       |    8 +-
 .../llap/vectorization_short_regress.q.out      |   72 +-
 .../llap/vectorized_bucketmapjoin1.q.out        |   54 +-
 .../llap/vectorized_context.q.out               |  109 +-
 .../llap/vectorized_date_funcs.q.out            |   42 +-
 .../llap/vectorized_distinct_gby.q.out          |   14 +-
 .../vectorized_dynamic_partition_pruning.q.out  |  649 +--
 .../vectorized_dynamic_semijoin_reduction.q.out |  238 +-
 .../clientpositive/llap/vectorized_join46.q.out |  250 +-
 .../llap/vectorized_parquet.q.out               |   12 +-
 .../llap/vectorized_parquet_types.q.out         |   16 +-
 .../clientpositive/llap/vectorized_ptf.q.out    |  538 +--
 .../llap/vectorized_timestamp.q.out             |   34 +-
 .../llap/vectorized_timestamp_funcs.q.out       |   80 +-
 .../results/clientpositive/merge_join_1.q.out   |   68 +-
 .../test/results/clientpositive/mergejoin.q.out |  132 +-
 .../clientpositive/mergejoins_mixed.q.out       |  237 +-
 ql/src/test/results/clientpositive/mm_all.q.out |   35 +-
 .../results/clientpositive/perf/query14.q.out   |    2 +-
 .../test/results/clientpositive/ppd_join5.q.out |  122 +-
 .../clientpositive/ppd_outer_join5.q.out        |   64 +-
 .../results/clientpositive/smb_mapjoin_47.q.out |  120 +-
 .../spark/auto_join_reordering_values.q.out     |   10 +-
 .../clientpositive/spark/auto_join_stats.q.out  |   34 +-
 .../clientpositive/spark/auto_join_stats2.q.out |   95 +-
 .../spark/auto_smb_mapjoin_14.q.out             |    2 +-
 .../spark/auto_sortmerge_join_12.q.out          |    8 +-
 .../spark/auto_sortmerge_join_6.q.out           |  464 +-
 .../spark/auto_sortmerge_join_9.q.out           |    4 +-
 .../spark/bucket_map_join_tez1.q.out            |  214 +-
 .../spark/bucket_map_join_tez2.q.out            |   28 +-
 .../spark/column_access_stats.q.out             |   38 +-
 .../results/clientpositive/spark/join19.q.out   |  136 +-
 .../spark/join_cond_pushdown_unqual1.q.out      |  274 +-
 .../spark/join_cond_pushdown_unqual2.q.out      |  152 +-
 .../spark/join_cond_pushdown_unqual3.q.out      |  274 +-
 .../spark/join_cond_pushdown_unqual4.q.out      |  152 +-
 .../clientpositive/spark/join_hive_626.q.out    |   64 +-
 .../clientpositive/spark/join_star.q.out        |  137 +-
 .../clientpositive/spark/mergejoins_mixed.q.out |  171 +-
 .../clientpositive/spark/ppd_join5.q.out        |  130 +-
 .../clientpositive/spark/ppd_outer_join5.q.out  |   64 +-
 .../spark/skewjoin_union_remove_1.q.out         |   16 +-
 .../spark/skewjoin_union_remove_2.q.out         |    6 +-
 .../clientpositive/spark/skewjoinopt1.q.out     |   16 +-
 .../clientpositive/spark/skewjoinopt10.q.out    |    8 +-
 .../clientpositive/spark/skewjoinopt11.q.out    |    8 +-
 .../clientpositive/spark/skewjoinopt12.q.out    |    4 +-
 .../clientpositive/spark/skewjoinopt14.q.out    |    8 +-
 .../clientpositive/spark/skewjoinopt15.q.out    |   16 +-
 .../clientpositive/spark/skewjoinopt16.q.out    |    4 +-
 .../clientpositive/spark/skewjoinopt17.q.out    |   12 +-
 .../clientpositive/spark/skewjoinopt19.q.out    |    8 +-
 .../clientpositive/spark/skewjoinopt2.q.out     |   16 +-
 .../clientpositive/spark/skewjoinopt20.q.out    |    8 +-
 .../clientpositive/spark/skewjoinopt3.q.out     |    8 +-
 .../clientpositive/spark/skewjoinopt4.q.out     |   16 +-
 .../clientpositive/spark/skewjoinopt5.q.out     |    4 +-
 .../clientpositive/spark/skewjoinopt6.q.out     |    4 +-
 .../clientpositive/spark/skewjoinopt7.q.out     |    6 +-
 .../clientpositive/spark/skewjoinopt8.q.out     |    6 +-
 .../spark/spark_dynamic_partition_pruning.q.out |  108 +-
 .../spark_dynamic_partition_pruning_3.q.out     |  118 +-
 ...dynamic_partition_pruning_mapjoin_only.q.out |  192 +-
 ...ic_partition_pruning_recursive_mapjoin.q.out | 1097 +++++
 .../spark/spark_explainuser_1.q.out             |  352 +-
 .../spark/spark_use_op_stats.q.out              |   92 +-
 ...k_vectorized_dynamic_partition_pruning.q.out | 4184 ++++++++++--------
 .../clientpositive/spark/stats_only_null.q.out  |   24 +-
 .../spark/table_access_keys_stats.q.out         |    8 +-
 .../clientpositive/stats_only_null.q.out        |   24 +-
 .../clientpositive/stats_partial_size.q.out     |   10 +-
 .../results/clientpositive/stats_ppr_all.q.out  |    6 +-
 .../clientpositive/tez/explainanalyze_2.q.out   |  208 +-
 .../clientpositive/tez/explainanalyze_3.q.out   |   47 +-
 .../clientpositive/tez/explainanalyze_5.q.out   |   14 +-
 .../clientpositive/tez/explainuser_3.q.out      |   43 +-
 .../tez/hybridgrace_hashjoin_1.q.out            |  195 +-
 .../tez/multi_count_distinct.q.out              |   30 +-
 .../results/clientpositive/tez/tez-tag.q.out    |   59 +-
 .../tez/vector_join_part_col_char.q.out         |   16 +-
 .../tez/vector_non_string_partition.q.out       |   28 +-
 .../vector_mr_diff_schema_alias.q.out           |   39 +-
 .../clientpositive/vector_outer_join6.q.out     |    4 +-
 .../clientpositive/vectorized_context.q.out     |   66 +-
 .../apache/hive/service/auth/HttpAuthUtils.java |    4 +-
 .../service/cli/session/HiveSessionImpl.java    |   28 +-
 .../service/cli/thrift/ThriftHttpServlet.java   |    4 +-
 448 files changed, 28238 insertions(+), 22347 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 021507f,230ca47..65c7477
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@@ -1724,11 -1703,11 +1723,11 @@@ public class DDLSemanticAnalyzer extend
        TableDesc tblDesc = Utilities.getTableDesc(tblObj);
        Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc);
        mergeDesc.setOutputDir(queryTmpdir);
 +      // No need to handle MM tables - unsupported path.
        LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc,
-           partSpec == null ? new HashMap<String, String>() : partSpec, null);
 -          partSpec == null ? new HashMap<>() : partSpec);
++          partSpec == null ? new HashMap<>() : partSpec, null);
        ltd.setLbCtx(lbCtx);
 -      Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false),
 -          conf);
 +      Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false), conf);
        mergeTask.addDependentTask(moveTsk);
  
        if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/18d13385/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
----------------------------------------------------------------------

[24/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bb1c0f25
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bb1c0f25
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bb1c0f25

Branch: refs/heads/master
Commit: bb1c0f25fc17d8c5ced8855a05b6651358c5fd96
Parents: 00e7510 5df1540
Author: Wei Zheng <we...@apache.org>
Authored: Fri Sep 8 15:35:21 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Fri Sep 8 15:35:21 2017 -0700

----------------------------------------------------------------------
 druid-handler/pom.xml                           |  48 +-
 .../hive/druid/DruidStorageHandlerUtils.java    |   7 +
 .../hadoop/hive/druid/io/DruidOutputFormat.java |   9 +-
 .../druid/io/DruidQueryBasedInputFormat.java    |   8 +-
 .../hadoop/hive/druid/io/DruidRecordWriter.java |   3 +-
 .../serde/DruidGroupByQueryRecordReader.java    |   5 +-
 .../serde/DruidSelectQueryRecordReader.java     |   5 +-
 .../hadoop/hive/druid/serde/DruidSerDe.java     |  15 +-
 .../serde/DruidTimeseriesQueryRecordReader.java |   5 +-
 .../druid/serde/DruidTopNQueryRecordReader.java |   5 +-
 .../TestHiveDruidQueryBasedInputFormat.java     |   6 +-
 .../hive/ql/io/TestDruidRecordWriter.java       |  19 +-
 .../test/resources/testconfiguration.properties |   1 +
 .../hadoop/hive/metastore/HiveAlterHandler.java |  24 +-
 .../hive/metastore/MetaStoreDirectSql.java      |   7 +-
 .../hadoop/hive/metastore/ObjectStore.java      |  11 +-
 .../metastore/PartitionExpressionProxy.java     |  10 +-
 .../hive/metastore/cache/CachedStore.java       |  12 +-
 .../hive/metastore/parser/ExpressionTree.java   |   6 +-
 .../MockPartitionExpressionForMetastore.java    |   5 +-
 .../hadoop/hive/metastore/TestObjectStore.java  |  11 +-
 .../hadoop/hive/metastore/TestOldSchema.java    |   6 +-
 .../hadoop/hive/ql/exec/repl/ReplDumpTask.java  |   6 +-
 .../hive/ql/optimizer/ConvertJoinMapJoin.java   |   4 +-
 .../ppr/PartitionExpressionForMetastore.java    |  14 +-
 .../hadoop/hive/ql/parse/ParseContext.java      |  10 +
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  32 +-
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |  11 +-
 .../hadoop/hive/ql/TestTxnCommandsBase.java     | 162 -----
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java |   2 +-
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java | 162 +++++
 .../test/queries/clientpositive/mapjoin_hint.q  |  69 ++
 ql/src/test/queries/clientpositive/masking_11.q |   6 +
 .../clientpositive/llap/mapjoin_hint.q.out      | 643 +++++++++++++++++++
 .../results/clientpositive/masking_11.q.out     |  20 +
 .../hadoop/hive/metastore/ColumnType.java       | 242 +++++++
 36 files changed, 1313 insertions(+), 298 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --cc metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 7573fb2,5448d0d..75eba79
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@@ -290,11 -272,11 +287,11 @@@ public class TestObjectStore 
    public void testPartitionOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
      Database db1 = new Database(DB1, "description", "locationurl", null);
      objectStore.createDatabase(db1);
 -    StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
 +    StorageDescriptor sd = createFakeSd("location");
      HashMap<String,String> tableParams = new HashMap<String,String>();
      tableParams.put("EXTERNAL", "false");
-     FieldSchema partitionKey1 = new FieldSchema("Country", serdeConstants.STRING_TYPE_NAME, "");
-     FieldSchema partitionKey2 = new FieldSchema("State", serdeConstants.STRING_TYPE_NAME, "");
+     FieldSchema partitionKey1 = new FieldSchema("Country", ColumnType.STRING_TYPE_NAME, "");
+     FieldSchema partitionKey2 = new FieldSchema("State", ColumnType.STRING_TYPE_NAME, "");
      Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, null, null, "MANAGED_TABLE");
      objectStore.createTable(tbl1);
      HashMap<String, String> partitionParams = new HashMap<String, String>();

http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/bb1c0f25/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------

[14/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/results/clientpositive/mm_loaddata.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/mm_loaddata.q.out
index 0000000,0000000..b849a88
new file mode 100644
--- /dev/null
+++ b/ql/src/test/results/clientpositive/mm_loaddata.q.out
@@@ -1,0 -1,0 +1,296 @@@
++PREHOOK: query: drop table load0_mm
++PREHOOK: type: DROPTABLE
++POSTHOOK: query: drop table load0_mm
++POSTHOOK: type: DROPTABLE
++PREHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++PREHOOK: type: CREATETABLE
++PREHOOK: Output: database:default
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++POSTHOOK: type: CREATETABLE
++POSTHOOK: Output: database:default
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: select count(1) from load0_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load0_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++500
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: select count(1) from load0_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load0_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++1000
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: select count(1) from load0_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load0_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load0_mm
++#### A masked pattern was here ####
++500
++PREHOOK: query: drop table load0_mm
++PREHOOK: type: DROPTABLE
++PREHOOK: Input: default@load0_mm
++PREHOOK: Output: default@load0_mm
++POSTHOOK: query: drop table load0_mm
++POSTHOOK: type: DROPTABLE
++POSTHOOK: Input: default@load0_mm
++POSTHOOK: Output: default@load0_mm
++PREHOOK: query: drop table intermediate2
++PREHOOK: type: DROPTABLE
++POSTHOOK: query: drop table intermediate2
++POSTHOOK: type: DROPTABLE
++PREHOOK: query: create table intermediate2 (key string, value string) stored as textfile
++#### A masked pattern was here ####
++PREHOOK: type: CREATETABLE
++#### A masked pattern was here ####
++PREHOOK: Output: database:default
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: create table intermediate2 (key string, value string) stored as textfile
++#### A masked pattern was here ####
++POSTHOOK: type: CREATETABLE
++#### A masked pattern was here ####
++POSTHOOK: Output: database:default
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: drop table load1_mm
++PREHOOK: type: DROPTABLE
++POSTHOOK: query: drop table load1_mm
++POSTHOOK: type: DROPTABLE
++PREHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++PREHOOK: type: CREATETABLE
++PREHOOK: Output: database:default
++PREHOOK: Output: default@load1_mm
++POSTHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++POSTHOOK: type: CREATETABLE
++POSTHOOK: Output: database:default
++POSTHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load1_mm
++PREHOOK: query: select count(1) from load1_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load1_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++1000
++PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++#### A masked pattern was here ####
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load1_mm
++PREHOOK: query: select count(1) from load1_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load1_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++1050
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++#### A masked pattern was here ####
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load1_mm
++PREHOOK: query: select count(1) from load1_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load1_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load1_mm
++#### A masked pattern was here ####
++500
++PREHOOK: query: drop table load1_mm
++PREHOOK: type: DROPTABLE
++PREHOOK: Input: default@load1_mm
++PREHOOK: Output: default@load1_mm
++POSTHOOK: query: drop table load1_mm
++POSTHOOK: type: DROPTABLE
++POSTHOOK: Input: default@load1_mm
++POSTHOOK: Output: default@load1_mm
++PREHOOK: query: drop table load2_mm
++PREHOOK: type: DROPTABLE
++POSTHOOK: query: drop table load2_mm
++POSTHOOK: type: DROPTABLE
++PREHOOK: query: create table load2_mm (key string, value string)
++  partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++PREHOOK: type: CREATETABLE
++PREHOOK: Output: database:default
++PREHOOK: Output: default@load2_mm
++POSTHOOK: query: create table load2_mm (key string, value string)
++  partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
++POSTHOOK: type: CREATETABLE
++POSTHOOK: Output: database:default
++POSTHOOK: Output: default@load2_mm
++PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@intermediate2
++#### A masked pattern was here ####
++PREHOOK: type: LOAD
++#### A masked pattern was here ####
++PREHOOK: Output: default@load2_mm
++#### A masked pattern was here ####
++POSTHOOK: type: LOAD
++#### A masked pattern was here ####
++POSTHOOK: Output: default@load2_mm
++POSTHOOK: Output: default@load2_mm@k=5/l=5
++PREHOOK: query: select count(1) from load2_mm
++PREHOOK: type: QUERY
++PREHOOK: Input: default@load2_mm
++PREHOOK: Input: default@load2_mm@k=5/l=5
++#### A masked pattern was here ####
++POSTHOOK: query: select count(1) from load2_mm
++POSTHOOK: type: QUERY
++POSTHOOK: Input: default@load2_mm
++POSTHOOK: Input: default@load2_mm@k=5/l=5
++#### A masked pattern was here ####
++1025
++PREHOOK: query: drop table load2_mm
++PREHOOK: type: DROPTABLE
++PREHOOK: Input: default@load2_mm
++PREHOOK: Output: default@load2_mm
++POSTHOOK: query: drop table load2_mm
++POSTHOOK: type: DROPTABLE
++POSTHOOK: Input: default@load2_mm
++POSTHOOK: Output: default@load2_mm
++PREHOOK: query: drop table intermediate2
++PREHOOK: type: DROPTABLE
++PREHOOK: Input: default@intermediate2
++PREHOOK: Output: default@intermediate2
++POSTHOOK: query: drop table intermediate2
++POSTHOOK: type: DROPTABLE
++POSTHOOK: Input: default@intermediate2
++POSTHOOK: Output: default@intermediate2

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
----------------------------------------------------------------------

[46/50] [abbrv] hive git commit: HIVE-17693 : remove the logic to convert from MM to plain hive table (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-17693 : remove the logic to convert from MM to plain hive table (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ae4ac4b1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ae4ac4b1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ae4ac4b1

Branch: refs/heads/master
Commit: ae4ac4b124655d6a674ff5077a3a63bc6a87a1d8
Parents: 42335b4
Author: sergey <se...@apache.org>
Authored: Mon Oct 9 17:51:08 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Mon Oct 9 17:51:08 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |  62 +--
 .../queries/clientpositive/mm_conversions.q     |  44 --
 .../clientpositive/llap/mm_conversions.q.out    | 544 -------------------
 .../results/clientpositive/mm_conversions.q.out | 544 -------------------
 4 files changed, 4 insertions(+), 1190 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ae4ac4b1/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 75a1c87..335ea63 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4109,7 +4109,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
       boolean isFromMmTable = AcidUtils.isInsertOnlyTable(tbl.getParameters()),
           isRemoved = AcidUtils.isRemovedInsertOnlyTable(removedSet);
       if (isFromMmTable && isRemoved) {
-        result = generateRemoveMmTasks(tbl);
+        throw new HiveException("Cannot convert an ACID table to non-ACID");
       }
     }
     Iterator<String> keyItr = alterTbl.getProps().keySet().iterator();
@@ -4123,72 +4123,18 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
     return result;
   }
 
-  private List<Task<?>> generateRemoveMmTasks(Table tbl) throws HiveException {
-    // To avoid confusion from nested MM directories when table is converted back and forth, we
-    // want to rename mm_ dirs to remove the prefix; however, given the unpredictable nested
-    // directory handling in Hive/MR, we will instead move all the files into the root directory.
-    // We will also delete any directories that are not committed. 
-    // Note that this relies on locks. Note also that we only do the renames AFTER the metastore
-    // operation commits. Deleting uncommitted things is safe, but moving stuff before we convert
-    // could cause data loss.
-    List<Path> allMmDirs = new ArrayList<>();
-    checkMmLb(tbl);
-    List<String> bucketCols = tbl.getBucketCols();
-    if (bucketCols != null && !bucketCols.isEmpty()
-        && HiveConf.getBoolVar(conf, ConfVars.HIVE_STRICT_CHECKS_BUCKETING)) {
-      throw new HiveException("Converting bucketed tables from MM is not supported by default; "
-          + "copying files from multiple MM directories may potentially break the buckets. You "
-          + "can set " + ConfVars.HIVE_STRICT_CHECKS_BUCKETING.varname
-          + " to false for this query if you want to force the conversion.");
-    }
-    Hive db = getHive();
-    String value = conf.get(ValidTxnList.VALID_TXNS_KEY);
-    ValidTxnList validTxnList = value == null ? new ValidReadTxnList() : new ValidReadTxnList(value);
-    if (tbl.getPartitionKeys().size() > 0) {
-      PartitionIterable parts = new PartitionIterable(db, tbl, null,
-          HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
-      Iterator<Partition> partIter = parts.iterator();
-      while (partIter.hasNext()) {
-        Partition part = partIter.next();
-        checkMmLb(part);
-        handleRemoveMm(part.getDataLocation(), validTxnList, allMmDirs);
-      }
-    } else {
-      checkMmLb(tbl);
-      handleRemoveMm(tbl.getDataLocation(), validTxnList, allMmDirs);
-    }
-    List<Path> targetPaths = new ArrayList<>(allMmDirs.size());
-    List<String> targetPrefix = new ArrayList<>(allMmDirs.size());
-    int prefixLen = JavaUtils.DELTA_PREFIX.length();
-    for (int i = 0; i < allMmDirs.size(); ++i) {
-      Path src = allMmDirs.get(i);
-      Path tgt = src.getParent();
-      String prefix = src.getName().substring(prefixLen + 1) + "_";
-      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
-        Utilities.FILE_OP_LOGGER.trace("Will move " + src + " to " + tgt + " (prefix " + prefix + ")");
-      }
-      targetPaths.add(tgt);
-      targetPrefix.add(prefix);
-    }
-    // Don't set inputs and outputs - the locks have already been taken so it's pointless.
-    MoveWork mw = new MoveWork(null, null, null, null, false, SessionState.get().getLineageState());
-    mw.setMultiFilesDesc(new LoadMultiFilesDesc(
-        allMmDirs, targetPaths, targetPrefix, true, null, null));
-    return Lists.<Task<?>>newArrayList(TaskFactory.get(mw, conf));
-  }
-
   private void checkMmLb(Table tbl) throws HiveException {
     if (!tbl.isStoredAsSubDirectories()) return;
     // TODO [MM gap?]: by design; no-one seems to use LB tables. They will work, but not convert.
     //                 It's possible to work around this by re-creating and re-inserting the table.
     throw new HiveException("Converting list bucketed tables stored as subdirectories "
-        + " to and from MM is not supported. Please re-create a table in the desired format.");
+        + " to MM is not supported. Please re-create a table in the desired format.");
   }
 
   private void checkMmLb(Partition part) throws HiveException {
     if (!part.isStoredAsSubDirectories()) return;
     throw new HiveException("Converting list bucketed tables stored as subdirectories "
-        + " to and from MM is not supported. Please re-create a table in the desired format.");
+        + " to MM is not supported. Please re-create a table in the desired format.");
   }
 
   private void handleRemoveMm(
@@ -4299,7 +4245,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
         if (!isFromMmTable && isToMmTable) {
           result = generateAddMmTasks(tbl);
         } else if (isFromMmTable && !isToMmTable) {
-          result = generateRemoveMmTasks(tbl);
+          throw new HiveException("Cannot convert an ACID table to non-ACID");
         }
       }
       tbl.getTTable().getParameters().putAll(alterTbl.getProps());

http://git-wip-us.apache.org/repos/asf/hive/blob/ae4ac4b1/ql/src/test/queries/clientpositive/mm_conversions.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_conversions.q b/ql/src/test/queries/clientpositive/mm_conversions.q
index 62faeac..14d16ab 100644
--- a/ql/src/test/queries/clientpositive/mm_conversions.q
+++ b/ql/src/test/queries/clientpositive/mm_conversions.q
@@ -14,27 +14,6 @@ insert into table intermediate partition(p='455') select distinct key from src w
 insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 1;
 insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 1;
 
-drop table simple_from_mm1;
-create table simple_from_mm1(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only");
-insert into table simple_from_mm1 select key from intermediate;
-insert into table simple_from_mm1 select key from intermediate;
-select * from simple_from_mm1 s1 order by key;
-alter table simple_from_mm1 unset tblproperties('transactional_properties', 'transactional');
-select * from simple_from_mm1 s2 order by key;
-insert into table simple_from_mm1 select key from intermediate;
-select * from simple_from_mm1 s3 order by key;
-drop table simple_from_mm1;
-
-drop table simple_from_mm2;
-create table simple_from_mm2(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only");
-insert into table simple_from_mm2 select key from intermediate;
-insert into table simple_from_mm2 select key from intermediate;
-select * from simple_from_mm2 s1 order by key;
-alter table simple_from_mm2 set tblproperties("transactional"="false", 'transactional_properties'='false');
-select * from simple_from_mm2 s2 order by key;
-insert into table simple_from_mm2 select key from intermediate;
-select * from simple_from_mm2 s3 order by key;
-drop table simple_from_mm2;
 
 drop table simple_to_mm;
 create table simple_to_mm(key int) stored as orc;
@@ -47,29 +26,6 @@ insert into table simple_to_mm select key from intermediate;
 select * from simple_to_mm s3 order by key;
 drop table simple_to_mm;
 
-drop table part_from_mm1;
-create table part_from_mm1(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only");
-insert into table part_from_mm1 partition(key_mm='455') select key from intermediate;
-insert into table part_from_mm1 partition(key_mm='455') select key from intermediate;
-insert into table part_from_mm1 partition(key_mm='456') select key from intermediate;
-select * from part_from_mm1 s1 order by key, key_mm;
-alter table part_from_mm1 unset tblproperties('transactional_properties', 'transactional');
-select * from part_from_mm1 s2 order by key, key_mm;
-insert into table part_from_mm1 partition(key_mm='456') select key from intermediate;
-insert into table part_from_mm1 partition(key_mm='457') select key from intermediate;
-select * from part_from_mm1 s3 order by key, key_mm;
-drop table part_from_mm1;
-
-drop table part_from_mm2;
-create table part_from_mm2(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only");
-insert into table part_from_mm2 partition(key_mm='456') select key from intermediate;--fails here
-insert into table part_from_mm2 partition(key_mm='455') select key from intermediate;
-select * from part_from_mm2 s1 order by key, key_mm;
-alter table part_from_mm2 set tblproperties("transactional"="false", 'transactional_properties'='false');
-select * from part_from_mm2 s2 order by key, key_mm;
-insert into table part_from_mm2 partition(key_mm='457') select key from intermediate;
-select * from part_from_mm2 s3 order by key, key_mm;
-drop table part_from_mm2;
 
 drop table part_to_mm;
 create table part_to_mm(key int) partitioned by (key_mm int) stored as orc;

http://git-wip-us.apache.org/repos/asf/hive/blob/ae4ac4b1/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_conversions.q.out b/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
index d95a70e..4754710 100644
--- a/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
@@ -37,242 +37,6 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src
 POSTHOOK: Output: default@intermediate@p=457
 POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: drop table simple_from_mm1
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table simple_from_mm1
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table simple_from_mm1(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: create table simple_from_mm1(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@simple_from_mm1
-PREHOOK: query: insert into table simple_from_mm1 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: insert into table simple_from_mm1 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm1
-POSTHOOK: Lineage: simple_from_mm1.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table simple_from_mm1 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: insert into table simple_from_mm1 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm1
-POSTHOOK: Lineage: simple_from_mm1.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_from_mm1 s1 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm1 s1 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-0
-0
-98
-98
-100
-100
-PREHOOK: query: alter table simple_from_mm1 unset tblproperties('transactional_properties', 'transactional')
-PREHOOK: type: ALTERTABLE_PROPERTIES
-PREHOOK: Input: default@simple_from_mm1
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: alter table simple_from_mm1 unset tblproperties('transactional_properties', 'transactional')
-POSTHOOK: type: ALTERTABLE_PROPERTIES
-POSTHOOK: Input: default@simple_from_mm1
-POSTHOOK: Output: default@simple_from_mm1
-PREHOOK: query: select * from simple_from_mm1 s2 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm1 s2 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-0
-0
-98
-98
-100
-100
-PREHOOK: query: insert into table simple_from_mm1 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: insert into table simple_from_mm1 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm1
-POSTHOOK: Lineage: simple_from_mm1.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_from_mm1 s3 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm1 s3 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-0
-0
-0
-98
-98
-98
-100
-100
-100
-PREHOOK: query: drop table simple_from_mm1
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@simple_from_mm1
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: drop table simple_from_mm1
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@simple_from_mm1
-POSTHOOK: Output: default@simple_from_mm1
-PREHOOK: query: drop table simple_from_mm2
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table simple_from_mm2
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table simple_from_mm2(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: create table simple_from_mm2(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@simple_from_mm2
-PREHOOK: query: insert into table simple_from_mm2 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: insert into table simple_from_mm2 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm2
-POSTHOOK: Lineage: simple_from_mm2.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table simple_from_mm2 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: insert into table simple_from_mm2 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm2
-POSTHOOK: Lineage: simple_from_mm2.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_from_mm2 s1 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm2 s1 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-0
-0
-98
-98
-100
-100
-PREHOOK: query: alter table simple_from_mm2 set tblproperties("transactional"="false", 'transactional_properties'='false')
-PREHOOK: type: ALTERTABLE_PROPERTIES
-PREHOOK: Input: default@simple_from_mm2
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: alter table simple_from_mm2 set tblproperties("transactional"="false", 'transactional_properties'='false')
-POSTHOOK: type: ALTERTABLE_PROPERTIES
-POSTHOOK: Input: default@simple_from_mm2
-POSTHOOK: Output: default@simple_from_mm2
-PREHOOK: query: select * from simple_from_mm2 s2 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm2 s2 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-0
-0
-98
-98
-100
-100
-PREHOOK: query: insert into table simple_from_mm2 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: insert into table simple_from_mm2 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm2
-POSTHOOK: Lineage: simple_from_mm2.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_from_mm2 s3 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm2 s3 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-0
-0
-0
-98
-98
-98
-100
-100
-100
-PREHOOK: query: drop table simple_from_mm2
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@simple_from_mm2
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: drop table simple_from_mm2
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@simple_from_mm2
-POSTHOOK: Output: default@simple_from_mm2
 PREHOOK: query: drop table simple_to_mm
 PREHOOK: type: DROPTABLE
 POSTHOOK: query: drop table simple_to_mm
@@ -385,314 +149,6 @@ POSTHOOK: query: drop table simple_to_mm
 POSTHOOK: type: DROPTABLE
 POSTHOOK: Input: default@simple_to_mm
 POSTHOOK: Output: default@simple_to_mm
-PREHOOK: query: drop table part_from_mm1
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table part_from_mm1
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table part_from_mm1(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@part_from_mm1
-POSTHOOK: query: create table part_from_mm1(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@part_from_mm1
-PREHOOK: query: insert into table part_from_mm1 partition(key_mm='455') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm1@key_mm=455
-POSTHOOK: query: insert into table part_from_mm1 partition(key_mm='455') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm1@key_mm=455
-POSTHOOK: Lineage: part_from_mm1 PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table part_from_mm1 partition(key_mm='455') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm1@key_mm=455
-POSTHOOK: query: insert into table part_from_mm1 partition(key_mm='455') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm1@key_mm=455
-POSTHOOK: Lineage: part_from_mm1 PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table part_from_mm1 partition(key_mm='456') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm1@key_mm=456
-POSTHOOK: query: insert into table part_from_mm1 partition(key_mm='456') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm1@key_mm=456
-POSTHOOK: Lineage: part_from_mm1 PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from part_from_mm1 s1 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm1
-PREHOOK: Input: default@part_from_mm1@key_mm=455
-PREHOOK: Input: default@part_from_mm1@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm1 s1 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm1
-POSTHOOK: Input: default@part_from_mm1@key_mm=455
-POSTHOOK: Input: default@part_from_mm1@key_mm=456
-#### A masked pattern was here ####
-0	455
-0	455
-0	456
-98	455
-98	455
-98	456
-100	455
-100	455
-100	456
-PREHOOK: query: alter table part_from_mm1 unset tblproperties('transactional_properties', 'transactional')
-PREHOOK: type: ALTERTABLE_PROPERTIES
-PREHOOK: Input: default@part_from_mm1
-PREHOOK: Output: default@part_from_mm1
-POSTHOOK: query: alter table part_from_mm1 unset tblproperties('transactional_properties', 'transactional')
-POSTHOOK: type: ALTERTABLE_PROPERTIES
-POSTHOOK: Input: default@part_from_mm1
-POSTHOOK: Output: default@part_from_mm1
-PREHOOK: query: select * from part_from_mm1 s2 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm1
-PREHOOK: Input: default@part_from_mm1@key_mm=455
-PREHOOK: Input: default@part_from_mm1@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm1 s2 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm1
-POSTHOOK: Input: default@part_from_mm1@key_mm=455
-POSTHOOK: Input: default@part_from_mm1@key_mm=456
-#### A masked pattern was here ####
-0	455
-0	455
-0	456
-98	455
-98	455
-98	456
-100	455
-100	455
-100	456
-PREHOOK: query: insert into table part_from_mm1 partition(key_mm='456') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm1@key_mm=456
-POSTHOOK: query: insert into table part_from_mm1 partition(key_mm='456') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm1@key_mm=456
-POSTHOOK: Lineage: part_from_mm1 PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table part_from_mm1 partition(key_mm='457') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm1@key_mm=457
-POSTHOOK: query: insert into table part_from_mm1 partition(key_mm='457') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm1@key_mm=457
-POSTHOOK: Lineage: part_from_mm1 PARTITION(key_mm=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from part_from_mm1 s3 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm1
-PREHOOK: Input: default@part_from_mm1@key_mm=455
-PREHOOK: Input: default@part_from_mm1@key_mm=456
-PREHOOK: Input: default@part_from_mm1@key_mm=457
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm1 s3 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm1
-POSTHOOK: Input: default@part_from_mm1@key_mm=455
-POSTHOOK: Input: default@part_from_mm1@key_mm=456
-POSTHOOK: Input: default@part_from_mm1@key_mm=457
-#### A masked pattern was here ####
-0	455
-0	455
-0	456
-0	456
-0	457
-98	455
-98	455
-98	456
-98	456
-98	457
-100	455
-100	455
-100	456
-100	456
-100	457
-PREHOOK: query: drop table part_from_mm1
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@part_from_mm1
-PREHOOK: Output: default@part_from_mm1
-POSTHOOK: query: drop table part_from_mm1
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@part_from_mm1
-POSTHOOK: Output: default@part_from_mm1
-PREHOOK: query: drop table part_from_mm2
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table part_from_mm2
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table part_from_mm2(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@part_from_mm2
-POSTHOOK: query: create table part_from_mm2(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@part_from_mm2
-PREHOOK: query: insert into table part_from_mm2 partition(key_mm='456') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm2@key_mm=456
-POSTHOOK: query: insert into table part_from_mm2 partition(key_mm='456') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm2@key_mm=456
-POSTHOOK: Lineage: part_from_mm2 PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: --fails here
-insert into table part_from_mm2 partition(key_mm='455') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm2@key_mm=455
-POSTHOOK: query: --fails here
-insert into table part_from_mm2 partition(key_mm='455') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm2@key_mm=455
-POSTHOOK: Lineage: part_from_mm2 PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from part_from_mm2 s1 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm2
-PREHOOK: Input: default@part_from_mm2@key_mm=455
-PREHOOK: Input: default@part_from_mm2@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm2 s1 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm2
-POSTHOOK: Input: default@part_from_mm2@key_mm=455
-POSTHOOK: Input: default@part_from_mm2@key_mm=456
-#### A masked pattern was here ####
-0	455
-0	456
-98	455
-98	456
-100	455
-100	456
-PREHOOK: query: alter table part_from_mm2 set tblproperties("transactional"="false", 'transactional_properties'='false')
-PREHOOK: type: ALTERTABLE_PROPERTIES
-PREHOOK: Input: default@part_from_mm2
-PREHOOK: Output: default@part_from_mm2
-POSTHOOK: query: alter table part_from_mm2 set tblproperties("transactional"="false", 'transactional_properties'='false')
-POSTHOOK: type: ALTERTABLE_PROPERTIES
-POSTHOOK: Input: default@part_from_mm2
-POSTHOOK: Output: default@part_from_mm2
-PREHOOK: query: select * from part_from_mm2 s2 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm2
-PREHOOK: Input: default@part_from_mm2@key_mm=455
-PREHOOK: Input: default@part_from_mm2@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm2 s2 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm2
-POSTHOOK: Input: default@part_from_mm2@key_mm=455
-POSTHOOK: Input: default@part_from_mm2@key_mm=456
-#### A masked pattern was here ####
-0	455
-0	456
-98	455
-98	456
-100	455
-100	456
-PREHOOK: query: insert into table part_from_mm2 partition(key_mm='457') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm2@key_mm=457
-POSTHOOK: query: insert into table part_from_mm2 partition(key_mm='457') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm2@key_mm=457
-POSTHOOK: Lineage: part_from_mm2 PARTITION(key_mm=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from part_from_mm2 s3 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm2
-PREHOOK: Input: default@part_from_mm2@key_mm=455
-PREHOOK: Input: default@part_from_mm2@key_mm=456
-PREHOOK: Input: default@part_from_mm2@key_mm=457
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm2 s3 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm2
-POSTHOOK: Input: default@part_from_mm2@key_mm=455
-POSTHOOK: Input: default@part_from_mm2@key_mm=456
-POSTHOOK: Input: default@part_from_mm2@key_mm=457
-#### A masked pattern was here ####
-0	455
-0	456
-0	457
-98	455
-98	456
-98	457
-100	455
-100	456
-100	457
-PREHOOK: query: drop table part_from_mm2
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@part_from_mm2
-PREHOOK: Output: default@part_from_mm2
-POSTHOOK: query: drop table part_from_mm2
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@part_from_mm2
-POSTHOOK: Output: default@part_from_mm2
 PREHOOK: query: drop table part_to_mm
 PREHOOK: type: DROPTABLE
 POSTHOOK: query: drop table part_to_mm

http://git-wip-us.apache.org/repos/asf/hive/blob/ae4ac4b1/ql/src/test/results/clientpositive/mm_conversions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/mm_conversions.q.out b/ql/src/test/results/clientpositive/mm_conversions.q.out
index d95a70e..4754710 100644
--- a/ql/src/test/results/clientpositive/mm_conversions.q.out
+++ b/ql/src/test/results/clientpositive/mm_conversions.q.out
@@ -37,242 +37,6 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src
 POSTHOOK: Output: default@intermediate@p=457
 POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: drop table simple_from_mm1
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table simple_from_mm1
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table simple_from_mm1(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: create table simple_from_mm1(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@simple_from_mm1
-PREHOOK: query: insert into table simple_from_mm1 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: insert into table simple_from_mm1 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm1
-POSTHOOK: Lineage: simple_from_mm1.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table simple_from_mm1 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: insert into table simple_from_mm1 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm1
-POSTHOOK: Lineage: simple_from_mm1.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_from_mm1 s1 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm1 s1 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-0
-0
-98
-98
-100
-100
-PREHOOK: query: alter table simple_from_mm1 unset tblproperties('transactional_properties', 'transactional')
-PREHOOK: type: ALTERTABLE_PROPERTIES
-PREHOOK: Input: default@simple_from_mm1
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: alter table simple_from_mm1 unset tblproperties('transactional_properties', 'transactional')
-POSTHOOK: type: ALTERTABLE_PROPERTIES
-POSTHOOK: Input: default@simple_from_mm1
-POSTHOOK: Output: default@simple_from_mm1
-PREHOOK: query: select * from simple_from_mm1 s2 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm1 s2 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-0
-0
-98
-98
-100
-100
-PREHOOK: query: insert into table simple_from_mm1 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: insert into table simple_from_mm1 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm1
-POSTHOOK: Lineage: simple_from_mm1.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_from_mm1 s3 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm1 s3 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm1
-#### A masked pattern was here ####
-0
-0
-0
-98
-98
-98
-100
-100
-100
-PREHOOK: query: drop table simple_from_mm1
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@simple_from_mm1
-PREHOOK: Output: default@simple_from_mm1
-POSTHOOK: query: drop table simple_from_mm1
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@simple_from_mm1
-POSTHOOK: Output: default@simple_from_mm1
-PREHOOK: query: drop table simple_from_mm2
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table simple_from_mm2
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table simple_from_mm2(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: create table simple_from_mm2(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@simple_from_mm2
-PREHOOK: query: insert into table simple_from_mm2 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: insert into table simple_from_mm2 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm2
-POSTHOOK: Lineage: simple_from_mm2.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table simple_from_mm2 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: insert into table simple_from_mm2 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm2
-POSTHOOK: Lineage: simple_from_mm2.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_from_mm2 s1 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm2 s1 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-0
-0
-98
-98
-100
-100
-PREHOOK: query: alter table simple_from_mm2 set tblproperties("transactional"="false", 'transactional_properties'='false')
-PREHOOK: type: ALTERTABLE_PROPERTIES
-PREHOOK: Input: default@simple_from_mm2
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: alter table simple_from_mm2 set tblproperties("transactional"="false", 'transactional_properties'='false')
-POSTHOOK: type: ALTERTABLE_PROPERTIES
-POSTHOOK: Input: default@simple_from_mm2
-POSTHOOK: Output: default@simple_from_mm2
-PREHOOK: query: select * from simple_from_mm2 s2 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm2 s2 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-0
-0
-98
-98
-100
-100
-PREHOOK: query: insert into table simple_from_mm2 select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: insert into table simple_from_mm2 select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_from_mm2
-POSTHOOK: Lineage: simple_from_mm2.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_from_mm2 s3 order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_from_mm2 s3 order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_from_mm2
-#### A masked pattern was here ####
-0
-0
-0
-98
-98
-98
-100
-100
-100
-PREHOOK: query: drop table simple_from_mm2
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@simple_from_mm2
-PREHOOK: Output: default@simple_from_mm2
-POSTHOOK: query: drop table simple_from_mm2
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@simple_from_mm2
-POSTHOOK: Output: default@simple_from_mm2
 PREHOOK: query: drop table simple_to_mm
 PREHOOK: type: DROPTABLE
 POSTHOOK: query: drop table simple_to_mm
@@ -385,314 +149,6 @@ POSTHOOK: query: drop table simple_to_mm
 POSTHOOK: type: DROPTABLE
 POSTHOOK: Input: default@simple_to_mm
 POSTHOOK: Output: default@simple_to_mm
-PREHOOK: query: drop table part_from_mm1
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table part_from_mm1
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table part_from_mm1(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@part_from_mm1
-POSTHOOK: query: create table part_from_mm1(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@part_from_mm1
-PREHOOK: query: insert into table part_from_mm1 partition(key_mm='455') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm1@key_mm=455
-POSTHOOK: query: insert into table part_from_mm1 partition(key_mm='455') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm1@key_mm=455
-POSTHOOK: Lineage: part_from_mm1 PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table part_from_mm1 partition(key_mm='455') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm1@key_mm=455
-POSTHOOK: query: insert into table part_from_mm1 partition(key_mm='455') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm1@key_mm=455
-POSTHOOK: Lineage: part_from_mm1 PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table part_from_mm1 partition(key_mm='456') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm1@key_mm=456
-POSTHOOK: query: insert into table part_from_mm1 partition(key_mm='456') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm1@key_mm=456
-POSTHOOK: Lineage: part_from_mm1 PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from part_from_mm1 s1 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm1
-PREHOOK: Input: default@part_from_mm1@key_mm=455
-PREHOOK: Input: default@part_from_mm1@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm1 s1 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm1
-POSTHOOK: Input: default@part_from_mm1@key_mm=455
-POSTHOOK: Input: default@part_from_mm1@key_mm=456
-#### A masked pattern was here ####
-0	455
-0	455
-0	456
-98	455
-98	455
-98	456
-100	455
-100	455
-100	456
-PREHOOK: query: alter table part_from_mm1 unset tblproperties('transactional_properties', 'transactional')
-PREHOOK: type: ALTERTABLE_PROPERTIES
-PREHOOK: Input: default@part_from_mm1
-PREHOOK: Output: default@part_from_mm1
-POSTHOOK: query: alter table part_from_mm1 unset tblproperties('transactional_properties', 'transactional')
-POSTHOOK: type: ALTERTABLE_PROPERTIES
-POSTHOOK: Input: default@part_from_mm1
-POSTHOOK: Output: default@part_from_mm1
-PREHOOK: query: select * from part_from_mm1 s2 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm1
-PREHOOK: Input: default@part_from_mm1@key_mm=455
-PREHOOK: Input: default@part_from_mm1@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm1 s2 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm1
-POSTHOOK: Input: default@part_from_mm1@key_mm=455
-POSTHOOK: Input: default@part_from_mm1@key_mm=456
-#### A masked pattern was here ####
-0	455
-0	455
-0	456
-98	455
-98	455
-98	456
-100	455
-100	455
-100	456
-PREHOOK: query: insert into table part_from_mm1 partition(key_mm='456') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm1@key_mm=456
-POSTHOOK: query: insert into table part_from_mm1 partition(key_mm='456') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm1@key_mm=456
-POSTHOOK: Lineage: part_from_mm1 PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table part_from_mm1 partition(key_mm='457') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm1@key_mm=457
-POSTHOOK: query: insert into table part_from_mm1 partition(key_mm='457') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm1@key_mm=457
-POSTHOOK: Lineage: part_from_mm1 PARTITION(key_mm=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from part_from_mm1 s3 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm1
-PREHOOK: Input: default@part_from_mm1@key_mm=455
-PREHOOK: Input: default@part_from_mm1@key_mm=456
-PREHOOK: Input: default@part_from_mm1@key_mm=457
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm1 s3 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm1
-POSTHOOK: Input: default@part_from_mm1@key_mm=455
-POSTHOOK: Input: default@part_from_mm1@key_mm=456
-POSTHOOK: Input: default@part_from_mm1@key_mm=457
-#### A masked pattern was here ####
-0	455
-0	455
-0	456
-0	456
-0	457
-98	455
-98	455
-98	456
-98	456
-98	457
-100	455
-100	455
-100	456
-100	456
-100	457
-PREHOOK: query: drop table part_from_mm1
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@part_from_mm1
-PREHOOK: Output: default@part_from_mm1
-POSTHOOK: query: drop table part_from_mm1
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@part_from_mm1
-POSTHOOK: Output: default@part_from_mm1
-PREHOOK: query: drop table part_from_mm2
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table part_from_mm2
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table part_from_mm2(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@part_from_mm2
-POSTHOOK: query: create table part_from_mm2(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@part_from_mm2
-PREHOOK: query: insert into table part_from_mm2 partition(key_mm='456') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm2@key_mm=456
-POSTHOOK: query: insert into table part_from_mm2 partition(key_mm='456') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm2@key_mm=456
-POSTHOOK: Lineage: part_from_mm2 PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: --fails here
-insert into table part_from_mm2 partition(key_mm='455') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm2@key_mm=455
-POSTHOOK: query: --fails here
-insert into table part_from_mm2 partition(key_mm='455') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm2@key_mm=455
-POSTHOOK: Lineage: part_from_mm2 PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from part_from_mm2 s1 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm2
-PREHOOK: Input: default@part_from_mm2@key_mm=455
-PREHOOK: Input: default@part_from_mm2@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm2 s1 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm2
-POSTHOOK: Input: default@part_from_mm2@key_mm=455
-POSTHOOK: Input: default@part_from_mm2@key_mm=456
-#### A masked pattern was here ####
-0	455
-0	456
-98	455
-98	456
-100	455
-100	456
-PREHOOK: query: alter table part_from_mm2 set tblproperties("transactional"="false", 'transactional_properties'='false')
-PREHOOK: type: ALTERTABLE_PROPERTIES
-PREHOOK: Input: default@part_from_mm2
-PREHOOK: Output: default@part_from_mm2
-POSTHOOK: query: alter table part_from_mm2 set tblproperties("transactional"="false", 'transactional_properties'='false')
-POSTHOOK: type: ALTERTABLE_PROPERTIES
-POSTHOOK: Input: default@part_from_mm2
-POSTHOOK: Output: default@part_from_mm2
-PREHOOK: query: select * from part_from_mm2 s2 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm2
-PREHOOK: Input: default@part_from_mm2@key_mm=455
-PREHOOK: Input: default@part_from_mm2@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm2 s2 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm2
-POSTHOOK: Input: default@part_from_mm2@key_mm=455
-POSTHOOK: Input: default@part_from_mm2@key_mm=456
-#### A masked pattern was here ####
-0	455
-0	456
-98	455
-98	456
-100	455
-100	456
-PREHOOK: query: insert into table part_from_mm2 partition(key_mm='457') select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_from_mm2@key_mm=457
-POSTHOOK: query: insert into table part_from_mm2 partition(key_mm='457') select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_from_mm2@key_mm=457
-POSTHOOK: Lineage: part_from_mm2 PARTITION(key_mm=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from part_from_mm2 s3 order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_from_mm2
-PREHOOK: Input: default@part_from_mm2@key_mm=455
-PREHOOK: Input: default@part_from_mm2@key_mm=456
-PREHOOK: Input: default@part_from_mm2@key_mm=457
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_from_mm2 s3 order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_from_mm2
-POSTHOOK: Input: default@part_from_mm2@key_mm=455
-POSTHOOK: Input: default@part_from_mm2@key_mm=456
-POSTHOOK: Input: default@part_from_mm2@key_mm=457
-#### A masked pattern was here ####
-0	455
-0	456
-0	457
-98	455
-98	456
-98	457
-100	455
-100	456
-100	457
-PREHOOK: query: drop table part_from_mm2
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@part_from_mm2
-PREHOOK: Output: default@part_from_mm2
-POSTHOOK: query: drop table part_from_mm2
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@part_from_mm2
-POSTHOOK: Output: default@part_from_mm2
 PREHOOK: query: drop table part_to_mm
 PREHOOK: type: DROPTABLE
 POSTHOOK: query: drop table part_to_mm

[05/50] [abbrv] hive git commit: HIVE-16963 : rely on AcidUtils.getAcidState() for read path (Wei Zheng)

Posted by se...@apache.org.

HIVE-16963 : rely on AcidUtils.getAcidState() for read path (Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/81853c12
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/81853c12
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/81853c12

Branch: refs/heads/master
Commit: 81853c12e3ad2ec3f00c4631bbfeadf1c89ae4e4
Parents: 8f36545
Author: Wei Zheng <we...@apache.org>
Authored: Mon Jun 26 10:25:34 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Mon Jun 26 10:25:34 2017 -0700

----------------------------------------------------------------------
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |  3 ++
 .../hadoop/hive/ql/io/HiveInputFormat.java      | 54 ++++++++++----------
 .../hive/ql/parse/ImportSemanticAnalyzer.java   | 31 +++++------
 .../hadoop/hive/ql/plan/FileSinkDesc.java       |  7 ++-
 4 files changed, 48 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/81853c12/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 004aaf7..7e1b8fa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -730,6 +730,9 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
       Utilities.copyTableJobPropertiesToConf(conf.getTableInfo(), jc);
       // only create bucket files only if no dynamic partitions,
       // buckets of dynamic partitions will be created for each newly created partition
+      //todo IOW integration. Full Acid uses the else if block to create Acid's RecordUpdater (HiveFileFormatUtils)
+      // and that will set writingBase(conf.getInsertOverwrite())
+      // If MM wants to create a new base for IOW (instead of delta dir), it should specify it here
       if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID || conf.isMmTable()) {
         Path outPath = fsp.outPaths[filesIdx];
         if (conf.isMmTable()

http://git-wip-us.apache.org/repos/asf/hive/blob/81853c12/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index 71d01ed..61e0d9b 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hive.common.ValidReadTxnList;
 import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
+import org.apache.hive.common.util.Ref;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configurable;
@@ -497,40 +498,37 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
       finalPaths.add(dir);
       return;
     }
-    FileStatus[] files = fs.listStatus(dir); // TODO: batch?
+
+    // Tez require the use of recursive input dirs for union processing, so we have to look into the
+    // directory to find out
     LinkedList<Path> subdirs = new LinkedList<>();
-    for (FileStatus file : files) {
-      handleNonMmDirChild(file, validTxnList, subdirs, finalPaths);
-    }
+    subdirs.add(dir); // add itself as a starting point
     while (!subdirs.isEmpty()) {
-      Path subdir = subdirs.poll();
-      for (FileStatus file : fs.listStatus(subdir)) {
-        handleNonMmDirChild(file, validTxnList, subdirs, finalPaths);
+      Path currDir = subdirs.poll();
+      FileStatus[] files = fs.listStatus(currDir);
+      boolean hadAcidState = false;   // whether getAcidState has been called for currDir
+      for (FileStatus file : files) {
+        Path path = file.getPath();
+        Utilities.LOG14535.warn("Checking " + path + " for inputs");
+        if (!file.isDirectory()) {
+          Utilities.LOG14535.warn("Ignoring a file not in MM directory " + path);
+        } else if (JavaUtils.extractTxnId(path) == null) {
+          subdirs.add(path);
+        } else {
+          if (!hadAcidState) {
+            AcidUtils.Directory dirInfo = AcidUtils.getAcidState(currDir, conf, validTxnList, Ref.from(false), true, null);
+            hadAcidState = true;
+            // todo for IOW, we also need to count in base dir, if any
+            for (AcidUtils.ParsedDelta delta : dirInfo.getCurrentDirectories()) {
+              Utilities.LOG14535.info("Adding input " + delta.getPath());
+              finalPaths.add(delta.getPath());
+            }
+          }
+        }
       }
     }
   }
 
-  private static void handleNonMmDirChild(FileStatus file, ValidTxnList validTxnList,
-      LinkedList<Path> subdirs, List<Path> finalPaths) {
-    Path path = file.getPath();
-    Utilities.LOG14535.warn("Checking " + path + " for inputs");
-    if (!file.isDirectory()) {
-      Utilities.LOG14535.warn("Ignoring a file not in MM directory " + path);
-      return;
-    }
-    Long txnId = JavaUtils.extractTxnId(path);
-    if (txnId == null) {
-      subdirs.add(path);
-      return;
-    }
-    if (!validTxnList.isTxnValid(txnId)) {
-      Utilities.LOG14535.warn("Ignoring an uncommitted directory " + path);
-      return;
-    }
-    Utilities.LOG14535.info("Adding input " + path);
-    finalPaths.add(path);
-  }
-
   Path[] getInputPaths(JobConf job) throws IOException {
     Path[] dirs;
     if (HiveConf.getVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {

http://git-wip-us.apache.org/repos/asf/hive/blob/81853c12/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index ae52d15..5ac15af 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -302,15 +302,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       tableExists = true;
     }
 
-    Long txnId = null;
+    Long txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
     int stmtId = 0;
-    if (table != null && MetaStoreUtils.isInsertOnlyTable(table.getParameters())) {
-      txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
-    } else if (table == null && isSourceMm) {
-      // We could import everything as is - directories and IDs, but that won't work with ACID
-      // txn ids in future. So, let's import everything into the new MM directory with ID == 0.
-      txnId = 0l;
-    }
     //todo due to the master merge, tblDesc is no longer CreateTableDesc, but ImportTableDesc
     /*
     if (txnId != null) {
@@ -363,7 +356,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       ReplicationSpec replicationSpec, EximUtil.SemanticAnalyzerWrapperContext x,
       Long txnId, int stmtId, boolean isSourceMm) {
     Path dataPath = new Path(fromURI.toString(), EximUtil.DATA_PATH_NAME);
-    Path destPath = txnId == null ? x.getCtx().getExternalTmpPath(tgtPath)
+    Path destPath = !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtPath)
         : new Path(tgtPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
     Utilities.LOG14535.info("adding import work for table with source location: "
         + dataPath + "; table: " + tgtPath + "; copy destination " + destPath + "; mm "
@@ -453,9 +446,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
           + partSpecToString(partSpec.getPartSpec())
           + " with source location: " + srcLocation);
       Path tgtLocation = new Path(partSpec.getLocation());
-      Path destPath = txnId == null ? x.getCtx().getExternalTmpPath(tgtLocation)
+      Path destPath = !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtLocation)
           : new Path(tgtLocation, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
-      Path moveTaskSrc =  txnId == null ? destPath : tgtLocation;
+      Path moveTaskSrc =  !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? destPath : tgtLocation;
       Utilities.LOG14535.info("adding import work for partition with source location: "
           + srcLocation + "; target: " + tgtLocation + "; copy dest " + destPath + "; mm "
           + txnId + " (src " + isSourceMm + ") for " + partSpecToString(partSpec.getPartSpec()));
@@ -788,7 +781,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       if (table.isPartitioned()) {
         x.getLOG().debug("table partitioned");
         Task<?> ict = createImportCommitTask(
-            table.getDbName(), table.getTableName(), txnId, stmtId, x.getConf());
+            table.getDbName(), table.getTableName(), txnId, stmtId, x.getConf(),
+            MetaStoreUtils.isInsertOnlyTable(table.getParameters()));
 
         for (AddPartitionDesc addPartitionDesc : partitionDescs) {
           Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
@@ -824,7 +818,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
 
       if (isPartitioned(tblDesc)) {
         Task<?> ict = createImportCommitTask(
-            tblDesc.getDatabaseName(), tblDesc.getTableName(), txnId, stmtId, x.getConf());
+            tblDesc.getDatabaseName(), tblDesc.getTableName(), txnId, stmtId, x.getConf(),
+            MetaStoreUtils.isInsertOnlyTable(tblDesc.getTblProps()));
         for (AddPartitionDesc addPartitionDesc : partitionDescs) {
           t.addDependentTask(addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc,
             replicationSpec, x, txnId, stmtId, isSourceMm, ict));
@@ -858,9 +853,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
   }
 
   private static Task<?> createImportCommitTask(
-      String dbName, String tblName, Long txnId, int stmtId, HiveConf conf) {
+      String dbName, String tblName, Long txnId, int stmtId, HiveConf conf, boolean isMmTable) {
     @SuppressWarnings("unchecked")
-    Task<ImportCommitWork> ict = (txnId == null) ? null : TaskFactory.get(
+    Task<ImportCommitWork> ict = (!isMmTable) ? null : TaskFactory.get(
         new ImportCommitWork(dbName, tblName, txnId, stmtId), conf);
     return ict;
   }
@@ -942,7 +937,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       if (!replicationSpec.isMetadataOnly()) {
         if (isPartitioned(tblDesc)) {
           Task<?> ict = createImportCommitTask(
-              tblDesc.getDatabaseName(), tblDesc.getTableName(), txnId, stmtId, x.getConf());
+              tblDesc.getDatabaseName(), tblDesc.getTableName(), txnId, stmtId, x.getConf(),
+              MetaStoreUtils.isInsertOnlyTable(tblDesc.getTblProps()));
           for (AddPartitionDesc addPartitionDesc : partitionDescs) {
             addPartitionDesc.setReplicationSpec(replicationSpec);
             t.addDependentTask(
@@ -970,7 +966,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
           Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
           org.apache.hadoop.hive.ql.metadata.Partition ptn = null;
           Task<?> ict = replicationSpec.isMetadataOnly() ? null : createImportCommitTask(
-              tblDesc.getDatabaseName(), tblDesc.getTableName(), txnId, stmtId, x.getConf());
+              tblDesc.getDatabaseName(), tblDesc.getTableName(), txnId, stmtId, x.getConf(),
+              MetaStoreUtils.isInsertOnlyTable(tblDesc.getTblProps()));
           if ((ptn = x.getHive().getPartition(table, partSpec, false)) == null) {
             if (!replicationSpec.isMetadataOnly()){
               x.getTasks().add(addSinglePartition(

http://git-wip-us.apache.org/repos/asf/hive/blob/81853c12/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index a26047d..955e0e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -202,8 +202,11 @@ public class FileSinkDesc extends AbstractOperatorDesc {
   /** getFinalDirName that takes into account MM, but not DP, LB or buckets. */
   public Path getMergeInputDirName() {
     Path root = getFinalDirName();
-    if (mmWriteId == null) return root;
-    return new Path(root, AcidUtils.deltaSubdir(txnId, txnId, 0));
+    if (isMmTable()) {
+      return new Path(root, AcidUtils.deltaSubdir(txnId, txnId, 0));
+    } else {
+      return root;
+    }
   }
 
   @Explain(displayName = "table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })

[06/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/main/thrift/hive_metastore.thrift
----------------------------------------------------------------------
diff --cc standalone-metastore/src/main/thrift/hive_metastore.thrift
index 0000000,7268d53..322e0ec
mode 000000,100644..100644
--- a/standalone-metastore/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift
@@@ -1,0 -1,1612 +1,1613 @@@
+ #!/usr/local/bin/thrift -java
+ 
+ /**
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ 
+ #
+ # Thrift Service that the MetaStore is built on
+ #
+ 
+ include "share/fb303/if/fb303.thrift"
+ 
+ namespace java org.apache.hadoop.hive.metastore.api
+ namespace php metastore
+ namespace cpp Apache.Hadoop.Hive
+ 
+ const string DDL_TIME = "transient_lastDdlTime"
+ 
+ struct Version {
+   1: string version,
+   2: string comments
+ }
+ 
+ struct FieldSchema {
+   1: string name, // name of the field
+   2: string type, // type of the field. primitive types defined above, specify list<TYPE_NAME>, map<TYPE_NAME, TYPE_NAME> for lists & maps
+   3: string comment
+ }
+ 
+ struct SQLPrimaryKey {
+   1: string table_db,    // table schema
+   2: string table_name,  // table name
+   3: string column_name, // column name
+   4: i32 key_seq,        // sequence number within primary key
+   5: string pk_name,     // primary key name
+   6: bool enable_cstr,   // Enable/Disable
+   7: bool validate_cstr,  // Validate/No validate
+   8: bool rely_cstr      // Rely/No Rely
+ }
+ 
+ struct SQLForeignKey {
+   1: string pktable_db,    // primary key table schema
+   2: string pktable_name,  // primary key table name
+   3: string pkcolumn_name, // primary key column name
+   4: string fktable_db,    // foreign key table schema
+   5: string fktable_name,  // foreign key table name
+   6: string fkcolumn_name, // foreign key column name
+   7: i32 key_seq,          // sequence within foreign key
+   8: i32 update_rule,      // what happens to foreign key when parent key is updated
+   9: i32 delete_rule,      // what happens to foreign key when parent key is deleted
+   10: string fk_name,      // foreign key name
+   11: string pk_name,      // primary key name
+   12: bool enable_cstr,    // Enable/Disable
+   13: bool validate_cstr,  // Validate/No validate
+   14: bool rely_cstr       // Rely/No Rely
+ }
+ 
+ struct SQLUniqueConstraint {
+   1: string table_db,    // table schema
+   2: string table_name,  // table name
+   3: string column_name, // column name
+   4: i32 key_seq,        // sequence number within unique constraint
+   5: string uk_name,     // unique key name
+   6: bool enable_cstr,   // Enable/Disable
+   7: bool validate_cstr, // Validate/No validate
+   8: bool rely_cstr      // Rely/No Rely
+ }
+ 
+ struct SQLNotNullConstraint {
+   1: string table_db,    // table schema
+   2: string table_name,  // table name
+   3: string column_name, // column name
+   4: string nn_name,     // not null name
+   5: bool enable_cstr,   // Enable/Disable
+   6: bool validate_cstr, // Validate/No validate
+   7: bool rely_cstr      // Rely/No Rely
+ }
+ 
+ struct Type {
+   1: string          name,             // one of the types in PrimitiveTypes or CollectionTypes or User defined types
+   2: optional string type1,            // object type if the name is 'list' (LIST_TYPE), key type if the name is 'map' (MAP_TYPE)
+   3: optional string type2,            // val type if the name is 'map' (MAP_TYPE)
+   4: optional list<FieldSchema> fields // if the name is one of the user defined types
+ }
+ 
+ enum HiveObjectType {
+   GLOBAL = 1,
+   DATABASE = 2,
+   TABLE = 3,
+   PARTITION = 4,
+   COLUMN = 5,
+ }
+ 
+ enum PrincipalType {
+   USER = 1,
+   ROLE = 2,
+   GROUP = 3,
+ }
+ 
+ const string HIVE_FILTER_FIELD_OWNER = "hive_filter_field_owner__"
+ const string HIVE_FILTER_FIELD_PARAMS = "hive_filter_field_params__"
+ const string HIVE_FILTER_FIELD_LAST_ACCESS = "hive_filter_field_last_access__"
+ 
+ enum PartitionEventType {
+   LOAD_DONE = 1,
+ }
+ 
+ // Enums for transaction and lock management 
+ enum TxnState {
+     COMMITTED = 1,
+     ABORTED = 2,
+     OPEN = 3,
+ }
+ 
+ enum LockLevel {
+     DB = 1,
+     TABLE = 2,
+     PARTITION = 3,
+ }
+ 
+ enum LockState {
+     ACQUIRED = 1,       // requester has the lock
+     WAITING = 2,        // requester is waiting for the lock and should call checklock at a later point to see if the lock has been obtained.
+     ABORT = 3,          // the lock has been aborted, most likely due to timeout
+     NOT_ACQUIRED = 4,   // returned only with lockNoWait, indicates the lock was not available and was not acquired
+ }
+ 
+ enum LockType {
+     SHARED_READ = 1,
+     SHARED_WRITE = 2,
+     EXCLUSIVE = 3,
+ }
+ 
+ enum CompactionType {
+     MINOR = 1,
+     MAJOR = 2,
+ }
+ 
+ enum GrantRevokeType {
+     GRANT = 1,
+     REVOKE = 2,
+ }
+ 
+ enum DataOperationType {
+     SELECT = 1,
+     INSERT = 2
+     UPDATE = 3,
+     DELETE = 4,
+     UNSET = 5,//this is the default to distinguish from NULL from old clients
+     NO_TXN = 6,//drop table, insert overwrite, etc - something non-transactional
+ }
+ 
+ // Types of events the client can request that the metastore fire.  For now just support DML operations, as the metastore knows
+ // about DDL operations and there's no reason for the client to request such an event.
+ enum EventRequestType {
+     INSERT = 1,
+     UPDATE = 2,
+     DELETE = 3,
+ }
+ 
+ struct HiveObjectRef{
+   1: HiveObjectType objectType,
+   2: string dbName,
+   3: string objectName,
+   4: list<string> partValues,
+   5: string columnName,
+ }
+ 
+ struct PrivilegeGrantInfo {
+   1: string privilege,
+   2: i32 createTime,
+   3: string grantor,
+   4: PrincipalType grantorType,
+   5: bool grantOption,
+ }
+ 
+ struct HiveObjectPrivilege {
+   1: HiveObjectRef  hiveObject,
+   2: string principalName,
+   3: PrincipalType principalType,
+   4: PrivilegeGrantInfo grantInfo,
+ }
+ 
+ struct PrivilegeBag {
+   1: list<HiveObjectPrivilege> privileges,
+ }
+ 
+ struct PrincipalPrivilegeSet {
+   1: map<string, list<PrivilegeGrantInfo>> userPrivileges, // user name -> privilege grant info
+   2: map<string, list<PrivilegeGrantInfo>> groupPrivileges, // group name -> privilege grant info
+   3: map<string, list<PrivilegeGrantInfo>> rolePrivileges, //role name -> privilege grant info
+ }
+ 
+ struct GrantRevokePrivilegeRequest {
+   1: GrantRevokeType requestType;
+   2: PrivilegeBag privileges;
+   3: optional bool revokeGrantOption;  // Only for revoke request
+ }
+ 
+ struct GrantRevokePrivilegeResponse {
+   1: optional bool success;
+ }
+ 
+ struct Role {
+   1: string roleName,
+   2: i32 createTime,
+   3: string ownerName,
+ }
+ 
+ // Representation of a grant for a principal to a role
+ struct RolePrincipalGrant {
+   1: string roleName,
+   2: string principalName,
+   3: PrincipalType principalType,
+   4: bool grantOption,
+   5: i32 grantTime,
+   6: string grantorName,
+   7: PrincipalType grantorPrincipalType
+ }
+ 
+ struct GetRoleGrantsForPrincipalRequest {
+   1: required string principal_name,
+   2: required PrincipalType principal_type
+ }
+ 
+ struct GetRoleGrantsForPrincipalResponse {
+   1: required list<RolePrincipalGrant> principalGrants;
+ }
+ 
+ struct GetPrincipalsInRoleRequest {
+   1: required string roleName;
+ }
+ 
+ struct GetPrincipalsInRoleResponse {
+   1: required list<RolePrincipalGrant> principalGrants;
+ }
+ 
+ struct GrantRevokeRoleRequest {
+   1: GrantRevokeType requestType;
+   2: string roleName;
+   3: string principalName;
+   4: PrincipalType principalType;
+   5: optional string grantor;            // Needed for grant
+   6: optional PrincipalType grantorType; // Needed for grant
+   7: optional bool grantOption;
+ }
+ 
+ struct GrantRevokeRoleResponse {
+   1: optional bool success;
+ }
+ 
+ // namespace for tables
+ struct Database {
+   1: string name,
+   2: string description,
+   3: string locationUri,
+   4: map<string, string> parameters, // properties associated with the database
+   5: optional PrincipalPrivilegeSet privileges,
+   6: optional string ownerName,
+   7: optional PrincipalType ownerType
+ }
+ 
+ // This object holds the information needed by SerDes
+ struct SerDeInfo {
+   1: string name,                   // name of the serde, table name by default
+   2: string serializationLib,       // usually the class that implements the extractor & loader
+   3: map<string, string> parameters // initialization parameters
+ }
+ 
+ // sort order of a column (column name along with asc(1)/desc(0))
+ struct Order {
+   1: string col,      // sort column name
+   2: i32    order     // asc(1) or desc(0)
+ }
+ 
+ // this object holds all the information about skewed table
+ struct SkewedInfo {
+   1: list<string> skewedColNames, // skewed column names
+   2: list<list<string>> skewedColValues, //skewed values
+   3: map<list<string>, string> skewedColValueLocationMaps, //skewed value to location mappings
+ }
+ 
+ // this object holds all the information about physical storage of the data belonging to a table
+ struct StorageDescriptor {
+   1: list<FieldSchema> cols,  // required (refer to types defined above)
+   2: string location,         // defaults to <warehouse loc>/<db loc>/tablename
+   3: string inputFormat,      // SequenceFileInputFormat (binary) or TextInputFormat`  or custom format
+   4: string outputFormat,     // SequenceFileOutputFormat (binary) or IgnoreKeyTextOutputFormat or custom format
+   5: bool   compressed,       // compressed or not
+   6: i32    numBuckets,       // this must be specified if there are any dimension columns
+   7: SerDeInfo    serdeInfo,  // serialization and deserialization information
+   8: list<string> bucketCols, // reducer grouping columns and clustering columns and bucketing columns`
+   9: list<Order>  sortCols,   // sort order of the data in each bucket
+   10: map<string, string> parameters, // any user supplied key value hash
+   11: optional SkewedInfo skewedInfo, // skewed information
+   12: optional bool   storedAsSubDirectories       // stored as subdirectories or not
+ }
+ 
+ // table information
+ struct Table {
+   1: string tableName,                // name of the table
+   2: string dbName,                   // database name ('default')
+   3: string owner,                    // owner of this table
+   4: i32    createTime,               // creation time of the table
+   5: i32    lastAccessTime,           // last access time (usually this will be filled from HDFS and shouldn't be relied on)
+   6: i32    retention,                // retention time
+   7: StorageDescriptor sd,            // storage descriptor of the table
+   8: list<FieldSchema> partitionKeys, // partition keys of the table. only primitive types are supported
+   9: map<string, string> parameters,   // to store comments or any other user level parameters
+   10: string viewOriginalText,         // original view text, null for non-view
+   11: string viewExpandedText,         // expanded view text, null for non-view
+   12: string tableType,                // table type enum, e.g. EXTERNAL_TABLE
+   13: optional PrincipalPrivilegeSet privileges,
+   14: optional bool temporary=false,
 -  15: optional bool rewriteEnabled     // rewrite enabled or not
++  15: optional bool rewriteEnabled,     // rewrite enabled or not
+ }
+ 
+ struct Partition {
+   1: list<string> values // string value is converted to appropriate partition key type
+   2: string       dbName,
+   3: string       tableName,
+   4: i32          createTime,
+   5: i32          lastAccessTime,
+   6: StorageDescriptor   sd,
+   7: map<string, string> parameters,
+   8: optional PrincipalPrivilegeSet privileges
+ }
+ 
+ struct PartitionWithoutSD {
+   1: list<string> values // string value is converted to appropriate partition key type
+   2: i32          createTime,
+   3: i32          lastAccessTime,
+   4: string       relativePath,
+   5: map<string, string> parameters,
+   6: optional PrincipalPrivilegeSet privileges
+ }
+ 
+ struct PartitionSpecWithSharedSD {
+   1: list<PartitionWithoutSD> partitions,
+   2: StorageDescriptor sd,
+ }
+ 
+ struct PartitionListComposingSpec {
+   1: list<Partition> partitions
+ }
+ 
+ struct PartitionSpec {
+   1: string dbName,
+   2: string tableName,
+   3: string rootPath,
+   4: optional PartitionSpecWithSharedSD sharedSDPartitionSpec,
+   5: optional PartitionListComposingSpec partitionList
+ }
+ 
+ struct Index {
+   1: string       indexName, // unique with in the whole database namespace
+   2: string       indexHandlerClass, // reserved
+   3: string       dbName,
+   4: string       origTableName,
+   5: i32          createTime,
+   6: i32          lastAccessTime,
+   7: string       indexTableName,
+   8: StorageDescriptor   sd,
+   9: map<string, string> parameters,
+   10: bool         deferredRebuild
+ }
+ 
+ // column statistics
+ struct BooleanColumnStatsData {
+ 1: required i64 numTrues,
+ 2: required i64 numFalses,
+ 3: required i64 numNulls,
+ 4: optional binary bitVectors
+ }
+ 
+ struct DoubleColumnStatsData {
+ 1: optional double lowValue,
+ 2: optional double highValue,
+ 3: required i64 numNulls,
+ 4: required i64 numDVs,
+ 5: optional binary bitVectors
+ }
+ 
+ struct LongColumnStatsData {
+ 1: optional i64 lowValue,
+ 2: optional i64 highValue,
+ 3: required i64 numNulls,
+ 4: required i64 numDVs,
+ 5: optional binary bitVectors
+ }
+ 
+ struct StringColumnStatsData {
+ 1: required i64 maxColLen,
+ 2: required double avgColLen,
+ 3: required i64 numNulls,
+ 4: required i64 numDVs,
+ 5: optional binary bitVectors
+ }
+ 
+ struct BinaryColumnStatsData {
+ 1: required i64 maxColLen,
+ 2: required double avgColLen,
+ 3: required i64 numNulls,
+ 4: optional binary bitVectors
+ }
+ 
+ 
+ struct Decimal {
+ 1: required binary unscaled,
+ 3: required i16 scale
+ }
+ 
+ struct DecimalColumnStatsData {
+ 1: optional Decimal lowValue,
+ 2: optional Decimal highValue,
+ 3: required i64 numNulls,
+ 4: required i64 numDVs,
+ 5: optional binary bitVectors
+ }
+ 
+ struct Date {
+ 1: required i64 daysSinceEpoch
+ }
+ 
+ struct DateColumnStatsData {
+ 1: optional Date lowValue,
+ 2: optional Date highValue,
+ 3: required i64 numNulls,
+ 4: required i64 numDVs,
+ 5: optional binary bitVectors
+ }
+ 
+ union ColumnStatisticsData {
+ 1: BooleanColumnStatsData booleanStats,
+ 2: LongColumnStatsData longStats,
+ 3: DoubleColumnStatsData doubleStats,
+ 4: StringColumnStatsData stringStats,
+ 5: BinaryColumnStatsData binaryStats,
+ 6: DecimalColumnStatsData decimalStats,
+ 7: DateColumnStatsData dateStats
+ }
+ 
+ struct ColumnStatisticsObj {
+ 1: required string colName,
+ 2: required string colType,
+ 3: required ColumnStatisticsData statsData
+ }
+ 
+ struct ColumnStatisticsDesc {
+ 1: required bool isTblLevel,
+ 2: required string dbName,
+ 3: required string tableName,
+ 4: optional string partName,
+ 5: optional i64 lastAnalyzed
+ }
+ 
+ struct ColumnStatistics {
+ 1: required ColumnStatisticsDesc statsDesc,
+ 2: required list<ColumnStatisticsObj> statsObj;
+ }
+ 
+ struct AggrStats {
+ 1: required list<ColumnStatisticsObj> colStats,
+ 2: required i64 partsFound // number of partitions for which stats were found
+ }
+ 
+ struct SetPartitionsStatsRequest {
+ 1: required list<ColumnStatistics> colStats,
+ 2: optional bool needMerge //stats need to be merged with the existing stats
+ }
+ 
+ // schema of the table/query results etc.
+ struct Schema {
+  // column names, types, comments
+  1: list<FieldSchema> fieldSchemas,  // delimiters etc
+  2: map<string, string> properties
+ }
+ 
+ // Key-value store to be used with selected
+ // Metastore APIs (create, alter methods).
+ // The client can pass environment properties / configs that can be
+ // accessed in hooks.
+ struct EnvironmentContext {
+   1: map<string, string> properties
+ }
+ 
+ struct PrimaryKeysRequest {
+   1: required string db_name,
+   2: required string tbl_name
+ }
+ 
+ struct PrimaryKeysResponse {
+   1: required list<SQLPrimaryKey> primaryKeys
+ }
+ 
+ struct ForeignKeysRequest {
+   1: string parent_db_name,
+   2: string parent_tbl_name,
+   3: string foreign_db_name,
+   4: string foreign_tbl_name
+ }
+ 
+ struct ForeignKeysResponse {
+   1: required list<SQLForeignKey> foreignKeys
+ }
+ 
+ struct UniqueConstraintsRequest {
+   1: required string db_name,
+   2: required string tbl_name
+ }
+ 
+ struct UniqueConstraintsResponse {
+   1: required list<SQLUniqueConstraint> uniqueConstraints
+ }
+ 
+ struct NotNullConstraintsRequest {
+   1: required string db_name,
+   2: required string tbl_name
+ }
+ 
+ struct NotNullConstraintsResponse {
+   1: required list<SQLNotNullConstraint> notNullConstraints
+ }
+ 
+ struct DropConstraintRequest {
+   1: required string dbname, 
+   2: required string tablename,
+   3: required string constraintname
+ }
+ 
+ struct AddPrimaryKeyRequest {
+   1: required list<SQLPrimaryKey> primaryKeyCols
+ }
+ 
+ struct AddForeignKeyRequest {
+   1: required list<SQLForeignKey> foreignKeyCols
+ }
+ 
+ struct AddUniqueConstraintRequest {
+   1: required list<SQLUniqueConstraint> uniqueConstraintCols
+ }
+ 
+ struct AddNotNullConstraintRequest {
+   1: required list<SQLNotNullConstraint> notNullConstraintCols
+ }
+ 
+ // Return type for get_partitions_by_expr
+ struct PartitionsByExprResult {
+   1: required list<Partition> partitions,
+   // Whether the results has any (currently, all) partitions which may or may not match
+   2: required bool hasUnknownPartitions
+ }
+ 
+ struct PartitionsByExprRequest {
+   1: required string dbName,
+   2: required string tblName,
+   3: required binary expr,
+   4: optional string defaultPartitionName,
+   5: optional i16 maxParts=-1
+ }
+ 
+ struct TableStatsResult {
+   1: required list<ColumnStatisticsObj> tableStats
+ }
+ 
+ struct PartitionsStatsResult {
+   1: required map<string, list<ColumnStatisticsObj>> partStats
+ }
+ 
+ struct TableStatsRequest {
+  1: required string dbName,
+  2: required string tblName,
+  3: required list<string> colNames
+ }
+ 
+ struct PartitionsStatsRequest {
+  1: required string dbName,
+  2: required string tblName,
+  3: required list<string> colNames,
+  4: required list<string> partNames
+ }
+ 
+ // Return type for add_partitions_req
+ struct AddPartitionsResult {
+   1: optional list<Partition> partitions,
+ }
+ 
+ // Request type for add_partitions_req
+ struct AddPartitionsRequest {
+   1: required string dbName,
+   2: required string tblName,
+   3: required list<Partition> parts,
+   4: required bool ifNotExists,
+   5: optional bool needResult=true
+ }
+ 
+ // Return type for drop_partitions_req
+ struct DropPartitionsResult {
+   1: optional list<Partition> partitions,
+ }
+ 
+ struct DropPartitionsExpr {
+   1: required binary expr;
+   2: optional i32 partArchiveLevel;
+ }
+ 
+ union RequestPartsSpec {
+   1: list<string> names;
+   2: list<DropPartitionsExpr> exprs;
+ }
+ 
+ // Request type for drop_partitions_req
+ // TODO: we might want to add "bestEffort" flag; where a subset can fail
+ struct DropPartitionsRequest {
+   1: required string dbName,
+   2: required string tblName,
+   3: required RequestPartsSpec parts,
+   4: optional bool deleteData,
+   5: optional bool ifExists=true, // currently verified on client
+   6: optional bool ignoreProtection,
+   7: optional EnvironmentContext environmentContext,
+   8: optional bool needResult=true
+ }
+ 
+ enum FunctionType {
+   JAVA = 1,
+ }
+ 
+ enum ResourceType {
+   JAR     = 1,
+   FILE    = 2,
+   ARCHIVE = 3,
+ }
+ 
+ struct ResourceUri {
+   1: ResourceType resourceType,
+   2: string       uri,
+ }
+ 
+ // User-defined function
+ struct Function {
+   1: string           functionName,
+   2: string           dbName,
+   3: string           className,
+   4: string           ownerName,
+   5: PrincipalType    ownerType,
+   6: i32              createTime,
+   7: FunctionType     functionType,
+   8: list<ResourceUri> resourceUris,
+ }
+ 
+ // Structs for transaction and locks
+ struct TxnInfo {
+     1: required i64 id,
+     2: required TxnState state,
+     3: required string user,        // used in 'show transactions' to help admins find who has open transactions
+     4: required string hostname,    // used in 'show transactions' to help admins find who has open transactions
+     5: optional string agentInfo = "Unknown",
+     6: optional i32 heartbeatCount=0,
+     7: optional string metaInfo,
+     8: optional i64 startedTime,
+     9: optional i64 lastHeartbeatTime,
+ }
+ 
+ struct GetOpenTxnsInfoResponse {
+     1: required i64 txn_high_water_mark,
+     2: required list<TxnInfo> open_txns,
+ }
+ 
+ struct GetOpenTxnsResponse {
+     1: required i64 txn_high_water_mark,
+     2: required list<i64> open_txns,  // set<i64> changed to list<i64> since 3.0
+     3: optional i64 min_open_txn, //since 1.3,2.2
+     4: required binary abortedBits,   // since 3.0
+ }
+ 
+ struct OpenTxnRequest {
+     1: required i32 num_txns,
+     2: required string user,
+     3: required string hostname,
+     4: optional string agentInfo = "Unknown",
+ }
+ 
+ struct OpenTxnsResponse {
+     1: required list<i64> txn_ids,
+ }
+ 
+ struct AbortTxnRequest {
+     1: required i64 txnid,
+ }
+ 
+ struct AbortTxnsRequest {
+     1: required list<i64> txn_ids,
+ }
+ 
+ struct CommitTxnRequest {
+     1: required i64 txnid,
+ }
+ 
+ struct LockComponent {
+     1: required LockType type,
+     2: required LockLevel level,
+     3: required string dbname,
+     4: optional string tablename,
+     5: optional string partitionname,
+     6: optional DataOperationType operationType = DataOperationType.UNSET,
+     7: optional bool isAcid = false,
+     8: optional bool isDynamicPartitionWrite = false
+ }
+ 
+ struct LockRequest {
+     1: required list<LockComponent> component,
+     2: optional i64 txnid,
+     3: required string user,     // used in 'show locks' to help admins find who has open locks
+     4: required string hostname, // used in 'show locks' to help admins find who has open locks
+     5: optional string agentInfo = "Unknown",
+ }
+ 
+ struct LockResponse {
+     1: required i64 lockid,
+     2: required LockState state,
+ }
+ 
+ struct CheckLockRequest {
+     1: required i64 lockid,
+     2: optional i64 txnid,
+     3: optional i64 elapsed_ms,
+ }
+ 
+ struct UnlockRequest {
+     1: required i64 lockid,
+ }
+ 
+ struct ShowLocksRequest {
+     1: optional string dbname,
+     2: optional string tablename,
+     3: optional string partname,
+     4: optional bool isExtended=false,
+ }
+ 
+ struct ShowLocksResponseElement {
+     1: required i64 lockid,
+     2: required string dbname,
+     3: optional string tablename,
+     4: optional string partname,
+     5: required LockState state,
+     6: required LockType type,
+     7: optional i64 txnid,
+     8: required i64 lastheartbeat,
+     9: optional i64 acquiredat,
+     10: required string user,
+     11: required string hostname,
+     12: optional i32 heartbeatCount = 0,
+     13: optional string agentInfo,
+     14: optional i64 blockedByExtId,
+     15: optional i64 blockedByIntId,
+     16: optional i64 lockIdInternal,
+ }
+ 
+ struct ShowLocksResponse {
+     1: list<ShowLocksResponseElement> locks,
+ }
+ 
+ struct HeartbeatRequest {
+     1: optional i64 lockid,
+     2: optional i64 txnid
+ }
+ 
+ struct HeartbeatTxnRangeRequest {
+     1: required i64 min,
+     2: required i64 max
+ }
+ 
+ struct HeartbeatTxnRangeResponse {
+     1: required set<i64> aborted,
+     2: required set<i64> nosuch
+ }
+ 
+ struct CompactionRequest {
+     1: required string dbname,
+     2: required string tablename,
+     3: optional string partitionname,
+     4: required CompactionType type,
+     5: optional string runas,
+     6: optional map<string, string> properties
+ }
+ 
+ struct CompactionResponse {
+     1: required i64 id,
+     2: required string state,
+     3: required bool accepted
+ }
+ 
+ struct ShowCompactRequest {
+ }
+ 
+ struct ShowCompactResponseElement {
+     1: required string dbname,
+     2: required string tablename,
+     3: optional string partitionname,
+     4: required CompactionType type,
+     5: required string state,
+     6: optional string workerid,
+     7: optional i64 start,
+     8: optional string runAs,
+     9: optional i64 hightestTxnId, // Highest Txn ID handled by this compaction
+     10: optional string metaInfo,
+     11: optional i64 endTime,
+     12: optional string hadoopJobId = "None",
+     13: optional i64 id,
+ }
+ 
+ struct ShowCompactResponse {
+     1: required list<ShowCompactResponseElement> compacts,
+ }
+ 
+ struct AddDynamicPartitions {
+     1: required i64 txnid,
+     2: required string dbname,
+     3: required string tablename,
+     4: required list<string> partitionnames,
+     5: optional DataOperationType operationType = DataOperationType.UNSET
+ }
+ 
+ struct NotificationEventRequest {
+     1: required i64 lastEvent,
+     2: optional i32 maxEvents,
+ }
+ 
+ struct NotificationEvent {
+     1: required i64 eventId,
+     2: required i32 eventTime,
+     3: required string eventType,
+     4: optional string dbName,
+     5: optional string tableName,
+     6: required string message,
+     7: optional string messageFormat,
+ }
+ 
+ struct NotificationEventResponse {
+     1: required list<NotificationEvent> events,
+ }
+ 
+ struct CurrentNotificationEventId {
+     1: required i64 eventId,
+ }
+ 
+ struct NotificationEventsCountRequest {
+     1: required i64 fromEventId,
+     2: required string dbName,
+ }
+ 
+ struct NotificationEventsCountResponse {
+     1: required i64 eventsCount,
+ }
+ 
+ struct InsertEventRequestData {
+     1: optional bool replace,
+     2: required list<string> filesAdded,
+     // Checksum of files (hex string of checksum byte payload)
+     3: optional list<string> filesAddedChecksum,
+ }
+ 
+ union FireEventRequestData {
+     1: InsertEventRequestData insertData
+ }
+ 
+ struct FireEventRequest {
+     1: required bool successful,
+     2: required FireEventRequestData data
+     // dbname, tablename, and partition vals are included as optional in the top level event rather than placed in each type of
+     // subevent as I assume they'll be used across most event types.
+     3: optional string dbName,
+     4: optional string tableName,
+     5: optional list<string> partitionVals,
+ }
+ 
+ struct FireEventResponse {
+     // NOP for now, this is just a place holder for future responses
+ }
+     
+ struct MetadataPpdResult {
+   1: optional binary metadata,
+   2: optional binary includeBitset
+ }
+ 
+ // Return type for get_file_metadata_by_expr
+ struct GetFileMetadataByExprResult {
+   1: required map<i64, MetadataPpdResult> metadata,
+   2: required bool isSupported
+ }
+ 
+ enum FileMetadataExprType {
+   ORC_SARG = 1
+ }
+ 
+ 
+ // Request type for get_file_metadata_by_expr
+ struct GetFileMetadataByExprRequest {
+   1: required list<i64> fileIds,
+   2: required binary expr,
+   3: optional bool doGetFooters,
+   4: optional FileMetadataExprType type
+ }
+ 
+ // Return type for get_file_metadata
+ struct GetFileMetadataResult {
+   1: required map<i64, binary> metadata,
+   2: required bool isSupported
+ }
+ 
+ // Request type for get_file_metadata
+ struct GetFileMetadataRequest {
+   1: required list<i64> fileIds
+ }
+ 
+ // Return type for put_file_metadata
+ struct PutFileMetadataResult {
+ }
+ 
+ // Request type for put_file_metadata
+ struct PutFileMetadataRequest {
+   1: required list<i64> fileIds,
+   2: required list<binary> metadata,
+   3: optional FileMetadataExprType type
+ }
+ 
+ // Return type for clear_file_metadata
+ struct ClearFileMetadataResult {
+ }
+ 
+ // Request type for clear_file_metadata
+ struct ClearFileMetadataRequest {
+   1: required list<i64> fileIds
+ }
+ 
+ // Return type for cache_file_metadata
+ struct CacheFileMetadataResult {
+   1: required bool isSupported
+ }
+ 
+ // Request type for cache_file_metadata
+ struct CacheFileMetadataRequest {
+   1: required string dbName,
+   2: required string tblName,
+   3: optional string partName,
+   4: optional bool isAllParts
+ }
+ 
+ struct GetAllFunctionsResponse {
+   1: optional list<Function> functions
+ }
+ 
+ enum ClientCapability {
 -  TEST_CAPABILITY = 1
++  TEST_CAPABILITY = 1,
++  INSERT_ONLY_TABLES = 2
+ }
+ 
+ 
+ struct ClientCapabilities {
+   1: required list<ClientCapability> values
+ }
+ 
+ struct GetTableRequest {
+   1: required string dbName,
+   2: required string tblName,
+   3: optional ClientCapabilities capabilities
+ }
+ 
+ struct GetTableResult {
+   1: required Table table
+ }
+ 
+ struct GetTablesRequest {
+   1: required string dbName,
+   2: optional list<string> tblNames,
+   3: optional ClientCapabilities capabilities
+ }
+ 
+ struct GetTablesResult {
+   1: required list<Table> tables
+ }
+ 
+ // Request type for cm_recycle
+ struct CmRecycleRequest {
+   1: required string dataPath,
+   2: required bool purge
+ }
+ 
+ // Response type for cm_recycle
+ struct CmRecycleResponse {
+ }
+ 
+ struct TableMeta {
+   1: required string dbName;
+   2: required string tableName;
+   3: required string tableType;
+   4: optional string comments;
+ }
+ 
+ exception MetaException {
+   1: string message
+ }
+ 
+ exception UnknownTableException {
+   1: string message
+ }
+ 
+ exception UnknownDBException {
+   1: string message
+ }
+ 
+ exception AlreadyExistsException {
+   1: string message
+ }
+ 
+ exception InvalidPartitionException {
+   1: string message
+ }
+ 
+ exception UnknownPartitionException {
+   1: string message
+ }
+ 
+ exception InvalidObjectException {
+   1: string message
+ }
+ 
+ exception NoSuchObjectException {
+   1: string message
+ }
+ 
+ exception IndexAlreadyExistsException {
+   1: string message
+ }
+ 
+ exception InvalidOperationException {
+   1: string message
+ }
+ 
+ exception ConfigValSecurityException {
+   1: string message
+ }
+ 
+ exception InvalidInputException {
+   1: string message
+ }
+ 
+ // Transaction and lock exceptions
+ exception NoSuchTxnException {
+     1: string message
+ }
+ 
+ exception TxnAbortedException {
+     1: string message
+ }
+ 
+ exception TxnOpenException {
+     1: string message
+ }
+ 
+ exception NoSuchLockException {
+     1: string message
+ }
+ 
+ /**
+ * This interface is live.
+ */
+ service ThriftHiveMetastore extends fb303.FacebookService
+ {
+   string getMetaConf(1:string key) throws(1:MetaException o1)
+   void setMetaConf(1:string key, 2:string value) throws(1:MetaException o1)
+ 
+   void create_database(1:Database database) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
+   Database get_database(1:string name) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+   void drop_database(1:string name, 2:bool deleteData, 3:bool cascade) throws(1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3)
+   list<string> get_databases(1:string pattern) throws(1:MetaException o1)
+   list<string> get_all_databases() throws(1:MetaException o1)
+   void alter_database(1:string dbname, 2:Database db) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   // returns the type with given name (make seperate calls for the dependent types if needed)
+   Type get_type(1:string name)  throws(1:MetaException o1, 2:NoSuchObjectException o2)
+   bool create_type(1:Type type) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3)
+   bool drop_type(1:string type) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+   map<string, Type> get_type_all(1:string name)
+                                 throws(1:MetaException o2)
+ 
+   // Gets a list of FieldSchemas describing the columns of a particular table
+   list<FieldSchema> get_fields(1: string db_name, 2: string table_name) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3),
+   list<FieldSchema> get_fields_with_environment_context(1: string db_name, 2: string table_name, 3:EnvironmentContext environment_context) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3)
+ 
+   // Gets a list of FieldSchemas describing both the columns and the partition keys of a particular table
+   list<FieldSchema> get_schema(1: string db_name, 2: string table_name) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3)
+   list<FieldSchema> get_schema_with_environment_context(1: string db_name, 2: string table_name, 3:EnvironmentContext environment_context) throws (1: MetaException o1, 2: UnknownTableException o2, 3: UnknownDBException o3)
+ 
+   // create a Hive table. Following fields must be set
+   // tableName
+   // database        (only 'default' for now until Hive QL supports databases)
+   // owner           (not needed, but good to have for tracking purposes)
+   // sd.cols         (list of field schemas)
+   // sd.inputFormat  (SequenceFileInputFormat (binary like falcon tables or u_full) or TextInputFormat)
+   // sd.outputFormat (SequenceFileInputFormat (binary) or TextInputFormat)
+   // sd.serdeInfo.serializationLib (SerDe class name eg org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe
+   // * See notes on DDL_TIME
+   void create_table(1:Table tbl) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:NoSuchObjectException o4)
+   void create_table_with_environment_context(1:Table tbl,
+       2:EnvironmentContext environment_context)
+       throws (1:AlreadyExistsException o1,
+               2:InvalidObjectException o2, 3:MetaException o3,
+               4:NoSuchObjectException o4)
+   void create_table_with_constraints(1:Table tbl, 2: list<SQLPrimaryKey> primaryKeys, 3: list<SQLForeignKey> foreignKeys,
+   4: list<SQLUniqueConstraint> uniqueConstraints, 5: list<SQLNotNullConstraint> notNullConstraints)
+       throws (1:AlreadyExistsException o1,
+               2:InvalidObjectException o2, 3:MetaException o3,
+               4:NoSuchObjectException o4)
+   void drop_constraint(1:DropConstraintRequest req)
+       throws(1:NoSuchObjectException o1, 2:MetaException o3)
+   void add_primary_key(1:AddPrimaryKeyRequest req)
+       throws(1:NoSuchObjectException o1, 2:MetaException o2)
+   void add_foreign_key(1:AddForeignKeyRequest req)
+       throws(1:NoSuchObjectException o1, 2:MetaException o2)  
+   void add_unique_constraint(1:AddUniqueConstraintRequest req)
+       throws(1:NoSuchObjectException o1, 2:MetaException o2)
+   void add_not_null_constraint(1:AddNotNullConstraintRequest req)
+       throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ 
+   // drops the table and all the partitions associated with it if the table has partitions
+   // delete data (including partitions) if deleteData is set to true
+   void drop_table(1:string dbname, 2:string name, 3:bool deleteData)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o3)
+   void drop_table_with_environment_context(1:string dbname, 2:string name, 3:bool deleteData,
+       4:EnvironmentContext environment_context)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o3)
+   void truncate_table(1:string dbName, 2:string tableName, 3:list<string> partNames)
+                           throws(1:MetaException o1)
+   list<string> get_tables(1: string db_name, 2: string pattern) throws (1: MetaException o1)
+   list<string> get_tables_by_type(1: string db_name, 2: string pattern, 3: string tableType) throws (1: MetaException o1)
+   list<TableMeta> get_table_meta(1: string db_patterns, 2: string tbl_patterns, 3: list<string> tbl_types)
+                        throws (1: MetaException o1)
+   list<string> get_all_tables(1: string db_name) throws (1: MetaException o1)
+ 
+   Table get_table(1:string dbname, 2:string tbl_name)
+                        throws (1:MetaException o1, 2:NoSuchObjectException o2)
+   list<Table> get_table_objects_by_name(1:string dbname, 2:list<string> tbl_names)
+   GetTableResult get_table_req(1:GetTableRequest req) throws (1:MetaException o1, 2:NoSuchObjectException o2)
+   GetTablesResult get_table_objects_by_name_req(1:GetTablesRequest req)
+ 				   throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
+ 
+   // Get a list of table names that match a filter.
+   // The filter operators are LIKE, <, <=, >, >=, =, <>
+   //
+   // In the filter statement, values interpreted as strings must be enclosed in quotes,
+   // while values interpreted as integers should not be.  Strings and integers are the only
+   // supported value types.
+   //
+   // The currently supported key names in the filter are:
+   // Constants.HIVE_FILTER_FIELD_OWNER, which filters on the tables' owner's name
+   //   and supports all filter operators
+   // Constants.HIVE_FILTER_FIELD_LAST_ACCESS, which filters on the last access times
+   //   and supports all filter operators except LIKE
+   // Constants.HIVE_FILTER_FIELD_PARAMS, which filters on the tables' parameter keys and values
+   //   and only supports the filter operators = and <>.
+   //   Append the parameter key name to HIVE_FILTER_FIELD_PARAMS in the filter statement.
+   //   For example, to filter on parameter keys called "retention", the key name in the filter
+   //   statement should be Constants.HIVE_FILTER_FIELD_PARAMS + "retention"
+   //   Also, = and <> only work for keys that exist
+   //   in the tables. E.g., if you are looking for tables where key1 <> value, it will only
+   //   look at tables that have a value for the parameter key1.
+   // Some example filter statements include:
+   // filter = Constants.HIVE_FILTER_FIELD_OWNER + " like \".*test.*\" and " +
+   //   Constants.HIVE_FILTER_FIELD_LAST_ACCESS + " = 0";
+   // filter = Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"30\" or " +
+   //   Constants.HIVE_FILTER_FIELD_PARAMS + "retention = \"90\""
+   // @param dbName
+   //          The name of the database from which you will retrieve the table names
+   // @param filterType
+   //          The type of filter
+   // @param filter
+   //          The filter string
+   // @param max_tables
+   //          The maximum number of tables returned
+   // @return  A list of table names that match the desired filter
+   list<string> get_table_names_by_filter(1:string dbname, 2:string filter, 3:i16 max_tables=-1)
+                        throws (1:MetaException o1, 2:InvalidOperationException o2, 3:UnknownDBException o3)
+ 
+   // alter table applies to only future partitions not for existing partitions
+   // * See notes on DDL_TIME
+   void alter_table(1:string dbname, 2:string tbl_name, 3:Table new_tbl)
+                        throws (1:InvalidOperationException o1, 2:MetaException o2)
+   void alter_table_with_environment_context(1:string dbname, 2:string tbl_name,
+       3:Table new_tbl, 4:EnvironmentContext environment_context)
+       throws (1:InvalidOperationException o1, 2:MetaException o2)
+   // alter table not only applies to future partitions but also cascade to existing partitions
+   void alter_table_with_cascade(1:string dbname, 2:string tbl_name, 3:Table new_tbl, 4:bool cascade)
+                        throws (1:InvalidOperationException o1, 2:MetaException o2)
+   // the following applies to only tables that have partitions
+   // * See notes on DDL_TIME
+   Partition add_partition(1:Partition new_part)
+                        throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+   Partition add_partition_with_environment_context(1:Partition new_part,
+       2:EnvironmentContext environment_context)
+       throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2,
+       3:MetaException o3)
+   i32 add_partitions(1:list<Partition> new_parts)
+                        throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+   i32 add_partitions_pspec(1:list<PartitionSpec> new_parts)
+                        throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+   Partition append_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
+                        throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+   AddPartitionsResult add_partitions_req(1:AddPartitionsRequest request)
+                        throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+   Partition append_partition_with_environment_context(1:string db_name, 2:string tbl_name,
+       3:list<string> part_vals, 4:EnvironmentContext environment_context)
+                        throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+   Partition append_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name)
+                        throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+   Partition append_partition_by_name_with_environment_context(1:string db_name, 2:string tbl_name,
+       3:string part_name, 4:EnvironmentContext environment_context)
+                        throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+   bool drop_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:bool deleteData)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o2)
+   bool drop_partition_with_environment_context(1:string db_name, 2:string tbl_name,
+       3:list<string> part_vals, 4:bool deleteData, 5:EnvironmentContext environment_context)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o2)
+   bool drop_partition_by_name(1:string db_name, 2:string tbl_name, 3:string part_name, 4:bool deleteData)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o2)
+   bool drop_partition_by_name_with_environment_context(1:string db_name, 2:string tbl_name,
+       3:string part_name, 4:bool deleteData, 5:EnvironmentContext environment_context)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o2)
+   DropPartitionsResult drop_partitions_req(1: DropPartitionsRequest req)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ 
+   Partition get_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+   Partition exchange_partition(1:map<string, string> partitionSpecs, 2:string source_db,
+       3:string source_table_name, 4:string dest_db, 5:string dest_table_name)
+       throws(1:MetaException o1, 2:NoSuchObjectException o2, 3:InvalidObjectException o3,
+       4:InvalidInputException o4)
+ 
+   list<Partition> exchange_partitions(1:map<string, string> partitionSpecs, 2:string source_db,
+       3:string source_table_name, 4:string dest_db, 5:string dest_table_name)
+       throws(1:MetaException o1, 2:NoSuchObjectException o2, 3:InvalidObjectException o3,
+       4:InvalidInputException o4)
+ 
+   Partition get_partition_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals,
+       4: string user_name, 5: list<string> group_names) throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   Partition get_partition_by_name(1:string db_name 2:string tbl_name, 3:string part_name)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   // returns all the partitions for this table in reverse chronological order.
+   // If max parts is given then it will return only that many.
+   list<Partition> get_partitions(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o2)
+   list<Partition> get_partitions_with_auth(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1,
+      4: string user_name, 5: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ 
+   list<PartitionSpec> get_partitions_pspec(1:string db_name, 2:string tbl_name, 3:i32 max_parts=-1)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ 
+   list<string> get_partition_names(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
+                        throws(1:MetaException o2)
+ 
+   // get_partition*_ps methods allow filtering by a partial partition specification,
+   // as needed for dynamic partitions. The values that are not restricted should
+   // be empty strings. Nulls were considered (instead of "") but caused errors in
+   // generated Python code. The size of part_vals may be smaller than the
+   // number of partition columns - the unspecified values are considered the same
+   // as "".
+   list<Partition> get_partitions_ps(1:string db_name 2:string tbl_name
+   	3:list<string> part_vals, 4:i16 max_parts=-1)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+   list<Partition> get_partitions_ps_with_auth(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1,
+      5: string user_name, 6: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ 
+   list<string> get_partition_names_ps(1:string db_name,
+   	2:string tbl_name, 3:list<string> part_vals, 4:i16 max_parts=-1)
+   	                   throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   // get the partitions matching the given partition filter
+   list<Partition> get_partitions_by_filter(1:string db_name 2:string tbl_name
+     3:string filter, 4:i16 max_parts=-1)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   // List partitions as PartitionSpec instances.
+   list<PartitionSpec> get_part_specs_by_filter(1:string db_name 2:string tbl_name
+     3:string filter, 4:i32 max_parts=-1)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   // get the partitions matching the given partition filter
+   // unlike get_partitions_by_filter, takes serialized hive expression, and with that can work
+   // with any filter (get_partitions_by_filter only works if the filter can be pushed down to JDOQL.
+   PartitionsByExprResult get_partitions_by_expr(1:PartitionsByExprRequest req)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   // get the partitions matching the given partition filter
+   i32 get_num_partitions_by_filter(1:string db_name 2:string tbl_name 3:string filter)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   // get partitions give a list of partition names
+   list<Partition> get_partitions_by_names(1:string db_name 2:string tbl_name 3:list<string> names)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   // changes the partition to the new partition object. partition is identified from the part values
+   // in the new_part
+   // * See notes on DDL_TIME
+   void alter_partition(1:string db_name, 2:string tbl_name, 3:Partition new_part)
+                        throws (1:InvalidOperationException o1, 2:MetaException o2)
+ 
+   // change a list of partitions. All partitions are altered atomically and all
+   // prehooks are fired together followed by all post hooks
+   void alter_partitions(1:string db_name, 2:string tbl_name, 3:list<Partition> new_parts)
+                        throws (1:InvalidOperationException o1, 2:MetaException o2)
+   void alter_partitions_with_environment_context(1:string db_name, 2:string tbl_name, 3:list<Partition> new_parts, 4:EnvironmentContext environment_context) throws (1:InvalidOperationException o1, 2:MetaException o2)
+ 
+   void alter_partition_with_environment_context(1:string db_name,
+       2:string tbl_name, 3:Partition new_part,
+       4:EnvironmentContext environment_context)
+       throws (1:InvalidOperationException o1, 2:MetaException o2)
+ 
+   // rename the old partition to the new partition object by changing old part values to the part values
+   // in the new_part. old partition is identified from part_vals.
+   // partition keys in new_part should be the same as those in old partition.
+   void rename_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals, 4:Partition new_part)
+                        throws (1:InvalidOperationException o1, 2:MetaException o2)
+ 
+   // returns whether or not the partition name is valid based on the value of the config
+   // hive.metastore.partition.name.whitelist.pattern
+   bool partition_name_has_valid_characters(1:list<string> part_vals, 2:bool throw_exception)
+  	throws(1: MetaException o1)
+ 
+   // gets the value of the configuration key in the metastore server. returns
+   // defaultValue if the key does not exist. if the configuration key does not
+   // begin with "hive", "mapred", or "hdfs", a ConfigValSecurityException is
+   // thrown.
+   string get_config_value(1:string name, 2:string defaultValue)
+                           throws(1:ConfigValSecurityException o1)
+ 
+   // converts a partition name into a partition values array
+   list<string> partition_name_to_vals(1: string part_name)
+                           throws(1: MetaException o1)
+   // converts a partition name into a partition specification (a mapping from
+   // the partition cols to the values)
+   map<string, string> partition_name_to_spec(1: string part_name)
+                           throws(1: MetaException o1)
+ 
+   void markPartitionForEvent(1:string db_name, 2:string tbl_name, 3:map<string,string> part_vals,
+                   4:PartitionEventType eventType) throws (1: MetaException o1, 2: NoSuchObjectException o2,
+                   3: UnknownDBException o3, 4: UnknownTableException o4, 5: UnknownPartitionException o5,
+                   6: InvalidPartitionException o6)
+   bool isPartitionMarkedForEvent(1:string db_name, 2:string tbl_name, 3:map<string,string> part_vals,
+                   4: PartitionEventType eventType) throws (1: MetaException o1, 2:NoSuchObjectException o2,
+                   3: UnknownDBException o3, 4: UnknownTableException o4, 5: UnknownPartitionException o5,
+                   6: InvalidPartitionException o6)
+ 
+   //index
+   Index add_index(1:Index new_index, 2: Table index_table)
+                        throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+   void alter_index(1:string dbname, 2:string base_tbl_name, 3:string idx_name, 4:Index new_idx)
+                        throws (1:InvalidOperationException o1, 2:MetaException o2)
+   bool drop_index_by_name(1:string db_name, 2:string tbl_name, 3:string index_name, 4:bool deleteData)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o2)
+   Index get_index_by_name(1:string db_name 2:string tbl_name, 3:string index_name)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   list<Index> get_indexes(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
+                        throws(1:NoSuchObjectException o1, 2:MetaException o2)
+   list<string> get_index_names(1:string db_name, 2:string tbl_name, 3:i16 max_indexes=-1)
+                        throws(1:MetaException o2)
+ 
+   //primary keys and foreign keys
+   PrimaryKeysResponse get_primary_keys(1:PrimaryKeysRequest request)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+   ForeignKeysResponse get_foreign_keys(1:ForeignKeysRequest request)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+   // other constraints
+   UniqueConstraintsResponse get_unique_constraints(1:UniqueConstraintsRequest request)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+   NotNullConstraintsResponse get_not_null_constraints(1:NotNullConstraintsRequest request)
+                        throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   // column statistics interfaces
+ 
+   // update APIs persist the column statistics object(s) that are passed in. If statistics already
+   // exists for one or more columns, the existing statistics will be overwritten. The update APIs
+   // validate that the dbName, tableName, partName, colName[] passed in as part of the ColumnStatistics
+   // struct are valid, throws InvalidInputException/NoSuchObjectException if found to be invalid
+   bool update_table_column_statistics(1:ColumnStatistics stats_obj) throws (1:NoSuchObjectException o1,
+               2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
+   bool update_partition_column_statistics(1:ColumnStatistics stats_obj) throws (1:NoSuchObjectException o1,
+               2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
+ 
+   // get APIs return the column statistics corresponding to db_name, tbl_name, [part_name], col_name if
+   // such statistics exists. If the required statistics doesn't exist, get APIs throw NoSuchObjectException
+   // For instance, if get_table_column_statistics is called on a partitioned table for which only
+   // partition level column stats exist, get_table_column_statistics will throw NoSuchObjectException
+   ColumnStatistics get_table_column_statistics(1:string db_name, 2:string tbl_name, 3:string col_name) throws
+               (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidInputException o3, 4:InvalidObjectException o4)
+   ColumnStatistics get_partition_column_statistics(1:string db_name, 2:string tbl_name, 3:string part_name,
+                4:string col_name) throws (1:NoSuchObjectException o1, 2:MetaException o2,
+                3:InvalidInputException o3, 4:InvalidObjectException o4)
+   TableStatsResult get_table_statistics_req(1:TableStatsRequest request) throws
+               (1:NoSuchObjectException o1, 2:MetaException o2)
+   PartitionsStatsResult get_partitions_statistics_req(1:PartitionsStatsRequest request) throws
+               (1:NoSuchObjectException o1, 2:MetaException o2)
+   AggrStats get_aggr_stats_for(1:PartitionsStatsRequest request) throws
+               (1:NoSuchObjectException o1, 2:MetaException o2)
+   bool set_aggr_stats_for(1:SetPartitionsStatsRequest request) throws
+               (1:NoSuchObjectException o1, 2:InvalidObjectException o2, 3:MetaException o3, 4:InvalidInputException o4)
+ 
+ 
+   // delete APIs attempt to delete column statistics, if found, associated with a given db_name, tbl_name, [part_name]
+   // and col_name. If the delete API doesn't find the statistics record in the metastore, throws NoSuchObjectException
+   // Delete API validates the input and if the input is invalid throws InvalidInputException/InvalidObjectException.
+   bool delete_partition_column_statistics(1:string db_name, 2:string tbl_name, 3:string part_name, 4:string col_name) throws
+               (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
+                4:InvalidInputException o4)
+   bool delete_table_column_statistics(1:string db_name, 2:string tbl_name, 3:string col_name) throws
+               (1:NoSuchObjectException o1, 2:MetaException o2, 3:InvalidObjectException o3,
+                4:InvalidInputException o4)
+ 
+   //
+   // user-defined functions
+   //
+ 
+   void create_function(1:Function func)
+       throws (1:AlreadyExistsException o1,
+               2:InvalidObjectException o2,
+               3:MetaException o3,
+               4:NoSuchObjectException o4)
+ 
+   void drop_function(1:string dbName, 2:string funcName)
+       throws (1:NoSuchObjectException o1, 2:MetaException o3)
+ 
+   void alter_function(1:string dbName, 2:string funcName, 3:Function newFunc)
+       throws (1:InvalidOperationException o1, 2:MetaException o2)
+ 
+   list<string> get_functions(1:string dbName, 2:string pattern)
+       throws (1:MetaException o1)
+   Function get_function(1:string dbName, 2:string funcName)
+       throws (1:MetaException o1, 2:NoSuchObjectException o2)
+ 
+   GetAllFunctionsResponse get_all_functions() throws (1:MetaException o1)
+ 
+   //authorization privileges
+ 
+   bool create_role(1:Role role) throws(1:MetaException o1)
+   bool drop_role(1:string role_name) throws(1:MetaException o1)
+   list<string> get_role_names() throws(1:MetaException o1)
+   // Deprecated, use grant_revoke_role()
+   bool grant_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type,
+     4:string grantor, 5:PrincipalType grantorType, 6:bool grant_option) throws(1:MetaException o1)
+   // Deprecated, use grant_revoke_role()
+   bool revoke_role(1:string role_name, 2:string principal_name, 3:PrincipalType principal_type)
+                         throws(1:MetaException o1)
+   list<Role> list_roles(1:string principal_name, 2:PrincipalType principal_type) throws(1:MetaException o1)
+   GrantRevokeRoleResponse grant_revoke_role(1:GrantRevokeRoleRequest request) throws(1:MetaException o1)
+ 
+   // get all role-grants for users/roles that have been granted the given role
+   // Note that in the returned list of RolePrincipalGrants, the roleName is
+   // redundant as it would match the role_name argument of this function
+   GetPrincipalsInRoleResponse get_principals_in_role(1: GetPrincipalsInRoleRequest request) throws(1:MetaException o1)
+ 
+   // get grant information of all roles granted to the given principal
+   // Note that in the returned list of RolePrincipalGrants, the principal name,type is
+   // redundant as it would match the principal name,type arguments of this function
+   GetRoleGrantsForPrincipalResponse get_role_grants_for_principal(1: GetRoleGrantsForPrincipalRequest request) throws(1:MetaException o1)
+ 
+   PrincipalPrivilegeSet get_privilege_set(1:HiveObjectRef hiveObject, 2:string user_name,
+     3: list<string> group_names) throws(1:MetaException o1)
+   list<HiveObjectPrivilege> list_privileges(1:string principal_name, 2:PrincipalType principal_type,
+     3: HiveObjectRef hiveObject) throws(1:MetaException o1)
+ 
+   // Deprecated, use grant_revoke_privileges()
+   bool grant_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
+   // Deprecated, use grant_revoke_privileges()
+   bool revoke_privileges(1:PrivilegeBag privileges) throws(1:MetaException o1)
+   GrantRevokePrivilegeResponse grant_revoke_privileges(1:GrantRevokePrivilegeRequest request) throws(1:MetaException o1);
+ 
+   // this is used by metastore client to send UGI information to metastore server immediately
+   // after setting up a connection.
+   list<string> set_ugi(1:string user_name, 2:list<string> group_names) throws (1:MetaException o1)
+ 
+   //Authentication (delegation token) interfaces
+ 
+   // get metastore server delegation token for use from the map/reduce tasks to authenticate
+   // to metastore server
+   string get_delegation_token(1:string token_owner, 2:string renewer_kerberos_principal_name)
+     throws (1:MetaException o1)
+ 
+   // method to renew delegation token obtained from metastore server
+   i64 renew_delegation_token(1:string token_str_form) throws (1:MetaException o1)
+ 
+   // method to cancel delegation token obtained from metastore server
+   void cancel_delegation_token(1:string token_str_form) throws (1:MetaException o1)
+ 
+   // add a delegation token
+   bool add_token(1:string token_identifier, 2:string delegation_token)
+ 
+   // remove a delegation token
+   bool remove_token(1:string token_identifier)
+ 
+   // get a delegation token by identifier
+   string get_token(1:string token_identifier)
+ 
+   // get all delegation token identifiers
+   list<string> get_all_token_identifiers()
+ 
+   // add master key
+   i32 add_master_key(1:string key) throws (1:MetaException o1)
+ 
+   // update master key
+   void update_master_key(1:i32 seq_number, 2:string key) throws (1:NoSuchObjectException o1, 2:MetaException o2)
+ 
+   // remove master key
+   bool remove_master_key(1:i32 key_seq)
+ 
+   // get master keys
+   list<string> get_master_keys()
+ 
+   // Transaction and lock management calls
+   // Get just list of open transactions
+   GetOpenTxnsResponse get_open_txns()
+   // Get list of open transactions with state (open, aborted)
+   GetOpenTxnsInfoResponse get_open_txns_info()
+   OpenTxnsResponse open_txns(1:OpenTxnRequest rqst)
+   void abort_txn(1:AbortTxnRequest rqst) throws (1:NoSuchTxnException o1)
+   void abort_txns(1:AbortTxnsRequest rqst) throws (1:NoSuchTxnException o1)
+   void commit_txn(1:CommitTxnRequest rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2)
+   LockResponse lock(1:LockRequest rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2)
+   LockResponse check_lock(1:CheckLockRequest rqst)
+     throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2, 3:NoSuchLockException o3)
+   void unlock(1:UnlockRequest rqst) throws (1:NoSuchLockException o1, 2:TxnOpenException o2)
+   ShowLocksResponse show_locks(1:ShowLocksRequest rqst)
+   void heartbeat(1:HeartbeatRequest ids) throws (1:NoSuchLockException o1, 2:NoSuchTxnException o2, 3:TxnAbortedException o3)
+   HeartbeatTxnRangeResponse heartbeat_txn_range(1:HeartbeatTxnRangeRequest txns)
+   void compact(1:CompactionRequest rqst) 
+   CompactionResponse compact2(1:CompactionRequest rqst) 
+   ShowCompactResponse show_compact(1:ShowCompactRequest rqst)
+   void add_dynamic_partitions(1:AddDynamicPartitions rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2)
+ 
+   // Notification logging calls
+   NotificationEventResponse get_next_notification(1:NotificationEventRequest rqst) 
+   CurrentNotificationEventId get_current_notificationEventId()
+   NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst)
+   FireEventResponse fire_listener_event(1:FireEventRequest rqst)
+   void flushCache()
+ 
+   // Repl Change Management api
+   CmRecycleResponse cm_recycle(1:CmRecycleRequest request) throws(1:MetaException o1)
+ 
+   GetFileMetadataByExprResult get_file_metadata_by_expr(1:GetFileMetadataByExprRequest req)
+   GetFileMetadataResult get_file_metadata(1:GetFileMetadataRequest req)
+   PutFileMetadataResult put_file_metadata(1:PutFileMetadataRequest req)
+   ClearFileMetadataResult clear_file_metadata(1:ClearFileMetadataRequest req)
+   CacheFileMetadataResult cache_file_metadata(1:CacheFileMetadataRequest req)
+ 
+   // Metastore DB properties
+   string get_metastore_db_uuid() throws (1:MetaException o1)
+ }
+ 
+ // * Note about the DDL_TIME: When creating or altering a table or a partition,
+ // if the DDL_TIME is not set, the current time will be used.
+ 
+ // For storing info about archived partitions in parameters
+ 
+ // Whether the partition is archived
+ const string IS_ARCHIVED = "is_archived",
+ // The original location of the partition, before archiving. After archiving,
+ // this directory will contain the archive. When the partition
+ // is dropped, this directory will be deleted
+ const string ORIGINAL_LOCATION = "original_location",
+ 
+ // Whether or not the table is considered immutable - immutable tables can only be
+ // overwritten or created if unpartitioned, or if partitioned, partitions inside them
+ // can only be overwritten or created. Immutability supports write-once and replace
+ // semantics, but not append.
+ const string IS_IMMUTABLE = "immutable",
+ 
+ // these should be needed only for backward compatibility with filestore
+ const string META_TABLE_COLUMNS   = "columns",
+ const string META_TABLE_COLUMN_TYPES   = "columns.types",
+ const string BUCKET_FIELD_NAME    = "bucket_field_name",
+ const string BUCKET_COUNT         = "bucket_count",
+ const string FIELD_TO_DIMENSION   = "field_to_dimension",
+ const string META_TABLE_NAME      = "name",
+ const string META_TABLE_DB        = "db",
+ const string META_TABLE_LOCATION  = "location",
+ const string META_TABLE_SERDE     = "serde",
+ const string META_TABLE_PARTITION_COLUMNS = "partition_columns",
+ const string META_TABLE_PARTITION_COLUMN_TYPES = "partition_columns.types",
+ const string FILE_INPUT_FORMAT    = "file.inputformat",
+ const string FILE_OUTPUT_FORMAT   = "file.outputformat",
+ const string META_TABLE_STORAGE   = "storage_handler",
+ const string TABLE_IS_TRANSACTIONAL = "transactional",
+ const string TABLE_NO_AUTO_COMPACT = "no_auto_compaction",
+ const string TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties",
+ 
+

[32/50] [abbrv] hive git commit: cherry picked HIVE-15899 from master

Posted by se...@apache.org.

cherry picked HIVE-15899 from master


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5b3c7979
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5b3c7979
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5b3c7979

Branch: refs/heads/master
Commit: 5b3c79798e28fd3c7f5718e361efee0f77eec047
Parents: 6f0be79
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Sep 26 19:42:06 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Sep 26 19:48:41 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/TestAcidOnTez.java    | 329 +++++++++++++++----
 .../java/org/apache/hadoop/hive/ql/Context.java |   3 +
 .../java/org/apache/hadoop/hive/ql/Driver.java  |   6 +-
 .../apache/hadoop/hive/ql/exec/MoveTask.java    |  13 +-
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java |  12 +
 .../apache/hadoop/hive/ql/metadata/Hive.java    |  43 ++-
 .../ql/optimizer/QueryPlanPostProcessor.java    | 166 ++++++++++
 .../optimizer/unionproc/UnionProcFactory.java   |   3 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  35 +-
 .../apache/hadoop/hive/ql/plan/LoadDesc.java    |  22 +-
 .../hadoop/hive/ql/plan/LoadFileDesc.java       |  22 +-
 .../hadoop/hive/ql/plan/LoadTableDesc.java      |  29 +-
 .../hadoop/hive/ql/plan/TezEdgeProperty.java    |  10 +-
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 189 +++++++++--
 .../clientpositive/autoColumnStats_4.q.out      |   1 +
 .../clientpositive/llap/acid_no_buckets.q.out   |   8 +
 .../llap/dynamic_semijoin_reduction_3.q.out     |   7 +
 .../llap/dynpart_sort_optimization_acid.q.out   |  12 +
 .../results/clientpositive/llap/mm_all.q.out    |   1 +
 .../results/clientpositive/llap/sqlmerge.q.out  |   4 +
 ql/src/test/results/clientpositive/mm_all.q.out |   1 +
 21 files changed, 760 insertions(+), 156 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
index d0b5cf6..8b4b21f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
 import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator;
 import org.apache.hadoop.hive.ql.io.BucketCodec;
 import org.apache.hadoop.hive.ql.io.HiveInputFormat;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
@@ -88,7 +89,6 @@ public class TestAcidOnTez {
     hiveConf = new HiveConf(this.getClass());
     hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
     hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
-    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
     hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR);
     hiveConf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
     hiveConf.setVar(HiveConf.ConfVars.HIVEINPUTFORMAT, HiveInputFormat.class.getName());
@@ -179,14 +179,14 @@ public class TestAcidOnTez {
     runStatementOnDriver("create table " + Table.NONACIDNONBUCKET + " stored as ORC TBLPROPERTIES('transactional'='false') as " +
       "select a, b from " + Table.ACIDTBL + " where a <= 5 union all select a, b from " + Table.NONACIDORCTBL + " where a >= 5", confForTez);
 
-    List<String> rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by a, b, INPUT__FILE__NAME");
+    List<String> rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by a, b, INPUT__FILE__NAME", confForTez);
     String expected0[][] = {
-      {"1\t2", "/1/000000_0"},
-      {"3\t4", "/1/000000_0"},
-      {"5\t6", "/1/000000_0"},
-      {"5\t6", "/2/000000_0"},
-      {"7\t8", "/2/000000_0"},
-      {"9\t10", "/2/000000_0"},
+      {"1\t2", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"3\t4", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"5\t6", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"5\t6", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
+      {"7\t8", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
+      {"9\t10", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
     };
     Assert.assertEquals("Unexpected row count after ctas", expected0.length, rs.size());
     //verify data and layout
@@ -195,9 +195,9 @@ public class TestAcidOnTez {
       Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected0[i][1]));
     }
     //make the table ACID
-    runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " SET TBLPROPERTIES ('transactional'='true')");
+    runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " SET TBLPROPERTIES ('transactional'='true')", confForTez);
 
-    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID");
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", confForTez);
     LOG.warn("after ctas:");
     for (String s : rs) {
       LOG.warn(s);
@@ -206,12 +206,12 @@ public class TestAcidOnTez {
     /*
     * Expected result 0th entry i the RecordIdentifier + data.  1st entry file before compact*/
     String expected[][] = {
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6", "/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10", "/2/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t7\t8", "/2/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t5\t6", "/2/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t7\t8", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t5\t6", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
     };
     Assert.assertEquals("Unexpected row count after ctas", expected.length, rs.size());
     //verify data and layout
@@ -220,17 +220,17 @@ public class TestAcidOnTez {
       Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][1]));
     }
     //perform some Update/Delete
-    runStatementOnDriver("update " + Table.NONACIDNONBUCKET + " set a = 70, b  = 80 where a = 7");
-    runStatementOnDriver("delete from " + Table.NONACIDNONBUCKET + " where a = 5");
-    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID");
+    runStatementOnDriver("update " + Table.NONACIDNONBUCKET + " set a = 70, b  = 80 where a = 7", confForTez);
+    runStatementOnDriver("delete from " + Table.NONACIDNONBUCKET + " where a = 5", confForTez);
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", confForTez);
     LOG.warn("after update/delete:");
     for (String s : rs) {
       LOG.warn(s);
     }
     String[][] expected2 = {
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10", "/2/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
       {"{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t70\t80", "delta_0000021_0000021_0000/bucket_00000"}
     };
     Assert.assertEquals("Unexpected row count after update", expected2.length, rs.size());
@@ -255,9 +255,9 @@ public class TestAcidOnTez {
       Assert.assertNull("at " + i + " " + expectedDelDelta[i] + " not found on disk", expectedDelDelta[i]);
     }
     //run Minor compaction
-    runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " compact 'minor'");
+    runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " compact 'minor'", confForTez);
     TestTxnCommands2.runWorker(hiveConf);
-    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID");
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", confForTez);
     LOG.warn("after compact minor:");
     for (String s : rs) {
       LOG.warn(s);
@@ -285,9 +285,9 @@ public class TestAcidOnTez {
       Assert.assertNull("at " + i + " " + expectedDelDelta2[i] + " not found on disk", expectedDelDelta2[i]);
     }
     //run Major compaction
-    runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " compact 'major'");
+    runStatementOnDriver("alter table " + Table.NONACIDNONBUCKET + " compact 'major'", confForTez);
     TestTxnCommands2.runWorker(hiveConf);
-    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID");
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.NONACIDNONBUCKET + " order by ROW__ID", confForTez);
     LOG.warn("after compact major:");
     for (String s : rs) {
       LOG.warn(s);
@@ -306,8 +306,12 @@ public class TestAcidOnTez {
    * How to do this?  CTAS is the only way to create data files which are not immediate children
    * of the partition dir.  CTAS/Union/Tez doesn't support partition tables.  The only way is to copy
    * data files in directly.
+   *
+   * Actually Insert Into ... select ... union all ... with
+   * HIVE_OPTIMIZE_UNION_REMOVE (and HIVEFETCHTASKCONVERSION="none"?) will create subdirs
+   * but if writing to non acid table there is a merge task on MR (but not on Tez)
    */
-  @Ignore("HIVE-17214")
+  @Ignore("HIVE-17214")//this consistently works locally but never in ptest....
   @Test
   public void testNonStandardConversion02() throws Exception {
     HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf
@@ -320,13 +324,13 @@ public class TestAcidOnTez {
       "union all select a, b from " + Table.ACIDTBL + " where a = 5", confForTez);
 
     List<String> rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from " +
-      Table.NONACIDNONBUCKET + " order by a, b");
+      Table.NONACIDNONBUCKET + " order by a, b", confForTez);
     String expected0[][] = {
-      {"1\t2", "/1/000000_0"},
-      {"3\t4", "/1/000000_0"},
-      {"5\t6", "/3/000000_0"},
-      {"7\t8", "/2/000000_0"},
-      {"9\t10", "/2/000000_0"},
+      {"1\t2", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"3\t4", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"5\t6", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "3/000000_0"},
+      {"7\t8", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
+      {"9\t10", AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
     };
     Assert.assertEquals("Unexpected row count after ctas", expected0.length, rs.size());
     //verify data and layout
@@ -338,7 +342,7 @@ public class TestAcidOnTez {
     FileStatus[] status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
       (Table.NONACIDNONBUCKET).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
     //ensure there is partition dir
-    runStatementOnDriver("insert into " + Table.NONACIDPART + " partition (p=1) values (100,110)");
+    runStatementOnDriver("insert into " + Table.NONACIDPART + " partition (p=1) values (100,110)", confForTez);
     //creates more files in that partition
     for(FileStatus stat : status) {
       int limit = 5;
@@ -357,29 +361,29 @@ public class TestAcidOnTez {
     nonacidpart/
     └── p=1
     ├── 000000_0
-    ├── 1
+    ├── HIVE_UNION_SUBDIR__1
     │   └── 000000_0
-    ├── 2
+    ├── HIVE_UNION_SUBDIR_2
     │   └── 000000_0
-    └── 3
+    └── HIVE_UNION_SUBDIR_3
         └── 000000_0
 
 4 directories, 4 files
     **/
     //make the table ACID
-    runStatementOnDriver("alter table " + Table.NONACIDPART + " SET TBLPROPERTIES ('transactional'='true')");
-    rs = runStatementOnDriver("select ROW__ID, a, b, p, INPUT__FILE__NAME from " + Table.NONACIDPART + " order by ROW__ID");
+    runStatementOnDriver("alter table " + Table.NONACIDPART + " SET TBLPROPERTIES ('transactional'='true')", confForTez);
+    rs = runStatementOnDriver("select ROW__ID, a, b, p, INPUT__FILE__NAME from " + Table.NONACIDPART + " order by ROW__ID", confForTez);
     LOG.warn("after acid conversion:");
     for (String s : rs) {
       LOG.warn(s);
     }
     String[][] expected = {
       {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t100\t110\t1", "nonacidpart/p=1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t1", "nonacidpart/p=1/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t3\t4\t1", "nonacidpart/p=1/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10\t1", "nonacidpart/p=1/2/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t7\t8\t1", "nonacidpart/p=1/2/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t5\t6\t1", "nonacidpart/p=1/3/000000_0"}
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t1\t2\t1", "nonacidpart/p=1/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t3\t4\t1", "nonacidpart/p=1/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10\t1", "nonacidpart/p=1/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t7\t8\t1", "nonacidpart/p=1/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t5\t6\t1", "nonacidpart/p=1/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "3/000000_0"}
     };
     Assert.assertEquals("Wrong row count", expected.length, rs.size());
     //verify data and layout
@@ -389,9 +393,9 @@ public class TestAcidOnTez {
     }
 
     //run Major compaction
-    runStatementOnDriver("alter table " + Table.NONACIDPART + " partition (p=1) compact 'major'");
+    runStatementOnDriver("alter table " + Table.NONACIDPART + " partition (p=1) compact 'major'", confForTez);
     TestTxnCommands2.runWorker(hiveConf);
-    rs = runStatementOnDriver("select ROW__ID, a, b, p, INPUT__FILE__NAME from " + Table.NONACIDPART + " order by ROW__ID");
+    rs = runStatementOnDriver("select ROW__ID, a, b, p, INPUT__FILE__NAME from " + Table.NONACIDPART + " order by ROW__ID", confForTez);
     LOG.warn("after major compaction:");
     for (String s : rs) {
       LOG.warn(s);
@@ -406,33 +410,42 @@ public class TestAcidOnTez {
   }
   /**
    * CTAS + Tez + Union creates a non-standard layout in table dir
-   * Each leg of the union places data into a subdir of the table/partition.  Subdirs are named 1/, 2/, etc
-   * The way this currently works is that CTAS creates an Acid table but the insert statement writes
-   * the data in non-acid layout.  Then on read, it's treated like an non-acid to acid conversion.
-   * Longer term CTAS should create acid layout from the get-go.
+   * Each leg of the union places data into a subdir of the table/partition.
+   * Subdirs are named HIVE_UNION_SUBDIR_1/, HIVE_UNION_SUBDIR_2/, etc
+   * For Acid tables the writer for each dir must have a different statementId ensured by
+   * {@link org.apache.hadoop.hive.ql.optimizer.QueryPlanPostProcessor}.
+   * {@link org.apache.hadoop.hive.ql.metadata.Hive#moveAcidFiles(FileSystem, FileStatus[], Path, List)} drops the union subdirs
+   * since each delta file has a unique name.
    */
   @Test
   public void testCtasTezUnion() throws Exception {
     HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf
+    confForTez.setBoolVar(HiveConf.ConfVars.HIVE_EXPLAIN_USER, false);
     setupTez(confForTez);
     //CTAS with ACID target table
+    List<String> rs0 = runStatementOnDriver("explain create table " + Table.ACIDNOBUCKET + " stored as ORC TBLPROPERTIES('transactional'='true') as " +
+      "select a, b from " + Table.ACIDTBL + " where a <= 5 union all select a, b from " + Table.NONACIDORCTBL + " where a >= 5", confForTez);
+    LOG.warn("explain ctas:");//TezEdgeProperty.EdgeType
+    for (String s : rs0) {
+      LOG.warn(s);
+    }
     runStatementOnDriver("create table " + Table.ACIDNOBUCKET + " stored as ORC TBLPROPERTIES('transactional'='true') as " +
       "select a, b from " + Table.ACIDTBL + " where a <= 5 union all select a, b from " + Table.NONACIDORCTBL + " where a >= 5", confForTez);
-    List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID");
+    List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", confForTez);
     LOG.warn("after ctas:");
     for (String s : rs) {
       LOG.warn(s);
     }
     Assert.assertEquals(0, BucketCodec.determineVersion(536870912).decodeWriterId(536870912));
     /*
-    * Expected result 0th entry i the RecordIdentifier + data.  1st entry file before compact*/
+    * Expected result 0th entry is the RecordIdentifier + data.  1st entry file before compact*/
     String expected[][] = {
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6", "/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10", "/2/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t7\t8", "/2/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":5}\t5\t6", "/2/000000_0"},
+      {"{\"transactionid\":18,\"bucketid\":536870913,\"rowid\":0}\t1\t2", "/delta_0000018_0000018_0001/bucket_00000"},
+      {"{\"transactionid\":18,\"bucketid\":536870913,\"rowid\":1}\t3\t4", "/delta_0000018_0000018_0001/bucket_00000"},
+      {"{\"transactionid\":18,\"bucketid\":536870913,\"rowid\":2}\t5\t6", "/delta_0000018_0000018_0001/bucket_00000"},
+      {"{\"transactionid\":18,\"bucketid\":536870914,\"rowid\":0}\t9\t10", "/delta_0000018_0000018_0002/bucket_00000"},
+      {"{\"transactionid\":18,\"bucketid\":536870914,\"rowid\":1}\t7\t8", "/delta_0000018_0000018_0002/bucket_00000"},
+      {"{\"transactionid\":18,\"bucketid\":536870914,\"rowid\":2}\t5\t6", "/delta_0000018_0000018_0002/bucket_00000"},
     };
     Assert.assertEquals("Unexpected row count after ctas", expected.length, rs.size());
     //verify data and layout
@@ -441,18 +454,18 @@ public class TestAcidOnTez {
       Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][1]));
     }
     //perform some Update/Delete
-    runStatementOnDriver("update " + Table.ACIDNOBUCKET + " set a = 70, b  = 80 where a = 7");
-    runStatementOnDriver("delete from " + Table.ACIDNOBUCKET + " where a = 5");
-    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID");
+    runStatementOnDriver("update " + Table.ACIDNOBUCKET + " set a = 70, b  = 80 where a = 7", confForTez);
+    runStatementOnDriver("delete from " + Table.ACIDNOBUCKET + " where a = 5", confForTez);
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", confForTez);
     LOG.warn("after update/delete:");
     for (String s : rs) {
       LOG.warn(s);
     }
     String[][] expected2 = {
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/1/000000_0"},
-      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t9\t10", "/2/000000_0"},
-      {"{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\t70\t80", "delta_0000019_0000019_0000/bucket_00000"}
+      {"{\"transactionid\":18,\"bucketid\":536870913,\"rowid\":0}\t1\t2", "/delta_0000018_0000018_0001/bucket_00000"},
+      {"{\"transactionid\":18,\"bucketid\":536870913,\"rowid\":1}\t3\t4", "/delta_0000018_0000018_0001/bucket_00000"},
+      {"{\"transactionid\":18,\"bucketid\":536870914,\"rowid\":0}\t9\t10", "/delta_0000018_0000018_0002/bucket_00000"},
+      {"{\"transactionid\":20,\"bucketid\":536870912,\"rowid\":0}\t70\t80", "delta_0000020_0000020_0000/bucket_00000"}
     };
     Assert.assertEquals("Unexpected row count after update", expected2.length, rs.size());
     //verify data and layout
@@ -464,7 +477,7 @@ public class TestAcidOnTez {
     FileSystem fs = FileSystem.get(hiveConf);
     FileStatus[] status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
       (Table.ACIDNOBUCKET).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
-    String[] expectedDelDelta = {"delete_delta_0000019_0000019_0000", "delete_delta_0000020_0000020_0000"};
+    String[] expectedDelDelta = {"delete_delta_0000020_0000020_0000", "delete_delta_0000021_0000021_0000"};
     for(FileStatus stat : status) {
       for(int i = 0; i < expectedDelDelta.length; i++) {
         if(expectedDelDelta[i] != null && stat.getPath().toString().endsWith(expectedDelDelta[i])) {
@@ -476,9 +489,9 @@ public class TestAcidOnTez {
       Assert.assertNull("at " + i + " " + expectedDelDelta[i] + " not found on disk", expectedDelDelta[i]);
     }
     //run Minor compaction
-    runStatementOnDriver("alter table " + Table.ACIDNOBUCKET + " compact 'minor'");
+    runStatementOnDriver("alter table " + Table.ACIDNOBUCKET + " compact 'minor'", confForTez);
     TestTxnCommands2.runWorker(hiveConf);
-    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID");
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", confForTez);
     LOG.warn("after compact minor:");
     for (String s : rs) {
       LOG.warn(s);
@@ -493,7 +506,7 @@ public class TestAcidOnTez {
     //check we have right delete delta files after minor compaction
     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
       (Table.ACIDNOBUCKET).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
-    String[] expectedDelDelta2 = { "delete_delta_0000019_0000019_0000", "delete_delta_0000020_0000020_0000", "delete_delta_0000019_0000020"};
+    String[] expectedDelDelta2 = { "delete_delta_0000020_0000020_0000", "delete_delta_0000021_0000021_0000", "delete_delta_0000018_0000021"};
     for(FileStatus stat : status) {
       for(int i = 0; i < expectedDelDelta2.length; i++) {
         if(expectedDelDelta2[i] != null && stat.getPath().toString().endsWith(expectedDelDelta2[i])) {
@@ -506,9 +519,9 @@ public class TestAcidOnTez {
       Assert.assertNull("at " + i + " " + expectedDelDelta2[i] + " not found on disk", expectedDelDelta2[i]);
     }
     //run Major compaction
-    runStatementOnDriver("alter table " + Table.ACIDNOBUCKET + " compact 'major'");
+    runStatementOnDriver("alter table " + Table.ACIDNOBUCKET + " compact 'major'", confForTez);
     TestTxnCommands2.runWorker(hiveConf);
-    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID");
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from " + Table.ACIDNOBUCKET + " order by ROW__ID", confForTez);
     LOG.warn("after compact major:");
     for (String s : rs) {
       LOG.warn(s);
@@ -517,7 +530,178 @@ public class TestAcidOnTez {
     for(int i = 0; i < expected2.length; i++) {
       Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
       //everything is now in base/
-      Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith("base_0000020/bucket_00000"));
+      Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith("base_0000021/bucket_00000"));
+    }
+  }
+  /**
+   * 1. Insert into regular unbucketed table from Union all - union is removed and data is placed in
+   * subdirs of target table.
+   * 2. convert to acid table and check data
+   * 3. compact and check data
+   * Compare with {@link #testAcidInsertWithRemoveUnion()} where T is transactional=true
+   */
+  @Test
+  public void testInsertWithRemoveUnion() throws Exception {
+    int[][] values = {{1,2},{3,4},{5,6},{7,8},{9,10}};
+    HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf
+    setupTez(confForTez);
+    runStatementOnDriver("drop table if exists T", confForTez);
+    runStatementOnDriver("create table T (a int, b int) stored as ORC  TBLPROPERTIES ('transactional'='false')", confForTez);
+    /*
+ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  ~/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505502329802/warehouse/t/.hive-staging_hive_2017-09-15_12-07-33_224_7717909516029836949-1/
+/Users/ekoifman/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505502329802/warehouse/t/.hive-staging_hive_2017-09-15_12-07-33_224_7717909516029836949-1/
+└── -ext-10000
+    ├── HIVE_UNION_SUBDIR_1
+    │   └── 000000_0
+    ├── HIVE_UNION_SUBDIR_2
+    │   └── 000000_0
+    └── HIVE_UNION_SUBDIR_3
+        └── 000000_0
+
+4 directories, 3 files
+     */
+    runStatementOnDriver("insert into T(a,b) select a, b from " + Table.ACIDTBL + " where a between 1 and 3 group by a, b union all select a, b from " + Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + Table.ACIDTBL + " where a >= 9", confForTez);
+    List<String> rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from T order by a, b, INPUT__FILE__NAME", confForTez);
+    LOG.warn(testName.getMethodName() + ": before converting to acid");
+    for(String s : rs) {
+      LOG.warn(s);
+    }
+    String[][] expected = {
+      {"1\t2","warehouse/t/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"3\t4","warehouse/t/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "1/000000_0"},
+      {"5\t6","warehouse/t/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
+      {"7\t8","warehouse/t/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "2/000000_0"},
+      {"9\t10","warehouse/t/" + AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "3/000000_0"}
+    };
+    Assert.assertEquals("Unexpected row count after conversion", expected.length, rs.size());
+    for(int i = 0; i < expected.length; i++) {
+      Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+      Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][1]));
+    }
+    //make the table ACID
+    runStatementOnDriver("alter table T SET TBLPROPERTIES ('transactional'='true')", confForTez);
+    rs = runStatementOnDriver("select a,b from T order by a, b", confForTez);
+    Assert.assertEquals("After to Acid conversion", TestTxnCommands2.stringifyValues(values), rs);
+
+    //run Major compaction
+    runStatementOnDriver("alter table T compact 'major'", confForTez);
+    TestTxnCommands2.runWorker(hiveConf);
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by ROW__ID", confForTez);
+    LOG.warn(testName.getMethodName() + ": after compact major of T:");
+    for (String s : rs) {
+      LOG.warn(s);
+    }
+    String[][] expected2 = {
+       {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "warehouse/t/base_-9223372036854775808/bucket_00000"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "warehouse/t/base_-9223372036854775808/bucket_00000"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":2}\t5\t6", "warehouse/t/base_-9223372036854775808/bucket_00000"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":3}\t7\t8", "warehouse/t/base_-9223372036854775808/bucket_00000"},
+      {"{\"transactionid\":0,\"bucketid\":536870912,\"rowid\":4}\t9\t10", "warehouse/t/base_-9223372036854775808/bucket_00000"}
+    };
+    Assert.assertEquals("Unexpected row count after major compact", expected2.length, rs.size());
+    for(int i = 0; i < expected2.length; i++) {
+      Assert.assertTrue("Actual line " + i + " ac: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
+      Assert.assertTrue("Actual line(file) " + i + " ac: " + rs.get(i), rs.get(i).endsWith(expected2[i][1]));
+    }
+  }
+  /**
+   * 1. Insert into unbucketed acid table from Union all - union is removed and data is placed in
+   * subdirs of target table.
+   * 2. convert to acid table and check data
+   * 3. compact and check data
+   * Compare with {@link #testInsertWithRemoveUnion()} where T is transactional=false
+   */
+  @Test
+  public void testAcidInsertWithRemoveUnion() throws Exception {
+    HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf
+    setupTez(confForTez);
+    int[][] values = {{1,2},{3,4},{5,6},{7,8},{9,10}};
+    runStatementOnDriver("drop table if exists T", confForTez);
+    runStatementOnDriver("create table T (a int, b int) stored as ORC  TBLPROPERTIES ('transactional'='true')", confForTez);
+    /*On Tez, below (T is transactional), we get the following layout
+ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  ~/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505500035574/warehouse/t/.hive-staging_hive_2017-09-15_11-28-33_960_9111484239090506828-1/
+/Users/ekoifman/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505500035574/warehouse/t/.hive-staging_hive_2017-09-15_11-28-33_960_9111484239090506828-1/
+└── -ext-10000
+    ├── HIVE_UNION_SUBDIR_1
+    │   └── 000000_0
+    │       ├── _orc_acid_version
+    │       └── delta_0000019_0000019_0001
+    │           └── bucket_00000
+    ├── HIVE_UNION_SUBDIR_2
+    │   └── 000000_0
+    │       ├── _orc_acid_version
+    │       └── delta_0000019_0000019_0002
+    │           └── bucket_00000
+    └── HIVE_UNION_SUBDIR_3
+        └── 000000_0
+            ├── _orc_acid_version
+            └── delta_0000019_0000019_0003
+                └── bucket_00000
+
+10 directories, 6 files     */
+    runStatementOnDriver("insert into T(a,b) select a, b from " + Table.ACIDTBL + " where a between 1 and 3 union all select a, b from " + Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + Table.ACIDTBL + " where a >= 9", confForTez);
+    List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b", confForTez);
+    LOG.warn(testName.getMethodName() + ": reading acid table T");
+    for(String s : rs) {
+      LOG.warn(s);
+    }
+
+    String[][] expected2 = {
+      {"{\"transactionid\":19,\"bucketid\":536870913,\"rowid\":0}\t1\t2", "warehouse/t/delta_0000019_0000019_0001/bucket_00000"},
+      {"{\"transactionid\":19,\"bucketid\":536870913,\"rowid\":1}\t3\t4", "warehouse/t/delta_0000019_0000019_0001/bucket_00000"},
+      {"{\"transactionid\":19,\"bucketid\":536870914,\"rowid\":0}\t5\t6", "warehouse/t/delta_0000019_0000019_0002/bucket_00000"},
+      {"{\"transactionid\":19,\"bucketid\":536870914,\"rowid\":1}\t7\t8", "warehouse/t/delta_0000019_0000019_0002/bucket_00000"},
+      {"{\"transactionid\":19,\"bucketid\":536870915,\"rowid\":0}\t9\t10", "warehouse/t/delta_0000019_0000019_0003/bucket_00000"}
+    };
+    Assert.assertEquals("Unexpected row count", expected2.length, rs.size());
+    for(int i = 0; i < expected2.length; i++) {
+      Assert.assertTrue("Actual line " + i + " ac: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
+      Assert.assertTrue("Actual line(file) " + i + " ac: " + rs.get(i), rs.get(i).endsWith(expected2[i][1]));
+    }
+  }
+  @Test
+  public void testBucketedAcidInsertWithRemoveUnion() throws Exception {
+    HiveConf confForTez = new HiveConf(hiveConf); // make a clone of existing hive conf
+    setupTez(confForTez);
+    int[][] values = {{1,2},{2,4},{5,6},{6,8},{9,10}};
+    runStatementOnDriver("delete from " + Table.ACIDTBL, confForTez);
+    runStatementOnDriver("insert into " + Table.ACIDTBL + TestTxnCommands2.makeValuesClause(values));//make sure both buckets are not empty
+    runStatementOnDriver("drop table if exists T", confForTez);
+    /*
+    With bucketed target table Union All is not removed
+
+    ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree  ~/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505510130462/warehouse/t/.hive-staging_hive_2017-09-15_14-16-32_422_4626314315862498838-1/
+/Users/ekoifman/dev/hiverwgit/itests/hive-unit/target/tmp/org.apache.hadoop.hive.ql.TestAcidOnTez-1505510130462/warehouse/t/.hive-staging_hive_2017-09-15_14-16-32_422_4626314315862498838-1/
+└── -ext-10000
+    ├── 000000_0
+    │   ├── _orc_acid_version
+    │   └── delta_0000021_0000021_0000
+    │       └── bucket_00000
+    └── 000001_0
+        ├── _orc_acid_version
+        └── delta_0000021_0000021_0000
+            └── bucket_00001
+
+5 directories, 4 files
+*/
+    runStatementOnDriver("create table T (a int, b int) clustered by (a) into 2 buckets stored as ORC  TBLPROPERTIES ('transactional'='true')", confForTez);
+    runStatementOnDriver("insert into T(a,b) select a, b from " + Table.ACIDTBL + " where a between 1 and 3 union all select a, b from " + Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + Table.ACIDTBL + " where a >= 9", confForTez);
+    List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b", confForTez);
+    LOG.warn(testName.getMethodName() + ": reading bucketed acid table T");
+    for(String s : rs) {
+      LOG.warn(s);
+    }
+    String[][] expected2 = {
+      {"{\"transactionid\":21,\"bucketid\":536936448,\"rowid\":0}\t1\t2", "warehouse/t/delta_0000021_0000021_0000/bucket_00001"},
+      {"{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t2\t4", "warehouse/t/delta_0000021_0000021_0000/bucket_00000"},
+      {"{\"transactionid\":21,\"bucketid\":536936448,\"rowid\":1}\t5\t6", "warehouse/t/delta_0000021_0000021_0000/bucket_00001"},
+      {"{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":1}\t6\t8", "warehouse/t/delta_0000021_0000021_0000/bucket_00000"},
+      {"{\"transactionid\":21,\"bucketid\":536936448,\"rowid\":2}\t9\t10", "warehouse/t/delta_0000021_0000021_0000/bucket_00001"}
+    };
+    Assert.assertEquals("Unexpected row count", expected2.length, rs.size());
+    for(int i = 0; i < expected2.length; i++) {
+      Assert.assertTrue("Actual line " + i + " ac: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
+      Assert.assertTrue("Actual line(file) " + i + " ac: " + rs.get(i), rs.get(i).endsWith(expected2[i][1]));
     }
   }
   // Ideally test like this should be a qfile test. However, the explain output from qfile is always
@@ -613,6 +797,7 @@ public class TestAcidOnTez {
   private List<String> runStatementOnDriver(String stmt, HiveConf conf)
       throws Exception {
     Driver driver = new Driver(conf);
+    driver.setMaxRows(10000);
     CommandProcessorResponse cpr = driver.run(stmt);
     if(cpr.getResponseCode() != 0) {
       throw new RuntimeException(stmt + " failed: " + cpr);

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index c66934a..d074701 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -1002,4 +1002,7 @@ public class Context {
   public void setIsUpdateDeleteMerge(boolean isUpdate) {
     this.isUpdateDeleteMerge = isUpdate;
   }
+  public String getExecutionId() {
+    return executionId;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 1157e00..b35edfb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1198,7 +1198,11 @@ public class Driver implements CommandProcessor {
       }
       // Set the transaction id in all of the acid file sinks
       if (haveAcidWrite()) {
-        for (FileSinkDesc desc : plan.getAcidSinks()) {
+        List<FileSinkDesc> acidSinks = new ArrayList<>(plan.getAcidSinks());
+        //sorting makes tests easier to write since file names and ROW__IDs depend on statementId
+        //so this makes (file name -> data) mapping stable
+        acidSinks.sort((FileSinkDesc fsd1, FileSinkDesc fsd2) -> fsd1.getDirName().compareTo(fsd2.getDirName()));
+        for (FileSinkDesc desc : acidSinks) {
           desc.setTransactionId(txnMgr.getCurrentTxnId());
           //it's possible to have > 1 FileSink writing to the same table/partition
           //e.g. Merge stmt, multi-insert stmt when mixing DP and SP writes

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 739ce18..b000745 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -289,10 +289,19 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
           Utilities.LOG14535.info("MoveTask not moving LFD " + sourcePath);
         } else {
           Utilities.LOG14535.info("MoveTask moving LFD " + sourcePath + " to " + targetPath);
-          moveFile(sourcePath, targetPath, lfd.getIsDfsDir());
+          if(lfd.getWriteType() == AcidUtils.Operation.INSERT) {
+            //'targetPath' is table root of un-partitioned table/partition
+            //'sourcePath' result of 'select ...' part of CTAS statement
+            assert lfd.getIsDfsDir();
+            FileSystem srcFs = sourcePath.getFileSystem(conf);
+            List<Path> newFiles = new ArrayList<>();
+            Hive.moveAcidFiles(srcFs, srcFs.globStatus(sourcePath), targetPath, newFiles);
+          }
+          else {
+            moveFile(sourcePath, targetPath, lfd.getIsDfsDir());
+          }
         }
       }
-
       // Multi-file load is for dynamic partitions when some partitions do not
       // need to merge and they can simply be moved to the target directory.
       // This is also used for MM table conversion.

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 3de83c6..c2d4612 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -42,6 +42,8 @@ import org.apache.hadoop.hive.metastore.TransactionalValidationListener;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.io.orc.OrcRecordUpdater;
 import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
 import org.apache.hadoop.hive.shims.ShimLoader;
@@ -1195,6 +1197,16 @@ public class AcidUtils {
 
     return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
   }
+  public static boolean isAcidTable(CreateTableDesc table) {
+    if (table == null || table.getTblProps() == null) {
+      return false;
+    }
+    String tableIsTransactional = table.getTblProps().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+    if (tableIsTransactional == null) {
+      tableIsTransactional = table.getTblProps().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.toUpperCase());
+    }
+    return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
+  }
 
   public static boolean isFullAcidTable(Table table) {
     return isAcidTable(table) && !MetaStoreUtils.isInsertOnlyTable(table.getParameters());

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index e51e648..d44d081 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -134,6 +134,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest;
 import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.FunctionTask;
 import org.apache.hadoop.hive.ql.exec.FunctionUtils;
@@ -3612,12 +3613,14 @@ private void constructOneLBLocationMap(FileStatus fSta,
     }
   }
 
-  private static void moveAcidFiles(FileSystem fs, FileStatus[] stats, Path dst,
+  public static void moveAcidFiles(FileSystem fs, FileStatus[] stats, Path dst,
                                     List<Path> newFiles) throws HiveException {
     // The layout for ACID files is table|partname/base|delta|delete_delta/bucket
-    // We will always only be writing delta files.  In the buckets created by FileSinkOperator
-    // it will look like bucket/delta|delete_delta/bucket.  So we need to move that into
-    // the above structure. For the first mover there will be no delta directory,
+    // We will always only be writing delta files ( except IOW which writes base_X/ ).
+    // In the buckets created by FileSinkOperator
+    // it will look like original_bucket/delta|delete_delta/bucket
+    // (e.g. .../-ext-10004/000000_0/delta_0000014_0000014_0000/bucket_00000).  So we need to
+    // move that into the above structure. For the first mover there will be no delta directory,
     // so we can move the whole directory.
     // For everyone else we will need to just move the buckets under the existing delta
     // directory.
@@ -3632,6 +3635,36 @@ private void constructOneLBLocationMap(FileStatus fSta,
       FileStatus[] origBucketStats = null;
       try {
         origBucketStats = fs.listStatus(srcPath, AcidUtils.originalBucketFilter);
+        if(origBucketStats == null || origBucketStats.length == 0) {
+          /**
+           check if we are dealing with data with non-standard layout. For example a write
+           produced by a (optimized) Union All query
+           which looks like
+          └── -ext-10000
+            ├── HIVE_UNION_SUBDIR_1
+            │   └── 000000_0
+            │       ├── _orc_acid_version
+            │       └── delta_0000019_0000019_0001
+            │           └── bucket_00000
+            ├── HIVE_UNION_SUBDIR_2
+            │   └── 000000_0
+            │       ├── _orc_acid_version
+            │       └── delta_0000019_0000019_0002
+            │           └── bucket_00000
+           The assumption is that we either have all data in subdirs or root of srcPath
+           but not both.
+           For Union case, we expect delta dirs to have unique names which is assured by
+           {@link org.apache.hadoop.hive.ql.optimizer.QueryPlanPostProcessor}
+          */
+          FileStatus[] unionSubdirs = fs.globStatus(new Path(srcPath,
+            AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + "[0-9]*"));
+          List<FileStatus> buckets = new ArrayList<>();
+          for(FileStatus unionSubdir : unionSubdirs) {
+            Collections.addAll(buckets,
+              fs.listStatus(unionSubdir.getPath(), AcidUtils.originalBucketFilter));
+          }
+          origBucketStats = buckets.toArray(new FileStatus[buckets.size()]);
+        }
       } catch (IOException e) {
         String msg = "Unable to look for bucket files in src path " + srcPath.toUri().toString();
         LOG.error(msg);
@@ -3645,7 +3678,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
                 fs, dst, origBucketPath, createdDeltaDirs, newFiles);
         moveAcidFiles(AcidUtils.DELETE_DELTA_PREFIX, AcidUtils.deleteEventDeltaDirFilter,
                 fs, dst,origBucketPath, createdDeltaDirs, newFiles);
-        moveAcidFiles(AcidUtils.BASE_PREFIX, AcidUtils.baseFileFilter,
+        moveAcidFiles(AcidUtils.BASE_PREFIX, AcidUtils.baseFileFilter,//for Insert Overwrite
                 fs, dst, origBucketPath, createdDeltaDirs, newFiles);
       }
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/optimizer/QueryPlanPostProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/QueryPlanPostProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/QueryPlanPostProcessor.java
new file mode 100644
index 0000000..b5bc386
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/QueryPlanPostProcessor.java
@@ -0,0 +1,166 @@
+package org.apache.hadoop.hive.ql.optimizer;
+
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorUtils;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
+import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadWork;
+import org.apache.hadoop.hive.ql.index.IndexMetadataChangeWork;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.parse.GenTezProcContext;
+import org.apache.hadoop.hive.ql.parse.GenTezWork;
+import org.apache.hadoop.hive.ql.parse.spark.GenSparkWork;
+import org.apache.hadoop.hive.ql.plan.ArchiveWork;
+import org.apache.hadoop.hive.ql.plan.BaseWork;
+import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork;
+import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
+import org.apache.hadoop.hive.ql.plan.ConditionalWork;
+import org.apache.hadoop.hive.ql.plan.CopyWork;
+import org.apache.hadoop.hive.ql.plan.DDLWork;
+import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork;
+import org.apache.hadoop.hive.ql.plan.ExplainSQRewriteWork;
+import org.apache.hadoop.hive.ql.plan.ExplainWork;
+import org.apache.hadoop.hive.ql.plan.FetchWork;
+import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
+import org.apache.hadoop.hive.ql.plan.FunctionWork;
+import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
+import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.MoveWork;
+import org.apache.hadoop.hive.ql.plan.SparkWork;
+import org.apache.hadoop.hive.ql.plan.StatsNoJobWork;
+import org.apache.hadoop.hive.ql.plan.StatsWork;
+import org.apache.hadoop.hive.ql.plan.TezWork;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Finds Acid FileSinkDesc objects which can be created in the physical (disconnected) plan, e.g.
+ * {@link org.apache.hadoop.hive.ql.parse.GenTezUtils#removeUnionOperators(GenTezProcContext, BaseWork, int)}
+ * so that statementId can be properly assigned to ensure unique ROW__IDs
+ * {@link org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcFactory} is another example where
+ * Union All optimizations create new FileSinkDescS
+ */
+public class QueryPlanPostProcessor {
+  private static final Logger LOG = LoggerFactory.getLogger(QueryPlanPostProcessor.class);
+
+  public QueryPlanPostProcessor(List<Task<?>> rootTasks, Set<FileSinkDesc> acidSinks, String executionId) {
+    for(Task<?> t : rootTasks) {
+      //Work
+      Object work = t.getWork();
+      if(work instanceof TezWork) {
+        for(BaseWork bw : ((TezWork)work).getAllWorkUnsorted()) {
+          collectFileSinkDescs(bw.getAllLeafOperators(), acidSinks);
+        }
+      }
+      else if(work instanceof BaseWork) {
+        collectFileSinkDescs(((BaseWork)work).getAllLeafOperators(), acidSinks);
+      }
+      else if(work instanceof MapredWork) {
+        MapredWork w = (MapredWork)work;
+        if(w.getMapWork() != null) {
+          collectFileSinkDescs(w.getMapWork().getAllLeafOperators(), acidSinks);
+        }
+        if(w.getReduceWork() != null) {
+          collectFileSinkDescs(w.getReduceWork().getAllLeafOperators(), acidSinks);
+        }
+      }
+      else if(work instanceof SparkWork) {
+        for(BaseWork bw : ((SparkWork)work).getRoots()) {
+          collectFileSinkDescs(bw.getAllLeafOperators(), acidSinks);
+        }
+      }
+      else if(work instanceof MapredLocalWork) {
+        //I don't think this can have any FileSinkOperatorS - more future proofing
+        Set<FileSinkOperator> fileSinkOperatorSet = OperatorUtils.findOperators(((MapredLocalWork)work).getAliasToWork().values(), FileSinkOperator.class);
+        for(FileSinkOperator fsop : fileSinkOperatorSet) {
+          collectFileSinkDescs(fsop, acidSinks);
+        }
+      }
+      else if(work instanceof ExplainWork) {
+        new QueryPlanPostProcessor(((ExplainWork)work).getRootTasks(), acidSinks, executionId);
+      }
+      /*
+      ekoifman:~ ekoifman$ cd dev/hiverwgit/ql/src/java/org/apache/
+ekoifman:apache ekoifman$ find . -name *Work.java
+./hadoop/hive/ql/exec/repl/bootstrap/ReplLoadWork.java
+./hadoop/hive/ql/exec/repl/ReplDumpWork.java
+./hadoop/hive/ql/exec/repl/ReplStateLogWork.java
+./hadoop/hive/ql/index/IndexMetadataChangeWork.java
+./hadoop/hive/ql/io/merge/MergeFileWork.java - extends MapWork
+./hadoop/hive/ql/io/rcfile/stats/PartialScanWork.java - extends MapWork
+./hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateWork.java - extends MapWork
+./hadoop/hive/ql/parse/GenTezWork.java
+./hadoop/hive/ql/parse/spark/GenSparkWork.java
+./hadoop/hive/ql/plan/ArchiveWork.java
+./hadoop/hive/ql/plan/BaseWork.java
+./hadoop/hive/ql/plan/ColumnStatsUpdateWork.java
+./hadoop/hive/ql/plan/ColumnStatsWork.java
+./hadoop/hive/ql/plan/ConditionalWork.java
+./hadoop/hive/ql/plan/CopyWork.java
+./hadoop/hive/ql/plan/DDLWork.java
+./hadoop/hive/ql/plan/DependencyCollectionWork.java
+./hadoop/hive/ql/plan/ExplainSQRewriteWork.java
+./hadoop/hive/ql/plan/ExplainWork.java
+./hadoop/hive/ql/plan/FetchWork.java
+./hadoop/hive/ql/plan/FunctionWork.java
+./hadoop/hive/ql/plan/MapredLocalWork.java
+./hadoop/hive/ql/plan/MapredWork.java
+./hadoop/hive/ql/plan/MapWork.java - extends BaseWork
+./hadoop/hive/ql/plan/MergeJoinWork.java - extends BaseWork
+./hadoop/hive/ql/plan/MoveWork.java
+./hadoop/hive/ql/plan/ReduceWork.java
+./hadoop/hive/ql/plan/ReplCopyWork.java - extends CopyWork
+./hadoop/hive/ql/plan/SparkWork.java
+./hadoop/hive/ql/plan/StatsNoJobWork.java
+./hadoop/hive/ql/plan/StatsWork.java
+./hadoop/hive/ql/plan/TezWork.java
+./hadoop/hive/ql/plan/UnionWork.java - extends BaseWork
+      */
+      else if(work instanceof ReplLoadWork ||
+        work instanceof ReplStateLogWork ||
+        work instanceof IndexMetadataChangeWork ||
+        work instanceof GenTezWork ||
+        work instanceof GenSparkWork ||
+        work instanceof ArchiveWork ||
+        work instanceof ColumnStatsUpdateWork ||
+        work instanceof ColumnStatsWork ||
+        work instanceof ConditionalWork ||
+        work instanceof CopyWork ||
+        work instanceof DDLWork ||
+        work instanceof DependencyCollectionWork ||
+        work instanceof ExplainSQRewriteWork ||
+        work instanceof FetchWork ||
+        work instanceof FunctionWork ||
+        work instanceof MoveWork ||
+        work instanceof StatsNoJobWork ||
+        work instanceof StatsWork) {
+        LOG.debug("Found " + work.getClass().getName() + " - no FileSinkOperation can be present.  executionId=" + executionId);
+      }
+      else {
+        //if here, someone must have added new Work object - should it be walked to find FileSinks?
+        throw new IllegalArgumentException("Unexpected Work object: " + work.getClass() + " executionId=" + executionId);
+      }
+    }
+  }
+  private void collectFileSinkDescs(Operator<?> leaf, Set<FileSinkDesc> acidSinks) {
+    if(leaf instanceof FileSinkOperator) {
+      FileSinkDesc fsd = ((FileSinkOperator) leaf).getConf();
+      if(fsd.getWriteType() != AcidUtils.Operation.NOT_ACID) {
+        if(acidSinks.add(fsd)) {
+          if(LOG.isDebugEnabled()) {
+            LOG.debug("Found Acid Sink: " + fsd.getDirName());
+          }
+        }
+      }
+    }
+  }
+  private void collectFileSinkDescs(Set<Operator<?>> leaves, Set<FileSinkDesc> acidSinks) {
+    for(Operator<?> leaf : leaves) {
+      collectFileSinkDescs(leaf, acidSinks);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
index 3a38a6d..bc26c5e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcFactory.java
@@ -24,6 +24,7 @@ import java.util.Map;
 import java.util.Stack;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator;
 import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.OperatorFactory;
@@ -220,7 +221,7 @@ public final class UnionProcFactory {
 
         for (Operator<? extends OperatorDesc> parent : parents) {
           FileSinkDesc fileSinkDesc = (FileSinkDesc) fileSinkOp.getConf().clone();
-          fileSinkDesc.setDirName(new Path(parentDirName, parent.getIdentifier()));
+          fileSinkDesc.setDirName(new Path(parentDirName, AbstractFileMergeOperator.UNION_SUDBIR_PREFIX + parent.getIdentifier()));
           fileSinkDesc.setLinkedFileSink(true);
           Utilities.LOG14535.info("Created LinkedFileSink for union " + fileSinkDesc.getDirName() + "; parent " + parentDirName);
           parent.setChildOperators(null);

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index f50b57e..3475c7c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -131,6 +131,7 @@ import org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
 import org.apache.hadoop.hive.ql.optimizer.Optimizer;
+import org.apache.hadoop.hive.ql.optimizer.QueryPlanPostProcessor;
 import org.apache.hadoop.hive.ql.optimizer.Transform;
 import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
 import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
@@ -7003,6 +7004,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         viewDesc.setSchema(new ArrayList<FieldSchema>(field_schemas));
       }
 
+      destTableIsAcid = tblDesc != null && AcidUtils.isAcidTable(tblDesc);
+
       boolean isDestTempFile = true;
       if (!ctx.isMRTmpFileURI(dest_path.toUri().toString())) {
         idToTableNameMap.put(String.valueOf(destTableId), dest_path.toUri().toString());
@@ -7013,9 +7016,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
 
       boolean isDfsDir = (dest_type.intValue() == QBMetaData.DEST_DFS_FILE);
       // Create LFD even for MM CTAS - it's a no-op move, but it still seems to be used for stats.
-      loadFileWork.add(new LoadFileDesc(tblDesc, viewDesc,
-          queryTmpdir, dest_path, isDfsDir, cols, colTypes, isMmCtas));
-
+      loadFileWork.add(new LoadFileDesc(tblDesc, viewDesc, queryTmpdir, dest_path, isDfsDir, cols,
+        colTypes, destTableIsAcid ? Operation.INSERT : Operation.NOT_ACID, isMmCtas));
       if (tblDesc == null) {
         if (viewDesc != null) {
           table_desc = PlanUtils.getTableDesc(viewDesc, cols, colTypes);
@@ -7104,7 +7106,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     FileSinkDesc fileSinkDesc = createFileSinkDesc(dest, table_desc, dest_part,
         dest_path, currentTableId, destTableIsAcid, destTableIsTemporary,
         destTableIsMaterialization, queryTmpdir, rsCtx, dpCtx, lbCtx, fsRS,
-        canBeMerged, dest_tab, txnId, isMmCtas);
+        canBeMerged, dest_tab, txnId, isMmCtas, dest_type);
     if (isMmCtas) {
       // Add FSD so that the LoadTask compilation could fix up its path to avoid the move.
       tableDesc.setWriter(fileSinkDesc);
@@ -7212,7 +7214,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
       boolean destTableIsAcid, boolean destTableIsTemporary,
       boolean destTableIsMaterialization, Path queryTmpdir,
       SortBucketRSCtx rsCtx, DynamicPartitionCtx dpCtx, ListBucketingCtx lbCtx,
-      RowSchema fsRS, boolean canBeMerged, Table dest_tab, Long mmWriteId, boolean isMmCtas) throws SemanticException {
+      RowSchema fsRS, boolean canBeMerged, Table dest_tab, Long mmWriteId, boolean isMmCtas,
+      Integer dest_type) throws SemanticException {
     FileSinkDesc fileSinkDesc = new FileSinkDesc(queryTmpdir, table_desc,
         conf.getBoolVar(HiveConf.ConfVars.COMPRESSRESULT), currentTableId, rsCtx.isMultiFileSpray(),
         canBeMerged, rsCtx.getNumFiles(), rsCtx.getTotalFiles(), rsCtx.getPartnCols(), dpCtx,
@@ -7236,10 +7239,22 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
           (deleting(dest) ? AcidUtils.Operation.DELETE : AcidUtils.Operation.INSERT);
       fileSinkDesc.setWriteType(wt);
 
-      String destTableFullName = dest_tab.getCompleteName().replace('@', '.');
-      Map<String, ASTNode> iowMap = qb.getParseInfo().getInsertOverwriteTables();
-      if (iowMap.containsKey(destTableFullName)) {
-        fileSinkDesc.setInsertOverwrite(true);
+      switch (dest_type) {
+        case QBMetaData.DEST_PARTITION:
+          //fall through
+        case QBMetaData.DEST_TABLE:
+          //INSERT [OVERWRITE] path
+          String destTableFullName = dest_tab.getCompleteName().replace('@', '.');
+          Map<String, ASTNode> iowMap = qb.getParseInfo().getInsertOverwriteTables();
+          if (iowMap.containsKey(destTableFullName)) {
+            fileSinkDesc.setInsertOverwrite(true);
+          }
+          break;
+        case QBMetaData.DEST_DFS_FILE:
+          //CTAS path
+          break;
+        default:
+          throw new IllegalStateException("Unexpected dest_type=" + dest_tab);
       }
       acidFileSinks.add(fileSinkDesc);
     }
@@ -11555,6 +11570,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
       compiler.compile(pCtx, rootTasks, inputs, outputs);
       fetchTask = pCtx.getFetchTask();
     }
+    //find all Acid FileSinkOperatorS
+    QueryPlanPostProcessor qp = new QueryPlanPostProcessor(rootTasks, acidFileSinks, ctx.getExecutionId());
     LOG.info("Completed plan generation");
 
     // 10. put accessed columns to readEntity

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java
index d708df3..45d4fb0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.plan;
 import java.io.Serializable;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.plan.Explain.Level;
 
 /**
@@ -30,12 +31,15 @@ import org.apache.hadoop.hive.ql.plan.Explain.Level;
 public class LoadDesc implements Serializable {
   private static final long serialVersionUID = 1L;
   private Path sourcePath;
+  /**
+   * Need to remember whether this is an acid compliant operation, and if so whether it is an
+   * insert, update, or delete.
+   */
+  private final AcidUtils.Operation writeType;
 
-  public LoadDesc() {
-  }
-
-  public LoadDesc(final Path sourcePath) {
+  public LoadDesc(final Path sourcePath, AcidUtils.Operation writeType) {
     this.sourcePath = sourcePath;
+    this.writeType = writeType;
   }
 
   @Explain(displayName = "source", explainLevels = { Level.EXTENDED })
@@ -46,4 +50,14 @@ public class LoadDesc implements Serializable {
   public void setSourcePath(Path sourcePath) {
     this.sourcePath = sourcePath;
   }
+
+  public AcidUtils.Operation getWriteType() {
+    return writeType;
+  }
+
+  @Explain(displayName = "Write Type")
+  public String getWriteTypeString() {
+    //if acid write, add to plan output, else don't bother
+    return getWriteType() == AcidUtils.Operation.NOT_ACID ? null : getWriteType().toString();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
index 6fad710..0032648 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
@@ -21,8 +21,8 @@ package org.apache.hadoop.hive.ql.plan;
 import java.io.Serializable;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.PTFUtils;
 import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 
 /**
  * LoadFileDesc.
@@ -38,11 +38,8 @@ public class LoadFileDesc extends LoadDesc implements Serializable {
   private String destinationCreateTable;
   private boolean isMmCtas;
 
-  public LoadFileDesc() {
-  }
-
   public LoadFileDesc(final LoadFileDesc o) {
-    super(o.getSourcePath());
+    super(o.getSourcePath(), o.getWriteType());
 
     this.targetDir = o.targetDir;
     this.isDfsDir = o.isDfsDir;
@@ -54,8 +51,8 @@ public class LoadFileDesc extends LoadDesc implements Serializable {
 
   public LoadFileDesc(final CreateTableDesc createTableDesc, final CreateViewDesc  createViewDesc,
                       final Path sourcePath, final Path targetDir, final boolean isDfsDir,
-                      final String columns, final String columnTypes, boolean isMmCtas) {
-    this(sourcePath, targetDir, isDfsDir, columns, columnTypes, isMmCtas);
+                      final String columns, final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) {
+   this(sourcePath, targetDir, isDfsDir, columns, columnTypes, writeType, isMmCtas);
     if (createTableDesc != null && createTableDesc.getDatabaseName() != null
         && createTableDesc.getTableName() != null) {
       destinationCreateTable = (createTableDesc.getTableName().contains(".") ? "" : createTableDesc
@@ -68,9 +65,14 @@ public class LoadFileDesc extends LoadDesc implements Serializable {
     }
   }
 
-  public LoadFileDesc(final Path sourcePath, final Path targetDir, final boolean isDfsDir,
-      final String columns, final String columnTypes, boolean isMmCtas) {
-    super(sourcePath);
+  public LoadFileDesc(final Path sourcePath, final Path targetDir,
+                      final boolean isDfsDir, final String columns, final String columnTypes, boolean isMmCtas) {
+    this(sourcePath, targetDir, isDfsDir, columns, columnTypes, AcidUtils.Operation.NOT_ACID, isMmCtas);
+  }
+  private LoadFileDesc(final Path sourcePath, final Path targetDir,
+      final boolean isDfsDir, final String columns,
+      final String columnTypes, AcidUtils.Operation writeType, boolean isMmCtas) {
+    super(sourcePath, writeType);
     Utilities.LOG14535.info("creating LFD from " + sourcePath + " to " + targetDir);
     this.targetDir = targetDir;
     this.isDfsDir = isDfsDir;

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
index 3201dc9..e893ab5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
@@ -32,17 +32,13 @@ import org.apache.hadoop.hive.ql.plan.Explain.Level;
  * LoadTableDesc.
  *
  */
-public class LoadTableDesc extends org.apache.hadoop.hive.ql.plan.LoadDesc
-    implements Serializable {
+public class LoadTableDesc extends LoadDesc implements Serializable {
   private static final long serialVersionUID = 1L;
   private boolean replace;
   private DynamicPartitionCtx dpCtx;
   private ListBucketingCtx lbCtx;
   private boolean inheritTableSpecs = true; //For partitions, flag controlling whether the current
                                             //table specs are to be used
-  // Need to remember whether this is an acid compliant operation, and if so whether it is an
-  // insert, update, or delete.
-  private AcidUtils.Operation writeType;
   private Long txnId;
   private int stmtId;
 
@@ -52,13 +48,12 @@ public class LoadTableDesc extends org.apache.hadoop.hive.ql.plan.LoadDesc
   private boolean commitMmWriteId = true;
 
   public LoadTableDesc(final LoadTableDesc o) {
-    super(o.getSourcePath());
+    super(o.getSourcePath(), o.getWriteType());
 
     this.replace = o.replace;
     this.dpCtx = o.dpCtx;
     this.lbCtx = o.lbCtx;
     this.inheritTableSpecs = o.inheritTableSpecs;
-    this.writeType = o.writeType;
     this.table = o.table;
     this.partitionSpec = o.partitionSpec;
   }
@@ -67,11 +62,12 @@ public class LoadTableDesc extends org.apache.hadoop.hive.ql.plan.LoadDesc
       final org.apache.hadoop.hive.ql.plan.TableDesc table,
       final Map<String, String> partitionSpec,
       final boolean replace,
-      final AcidUtils.Operation writeType, Long txnId) {
-    super(sourcePath);
+      final AcidUtils.Operation writeType,
+      Long txnId) {
+    super(sourcePath, writeType);
     Utilities.LOG14535.info("creating part LTD from " + sourcePath + " to "
         + ((table.getProperties() == null) ? "null" : table.getTableName()));
-    init(table, partitionSpec, replace, writeType, txnId);
+    init(table, partitionSpec, replace, txnId);
   }
 
   /**
@@ -113,13 +109,13 @@ public class LoadTableDesc extends org.apache.hadoop.hive.ql.plan.LoadDesc
       final DynamicPartitionCtx dpCtx,
       final AcidUtils.Operation writeType,
       boolean isReplace, Long txnId) {
-    super(sourcePath);
+    super(sourcePath, writeType);
     Utilities.LOG14535.info("creating LTD from " + sourcePath + " to " + table.getTableName()/*, new Exception()*/);
     this.dpCtx = dpCtx;
     if (dpCtx != null && dpCtx.getPartSpec() != null && partitionSpec == null) {
-      init(table, dpCtx.getPartSpec(), isReplace, writeType, txnId);
+      init(table, dpCtx.getPartSpec(), isReplace, txnId);
     } else {
-      init(table, new LinkedHashMap<String, String>(), isReplace, writeType, txnId);
+      init(table, new LinkedHashMap<String, String>(), isReplace, txnId);
     }
   }
 
@@ -127,11 +123,10 @@ public class LoadTableDesc extends org.apache.hadoop.hive.ql.plan.LoadDesc
       final org.apache.hadoop.hive.ql.plan.TableDesc table,
       final Map<String, String> partitionSpec,
       final boolean replace,
-      AcidUtils.Operation writeType, Long txnId) {
+      Long txnId) {
     this.table = table;
     this.partitionSpec = partitionSpec;
     this.replace = replace;
-    this.writeType = writeType;
     this.txnId = txnId;
   }
 
@@ -201,10 +196,6 @@ public class LoadTableDesc extends org.apache.hadoop.hive.ql.plan.LoadDesc
     this.lbCtx = lbCtx;
   }
 
-  public AcidUtils.Operation getWriteType() {
-    return writeType;
-  }
-
   public Long getTxnId() {
     return txnId;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
index b695f0f..bbed9be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TezEdgeProperty.java
@@ -22,12 +22,12 @@ import org.apache.hadoop.hive.conf.HiveConf;
 
 public class TezEdgeProperty {
 
-  public enum EdgeType {
-    SIMPLE_EDGE,
+  public enum EdgeType {//todo: HIVE-15549
+    SIMPLE_EDGE,//SORT_PARTITION_EDGE
     BROADCAST_EDGE,
-    CONTAINS,
-    CUSTOM_EDGE,
-    CUSTOM_SIMPLE_EDGE,
+    CONTAINS,//used for union (all?)
+    CUSTOM_EDGE,//CO_PARTITION_EDGE
+    CUSTOM_SIMPLE_EDGE,//PARTITION_EDGE
     ONE_TO_ONE_EDGE
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/5b3c7979/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index 1f0c269..880329d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -5,7 +5,10 @@ import org.apache.hadoop.hive.ql.io.BucketCodec;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -20,6 +23,8 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests {
     File.separator + TestTxnNoBuckets.class.getCanonicalName()
     + "-" + System.currentTimeMillis()
   ).getPath().replaceAll("\\\\", "/");
+  @Rule
+  public TestName testName = new TestName();
   @Override
   String getTestDataDir() {
     return TEST_DATA_DIR;
@@ -65,17 +70,7 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests {
     Assert.assertTrue(rs.get(2), rs.get(2).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00001"));
     Assert.assertTrue(rs.get(3), rs.get(3).startsWith("{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":1}\t2\t2\t2\t"));
     Assert.assertTrue(rs.get(3), rs.get(3).endsWith("nobuckets/delta_0000019_0000019_0000/bucket_00001"));
-    /*todo: WTF?
-    RS for update seems to spray randomly... is that OK?  maybe as long as all resultant files have different names... will they?
-    Assuming we name them based on taskId, we should create bucketX and bucketY.
-    we delete events can be written to bucketX file it could be useful for filter delete for a split by file name since the insert
-    events seem to be written to a proper bucketX file.  In fact this may reduce the number of changes elsewhere like compactor... maybe
-    But this limits the parallelism - what is worse, you don't know what the parallelism should be until you have a list of all the
-    input files since bucket count is no longer a metadata property.  Also, with late Update split, the file name has already been determined
-    from taskId so the Insert part won't end up matching the bucketX property necessarily.
-    With early Update split, the Insert can still be an insert - i.e. go to appropriate bucketX.  But deletes will still go wherever (random shuffle)
-    unless you know all the bucketX files to be read - may not be worth the trouble.
-    * 2nd: something in FS fails.  ArrayIndexOutOfBoundsException: 1 at FileSinkOperator.process(FileSinkOperator.java:779)*/
+
     runStatementOnDriver("update nobuckets set c3 = 17 where c3 in(0,1)");
     rs = runStatementOnDriver("select ROW__ID, c1, c2, c3, INPUT__FILE__NAME from nobuckets order by INPUT__FILE__NAME, ROW__ID");
     LOG.warn("after update");
@@ -152,15 +147,6 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests {
   }
 
   /**
-   * all of these pass but don't do exactly the right thing
-   * files land as if it's not an acid table "warehouse/myctas4/000000_0"
-   * even though in {@link org.apache.hadoop.hive.metastore.TransactionalValidationListener} fires
-   * and sees it as transactional table
-   * look for QB.isCTAS() and CreateTableDesc() in SemanticAnalyzer
-   *
-   * On read, these files are treated like non acid to acid conversion
-   *
-   * see HIVE-15899
    * See CTAS tests in TestAcidOnTez
    */
   @Test
@@ -169,30 +155,177 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests {
     runStatementOnDriver("insert into " + Table.NONACIDORCTBL +  makeValuesClause(values));
     runStatementOnDriver("create table myctas stored as ORC TBLPROPERTIES ('transactional" +
       "'='true', 'transactional_properties'='default') as select a, b from " + Table.NONACIDORCTBL);
-    List<String> rs = runStatementOnDriver("select * from myctas order by a, b");
-    Assert.assertEquals(stringifyValues(values), rs);
+    List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas order by ROW__ID");
+    String expected[][] = {
+      {"{\"transactionid\":14,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas/delta_0000014_0000014_0000/bucket_00000"},
+      {"{\"transactionid\":14,\"bucketid\":536870912,\"rowid\":1}\t1\t2", "warehouse/myctas/delta_0000014_0000014_0000/bucket_00000"},
+    };
+    checkExpected(rs, expected, "Unexpected row count after ctas from non acid table");
 
     runStatementOnDriver("insert into " + Table.ACIDTBL + makeValuesClause(values));
     runStatementOnDriver("create table myctas2 stored as ORC TBLPROPERTIES ('transactional" +
       "'='true', 'transactional_properties'='default') as select a, b from " + Table.ACIDTBL);
-    rs = runStatementOnDriver("select * from myctas2 order by a, b");
-    Assert.assertEquals(stringifyValues(values), rs);
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas2 order by ROW__ID");
+    String expected2[][] = {
+      {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas2/delta_0000017_0000017_0000/bucket_00000"},
+      {"{\"transactionid\":17,\"bucketid\":536870912,\"rowid\":1}\t1\t2", "warehouse/myctas2/delta_0000017_0000017_0000/bucket_00000"},
+    };
+    checkExpected(rs, expected2, "Unexpected row count after ctas from acid table");
 
     runStatementOnDriver("create table myctas3 stored as ORC TBLPROPERTIES ('transactional" +
       "'='true', 'transactional_properties'='default') as select a, b from " + Table.NONACIDORCTBL +
       " union all select a, b from " + Table.ACIDTBL);
-    rs = runStatementOnDriver("select * from myctas3 order by a, b");
-    Assert.assertEquals(stringifyValues(new int[][] {{1,2},{1,2},{3,4},{3,4}}), rs);
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas3 order by ROW__ID");
+    String expected3[][] = {
+      {"{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas3/delta_0000019_0000019_0000/bucket_00000"},
+      {"{\"transactionid\":19,\"bucketid\":536870912,\"rowid\":1}\t1\t2", "warehouse/myctas3/delta_0000019_0000019_0000/bucket_00000"},
+      {"{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":0}\t3\t4", "warehouse/myctas3/delta_0000019_0000019_0000/bucket_00001"},
+      {"{\"transactionid\":19,\"bucketid\":536936448,\"rowid\":1}\t1\t2", "warehouse/myctas3/delta_0000019_0000019_0000/bucket_00001"},
+    };
+    checkExpected(rs, expected3, "Unexpected row count after ctas from union all query");
 
     runStatementOnDriver("create table myctas4 stored as ORC TBLPROPERTIES ('transactional" +
       "'='true', 'transactional_properties'='default') as select a, b from " + Table.NONACIDORCTBL +
       " union distinct select a, b from " + Table.ACIDTBL);
-    rs = runStatementOnDriver("select * from myctas4 order by a, b");
+    rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas4 order by ROW__ID");
+    String expected4[][] = {
+      {"{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "/delta_0000021_0000021_0000/bucket_00000"},
+      {"{\"transactionid\":21,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "/delta_0000021_0000021_0000/bucket_00000"},
+    };
+    checkExpected(rs, expected4, "Unexpected row count after ctas from union distinct query");
+  }
+  /**
+   * Insert into unbucketed acid table from union all query
+   * Union All is flattend so nested subdirs are created and acid move drops them since
+   * delta dirs have unique names
+   */
+  @Test
+  public void testInsertToAcidWithUnionRemove() throws Exception {
+    hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE, true);
+    hiveConf.setVar(HiveConf.ConfVars.HIVEFETCHTASKCONVERSION, "none");
+    d.close();
+    d = new Driver(hiveConf);
+    int[][] values = {{1,2},{3,4},{5,6},{7,8},{9,10}};
+    runStatementOnDriver("insert into " + TxnCommandsBaseForTests.Table.ACIDTBL + makeValuesClause(values));//HIVE-17138: this creates 1 delta_0000013_0000013_0000/bucket_00001
+    runStatementOnDriver("drop table if exists T");
+    runStatementOnDriver("create table T (a int, b int) stored as ORC  TBLPROPERTIES ('transactional'='true')");
+    /*
+    So Union All removal kicks in and we get 3 subdirs in staging.
+ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiverwgit/ql/target/tmp/org.apache.hadoop.hive.ql.TestTxnNoBuckets-1505516390532/warehouse/t/.hive-staging_hive_2017-09-15_16-05-06_895_1123322677843388168-1/
+└── -ext-10000
+    ├── HIVE_UNION_SUBDIR_19
+    │   └── 000000_0
+    │       ├── _orc_acid_version
+    │       └── delta_0000016_0000016_0001
+    ├── HIVE_UNION_SUBDIR_20
+    │   └── 000000_0
+    │       ├── _orc_acid_version
+    │       └── delta_0000016_0000016_0002
+    └── HIVE_UNION_SUBDIR_21
+        └── 000000_0
+            ├── _orc_acid_version
+            └── delta_0000016_0000016_0003*/
+    runStatementOnDriver("insert into T(a,b) select a, b from " + TxnCommandsBaseForTests.Table.ACIDTBL + " where a between 1 and 3 group by a, b union all select a, b from " + TxnCommandsBaseForTests.Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + TxnCommandsBaseForTests.Table.ACIDTBL + " where a >= 9");
+
+    List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by ROW__ID");
+
+    String expected[][] = {
+      {"{\"transactionid\":16,\"bucketid\":536870913,\"rowid\":0}\t1\t2", "/delta_0000016_0000016_0001/bucket_00000"},
+      {"{\"transactionid\":16,\"bucketid\":536870913,\"rowid\":1}\t3\t4", "/delta_0000016_0000016_0001/bucket_00000"},
+      {"{\"transactionid\":16,\"bucketid\":536870914,\"rowid\":0}\t7\t8", "/delta_0000016_0000016_0002/bucket_00000"},
+      {"{\"transactionid\":16,\"bucketid\":536870914,\"rowid\":1}\t5\t6", "/delta_0000016_0000016_0002/bucket_00000"},
+      {"{\"transactionid\":16,\"bucketid\":536870915,\"rowid\":0}\t9\t10", "/delta_0000016_0000016_0003/bucket_00000"},
+    };
+    checkExpected(rs, expected, "Unexpected row count after ctas");
+  }
+  private void checkExpected(List<String> rs, String[][] expected, String msg) {
+    LOG.warn(testName.getMethodName() + ": read data(" + msg + "): ");
+    for(String s : rs) {
+      LOG.warn(s);
+    }
+    Assert.assertEquals( testName.getMethodName() + ": " + msg, expected.length, rs.size());
+    //verify data and layout
+    for(int i = 0; i < expected.length; i++) {
+      Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected[i][0]));
+      Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith(expected[i][1]));
+    }
+  }
+  /**
+   * The idea here is to create a non acid table that was written by multiple writers, i.e.
+   * unbucketed table that has 000000_0 & 000001_0, for example.  Unfortunately this doesn't work
+   * due to 'merge' logic - see comments in the method
+   */
+  @Ignore
+  @Test
+  public void testToAcidConversionMultiBucket() throws Exception {
+    int[][] values = {{1,2},{2,4},{5,6},{6,8},{9,10}};
+    runStatementOnDriver("insert into " + Table.ACIDTBL + makeValuesClause(values));
+    runStatementOnDriver("drop table if exists T");
+    runStatementOnDriver("create table T (a int, b int) stored as ORC  TBLPROPERTIES ('transactional'='false')");
+    /*T non-acid + non bucketd - 3 writers are created and then followed by merge to create a single output file
+    though how the data from union is split between writers is a mystery
+    (bucketed tables don't do merge)
+   Processing data file file:/Users/ekoifman/dev/hiverwgit/ql/target/tmp/org.apache.hadoop.hive.ql.TestTxnNoBuckets-1505317179157/warehouse/t/.hive-staging_hive_2017-09-13_08-40-30_275_8623609103176711840-1/-ext-10000/000000_0 [length: 515]
+{"a":6,"b":8}
+{"a":9,"b":10}
+{"a":5,"b":6}
+{"a":1,"b":2}
+{"a":2,"b":4}
+________________________________________________________________________________________________________________________
+
+Processing data file file:/Users/ekoifman/dev/hiverwgit/ql/target/tmp/org.apache.hadoop.hive.ql.TestTxnNoBuckets-1505317179157/warehouse/t/.hive-staging_hive_2017-09-13_08-40-30_275_8623609103176711840-1/-ext-10003/000000_0 [length: 242]
+{"a":6,"b":8}
+________________________________________________________________________________________________________________________
+
+Processing data file file:/Users/ekoifman/dev/hiverwgit/ql/target/tmp/org.apache.hadoop.hive.ql.TestTxnNoBuckets-1505317179157/warehouse/t/.hive-staging_hive_2017-09-13_08-40-30_275_8623609103176711840-1/-ext-10003/000001_0 [length: 244]
+{"a":9,"b":10}
+{"a":5,"b":6}
+________________________________________________________________________________________________________________________
+
+Processing data file file:/Users/ekoifman/dev/hiverwgit/ql/target/tmp/org.apache.hadoop.hive.ql.TestTxnNoBuckets-1505317179157/warehouse/t/.hive-staging_hive_2017-09-13_08-40-30_275_8623609103176711840-1/-ext-10003/000002_0 [length: 242]
+{"a":1,"b":2}
+{"a":2,"b":4}
+ */
+    runStatementOnDriver("insert into T(a,b) select a, b from " + Table.ACIDTBL + " where a between 1 and 3 group by a, b union all select a, b from " + Table.ACIDTBL + " where a between 5 and 7 union all select a, b from " + Table.ACIDTBL + " where a >= 9");
+    List<String> rs = runStatementOnDriver("select a, b, INPUT__FILE__NAME from T order by a, b, INPUT__FILE__NAME");
+    LOG.warn("before converting to acid");
+    for(String s : rs) {
+      LOG.warn(s);
+    }
+  }
+  @Test
+  public void testInsertFromUnion() throws Exception {
+    int[][] values = {{1,2},{2,4},{5,6},{6,8},{9,10}};
+    runStatementOnDriver("insert into " + Table.NONACIDNONBUCKET + makeValuesClause(values));
+    runStatementOnDriver("drop table if exists T");
+    runStatementOnDriver("create table T (a int, b int) stored as ORC  TBLPROPERTIES ('transactional'='true')");
+    runStatementOnDriver("insert into T(a,b) select a, b from " + Table.NONACIDNONBUCKET + " where a between 1 and 3 group by a, b union all select a, b from " + Table.NONACIDNONBUCKET + " where a between 5 and 7 union all select a, b from " + Table.NONACIDNONBUCKET + " where a >= 9");
+    List<String> rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from T order by a, b, INPUT__FILE__NAME");
+    LOG.warn("before converting to acid");
+    for(String s : rs) {
+      LOG.warn(s);
+    }
+    /*
+    The number of writers seems to be based on number of MR jobs for the src query.  todo check number of FileSinks
+    warehouse/t/.hive-staging_hive_2017-09-13_08-59-28_141_6304543600372946004-1/-ext-10000/000000_0/delta_0000016_0000016_0000/bucket_00000 [length: 648]
+    {"operation":0,"originalTransaction":16,"bucket":536870912,"rowId":0,"currentTransaction":16,"row":{"_col0":1,"_col1":2}}
+    {"operation":0,"originalTransaction":16,"bucket":536870912,"rowId":1,"currentTransaction":16,"row":{"_col0":2,"_col1":4}}
+    ________________________________________________________________________________________________________________________
+    warehouse/t/.hive-staging_hive_2017-09-13_08-59-28_141_6304543600372946004-1/-ext-10000/000001_0/delta_0000016_0000016_0000/bucket_00001 [length: 658]
+    {"operation":0,"originalTransaction":16,"bucket":536936448,"rowId":0,"currentTransaction":16,"row":{"_col0":5,"_col1":6}}
+    {"operation":0,"originalTransaction":16,"bucket":536936448,"rowId":1,"currentTransaction":16,"row":{"_col0":6,"_col1":8}}
+    {"operation":0,"originalTransaction":16,"bucket":536936448,"rowId":2,"currentTransaction":16,"row":{"_col0":9,"_col1":10}}
+    */
+    rs = runStatementOnDriver("select a, b from T order by a, b");
     Assert.assertEquals(stringifyValues(values), rs);
+    rs = runStatementOnDriver("select ROW__ID from T group by ROW__ID having count(*) > 1");
+    if(rs.size() > 0) {
+      Assert.assertEquals("Duplicate ROW__IDs: " + rs.get(0), 0, rs.size());
+    }
   }
   /**
    * see HIVE-16177
-   * See also {@link TestTxnCommands2#testNonAcidToAcidConversion02()}  todo need test with > 1 bucket file
+   * See also {@link TestTxnCommands2#testNonAcidToAcidConversion02()}
    */
   @Test
   public void testToAcidConversion02() throws Exception {

[04/50] [abbrv] hive git commit: HIVE-14671 : Merge branch 'master' into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

HIVE-14671 : Merge branch 'master' into hive-14535 (Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8f36545c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8f36545c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8f36545c

Branch: refs/heads/master
Commit: 8f36545cc645f54f300b8c54a84f7d5c658bd97d
Parents: fae2556 4408b8d
Author: Wei Zheng <we...@apache.org>
Authored: Wed Jun 7 15:29:19 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Wed Jun 7 15:29:19 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/common/log/InPlaceUpdate.java | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------

[16/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/queries/clientpositive/mm_loaddata.q
----------------------------------------------------------------------
diff --cc ql/src/test/queries/clientpositive/mm_loaddata.q
index 0000000,0000000..7e5787f
new file mode 100644
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/mm_loaddata.q
@@@ -1,0 -1,0 +1,53 @@@
++set hive.mapred.mode=nonstrict;
++set hive.explain.user=false;
++set hive.fetch.task.conversion=none;
++set tez.grouping.min-size=1;
++set tez.grouping.max-size=2;
++set hive.exec.dynamic.partition.mode=nonstrict;
++set hive.support.concurrency=true;
++set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
++
++
++drop table load0_mm;
++create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
++load data local inpath '../../data/files/kv1.txt' into table load0_mm;
++select count(1) from load0_mm;
++load data local inpath '../../data/files/kv2.txt' into table load0_mm;
++select count(1) from load0_mm;
++load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm;
++select count(1) from load0_mm;
++drop table load0_mm;
++
++
++drop table intermediate2;
++create table intermediate2 (key string, value string) stored as textfile
++location 'file:${system:test.tmp.dir}/intermediate2';
++load data local inpath '../../data/files/kv1.txt' into table intermediate2;
++load data local inpath '../../data/files/kv2.txt' into table intermediate2;
++load data local inpath '../../data/files/kv3.txt' into table intermediate2;
++
++drop table load1_mm;
++create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
++load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' into table load1_mm;
++load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv1.txt' into table load1_mm;
++select count(1) from load1_mm;
++load data local inpath '../../data/files/kv1.txt' into table intermediate2;
++load data local inpath '../../data/files/kv2.txt' into table intermediate2;
++load data local inpath '../../data/files/kv3.txt' into table intermediate2;
++load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' overwrite into table load1_mm;
++select count(1) from load1_mm;
++load data local inpath '../../data/files/kv2.txt' into table intermediate2;
++load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv2.txt' overwrite into table load1_mm;
++select count(1) from load1_mm;
++drop table load1_mm;
++
++drop table load2_mm;
++create table load2_mm (key string, value string)
++  partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only");
++load data local inpath '../../data/files/kv1.txt' into table intermediate2;
++load data local inpath '../../data/files/kv2.txt' into table intermediate2;
++load data local inpath '../../data/files/kv3.txt' into table intermediate2;
++load data inpath 'file:${system:test.tmp.dir}/intermediate2/kv*.txt' into table load2_mm partition(k=5, l=5);
++select count(1) from load2_mm;
++drop table load2_mm;
++drop table intermediate2;

[27/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e3fb1167
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e3fb1167
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e3fb1167

Branch: refs/heads/master
Commit: e3fb11671ff15215bc0d1ca6b054586e48fb8d32
Parents: 543e0b0 d2d9170
Author: sergey <se...@apache.org>
Authored: Fri Sep 15 20:38:51 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Fri Sep 15 20:39:13 2017 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/ql/Context.java | 18 +++++++++++++++++-
 .../ql/parse/UpdateDeleteSemanticAnalyzer.java  | 20 ++++++++++----------
 .../hive/ql/udf/generic/GenericUDFMaskHash.java |  2 +-
 .../org/apache/hive/service/CookieSigner.java   |  2 +-
 4 files changed, 29 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e3fb1167/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------

[23/50] [abbrv] hive git commit: HIVE-14990 : run all tests for MM tables and fix the issues that are found (Wei Zheng)

Posted by se...@apache.org.

HIVE-14990 : run all tests for MM tables and fix the issues that are found (Wei Zheng)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/00e75100
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/00e75100
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/00e75100

Branch: refs/heads/master
Commit: 00e75100725cb3972555481c51e11e10bbcefbdf
Parents: 18d1338
Author: Wei Zheng <we...@apache.org>
Authored: Fri Sep 8 15:35:05 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Fri Sep 8 15:35:05 2017 -0700

----------------------------------------------------------------------
 ql/src/test/results/clientpositive/mm_all.q.out | 35 ++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/00e75100/ql/src/test/results/clientpositive/mm_all.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/mm_all.q.out b/ql/src/test/results/clientpositive/mm_all.q.out
index b69ac00..ea60414 100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@ -1637,21 +1637,34 @@ POSTHOOK: Input: default@intermediate@p=455
 POSTHOOK: Input: default@intermediate@p=456
 POSTHOOK: Input: default@intermediate@p=457
 POSTHOOK: Output: default@multi1_mm@p=1
+POSTHOOK: Output: default@multi1_mm@p=455
+POSTHOOK: Output: default@multi1_mm@p=456
+POSTHOOK: Output: default@multi1_mm@p=457
 POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
-POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
-POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
+POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 PREHOOK: type: QUERY
 PREHOOK: Input: default@multi1_mm
 PREHOOK: Input: default@multi1_mm@p=1
 PREHOOK: Input: default@multi1_mm@p=2
+PREHOOK: Input: default@multi1_mm@p=455
+PREHOOK: Input: default@multi1_mm@p=456
+PREHOOK: Input: default@multi1_mm@p=457
 #### A masked pattern was here ####
 POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@multi1_mm
 POSTHOOK: Input: default@multi1_mm@p=1
 POSTHOOK: Input: default@multi1_mm@p=2
+POSTHOOK: Input: default@multi1_mm@p=455
+POSTHOOK: Input: default@multi1_mm@p=456
+POSTHOOK: Input: default@multi1_mm@p=457
 #### A masked pattern was here ####
 0	456	1
 0	456	1
@@ -1673,16 +1686,22 @@ POSTHOOK: Input: default@multi1_mm@p=2
 103	457	2
 455	97	1
 455	97	2
+455	97	455
 455	98	1
 455	98	2
+455	98	455
 456	0	1
 456	0	2
+456	0	456
 456	10	1
 456	10	2
+456	10	456
 457	100	1
 457	100	2
+457	100	457
 457	103	1
 457	103	2
+457	103	457
 PREHOOK: query: from intermediate
 insert into table multi1_mm partition(p) select p, key, 1
 insert into table multi1_mm partition(p=1) select key, p
@@ -1711,12 +1730,18 @@ PREHOOK: type: QUERY
 PREHOOK: Input: default@multi1_mm
 PREHOOK: Input: default@multi1_mm@p=1
 PREHOOK: Input: default@multi1_mm@p=2
+PREHOOK: Input: default@multi1_mm@p=455
+PREHOOK: Input: default@multi1_mm@p=456
+PREHOOK: Input: default@multi1_mm@p=457
 #### A masked pattern was here ####
 POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@multi1_mm
 POSTHOOK: Input: default@multi1_mm@p=1
 POSTHOOK: Input: default@multi1_mm@p=2
+POSTHOOK: Input: default@multi1_mm@p=455
+POSTHOOK: Input: default@multi1_mm@p=456
+POSTHOOK: Input: default@multi1_mm@p=457
 #### A masked pattern was here ####
 0	456	1
 0	456	1
@@ -1745,21 +1770,27 @@ POSTHOOK: Input: default@multi1_mm@p=2
 455	97	1
 455	97	1
 455	97	2
+455	97	455
 455	98	1
 455	98	1
 455	98	2
+455	98	455
 456	0	1
 456	0	1
 456	0	2
+456	0	456
 456	10	1
 456	10	1
 456	10	2
+456	10	456
 457	100	1
 457	100	1
 457	100	2
+457	100	457
 457	103	1
 457	103	1
 457	103	2
+457	103	457
 PREHOOK: query: drop table multi1_mm
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@multi1_mm

[37/50] [abbrv] hive git commit: HIVE-15212 : merge branch into master - fix some tests (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-15212 : merge branch into master - fix some tests (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a0c546f7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a0c546f7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a0c546f7

Branch: refs/heads/master
Commit: a0c546f7848c91a8a17e896ed127a6548d173aa3
Parents: 55d9cee
Author: sergey <se...@apache.org>
Authored: Thu Sep 28 16:29:53 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Sep 28 16:29:53 2017 -0700

----------------------------------------------------------------------
 .../results/clientpositive/llap/mm_all.q.out    | 1436 ------------------
 .../clientpositive/tez/explainuser_3.q.out      |    4 +-
 2 files changed, 2 insertions(+), 1438 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a0c546f7/ql/src/test/results/clientpositive/llap/mm_all.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_all.q.out b/ql/src/test/results/clientpositive/llap/mm_all.q.out
deleted file mode 100644
index 0adbddb..0000000
--- a/ql/src/test/results/clientpositive/llap/mm_all.q.out
+++ /dev/null
@@ -1,1436 +0,0 @@
-PREHOOK: query: drop table intermediate
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table intermediate
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@intermediate
-POSTHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@intermediate
-PREHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Output: default@intermediate@p=455
-POSTHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Output: default@intermediate@p=455
-POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Output: default@intermediate@p=456
-POSTHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Output: default@intermediate@p=456
-POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Output: default@intermediate@p=457
-POSTHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Output: default@intermediate@p=457
-POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: drop table part_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table part_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@part_mm
-POSTHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@part_mm
-PREHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
-PREHOOK: type: QUERY
-POSTHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
-  Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
-  Stage-0 depends on stages: Stage-2
-  Stage-3 depends on stages: Stage-0
-
-STAGE PLANS:
-  Stage: Stage-1
-    Tez
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
-            Map Operator Tree:
-                TableScan
-                  alias: intermediate
-                  Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE
-                  Select Operator
-                    expressions: key (type: int)
-                    outputColumnNames: _col0
-                    Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE
-                    File Output Operator
-                      compressed: false
-                      Statistics: Num rows: 6 Data size: 72 Basic stats: COMPLETE Column stats: NONE
-                      table:
-                          input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-                          output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-                          serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-                          name: default.part_mm
-                      Write Type: INSERT
-            Execution mode: llap
-            LLAP IO: all inputs
-
-  Stage: Stage-2
-    Dependency Collection
-
-  Stage: Stage-0
-    Move Operator
-      tables:
-          partition:
-            key_mm 455
-          replace: false
-          table:
-              input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
-              output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
-              serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
-              name: default.part_mm
-          Write Type: INSERT
-          micromanaged table: true
-
-  Stage: Stage-3
-    Stats-Aggr Operator
-
-PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_mm@key_mm=455
-POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_mm@key_mm=455
-POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_mm@key_mm=456
-POSTHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_mm@key_mm=456
-POSTHOOK: Lineage: part_mm PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@part_mm@key_mm=455
-POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@part_mm@key_mm=455
-POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from part_mm order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_mm
-PREHOOK: Input: default@part_mm@key_mm=455
-PREHOOK: Input: default@part_mm@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_mm order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_mm
-POSTHOOK: Input: default@part_mm@key_mm=455
-POSTHOOK: Input: default@part_mm@key_mm=456
-#### A masked pattern was here ####
-0	455
-0	455
-0	456
-10	455
-10	455
-10	456
-97	455
-97	455
-97	456
-98	455
-98	455
-98	456
-100	455
-100	455
-100	456
-103	455
-103	455
-103	456
-PREHOOK: query: select * from part_mm order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_mm
-PREHOOK: Input: default@part_mm@key_mm=455
-PREHOOK: Input: default@part_mm@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_mm order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_mm
-POSTHOOK: Input: default@part_mm@key_mm=455
-POSTHOOK: Input: default@part_mm@key_mm=456
-#### A masked pattern was here ####
-0	455
-0	455
-0	456
-10	455
-10	455
-10	456
-97	455
-97	455
-97	456
-98	455
-98	455
-98	456
-100	455
-100	455
-100	456
-103	455
-103	455
-103	456
-PREHOOK: query: truncate table part_mm
-PREHOOK: type: TRUNCATETABLE
-PREHOOK: Output: default@part_mm@key_mm=455
-PREHOOK: Output: default@part_mm@key_mm=456
-POSTHOOK: query: truncate table part_mm
-POSTHOOK: type: TRUNCATETABLE
-POSTHOOK: Output: default@part_mm@key_mm=455
-POSTHOOK: Output: default@part_mm@key_mm=456
-PREHOOK: query: select * from part_mm order by key, key_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@part_mm
-PREHOOK: Input: default@part_mm@key_mm=455
-PREHOOK: Input: default@part_mm@key_mm=456
-#### A masked pattern was here ####
-POSTHOOK: query: select * from part_mm order by key, key_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@part_mm
-POSTHOOK: Input: default@part_mm@key_mm=455
-POSTHOOK: Input: default@part_mm@key_mm=456
-#### A masked pattern was here ####
-PREHOOK: query: drop table part_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@part_mm
-PREHOOK: Output: default@part_mm
-POSTHOOK: query: drop table part_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@part_mm
-POSTHOOK: Output: default@part_mm
-PREHOOK: query: drop table simple_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table simple_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@simple_mm
-POSTHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@simple_mm
-PREHOOK: query: insert into table simple_mm select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_mm
-POSTHOOK: query: insert into table simple_mm select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_mm
-POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_mm order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_mm order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_mm
-#### A masked pattern was here ####
-0
-10
-97
-98
-100
-103
-PREHOOK: query: insert into table simple_mm select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@simple_mm
-POSTHOOK: query: insert into table simple_mm select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@simple_mm
-POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from simple_mm order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_mm order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_mm
-#### A masked pattern was here ####
-0
-0
-10
-10
-97
-97
-98
-98
-100
-100
-103
-103
-PREHOOK: query: truncate table simple_mm
-PREHOOK: type: TRUNCATETABLE
-PREHOOK: Output: default@simple_mm
-POSTHOOK: query: truncate table simple_mm
-POSTHOOK: type: TRUNCATETABLE
-POSTHOOK: Output: default@simple_mm
-PREHOOK: query: select * from simple_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@simple_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from simple_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@simple_mm
-#### A masked pattern was here ####
-PREHOOK: query: drop table simple_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@simple_mm
-PREHOOK: Output: default@simple_mm
-POSTHOOK: query: drop table simple_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@simple_mm
-POSTHOOK: Output: default@simple_mm
-PREHOOK: query: drop table dp_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table dp_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
-  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@dp_mm
-POSTHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
-  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@dp_mm
-PREHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@dp_mm@key1=123
-POSTHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@dp_mm@key1=123/key2=0
-POSTHOOK: Output: default@dp_mm@key1=123/key2=10
-POSTHOOK: Output: default@dp_mm@key1=123/key2=100
-POSTHOOK: Output: default@dp_mm@key1=123/key2=103
-POSTHOOK: Output: default@dp_mm@key1=123/key2=97
-POSTHOOK: Output: default@dp_mm@key1=123/key2=98
-POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=0).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=100).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=103).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=10).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=97).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=98).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from dp_mm order by key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@dp_mm
-PREHOOK: Input: default@dp_mm@key1=123/key2=0
-PREHOOK: Input: default@dp_mm@key1=123/key2=10
-PREHOOK: Input: default@dp_mm@key1=123/key2=100
-PREHOOK: Input: default@dp_mm@key1=123/key2=103
-PREHOOK: Input: default@dp_mm@key1=123/key2=97
-PREHOOK: Input: default@dp_mm@key1=123/key2=98
-#### A masked pattern was here ####
-POSTHOOK: query: select * from dp_mm order by key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@dp_mm
-POSTHOOK: Input: default@dp_mm@key1=123/key2=0
-POSTHOOK: Input: default@dp_mm@key1=123/key2=10
-POSTHOOK: Input: default@dp_mm@key1=123/key2=100
-POSTHOOK: Input: default@dp_mm@key1=123/key2=103
-POSTHOOK: Input: default@dp_mm@key1=123/key2=97
-POSTHOOK: Input: default@dp_mm@key1=123/key2=98
-#### A masked pattern was here ####
-0	123	0
-10	123	10
-97	123	97
-98	123	98
-100	123	100
-103	123	103
-PREHOOK: query: drop table dp_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@dp_mm
-PREHOOK: Output: default@dp_mm
-POSTHOOK: query: drop table dp_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@dp_mm
-POSTHOOK: Output: default@dp_mm
-PREHOOK: query: create table union_mm(id int)  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@union_mm
-POSTHOOK: query: create table union_mm(id int)  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@union_mm
-PREHOOK: query: insert into table union_mm 
-select temps.p from (
-select key as p from intermediate 
-union all 
-select key + 1 as p from intermediate ) temps
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@union_mm
-POSTHOOK: query: insert into table union_mm 
-select temps.p from (
-select key as p from intermediate 
-union all 
-select key + 1 as p from intermediate ) temps
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@union_mm
-POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from union_mm order by id
-PREHOOK: type: QUERY
-PREHOOK: Input: default@union_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from union_mm order by id
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@union_mm
-#### A masked pattern was here ####
-0
-1
-10
-11
-97
-98
-98
-99
-100
-101
-103
-104
-PREHOOK: query: insert into table union_mm 
-select p from
-(
-select key + 1 as p from intermediate
-union all
-select key from intermediate
-) tab group by p
-union all
-select key + 2 as p from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@union_mm
-POSTHOOK: query: insert into table union_mm 
-select p from
-(
-select key + 1 as p from intermediate
-union all
-select key from intermediate
-) tab group by p
-union all
-select key + 2 as p from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@union_mm
-POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from union_mm order by id
-PREHOOK: type: QUERY
-PREHOOK: Input: default@union_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from union_mm order by id
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@union_mm
-#### A masked pattern was here ####
-0
-1
-2
-10
-11
-12
-97
-98
-99
-99
-100
-100
-101
-102
-103
-104
-105
-PREHOOK: query: insert into table union_mm
-SELECT p FROM
-(
-  SELECT key + 1 as p FROM intermediate
-  UNION ALL
-  SELECT key as p FROM ( 
-    SELECT distinct key FROM (
-      SELECT key FROM (
-        SELECT key + 2 as key FROM intermediate
-        UNION ALL
-        SELECT key FROM intermediate
-      )t1 
-    group by key)t2
-  )t3
-)t4
-group by p
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@union_mm
-POSTHOOK: query: insert into table union_mm
-SELECT p FROM
-(
-  SELECT key + 1 as p FROM intermediate
-  UNION ALL
-  SELECT key as p FROM ( 
-    SELECT distinct key FROM (
-      SELECT key FROM (
-        SELECT key + 2 as key FROM intermediate
-        UNION ALL
-        SELECT key FROM intermediate
-      )t1 
-    group by key)t2
-  )t3
-)t4
-group by p
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@union_mm
-POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from union_mm order by id
-PREHOOK: type: QUERY
-PREHOOK: Input: default@union_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from union_mm order by id
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@union_mm
-#### A masked pattern was here ####
-0
-0
-1
-1
-2
-2
-10
-10
-11
-11
-12
-12
-97
-97
-98
-98
-99
-99
-99
-100
-100
-100
-101
-101
-102
-102
-103
-103
-104
-104
-105
-105
-PREHOOK: query: drop table union_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@union_mm
-PREHOOK: Output: default@union_mm
-POSTHOOK: query: drop table union_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@union_mm
-POSTHOOK: Output: default@union_mm
-PREHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@partunion_mm
-POSTHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@partunion_mm
-PREHOOK: query: insert into table partunion_mm partition(key)
-select temps.* from (
-select key as p, key from intermediate 
-union all 
-select key + 1 as p, key + 1 from intermediate ) temps
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@partunion_mm
-POSTHOOK: query: insert into table partunion_mm partition(key)
-select temps.* from (
-select key as p, key from intermediate 
-union all 
-select key + 1 as p, key + 1 from intermediate ) temps
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-PREHOOK: query: select * from partunion_mm order by id
-PREHOOK: type: QUERY
-PREHOOK: Input: default@partunion_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from partunion_mm order by id
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@partunion_mm
-#### A masked pattern was here ####
-PREHOOK: query: drop table partunion_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@partunion_mm
-PREHOOK: Output: default@partunion_mm
-POSTHOOK: query: drop table partunion_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@partunion_mm
-POSTHOOK: Output: default@partunion_mm
-PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
- stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@skew_mm
-POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
- stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@skew_mm
-PREHOOK: query: insert into table skew_mm 
-select key, key, key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@skew_mm
-POSTHOOK: query: insert into table skew_mm 
-select key, key, key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@skew_mm
-POSTHOOK: Lineage: skew_mm.k1 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: skew_mm.k2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: skew_mm.k4 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from skew_mm order by k2, k1, k4
-PREHOOK: type: QUERY
-PREHOOK: Input: default@skew_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from skew_mm order by k2, k1, k4
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@skew_mm
-#### A masked pattern was here ####
-0	0	0
-10	10	10
-97	97	97
-98	98	98
-100	100	100
-103	103	103
-PREHOOK: query: drop table skew_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@skew_mm
-PREHOOK: Output: default@skew_mm
-POSTHOOK: query: drop table skew_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@skew_mm
-POSTHOOK: Output: default@skew_mm
-PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) 
-skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@skew_dp_union_mm
-POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) 
-skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@skew_dp_union_mm
-PREHOOK: query: insert into table skew_dp_union_mm partition (k3)
-select key as i, key as j, key as k, key as l from intermediate
-union all 
-select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@skew_dp_union_mm
-POSTHOOK: query: insert into table skew_dp_union_mm partition (k3)
-select key as i, key as j, key as k, key as l from intermediate
-union all 
-select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-PREHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
-PREHOOK: type: QUERY
-PREHOOK: Input: default@skew_dp_union_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@skew_dp_union_mm
-#### A masked pattern was here ####
-PREHOOK: query: drop table skew_dp_union_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@skew_dp_union_mm
-PREHOOK: Output: default@skew_dp_union_mm
-POSTHOOK: query: drop table skew_dp_union_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@skew_dp_union_mm
-POSTHOOK: Output: default@skew_dp_union_mm
-PREHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@merge0_mm
-POSTHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@merge0_mm
-PREHOOK: query: insert into table merge0_mm select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@merge0_mm
-POSTHOOK: query: insert into table merge0_mm select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@merge0_mm
-POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from merge0_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@merge0_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from merge0_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@merge0_mm
-#### A masked pattern was here ####
-98
-97
-100
-103
-0
-10
-PREHOOK: query: insert into table merge0_mm select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@merge0_mm
-POSTHOOK: query: insert into table merge0_mm select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@merge0_mm
-POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from merge0_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@merge0_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from merge0_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@merge0_mm
-#### A masked pattern was here ####
-98
-97
-100
-103
-0
-10
-98
-97
-100
-103
-0
-10
-PREHOOK: query: drop table merge0_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@merge0_mm
-PREHOOK: Output: default@merge0_mm
-POSTHOOK: query: drop table merge0_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@merge0_mm
-POSTHOOK: Output: default@merge0_mm
-PREHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@merge2_mm
-POSTHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@merge2_mm
-PREHOOK: query: insert into table merge2_mm select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@merge2_mm
-POSTHOOK: query: insert into table merge2_mm select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@merge2_mm
-POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from merge2_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@merge2_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from merge2_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@merge2_mm
-#### A masked pattern was here ####
-98
-97
-100
-103
-0
-10
-PREHOOK: query: insert into table merge2_mm select key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@merge2_mm
-POSTHOOK: query: insert into table merge2_mm select key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@merge2_mm
-POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from merge2_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@merge2_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select * from merge2_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@merge2_mm
-#### A masked pattern was here ####
-98
-97
-100
-103
-0
-10
-98
-97
-100
-103
-0
-10
-PREHOOK: query: drop table merge2_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@merge2_mm
-PREHOOK: Output: default@merge2_mm
-POSTHOOK: query: drop table merge2_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@merge2_mm
-POSTHOOK: Output: default@merge2_mm
-PREHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@merge1_mm
-POSTHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@merge1_mm
-PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@merge1_mm
-POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@merge1_mm@key=0
-POSTHOOK: Output: default@merge1_mm@key=10
-POSTHOOK: Output: default@merge1_mm@key=100
-POSTHOOK: Output: default@merge1_mm@key=103
-POSTHOOK: Output: default@merge1_mm@key=97
-POSTHOOK: Output: default@merge1_mm@key=98
-POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from merge1_mm order by id, key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@merge1_mm
-PREHOOK: Input: default@merge1_mm@key=0
-PREHOOK: Input: default@merge1_mm@key=10
-PREHOOK: Input: default@merge1_mm@key=100
-PREHOOK: Input: default@merge1_mm@key=103
-PREHOOK: Input: default@merge1_mm@key=97
-PREHOOK: Input: default@merge1_mm@key=98
-#### A masked pattern was here ####
-POSTHOOK: query: select * from merge1_mm order by id, key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@merge1_mm
-POSTHOOK: Input: default@merge1_mm@key=0
-POSTHOOK: Input: default@merge1_mm@key=10
-POSTHOOK: Input: default@merge1_mm@key=100
-POSTHOOK: Input: default@merge1_mm@key=103
-POSTHOOK: Input: default@merge1_mm@key=97
-POSTHOOK: Input: default@merge1_mm@key=98
-#### A masked pattern was here ####
-0	0
-10	10
-97	97
-98	98
-100	100
-103	103
-PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
-PREHOOK: type: QUERY
-PREHOOK: Input: default@intermediate
-PREHOOK: Input: default@intermediate@p=455
-PREHOOK: Input: default@intermediate@p=456
-PREHOOK: Input: default@intermediate@p=457
-PREHOOK: Output: default@merge1_mm
-POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Input: default@intermediate@p=455
-POSTHOOK: Input: default@intermediate@p=456
-POSTHOOK: Input: default@intermediate@p=457
-POSTHOOK: Output: default@merge1_mm@key=0
-POSTHOOK: Output: default@merge1_mm@key=10
-POSTHOOK: Output: default@merge1_mm@key=100
-POSTHOOK: Output: default@merge1_mm@key=103
-POSTHOOK: Output: default@merge1_mm@key=97
-POSTHOOK: Output: default@merge1_mm@key=98
-POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from merge1_mm order by id, key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@merge1_mm
-PREHOOK: Input: default@merge1_mm@key=0
-PREHOOK: Input: default@merge1_mm@key=10
-PREHOOK: Input: default@merge1_mm@key=100
-PREHOOK: Input: default@merge1_mm@key=103
-PREHOOK: Input: default@merge1_mm@key=97
-PREHOOK: Input: default@merge1_mm@key=98
-#### A masked pattern was here ####
-POSTHOOK: query: select * from merge1_mm order by id, key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@merge1_mm
-POSTHOOK: Input: default@merge1_mm@key=0
-POSTHOOK: Input: default@merge1_mm@key=10
-POSTHOOK: Input: default@merge1_mm@key=100
-POSTHOOK: Input: default@merge1_mm@key=103
-POSTHOOK: Input: default@merge1_mm@key=97
-POSTHOOK: Input: default@merge1_mm@key=98
-#### A masked pattern was here ####
-0	0
-0	0
-10	10
-10	10
-97	97
-97	97
-98	98
-98	98
-100	100
-100	100
-103	103
-103	103
-PREHOOK: query: drop table merge1_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@merge1_mm
-PREHOOK: Output: default@merge1_mm
-POSTHOOK: query: drop table merge1_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@merge1_mm
-POSTHOOK: Output: default@merge1_mm
-PREHOOK: query: drop table load0_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table load0_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@load0_mm
-POSTHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@load0_mm
-PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@load0_mm
-POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@load0_mm
-PREHOOK: query: select count(1) from load0_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@load0_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select count(1) from load0_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@load0_mm
-#### A masked pattern was here ####
-500
-PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@load0_mm
-POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@load0_mm
-PREHOOK: query: select count(1) from load0_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@load0_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select count(1) from load0_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@load0_mm
-#### A masked pattern was here ####
-1000
-PREHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@load0_mm
-POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@load0_mm
-PREHOOK: query: select count(1) from load0_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@load0_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select count(1) from load0_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@load0_mm
-#### A masked pattern was here ####
-500
-PREHOOK: query: drop table load0_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@load0_mm
-PREHOOK: Output: default@load0_mm
-POSTHOOK: query: drop table load0_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@load0_mm
-POSTHOOK: Output: default@load0_mm
-PREHOOK: query: drop table intermediate2
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table intermediate2
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table intermediate2 (key string, value string) stored as textfile
-#### A masked pattern was here ####
-PREHOOK: type: CREATETABLE
-#### A masked pattern was here ####
-PREHOOK: Output: database:default
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: create table intermediate2 (key string, value string) stored as textfile
-#### A masked pattern was here ####
-POSTHOOK: type: CREATETABLE
-#### A masked pattern was here ####
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@intermediate2
-PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@intermediate2
-PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@intermediate2
-PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@intermediate2
-PREHOOK: query: drop table load1_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table load1_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@load1_mm
-POSTHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@load1_mm
-#### A masked pattern was here ####
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@load1_mm
-#### A masked pattern was here ####
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@load1_mm
-#### A masked pattern was here ####
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@load1_mm
-#### A masked pattern was here ####
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@load1_mm
-PREHOOK: query: select count(1) from load1_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@load1_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select count(1) from load1_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@load1_mm
-#### A masked pattern was here ####
-1000
-PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@intermediate2
-PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@intermediate2
-PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@intermediate2
-#### A masked pattern was here ####
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@load1_mm
-#### A masked pattern was here ####
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@load1_mm
-PREHOOK: query: select count(1) from load1_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@load1_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select count(1) from load1_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@load1_mm
-#### A masked pattern was here ####
-1050
-PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@intermediate2
-#### A masked pattern was here ####
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@load1_mm
-#### A masked pattern was here ####
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@load1_mm
-PREHOOK: query: select count(1) from load1_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@load1_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select count(1) from load1_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@load1_mm
-#### A masked pattern was here ####
-500
-PREHOOK: query: drop table load1_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@load1_mm
-PREHOOK: Output: default@load1_mm
-POSTHOOK: query: drop table load1_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@load1_mm
-POSTHOOK: Output: default@load1_mm
-PREHOOK: query: drop table load2_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table load2_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table load2_mm (key string, value string)
-  partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@load2_mm
-POSTHOOK: query: create table load2_mm (key string, value string)
-  partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@load2_mm
-PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@intermediate2
-PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@intermediate2
-PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@intermediate2
-#### A masked pattern was here ####
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@load2_mm
-#### A masked pattern was here ####
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@load2_mm
-POSTHOOK: Output: default@load2_mm@k=5/l=5
-PREHOOK: query: select count(1) from load2_mm
-PREHOOK: type: QUERY
-PREHOOK: Input: default@load2_mm
-PREHOOK: Input: default@load2_mm@k=5/l=5
-#### A masked pattern was here ####
-POSTHOOK: query: select count(1) from load2_mm
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@load2_mm
-POSTHOOK: Input: default@load2_mm@k=5/l=5
-#### A masked pattern was here ####
-1025
-PREHOOK: query: drop table load2_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@load2_mm
-PREHOOK: Output: default@load2_mm
-POSTHOOK: query: drop table load2_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@load2_mm
-POSTHOOK: Output: default@load2_mm
-PREHOOK: query: drop table intermediate2
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@intermediate2
-PREHOOK: Output: default@intermediate2
-POSTHOOK: query: drop table intermediate2
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@intermediate2
-POSTHOOK: Output: default@intermediate2
-PREHOOK: query: drop table multi0_1_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table multi0_1_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: drop table multi0_2_mm
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table multi0_2_mm
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table multi0_1_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@multi0_1_mm
-POSTHOOK: query: create table multi0_1_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@multi0_1_mm
-PREHOOK: query: create table multi0_2_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@multi0_2_mm
-POSTHOOK: query: create table multi0_2_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@multi0_2_mm
-PREHOOK: query: CREATE TABLE parquet1_mm(id INT) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@parquet1_mm
-POSTHOOK: query: CREATE TABLE parquet1_mm(id INT) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@parquet1_mm
-PREHOOK: query: INSERT INTO parquet1_mm VALUES(1), (2)
-PREHOOK: type: QUERY
-PREHOOK: Output: default@parquet1_mm
-POSTHOOK: query: INSERT INTO parquet1_mm VALUES(1), (2)
-POSTHOOK: type: QUERY
-POSTHOOK: Output: default@parquet1_mm
-POSTHOOK: Lineage: parquet1_mm.id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-PREHOOK: query: CREATE TABLE parquet2_mm(id INT, value STRING) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@parquet2_mm
-POSTHOOK: query: CREATE TABLE parquet2_mm(id INT, value STRING) STORED AS PARQUET tblproperties ("transactional"="true", "transactional_properties"="insert_only")
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@parquet2_mm
-PREHOOK: query: INSERT INTO parquet2_mm VALUES(1, 'value1')
-PREHOOK: type: QUERY
-PREHOOK: Output: default@parquet2_mm
-POSTHOOK: query: INSERT INTO parquet2_mm VALUES(1, 'value1')
-POSTHOOK: type: QUERY
-POSTHOOK: Output: default@parquet2_mm
-POSTHOOK: Lineage: parquet2_mm.id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: parquet2_mm.value SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
-PREHOOK: query: INSERT INTO parquet2_mm VALUES(1, 'value2')
-PREHOOK: type: QUERY
-PREHOOK: Output: default@parquet2_mm
-POSTHOOK: query: INSERT INTO parquet2_mm VALUES(1, 'value2')
-POSTHOOK: type: QUERY
-POSTHOOK: Output: default@parquet2_mm
-POSTHOOK: Lineage: parquet2_mm.id EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
-POSTHOOK: Lineage: parquet2_mm.value SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
-PREHOOK: query: select parquet1_mm.id, t1.value, t2.value FROM parquet1_mm
-  JOIN parquet2_mm t1 ON parquet1_mm.id=t1.id
-  JOIN parquet2_mm t2 ON parquet1_mm.id=t2.id
-where t1.value = 'value1' and t2.value = 'value2'
-PREHOOK: type: QUERY
-PREHOOK: Input: default@parquet1_mm
-PREHOOK: Input: default@parquet2_mm
-#### A masked pattern was here ####
-POSTHOOK: query: select parquet1_mm.id, t1.value, t2.value FROM parquet1_mm
-  JOIN parquet2_mm t1 ON parquet1_mm.id=t1.id
-  JOIN parquet2_mm t2 ON parquet1_mm.id=t2.id
-where t1.value = 'value1' and t2.value = 'value2'
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@parquet1_mm
-POSTHOOK: Input: default@parquet2_mm
-#### A masked pattern was here ####
-1	value1	value2
-PREHOOK: query: drop table parquet1_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@parquet1_mm
-PREHOOK: Output: default@parquet1_mm
-POSTHOOK: query: drop table parquet1_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@parquet1_mm
-POSTHOOK: Output: default@parquet1_mm
-PREHOOK: query: drop table parquet2_mm
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@parquet2_mm
-PREHOOK: Output: default@parquet2_mm
-POSTHOOK: query: drop table parquet2_mm
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@parquet2_mm
-POSTHOOK: Output: default@parquet2_mm
-PREHOOK: query: drop table intermediate
-PREHOOK: type: DROPTABLE
-PREHOOK: Input: default@intermediate
-PREHOOK: Output: default@intermediate
-POSTHOOK: query: drop table intermediate
-POSTHOOK: type: DROPTABLE
-POSTHOOK: Input: default@intermediate
-POSTHOOK: Output: default@intermediate

http://git-wip-us.apache.org/repos/asf/hive/blob/a0c546f7/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
index 7e89478..cd17cfa 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
@@ -511,9 +511,9 @@ Stage-3
                       Map 1 vectorized
                       File Output Operator [FS_8]
                         table:{"name:":"default.orc_merge5"}
-                        Select Operator [SEL_9] (rows=1 width=352)
+                        Select Operator [SEL_7] (rows=1 width=352)
                           Output:["_col0","_col1","_col2","_col3","_col4"]
-                          Filter Operator [FIL_8] (rows=1 width=352)
+                          Filter Operator [FIL_6] (rows=1 width=352)
                             predicate:(userid <= 13)
                             TableScan [TS_0] (rows=1 width=352)
                               default@orc_merge5,orc_merge5,Tbl:COMPLETE,Col:NONE,Output:["userid","string1","subtype","decimal1","ts"]

[45/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42335b46
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42335b46
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42335b46

Branch: refs/heads/master
Commit: 42335b46cacb2ebcc4569e1081a1cc6369558146
Parents: 14c9482 c254557
Author: sergey <se...@apache.org>
Authored: Mon Oct 9 17:37:32 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Mon Oct 9 17:37:32 2017 -0700

----------------------------------------------------------------------
 .../mr/AccumuloIndexedOutputFormat.java         |   50 +-
 .../positive/accumulo_predicate_pushdown.q.out  |    2 +-
 .../accumulo_single_sourced_multi_insert.q.out  |    4 +-
 .../java/org/apache/hive/beeline/BeeLine.java   |  110 +-
 .../apache/hive/beeline/ClassNameCompleter.java |  254 --
 .../org/apache/hive/beeline/HiveSchemaTool.java |   36 +-
 .../hive/beeline/TestBeelineArgParsing.java     |   13 +-
 .../hive/beeline/TestClassNameCompleter.java    |   76 -
 bin/ext/tokentool.sh                            |   67 +
 .../hive/common/ValidCompactorTxnList.java      |   89 -
 .../hadoop/hive/common/ValidReadTxnList.java    |  237 --
 .../apache/hadoop/hive/common/ValidTxnList.java |  112 -
 .../hadoop/hive/common/log/LogRedirector.java   |   99 +
 .../org/apache/hadoop/hive/conf/HiveConf.java   |   62 +-
 .../apache/hive/common/util/HiveTestUtils.java  |   37 +-
 .../java/org/apache/hive/http/HttpServer.java   |   34 +-
 .../hive/common/TestValidReadTxnList.java       |  109 -
 data/conf/spark/local/hive-site.xml             |  258 ++
 data/files/birthdays                            |    9 +
 data/scripts/q_test_init.sql                    |   10 +-
 .../results/positive/external_table_ppd.q.out   |    1 -
 .../positive/hbase_binary_storage_queries.q.out |    2 -
 .../src/test/results/positive/hbase_ddl.q.out   |    2 -
 .../src/test/results/positive/hbasestats.q.out  |    5 -
 .../apache/hive/hcatalog/data/JsonSerDe.java    |   15 +-
 .../mapreduce/FileOutputCommitterContainer.java |   11 +-
 .../hive/hcatalog/mapreduce/Security.java       |    3 +
 .../hive/hcatalog/streaming/TestStreaming.java  |    4 +-
 .../streaming/mutate/StreamingTestUtils.java    |    4 +-
 .../hive/hcatalog/api/TestHCatClient.java       |    2 +-
 .../svr/src/main/config/webhcat-default.xml     |    8 +
 .../hive/hcatalog/templeton/AppConfig.java      |    1 +
 .../apache/hive/hcatalog/templeton/Main.java    |   43 +
 .../test/queries/clientpositive/multiple_agg.q  |    9 +
 ...rt_addpartition_blobstore_to_blobstore.q.out |    4 -
 ...import_addpartition_blobstore_to_local.q.out |    4 -
 ...rt_addpartition_blobstore_to_warehouse.q.out |    4 -
 ...import_addpartition_local_to_blobstore.q.out |    4 -
 .../import_blobstore_to_blobstore.q.out         |    2 -
 .../import_blobstore_to_local.q.out             |    2 -
 .../import_blobstore_to_warehouse.q.out         |    2 -
 .../import_local_to_blobstore.q.out             |    2 -
 .../results/clientpositive/multiple_agg.q.out   |   35 +
 .../hive/metastore/TestHiveMetaStoreTxns.java   |    4 +-
 .../apache/hadoop/hive/ql/TestAcidOnTez.java    |    5 +-
 .../hadoop/hive/ql/history/TestHiveHistory.java |    4 +-
 .../hive/ql/parse/TestReplicationScenarios.java |   63 +-
 .../hive/ql/txn/compactor/TestCompactor.java    |    4 +-
 .../hive/cli/TestLocalSparkCliDriver.java       |   62 +
 .../test/resources/testconfiguration.properties |    2 +
 .../hadoop/hive/cli/control/CliConfigs.java     |   22 +
 .../hive/cli/control/CoreBeeLineDriver.java     |   41 +-
 .../java/org/apache/hive/beeline/QFile.java     |  124 +-
 .../apache/hive/beeline/QFileBeeLineClient.java |  138 +-
 .../llap/io/decode/OrcEncodedDataConsumer.java  |    7 +-
 .../llap/io/encoded/SerDeEncodedDataReader.java |   15 +-
 .../hive/llap/cache/TestLowLevelCacheImpl.java  |    4 +-
 .../upgrade/derby/046-HIVE-17566.derby.sql      |   25 +
 .../upgrade/derby/hive-schema-3.0.0.derby.sql   |   41 +-
 .../derby/upgrade-2.3.0-to-3.0.0.derby.sql      |    1 +
 .../upgrade/hive/hive-schema-3.0.0.hive.sql     |   17 +
 .../upgrade/mssql/031-HIVE-17566.mssql.sql      |   76 +
 .../upgrade/mssql/hive-schema-3.0.0.mssql.sql   |   78 +-
 .../mssql/upgrade-2.3.0-to-3.0.0.mssql.sql      |    1 +
 .../upgrade/mysql/045-HIVE-16886.mysql.sql      |    2 +-
 .../upgrade/mysql/046-HIVE-17566.mysql.sql      |   57 +
 .../upgrade/mysql/hive-schema-3.0.0.mysql.sql   |   64 +-
 .../mysql/upgrade-2.3.0-to-3.0.0.mysql.sql      |    1 +
 .../upgrade/oracle/046-HIVE-17566.oracle.sql    |   76 +
 .../upgrade/oracle/hive-schema-3.0.0.oracle.sql |   78 +-
 .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql    |    1 +
 .../postgres/045-HIVE-17566.postgres.sql        |   88 +
 .../postgres/hive-schema-3.0.0.postgres.sql     |   91 +
 .../upgrade-2.3.0-to-3.0.0.postgres.sql         |    1 +
 .../hadoop/hive/metastore/HiveMetaStore.java    |   14 +
 .../hive/metastore/HiveMetaStoreClient.java     |    2 +
 .../hive/metastore/MetaStoreDirectSql.java      |    2 +
 .../hadoop/hive/metastore/ObjectStore.java      |    2 +-
 .../hadoop/hive/metastore/cache/CacheUtils.java |    8 +-
 .../hive/metastore/cache/CachedStore.java       |  545 ++-
 .../hive/metastore/cache/SharedCache.java       |  124 +-
 .../datasource/BoneCPDataSourceProvider.java    |   95 -
 .../datasource/DataSourceProvider.java          |   79 -
 .../datasource/DataSourceProviderFactory.java   |   47 -
 .../datasource/HikariCPDataSourceProvider.java  |   97 -
 .../hive/metastore/datasource/package-info.java |   23 -
 .../hive/metastore/tools/SQLGenerator.java      |  172 -
 .../hive/metastore/txn/CompactionInfo.java      |  151 -
 .../metastore/txn/CompactionTxnHandler.java     |  960 -----
 .../hadoop/hive/metastore/txn/TxnDbUtil.java    |  369 --
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 3667 ------------------
 .../hadoop/hive/metastore/txn/TxnStore.java     |  433 ---
 .../hadoop/hive/metastore/txn/TxnUtils.java     |  242 --
 .../hive/metastore/cache/TestCachedStore.java   |  113 +-
 .../TestDataSourceProviderFactory.java          |  173 -
 .../metastore/txn/TestTxnHandlerNegative.java   |   54 -
 .../hadoop/hive/metastore/txn/TestTxnUtils.java |  215 -
 .../txn/TestValidCompactorTxnList.java          |  136 -
 pom.xml                                         |    2 +
 .../hadoop/hive/llap/LlapCacheAwareFs.java      |    4 +-
 .../java/org/apache/hadoop/hive/ql/Driver.java  |    4 +-
 .../org/apache/hadoop/hive/ql/ErrorMsg.java     |   35 +-
 .../apache/hadoop/hive/ql/QueryProperties.java  |   13 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java |   20 +-
 .../apache/hadoop/hive/ql/exec/MoveTask.java    |    9 +-
 .../hadoop/hive/ql/exec/ReplCopyTask.java       |   18 +-
 .../hadoop/hive/ql/exec/StatsNoJobTask.java     |    2 +-
 .../apache/hadoop/hive/ql/exec/StatsTask.java   |    9 +-
 .../apache/hadoop/hive/ql/exec/TaskFactory.java |    4 -
 .../apache/hadoop/hive/ql/exec/Utilities.java   |   34 +-
 .../hadoop/hive/ql/exec/mr/MapredLocalTask.java |   24 +-
 .../bootstrap/load/table/LoadPartitions.java    |    4 +-
 .../repl/bootstrap/load/table/LoadTable.java    |    4 +-
 .../hadoop/hive/ql/exec/tez/DagUtils.java       |   40 +-
 .../hadoop/hive/ql/exec/tez/TezProcessor.java   |   67 +-
 .../org/apache/hadoop/hive/ql/hooks/Entity.java |   17 +-
 .../hadoop/hive/ql/hooks/WriteEntity.java       |    5 +-
 .../hive/ql/io/orc/encoded/CacheChunk.java      |   12 +-
 .../ql/io/orc/encoded/EncodedReaderImpl.java    |   69 +-
 .../ql/io/rcfile/stats/PartialScanMapper.java   |  180 -
 .../ql/io/rcfile/stats/PartialScanTask.java     |  381 --
 .../ql/io/rcfile/stats/PartialScanWork.java     |  119 -
 .../apache/hadoop/hive/ql/metadata/Hive.java    |   81 +-
 .../apache/hadoop/hive/ql/metadata/Table.java   |    1 +
 .../formatting/JsonMetaDataFormatter.java       |    5 +-
 .../formatting/MetaDataFormatUtils.java         |  304 +-
 .../metadata/formatting/MetaDataFormatter.java  |    2 +-
 .../formatting/MetaDataPrettyFormatUtils.java   |  244 --
 .../formatting/TextMetaDataFormatter.java       |   42 +-
 .../metadata/formatting/TextMetaDataTable.java  |   61 +
 .../hive/ql/optimizer/GenMRTableScan1.java      |   57 -
 .../hive/ql/optimizer/GenMapRedUtils.java       |   31 -
 .../ql/optimizer/QueryPlanPostProcessor.java    |    1 -
 .../hive/ql/optimizer/StatsOptimizer.java       |    5 +
 .../calcite/translator/RexNodeConverter.java    |   23 +-
 .../ql/parse/ColumnStatsSemanticAnalyzer.java   |    2 -
 .../hive/ql/parse/DDLSemanticAnalyzer.java      |   25 +-
 .../hive/ql/parse/FunctionSemanticAnalyzer.java |    9 +-
 .../org/apache/hadoop/hive/ql/parse/HiveLexer.g |    2 -
 .../apache/hadoop/hive/ql/parse/HiveParser.g    |    6 +-
 .../hadoop/hive/ql/parse/IdentifiersParser.g    |    2 +-
 .../hive/ql/parse/ImportSemanticAnalyzer.java   |    8 +-
 .../hive/ql/parse/LoadSemanticAnalyzer.java     |    4 +-
 .../hive/ql/parse/ProcessAnalyzeTable.java      |   49 -
 .../hadoop/hive/ql/parse/QBParseInfo.java       |   16 -
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  173 +-
 .../hadoop/hive/ql/parse/repl/CopyUtils.java    |    6 +-
 .../hadoop/hive/ql/parse/repl/dump/Utils.java   |    6 +
 .../parse/spark/SparkProcessAnalyzeTable.java   |   51 +-
 .../hadoop/hive/ql/plan/DescTableDesc.java      |   16 +-
 .../hadoop/hive/ql/plan/LoadTableDesc.java      |   44 +-
 .../apache/hadoop/hive/ql/plan/StatsWork.java   |   16 -
 .../plugin/HiveAuthorizerImpl.java              |    1 -
 .../plugin/HivePrivilegeObject.java             |   25 +-
 .../hadoop/hive/ql/session/SessionState.java    |   79 +-
 .../hive/ql/txn/AcidOpenTxnsCounterService.java |   75 -
 .../hadoop/hive/ql/util/ResourceDownloader.java |   11 +-
 .../metastore/txn/TestCompactionTxnHandler.java |    4 +-
 .../hive/metastore/txn/TestTxnHandler.java      |    4 +-
 .../txn/TestTxnHandlerNoConnectionPool.java     |    6 +-
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |   23 +-
 .../apache/hadoop/hive/ql/TestTxnCommands2.java |   16 +-
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java |    5 +-
 .../hadoop/hive/ql/exec/TestExecDriver.java     |    4 +-
 .../hive/ql/lockmgr/TestDbTxnManager.java       |   28 +-
 .../hive/ql/lockmgr/TestDbTxnManager2.java      |  356 +-
 .../hive/ql/metadata/TestHiveCopyFiles.java     |   12 +-
 .../TestSQLStdHiveAccessControllerHS2.java      |    3 +-
 .../hadoop/hive/ql/session/TestAddResource.java |   18 +-
 .../hive/ql/txn/compactor/CompactorTest.java    |    2 +-
 .../stats_partialscan_autogether.q              |   32 -
 .../stats_partialscan_non_external.q            |    5 -
 .../stats_partialscan_non_native.q              |    6 -
 .../clientnegative/stats_partscan_norcfile.q    |   12 -
 .../queries/clientpositive/describe_pretty.q    |   80 -
 .../queries/clientpositive/escape_comments.q    |    1 -
 .../test/queries/clientpositive/orc_analyze.q   |   14 -
 .../clientpositive/spark_local_queries.q        |   16 +
 .../queries/clientpositive/stats_noscan_2.q     |    3 +
 .../queries/clientpositive/stats_partscan_1.q   |   37 -
 .../clientpositive/stats_partscan_1_23.q        |   38 -
 .../clientnegative/alter_file_format.q.out      |    1 -
 .../alter_partition_coltype_2columns.q.out      |    1 -
 ...alter_partition_coltype_invalidcolname.q.out |    1 -
 .../alter_partition_coltype_invalidtype.q.out   |    1 -
 ...ter_partition_partial_spec_dyndisabled.q.out |    1 -
 .../alter_view_as_select_with_partition.q.out   |    2 -
 .../results/clientnegative/desc_failure2.q.out  |    1 -
 .../stats_partialscan_autogether.q.out          |   77 -
 .../stats_partialscan_non_external.q.out        |    9 -
 .../stats_partialscan_non_native.q.out          |   11 -
 .../stats_partscan_norcfile.q.out               |   36 -
 .../clientnegative/wrong_column_type.q.out      |    2 +-
 .../clientpositive/acid_table_stats.q.out       |   16 -
 ql/src/test/results/clientpositive/alter2.q.out |   12 -
 ql/src/test/results/clientpositive/alter3.q.out |    6 -
 ql/src/test/results/clientpositive/alter5.q.out |    4 -
 .../clientpositive/alterColumnStats.q.out       |   33 +-
 .../clientpositive/alterColumnStatsPart.q.out   |   11 -
 .../clientpositive/alter_file_format.q.out      |   19 -
 .../results/clientpositive/alter_merge_2.q.out  |    1 -
 .../clientpositive/alter_merge_2_orc.q.out      |    1 -
 .../clientpositive/alter_merge_stats.q.out      |    3 -
 ...lter_numbuckets_partitioned_table2_h23.q.out |   36 -
 ...alter_numbuckets_partitioned_table_h23.q.out |   24 -
 .../alter_partition_change_col.q.out            |   13 -
 .../alter_partition_clusterby_sortby.q.out      |    8 -
 .../alter_partition_coltype.q.out               |    4 -
 .../alter_partition_format_loc.q.out            |    6 -
 .../alter_partition_onto_nocurrent_db.q.out     |    2 -
 .../alter_partition_update_status.q.out         |   75 +-
 .../clientpositive/alter_skewed_table.q.out     |    6 -
 .../alter_table_add_partition.q.out             |   10 -
 .../clientpositive/alter_table_cascade.q.out    |   46 -
 .../alter_table_column_stats.q.out              | 1324 +++++--
 .../clientpositive/alter_table_not_sorted.q.out |    2 -
 .../clientpositive/alter_table_serde.q.out      |    5 -
 .../clientpositive/alter_table_serde2.q.out     |    8 -
 .../alter_table_stats_status.q.out              |   13 -
 .../alter_table_update_status.q.out             |  720 +++-
 ..._table_update_status_disable_bitvector.q.out |  720 +++-
 .../clientpositive/alter_view_as_select.q.out   |    3 -
 .../analyze_table_null_partition.q.out          |    1 -
 .../clientpositive/analyze_tbl_date.q.out       |    1 -
 .../clientpositive/analyze_tbl_part.q.out       |   45 +-
 .../clientpositive/annotate_stats_select.q.out  |   16 +-
 .../clientpositive/authorization_index.q.out    |    1 -
 .../clientpositive/autoColumnStats_3.q.out      |   14 -
 .../clientpositive/autoColumnStats_4.q.out      |    3 -
 .../clientpositive/autoColumnStats_5.q.out      |   76 +-
 .../clientpositive/autoColumnStats_8.q.out      |    1 -
 .../clientpositive/autoColumnStats_9.q.out      |   31 +-
 .../results/clientpositive/auto_join25.q.out    |    4 +
 .../auto_join_without_localtask.q.out           |    2 +
 .../clientpositive/avro_add_column3.q.out       |    1 -
 .../results/clientpositive/avro_decimal.q.out   |   15 +-
 .../clientpositive/avro_decimal_native.q.out    |   15 +-
 .../avro_schema_evolution_native.q.out          |    4 -
 .../beeline/colstats_all_nulls.q.out            |   91 +
 .../beeline/escape_comments.q.out               |   32 +-
 .../test/results/clientpositive/bitvector.q.out |   15 +-
 .../bucketsortoptimize_insert_8.q.out           |    2 +
 .../clientpositive/colstats_all_nulls.q.out     |   30 +-
 .../columnStatsUpdateForStatsOptimizer_2.q.out  |    4 -
 ...names_with_leading_and_trailing_spaces.q.out |   31 +-
 .../column_pruner_multiple_children.q.out       |   31 +-
 .../clientpositive/columnstats_infinity.q.out   |    3 -
 .../clientpositive/columnstats_partlvl.q.out    |   90 +-
 .../clientpositive/columnstats_partlvl_dp.q.out |  150 +-
 .../clientpositive/columnstats_tbllvl.q.out     |  105 +-
 .../test/results/clientpositive/combine3.q.out  |    2 -
 .../results/clientpositive/compustat_avro.q.out |   30 +-
 .../clientpositive/compute_stats_date.q.out     |   30 +-
 .../confirm_initial_tbl_stats.q.out             |  166 +-
 .../clientpositive/convert_enum_to_string.q.out |    1 -
 .../create_alter_list_bucketing_table1.q.out    |    7 -
 .../results/clientpositive/create_like.q.out    |   10 -
 .../results/clientpositive/create_like2.q.out   |    1 -
 .../clientpositive/create_like_tbl_props.q.out  |    5 -
 .../clientpositive/create_like_view.q.out       |    5 -
 .../clientpositive/create_or_replace_view.q.out |    9 -
 .../clientpositive/create_skewed_table1.q.out   |    3 -
 .../create_table_like_stats.q.out               |    6 -
 .../results/clientpositive/create_view.q.out    |   14 -
 .../create_view_defaultformats.q.out            |    2 -
 .../create_view_partitioned.q.out               |    7 -
 .../clientpositive/create_view_translate.q.out  |    2 -
 .../create_with_constraints.q.out               |   36 -
 ql/src/test/results/clientpositive/ctas.q.out   |    5 -
 .../results/clientpositive/ctas_colname.q.out   |    7 -
 .../ctas_uses_database_location.q.out           |    1 -
 .../clientpositive/database_location.q.out      |    2 -
 .../clientpositive/dbtxnmgr_query5.q.out        |    1 -
 .../clientpositive/decimal_precision2.q.out     |    8 +-
 .../results/clientpositive/decimal_serde.q.out  |    2 -
 .../results/clientpositive/decimal_stats.q.out  |   15 +-
 .../clientpositive/default_file_format.q.out    |   12 -
 .../results/clientpositive/deleteAnalyze.q.out  |   16 +-
 .../clientpositive/desc_tbl_part_cols.q.out     |    1 -
 .../describe_comment_indent.q.out               |    1 -
 .../describe_comment_nonascii.q.out             |    1 -
 .../describe_formatted_view_partitioned.q.out   |    2 -
 .../clientpositive/describe_pretty.q.out        |  323 --
 .../clientpositive/describe_syntax.q.out        |   61 +-
 .../results/clientpositive/describe_table.q.out |   84 +-
 .../display_colstats_tbllvl.q.out               |  180 +-
 .../clientpositive/drop_partitions_filter.q.out |    1 -
 .../drop_partitions_filter2.q.out               |    1 -
 .../drop_partitions_filter3.q.out               |    1 -
 .../results/clientpositive/druid_basic1.q.out   |    2 -
 .../results/clientpositive/druid_basic2.q.out   |    1 -
 .../clientpositive/druid_intervals.q.out        |    1 -
 .../clientpositive/druid_timeseries.q.out       |    1 -
 .../results/clientpositive/druid_topn.q.out     |    1 -
 .../encrypted/encryption_insert_values.q.out    |    1 -
 .../encrypted/encryption_move_tbl.q.out         |   60 +-
 .../clientpositive/escape_comments.q.out        |   24 -
 .../clientpositive/exim_02_00_part_empty.q.out  |    1 -
 .../results/clientpositive/exim_02_part.q.out   |    1 -
 .../clientpositive/exim_04_all_part.q.out       |    1 -
 .../clientpositive/exim_04_evolved_parts.q.out  |    3 -
 .../clientpositive/exim_05_some_part.q.out      |    1 -
 .../clientpositive/exim_06_one_part.q.out       |    1 -
 .../exim_07_all_part_over_nonoverlap.q.out      |    1 -
 .../exim_09_part_spec_nonoverlap.q.out          |    1 -
 .../clientpositive/exim_15_external_part.q.out  |    1 -
 .../clientpositive/exim_18_part_external.q.out  |    1 -
 .../exim_19_00_part_external_location.q.out     |    1 -
 .../exim_19_part_external_location.q.out        |    1 -
 .../exim_20_part_managed_location.q.out         |    1 -
 .../clientpositive/exim_hidden_files.q.out      |    2 -
 .../extrapolate_part_stats_full.q.out           |   15 +-
 .../extrapolate_part_stats_partial.q.out        |   30 +-
 .../test/results/clientpositive/fm-sketch.q.out |   75 +-
 ql/src/test/results/clientpositive/hll.q.out    |   75 +-
 .../clientpositive/index_skewtable.q.out        |    1 -
 .../clientpositive/infer_bucket_sort.q.out      |   50 -
 .../infer_bucket_sort_convert_join.q.out        |    5 +-
 .../infer_bucket_sort_dyn_part.q.out            |   16 -
 .../infer_bucket_sort_grouping_operators.q.out  |   12 -
 .../infer_bucket_sort_list_bucket.q.out         |    4 -
 .../infer_bucket_sort_map_operators.q.out       |    8 -
 .../infer_bucket_sort_merge.q.out               |    4 -
 .../infer_bucket_sort_multi_insert.q.out        |   16 -
 .../infer_bucket_sort_num_buckets.q.out         |    4 -
 .../infer_bucket_sort_reducers_power_two.q.out  |   12 -
 .../test/results/clientpositive/input10.q.out   |    1 -
 .../results/clientpositive/input_part10.q.out   |    1 -
 .../test/results/clientpositive/inputddl2.q.out |    1 -
 .../test/results/clientpositive/inputddl4.q.out |    2 -
 .../test/results/clientpositive/inputddl6.q.out |    2 -
 .../test/results/clientpositive/inputddl7.q.out |    2 -
 .../test/results/clientpositive/inputddl8.q.out |    1 -
 .../insert_values_orig_table_use_metadata.q.out |   13 -
 .../results/clientpositive/lb_fs_stats.q.out    |    2 -
 .../clientpositive/list_bucket_dml_1.q.out      |    4 -
 .../clientpositive/list_bucket_dml_11.q.out     |    2 -
 .../clientpositive/list_bucket_dml_12.q.out     |    2 -
 .../clientpositive/list_bucket_dml_13.q.out     |    2 -
 .../clientpositive/list_bucket_dml_14.q.out     |    1 -
 .../clientpositive/list_bucket_dml_2.q.out      |    2 -
 .../clientpositive/list_bucket_dml_3.q.out      |    2 -
 .../clientpositive/list_bucket_dml_4.q.out      |    4 -
 .../clientpositive/list_bucket_dml_5.q.out      |    4 -
 .../clientpositive/list_bucket_dml_6.q.out      |    8 -
 .../clientpositive/list_bucket_dml_7.q.out      |    8 -
 .../clientpositive/list_bucket_dml_8.q.out      |    6 -
 .../clientpositive/list_bucket_dml_9.q.out      |    4 -
 .../list_bucket_query_multiskew_1.q.out         |    2 -
 .../list_bucket_query_multiskew_2.q.out         |    2 -
 .../list_bucket_query_multiskew_3.q.out         |    6 -
 .../list_bucket_query_oneskew_1.q.out           |    2 -
 .../list_bucket_query_oneskew_2.q.out           |    2 -
 .../list_bucket_query_oneskew_3.q.out           |    2 -
 .../clientpositive/literal_decimal.q.out        |    6 +-
 .../clientpositive/llap/alter_merge_2_orc.q.out |    1 -
 .../llap/alter_merge_stats_orc.q.out            |    8 -
 .../clientpositive/llap/autoColumnStats_1.q.out |   26 -
 .../clientpositive/llap/autoColumnStats_2.q.out |  148 +-
 .../results/clientpositive/llap/bucket5.q.out   |    1 -
 .../clientpositive/llap/bucket_groupby.q.out    |    4 -
 .../columnStatsUpdateForStatsOptimizer_1.q.out  |   14 -
 ...names_with_leading_and_trailing_spaces.q.out |   31 +-
 .../llap/column_table_stats.q.out               |   26 -
 .../llap/column_table_stats_orc.q.out           |   18 -
 .../llap/columnstats_part_coltype.q.out         |  203 +-
 .../test/results/clientpositive/llap/ctas.q.out |    5 -
 .../clientpositive/llap/deleteAnalyze.q.out     |   16 +-
 .../llap/dynpart_sort_opt_vectorization.q.out   |   32 -
 .../llap/dynpart_sort_optimization.q.out        |   32 -
 .../llap/dynpart_sort_optimization2.q.out       |   24 -
 .../clientpositive/llap/explainuser_1.q.out     |    1 -
 .../extrapolate_part_stats_partial_ndv.q.out    |  330 +-
 .../llap/infer_bucket_sort_bucketed_table.q.out |    2 -
 .../llap/list_bucket_dml_10.q.out               |    2 -
 .../clientpositive/llap/load_dyn_part1.q.out    |    1 -
 .../clientpositive/llap/load_dyn_part2.q.out    |    1 -
 .../clientpositive/llap/load_dyn_part3.q.out    |    1 -
 .../clientpositive/llap/load_dyn_part5.q.out    |    1 -
 .../clientpositive/llap/orc_analyze.q.out       |  330 --
 .../clientpositive/llap/orc_create.q.out        |   10 -
 .../clientpositive/llap/orc_llap_counters.q.out |    1 -
 .../llap/orc_llap_counters1.q.out               |    1 -
 .../clientpositive/llap/schema_evol_stats.q.out |   12 -
 .../llap/selectDistinctStar.q.out               |    2 -
 .../results/clientpositive/llap/stats11.q.out   |    8 -
 .../clientpositive/llap/stats_noscan_1.q.out    |   18 -
 .../clientpositive/llap/stats_only_null.q.out   |   19 +-
 .../results/clientpositive/llap/sysdb.q.out     |   45 +-
 .../llap/tez_bmj_schema_evolution.q.out         |    1 -
 .../clientpositive/llap/tez_fsstat.q.out        |    2 -
 .../llap/tez_schema_evolution.q.out             |    1 -
 .../clientpositive/llap/union_fast_stats.q.out  |   12 +-
 .../clientpositive/llap/union_stats.q.out       |    5 -
 .../llap/vector_decimal_math_funcs.q.out        |    6 +-
 .../llap/vectorized_math_funcs.q.out            |    2 +-
 .../results/clientpositive/load_dyn_part1.q.out |    1 -
 .../clientpositive/load_dyn_part10.q.out        |    1 -
 .../clientpositive/load_dyn_part11.q.out        |    1 -
 .../clientpositive/load_dyn_part12.q.out        |    1 -
 .../clientpositive/load_dyn_part13.q.out        |    1 -
 .../clientpositive/load_dyn_part14.q.out        |    1 -
 .../results/clientpositive/load_dyn_part2.q.out |    1 -
 .../results/clientpositive/load_dyn_part3.q.out |    1 -
 .../results/clientpositive/load_dyn_part4.q.out |    1 -
 .../results/clientpositive/load_dyn_part6.q.out |    1 -
 .../results/clientpositive/load_dyn_part7.q.out |    1 -
 .../results/clientpositive/load_dyn_part8.q.out |    1 -
 .../results/clientpositive/load_dyn_part9.q.out |    1 -
 .../results/clientpositive/mapjoin_hook.q.out   |    3 +
 .../materialized_view_describe.q.out            |    4 -
 ql/src/test/results/clientpositive/merge3.q.out |    1 -
 .../clientpositive/named_column_join.q.out      |    3 -
 .../clientpositive/parallel_orderby.q.out       |    2 -
 .../clientpositive/parquet_analyze.q.out        |    2 -
 .../parquet_array_null_element.q.out            |    1 -
 .../results/clientpositive/parquet_create.q.out |    1 -
 .../parquet_mixed_partition_formats.q.out       |    6 -
 .../clientpositive/parquet_partitioned.q.out    |    2 -
 .../results/clientpositive/parquet_serde.q.out  |    6 -
 .../clientpositive/part_inherit_tbl_props.q.out |    2 -
 .../part_inherit_tbl_props_empty.q.out          |    2 -
 .../part_inherit_tbl_props_with_star.q.out      |    2 -
 .../clientpositive/partial_column_stats.q.out   |   15 +-
 .../partition_coltype_literals.q.out            |  107 +-
 .../clientpositive/partition_date2.q.out        |    2 -
 .../clientpositive/partition_schema1.q.out      |    3 -
 .../clientpositive/partition_timestamp2.q.out   |    2 -
 .../clientpositive/perf/spark/query14.q.out     | 3237 ----------------
 .../clientpositive/perf/spark/query21.q.out     |    2 +-
 .../clientpositive/perf/spark/query75.q.out     |   72 +-
 .../clientpositive/perf/tez/query14.q.out       |    2 +-
 .../clientpositive/perf/tez/query21.q.out       |    2 +-
 .../results/clientpositive/quotedid_stats.q.out |    2 -
 .../clientpositive/rcfile_default_format.q.out  |    8 -
 .../rename_external_partition_location.q.out    |   68 +-
 .../rename_table_update_column_stats.q.out      |  180 +-
 .../clientpositive/repl_2_exim_basic.q.out      |    4 -
 .../clientpositive/repl_3_exim_metadata.q.out   |    2 -
 .../sample_islocalmode_hook_use_metadata.q.out  |    1 -
 .../clientpositive/serde_reported_schema.q.out  |    2 -
 .../clientpositive/show_indexes_syntax.q.out    |    2 -
 .../spark/alter_merge_stats_orc.q.out           |    8 -
 .../spark/avro_decimal_native.q.out             |   15 +-
 .../results/clientpositive/spark/bucket5.q.out  |    1 -
 .../results/clientpositive/spark/ctas.q.out     |    5 -
 .../infer_bucket_sort_bucketed_table.q.out      |    2 -
 .../spark/infer_bucket_sort_map_operators.q.out |    8 -
 .../spark/infer_bucket_sort_merge.q.out         |    4 -
 .../spark/infer_bucket_sort_num_buckets.q.out   |    4 -
 .../infer_bucket_sort_reducers_power_two.q.out  |   12 -
 .../spark/list_bucket_dml_10.q.out              |    2 -
 .../spark/list_bucket_dml_2.q.out               |    2 -
 .../clientpositive/spark/load_dyn_part1.q.out   |    1 -
 .../clientpositive/spark/load_dyn_part10.q.out  |    1 -
 .../clientpositive/spark/load_dyn_part11.q.out  |    1 -
 .../clientpositive/spark/load_dyn_part12.q.out  |    1 -
 .../clientpositive/spark/load_dyn_part13.q.out  |    1 -
 .../clientpositive/spark/load_dyn_part14.q.out  |    1 -
 .../clientpositive/spark/load_dyn_part2.q.out   |    1 -
 .../clientpositive/spark/load_dyn_part3.q.out   |    1 -
 .../clientpositive/spark/load_dyn_part4.q.out   |    1 -
 .../clientpositive/spark/load_dyn_part5.q.out   |    1 -
 .../clientpositive/spark/load_dyn_part6.q.out   |    1 -
 .../clientpositive/spark/load_dyn_part7.q.out   |    1 -
 .../clientpositive/spark/load_dyn_part8.q.out   |    1 -
 .../clientpositive/spark/load_dyn_part9.q.out   |    1 -
 .../clientpositive/spark/parallel_orderby.q.out |    2 -
 .../spark/spark_explainuser_1.q.out             |    1 -
 .../spark/spark_local_queries.q.out             |  131 +
 .../results/clientpositive/spark/stats0.q.out   |    4 -
 .../results/clientpositive/spark/stats1.q.out   |    2 -
 .../results/clientpositive/spark/stats10.q.out  |    6 -
 .../results/clientpositive/spark/stats12.q.out  |   10 -
 .../results/clientpositive/spark/stats13.q.out  |   12 -
 .../results/clientpositive/spark/stats14.q.out  |    9 -
 .../results/clientpositive/spark/stats15.q.out  |    9 -
 .../results/clientpositive/spark/stats16.q.out  |    2 -
 .../results/clientpositive/spark/stats18.q.out  |    4 -
 .../results/clientpositive/spark/stats2.q.out   |    4 -
 .../results/clientpositive/spark/stats3.q.out   |    3 -
 .../results/clientpositive/spark/stats5.q.out   |    1 -
 .../results/clientpositive/spark/stats6.q.out   |   10 -
 .../results/clientpositive/spark/stats7.q.out   |    6 -
 .../results/clientpositive/spark/stats8.q.out   |   20 -
 .../results/clientpositive/spark/stats9.q.out   |    1 -
 .../clientpositive/spark/stats_noscan_1.q.out   |   18 -
 .../clientpositive/spark/stats_noscan_2.q.out   |   83 +-
 .../clientpositive/spark/stats_only_null.q.out  |   19 +-
 .../spark/stats_partscan_1_23.q.out             |  187 -
 .../results/clientpositive/spark/statsfs.q.out  |   14 -
 .../clientpositive/spark/union_remove_1.q.out   |    1 -
 .../clientpositive/spark/union_remove_10.q.out  |    1 -
 .../clientpositive/spark/union_remove_11.q.out  |    1 -
 .../clientpositive/spark/union_remove_12.q.out  |    1 -
 .../clientpositive/spark/union_remove_13.q.out  |    1 -
 .../clientpositive/spark/union_remove_14.q.out  |    1 -
 .../clientpositive/spark/union_remove_15.q.out  |    2 -
 .../clientpositive/spark/union_remove_16.q.out  |    2 -
 .../clientpositive/spark/union_remove_17.q.out  |    2 -
 .../clientpositive/spark/union_remove_18.q.out  |    2 -
 .../clientpositive/spark/union_remove_19.q.out  |    1 -
 .../clientpositive/spark/union_remove_2.q.out   |    1 -
 .../clientpositive/spark/union_remove_20.q.out  |    1 -
 .../clientpositive/spark/union_remove_21.q.out  |    1 -
 .../clientpositive/spark/union_remove_22.q.out  |    1 -
 .../clientpositive/spark/union_remove_23.q.out  |    1 -
 .../clientpositive/spark/union_remove_24.q.out  |    1 -
 .../clientpositive/spark/union_remove_25.q.out  |    6 -
 .../clientpositive/spark/union_remove_3.q.out   |    1 -
 .../clientpositive/spark/union_remove_4.q.out   |    1 -
 .../clientpositive/spark/union_remove_5.q.out   |    1 -
 .../clientpositive/spark/union_remove_7.q.out   |    1 -
 .../clientpositive/spark/union_remove_8.q.out   |    1 -
 .../clientpositive/spark/union_remove_9.q.out   |    1 -
 .../spark/vectorized_math_funcs.q.out           |    2 +-
 ql/src/test/results/clientpositive/stats0.q.out |    4 -
 ql/src/test/results/clientpositive/stats1.q.out |    2 -
 .../test/results/clientpositive/stats10.q.out   |    6 -
 .../test/results/clientpositive/stats12.q.out   |   10 -
 .../test/results/clientpositive/stats13.q.out   |   12 -
 .../test/results/clientpositive/stats14.q.out   |    9 -
 .../test/results/clientpositive/stats15.q.out   |    9 -
 .../test/results/clientpositive/stats16.q.out   |    2 -
 .../test/results/clientpositive/stats18.q.out   |    4 -
 ql/src/test/results/clientpositive/stats2.q.out |    4 -
 ql/src/test/results/clientpositive/stats3.q.out |    3 -
 ql/src/test/results/clientpositive/stats4.q.out |   12 -
 ql/src/test/results/clientpositive/stats5.q.out |    1 -
 ql/src/test/results/clientpositive/stats6.q.out |   10 -
 ql/src/test/results/clientpositive/stats7.q.out |    6 -
 ql/src/test/results/clientpositive/stats8.q.out |   20 -
 ql/src/test/results/clientpositive/stats9.q.out |    1 -
 .../clientpositive/stats_empty_partition.q.out  |    2 -
 .../clientpositive/stats_invalidation.q.out     |    2 -
 .../clientpositive/stats_list_bucket.q.out      |    3 -
 .../results/clientpositive/stats_noscan_1.q.out |   18 -
 .../results/clientpositive/stats_noscan_2.q.out |   77 +-
 .../clientpositive/stats_only_null.q.out        |   19 +-
 .../clientpositive/stats_partscan_1.q.out       |  215 -
 .../clientpositive/stats_partscan_1_23.q.out    |  190 -
 .../test/results/clientpositive/statsfs.q.out   |   14 -
 .../temp_table_display_colstats_tbllvl.q.out    |  180 +-
 .../clientpositive/tez/explainanalyze_5.q.out   |    1 -
 .../clientpositive/truncate_column.q.out        |   11 -
 .../results/clientpositive/tunable_ndv.q.out    |  120 +-
 .../clientpositive/unicode_comments.q.out       |    5 -
 .../clientpositive/unicode_notation.q.out       |    3 -
 .../clientpositive/union_pos_alias.q.out        |    1 -
 .../results/clientpositive/union_remove_1.q.out |    1 -
 .../clientpositive/union_remove_10.q.out        |    1 -
 .../clientpositive/union_remove_11.q.out        |    1 -
 .../clientpositive/union_remove_12.q.out        |    1 -
 .../clientpositive/union_remove_13.q.out        |    1 -
 .../clientpositive/union_remove_14.q.out        |    1 -
 .../clientpositive/union_remove_15.q.out        |    2 -
 .../clientpositive/union_remove_16.q.out        |    2 -
 .../clientpositive/union_remove_17.q.out        |    2 -
 .../clientpositive/union_remove_18.q.out        |    2 -
 .../clientpositive/union_remove_19.q.out        |    1 -
 .../results/clientpositive/union_remove_2.q.out |    1 -
 .../clientpositive/union_remove_20.q.out        |    1 -
 .../clientpositive/union_remove_21.q.out        |    1 -
 .../clientpositive/union_remove_22.q.out        |    1 -
 .../clientpositive/union_remove_23.q.out        |    1 -
 .../clientpositive/union_remove_24.q.out        |    1 -
 .../clientpositive/union_remove_25.q.out        |    6 -
 .../results/clientpositive/union_remove_3.q.out |    1 -
 .../results/clientpositive/union_remove_4.q.out |    1 -
 .../results/clientpositive/union_remove_5.q.out |    1 -
 .../results/clientpositive/union_remove_7.q.out |    1 -
 .../results/clientpositive/union_remove_8.q.out |    1 -
 .../results/clientpositive/union_remove_9.q.out |    1 -
 .../results/clientpositive/union_stats.q.out    |    5 -
 .../clientpositive/updateAccessTime.q.out       |    6 -
 .../vector_decimal_math_funcs.q.out             |    6 +-
 .../clientpositive/vectorized_math_funcs.q.out  |    2 +-
 .../results/clientpositive/view_alias.q.out     |    6 -
 .../test/results/clientpositive/view_cbo.q.out  |   10 -
 .../apache/hive/service/server/HiveServer2.java |    8 +
 .../hive-webapps/hiveserver2/hiveserver2.jsp    |   16 +-
 .../hive/spark/client/SparkClientImpl.java      |   63 +-
 standalone-metastore/pom.xml                    |   20 +
 .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp  | 2132 +++++-----
 .../gen/thrift/gen-cpp/hive_metastore_types.cpp |  860 +++-
 .../gen/thrift/gen-cpp/hive_metastore_types.h   |  288 ++
 .../hadoop/hive/metastore/api/WMMapping.java    |  804 ++++
 .../hadoop/hive/metastore/api/WMPool.java       |  908 +++++
 .../hive/metastore/api/WMResourcePlan.java      |  602 +++
 .../hadoop/hive/metastore/api/WMTrigger.java    |  706 ++++
 .../src/gen/thrift/gen-php/metastore/Types.php  |  622 +++
 .../gen/thrift/gen-py/hive_metastore/ttypes.py  |  458 +++
 .../gen/thrift/gen-rb/hive_metastore_types.rb   |  100 +
 .../hive/metastore/RunnableConfigurable.java    |   26 +
 .../hive/metastore/conf/MetastoreConf.java      |   63 +-
 .../datasource/BoneCPDataSourceProvider.java    |   94 +
 .../datasource/DataSourceProvider.java          |   79 +
 .../datasource/DataSourceProviderFactory.java   |   47 +
 .../datasource/HikariCPDataSourceProvider.java  |   96 +
 .../hive/metastore/datasource/package-info.java |   23 +
 .../hadoop/hive/metastore/model/MWMMapping.java |   82 +
 .../hadoop/hive/metastore/model/MWMPool.java    |   89 +
 .../hive/metastore/model/MWMResourcePlan.java   |   95 +
 .../hadoop/hive/metastore/model/MWMTrigger.java |   78 +
 .../metastore/security/DelegationTokenTool.java |  251 ++
 .../hive/metastore/tools/SQLGenerator.java      |  172 +
 .../txn/AcidOpenTxnsCounterService.java         |   63 +
 .../hive/metastore/txn/CompactionInfo.java      |  151 +
 .../metastore/txn/CompactionTxnHandler.java     |  961 +++++
 .../hadoop/hive/metastore/txn/TxnDbUtil.java    |  381 ++
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 3643 +++++++++++++++++
 .../hadoop/hive/metastore/txn/TxnStore.java     |  420 ++
 .../hadoop/hive/metastore/txn/TxnUtils.java     |  243 ++
 .../hadoop/hive/metastore/utils/JavaUtils.java  |   46 +
 .../hive/metastore/utils/StringableMap.java     |   80 +
 .../src/main/resources/package.jdo              |  112 +
 .../src/main/thrift/hive_metastore.thrift       |   30 +
 .../hive/metastore/conf/TestMetastoreConf.java  |   17 +
 .../TestDataSourceProviderFactory.java          |  170 +
 .../metastore/txn/TestTxnHandlerNegative.java   |   55 +
 .../hadoop/hive/metastore/txn/TestTxnUtils.java |  216 ++
 storage-api/pom.xml                             |    7 +
 .../hive/common/ValidCompactorTxnList.java      |   89 +
 .../hadoop/hive/common/ValidReadTxnList.java    |  235 ++
 .../apache/hadoop/hive/common/ValidTxnList.java |  112 +
 .../hive/common/TestValidCompactorTxnList.java  |  134 +
 .../hive/common/TestValidReadTxnList.java       |  109 +
 627 files changed, 23127 insertions(+), 19526 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
----------------------------------------------------------------------
diff --cc itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
index e519785,8a6fe3d..2f0efce
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
@@@ -104,7 -105,7 +105,8 @@@ public class TestHiveHistory extends Te
          db.dropTable(Warehouse.DEFAULT_DATABASE_NAME, src, true, true);
          db.createTable(src, cols, null, TextInputFormat.class,
              IgnoreKeyTextOutputFormat.class);
-         db.loadTable(hadoopDataFile[i], src, false, false, false, false, false, null, 0, false);
 -        db.loadTable(hadoopDataFile[i], src, LoadFileType.KEEP_EXISTING, false, false, false, false);
++        db.loadTable(hadoopDataFile[i], src,
++          LoadFileType.KEEP_EXISTING, false, false, false, false, null, 0, false);
          i++;
        }
  

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 497b5af,f8b79a0..ff2d988
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@@ -7944,8 -8008,16 +7950,16 @@@ public class HiveMetaStore extends Thri
      startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService"));
      startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidCompactionHistoryService"));
      startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidWriteSetService"));
+ 
+     ThreadPool.initialize(conf);
+     RunnableConfigurable rc = new AcidOpenTxnsCounterService();
+     rc.setConf(conf);
+     ThreadPool.getPool().scheduleAtFixedRate(rc, 100, MetastoreConf.getTimeVar(conf,
+         MetastoreConf.ConfVars.COUNT_OPEN_TXNS_INTERVAL, TimeUnit.MILLISECONDS),
+         TimeUnit.MILLISECONDS);
+ 
    }
 -  private static void startHouseKeeperService(HiveConf conf, Class c) throws Exception {
 +  private static void startHouseKeeperService(HiveConf conf, Class<?> c) throws Exception {
      //todo: when metastore adds orderly-shutdown logic, houseKeeper.stop()
      //should be called form it
      HouseKeeperService houseKeeper = (HouseKeeperService)c.newInstance();

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 7cc5c30,0dadd51..bef7433
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@@ -372,13 -377,10 +373,13 @@@ public class MoveTask extends Task<Move
          DataContainer dc = null;
          if (tbd.getPartitionSpec().size() == 0) {
            dc = new DataContainer(table.getTTable());
 +          if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
 +            Utilities.FILE_OP_LOGGER.trace("loadTable called from " + tbd.getSourcePath()
 +              + " into " + tbd.getTable().getTableName());
 +          }
-           db.loadTable(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getReplace(),
+           db.loadTable(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getLoadFileType(),
 -              work.isSrcLocal(), isSkewedStoredAsDirs(tbd),
 -              work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID,
 -              hasFollowingStatsTask());
 +              work.isSrcLocal(), isSkewedStoredAsDirs(tbd), isFullAcidOp, hasFollowingStatsTask(),
 +              tbd.getTxnId(), tbd.getStmtId(), tbd.isMmTable());
            if (work.getOutputs() != null) {
              DDLTask.addIfAbsentByName(new WriteEntity(table,
                getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs());
@@@ -445,230 -587,6 +446,230 @@@
        return (1);
      }
    }
 +
 +  private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd,
 +      TaskInformation ti) throws HiveException, IOException, InvalidOperationException {
 +    List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(),  tbd.getPartitionSpec());
 +    db.validatePartitionNameCharacters(partVals);
 +    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
 +      Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath()
 +        + " into " + tbd.getTable().getTableName());
 +    }
 +    db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(),
-         tbd.getPartitionSpec(), tbd.getReplace(),
++        tbd.getPartitionSpec(), tbd.getLoadFileType(),
 +        tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(),
 +        work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID &&
 +            !tbd.isMmTable(),
 +        hasFollowingStatsTask(), tbd.getTxnId(), tbd.getStmtId());
 +    Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);
 +
 +    // See the comment inside updatePartitionBucketSortColumns.
 +    if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) {
 +      updatePartitionBucketSortColumns(db, table, partn, ti.bucketCols,
 +          ti.numBuckets, ti.sortCols);
 +    }
 +
 +    DataContainer dc = new DataContainer(table.getTTable(), partn.getTPartition());
 +    // add this partition to post-execution hook
 +    if (work.getOutputs() != null) {
 +      DDLTask.addIfAbsentByName(new WriteEntity(partn,
 +        getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs());
 +    }
 +    return dc;
 +  }
 +
 +  private DataContainer handleDynParts(Hive db, Table table, LoadTableDesc tbd,
 +      TaskInformation ti, DynamicPartitionCtx dpCtx) throws HiveException,
 +      IOException, InvalidOperationException {
 +    DataContainer dc;
 +    List<LinkedHashMap<String, String>> dps = Utilities.getFullDPSpecs(conf, dpCtx);
 +
 +    console.printInfo(System.getProperty("line.separator"));
 +    long startTime = System.currentTimeMillis();
 +    // load the list of DP partitions and return the list of partition specs
 +    // TODO: In a follow-up to HIVE-1361, we should refactor loadDynamicPartitions
 +    // to use Utilities.getFullDPSpecs() to get the list of full partSpecs.
 +    // After that check the number of DPs created to not exceed the limit and
 +    // iterate over it and call loadPartition() here.
 +    // The reason we don't do inside HIVE-1361 is the latter is large and we
 +    // want to isolate any potential issue it may introduce.
 +    Map<Map<String, String>, Partition> dp =
 +      db.loadDynamicPartitions(
 +        tbd.getSourcePath(),
 +        tbd.getTable().getTableName(),
 +        tbd.getPartitionSpec(),
-         tbd.getReplace(),
++        tbd.getLoadFileType(),
 +        dpCtx.getNumDPCols(),
 +        (tbd.getLbCtx() == null) ? 0 : tbd.getLbCtx().calculateListBucketingLevel(),
 +        work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID &&
 +            !tbd.isMmTable(),
 +        work.getLoadTableWork().getTxnId(), tbd.getStmtId(), hasFollowingStatsTask(),
 +        work.getLoadTableWork().getWriteType());
 +
 +    // publish DP columns to its subscribers
 +    if (dps != null && dps.size() > 0) {
 +      pushFeed(FeedType.DYNAMIC_PARTITIONS, dp.values());
 +    }
 +
 +    String loadTime = "\t Time taken to load dynamic partitions: "  +
 +        (System.currentTimeMillis() - startTime)/1000.0 + " seconds";
 +    console.printInfo(loadTime);
 +    LOG.info(loadTime);
 +
 +    if (dp.size() == 0 && conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) {
 +      throw new HiveException("This query creates no partitions." +
 +          " To turn off this error, set hive.error.on.empty.partition=false.");
 +    }
 +
 +    startTime = System.currentTimeMillis();
 +    // for each partition spec, get the partition
 +    // and put it to WriteEntity for post-exec hook
 +    for(Map.Entry<Map<String, String>, Partition> entry : dp.entrySet()) {
 +      Partition partn = entry.getValue();
 +
 +      // See the comment inside updatePartitionBucketSortColumns.
 +      if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) {
 +        updatePartitionBucketSortColumns(
 +            db, table, partn, ti.bucketCols, ti.numBuckets, ti.sortCols);
 +      }
 +
 +      WriteEntity enty = new WriteEntity(partn,
 +        getWriteType(tbd, work.getLoadTableWork().getWriteType()));
 +      if (work.getOutputs() != null) {
 +        DDLTask.addIfAbsentByName(enty, work.getOutputs());
 +      }
 +      // Need to update the queryPlan's output as well so that post-exec hook get executed.
 +      // This is only needed for dynamic partitioning since for SP the the WriteEntity is
 +      // constructed at compile time and the queryPlan already contains that.
 +      // For DP, WriteEntity creation is deferred at this stage so we need to update
 +      // queryPlan here.
 +      if (queryPlan.getOutputs() == null) {
 +        queryPlan.setOutputs(new LinkedHashSet<WriteEntity>());
 +      }
 +      queryPlan.getOutputs().add(enty);
 +
 +      // update columnar lineage for each partition
 +      dc = new DataContainer(table.getTTable(), partn.getTPartition());
 +
 +      // Don't set lineage on delete as we don't have all the columns
 +      if (work.getLineagState() != null &&
 +          work.getLoadTableWork().getWriteType() != AcidUtils.Operation.DELETE &&
 +          work.getLoadTableWork().getWriteType() != AcidUtils.Operation.UPDATE) {
 +        work.getLineagState().setLineage(tbd.getSourcePath(), dc,
 +            table.getCols());
 +      }
 +      LOG.info("\tLoading partition " + entry.getKey());
 +    }
 +    console.printInfo("\t Time taken for adding to write entity : " +
 +        (System.currentTimeMillis() - startTime)/1000.0 + " seconds");
 +    dc = null; // reset data container to prevent it being added again.
 +    return dc;
 +  }
 +
 +  private void inferTaskInformation(TaskInformation ti) {
 +    // Find the first ancestor of this MoveTask which is some form of map reduce task
 +    // (Either standard, local, or a merge)
 +    while (ti.task.getParentTasks() != null && ti.task.getParentTasks().size() == 1) {
 +      ti.task = (Task)ti.task.getParentTasks().get(0);
 +      // If it was a merge task or a local map reduce task, nothing can be inferred
 +      if (ti.task instanceof MergeFileTask || ti.task instanceof MapredLocalTask) {
 +        break;
 +      }
 +
 +      // If it's a standard map reduce task, check what, if anything, it inferred about
 +      // the directory this move task is moving
 +      if (ti.task instanceof MapRedTask) {
 +        MapredWork work = (MapredWork)ti.task.getWork();
 +        MapWork mapWork = work.getMapWork();
 +        ti.bucketCols = mapWork.getBucketedColsByDirectory().get(ti.path);
 +        ti.sortCols = mapWork.getSortedColsByDirectory().get(ti.path);
 +        if (work.getReduceWork() != null) {
 +          ti.numBuckets = work.getReduceWork().getNumReduceTasks();
 +        }
 +
 +        if (ti.bucketCols != null || ti.sortCols != null) {
 +          // This must be a final map reduce task (the task containing the file sink
 +          // operator that writes the final output)
 +          assert work.isFinalMapRed();
 +        }
 +        break;
 +      }
 +
 +      // If it's a move task, get the path the files were moved from, this is what any
 +      // preceding map reduce task inferred information about, and moving does not invalidate
 +      // those assumptions
 +      // This can happen when a conditional merge is added before the final MoveTask, but the
 +      // condition for merging is not met, see GenMRFileSink1.
 +      if (ti.task instanceof MoveTask) {
 +        MoveTask mt = (MoveTask)ti.task;
 +        if (mt.getWork().getLoadFileWork() != null) {
 +          ti.path = mt.getWork().getLoadFileWork().getSourcePath().toUri().toString();
 +        }
 +      }
 +    }
 +  }
 +
 +  private void checkFileFormats(Hive db, LoadTableDesc tbd, Table table)
 +      throws HiveException {
 +    if (work.getCheckFileFormat()) {
 +      // Get all files from the src directory
 +      FileStatus[] dirs;
 +      ArrayList<FileStatus> files;
 +      FileSystem srcFs; // source filesystem
 +      try {
 +        srcFs = tbd.getSourcePath().getFileSystem(conf);
 +        dirs = srcFs.globStatus(tbd.getSourcePath());
 +        files = new ArrayList<FileStatus>();
 +        for (int i = 0; (dirs != null && i < dirs.length); i++) {
 +          files.addAll(Arrays.asList(srcFs.listStatus(dirs[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER)));
 +          // We only check one file, so exit the loop when we have at least
 +          // one.
 +          if (files.size() > 0) {
 +            break;
 +          }
 +        }
 +      } catch (IOException e) {
 +        throw new HiveException(
 +            "addFiles: filesystem error in check phase", e);
 +      }
 +
 +      // handle file format check for table level
 +      if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVECHECKFILEFORMAT)) {
 +        boolean flag = true;
 +        // work.checkFileFormat is set to true only for Load Task, so assumption here is
 +        // dynamic partition context is null
 +        if (tbd.getDPCtx() == null) {
 +          if (tbd.getPartitionSpec() == null || tbd.getPartitionSpec().isEmpty()) {
 +            // Check if the file format of the file matches that of the table.
 +            flag = HiveFileFormatUtils.checkInputFormat(
 +                srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
 +          } else {
 +            // Check if the file format of the file matches that of the partition
 +            Partition oldPart = db.getPartition(table, tbd.getPartitionSpec(), false);
 +            if (oldPart == null) {
 +              // this means we have just created a table and are specifying partition in the
 +              // load statement (without pre-creating the partition), in which case lets use
 +              // table input format class. inheritTableSpecs defaults to true so when a new
 +              // partition is created later it will automatically inherit input format
 +              // from table object
 +              flag = HiveFileFormatUtils.checkInputFormat(
 +                  srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
 +            } else {
 +              flag = HiveFileFormatUtils.checkInputFormat(
 +                  srcFs, conf, oldPart.getInputFormatClass(), files);
 +            }
 +          }
 +          if (!flag) {
 +            throw new HiveException(ErrorMsg.WRONG_FILE_FORMAT);
 +          }
 +        } else {
 +          LOG.warn("Skipping file format check as dpCtx is not null");
 +        }
 +      }
 +    }
 +  }
 +
 +
    /**
     * so to make sure we crate WriteEntity with the right WriteType.  This is (at this point) only
     * for consistency since LockManager (which is the only thing that pays attention to WriteType)

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
index 1725629,bdf3710..f11f163
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
@@@ -179,10 -180,11 +180,11 @@@ public class StatsTask extends Task<Sta
          // work.getLoadTableDesc().getReplace() is true means insert overwrite command 
          // work.getLoadFileDesc().getDestinationCreateTable().isEmpty() means CTAS etc.
          // acidTable will not have accurate stats unless it is set through analyze command.
 -        if (work.getTableSpecs() == null && AcidUtils.isAcidTable(table)) {
 +        if (work.getTableSpecs() == null && AcidUtils.isFullAcidTable(table)) {
            StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE);
          } else if (work.getTableSpecs() != null
-             || (work.getLoadTableDesc() != null && work.getLoadTableDesc().getReplace())
+             || (work.getLoadTableDesc() != null
+                 && (work.getLoadTableDesc().getLoadFileType() == LoadFileType.REPLACE_ALL))
              || (work.getLoadFileDesc() != null && !work.getLoadFileDesc()
                  .getDestinationCreateTable().isEmpty())) {
            StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE);
@@@ -283,10 -282,11 +285,11 @@@
            //
            org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition();
            Map<String, String> parameters = tPart.getParameters();
 -          if (work.getTableSpecs() == null && AcidUtils.isAcidTable(table)) {
 +          if (work.getTableSpecs() == null && AcidUtils.isFullAcidTable(table)) {
              StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE);
            } else if (work.getTableSpecs() != null
-               || (work.getLoadTableDesc() != null && work.getLoadTableDesc().getReplace())
+               || (work.getLoadTableDesc() != null
+                   && (work.getLoadTableDesc().getLoadFileType() == LoadFileType.REPLACE_ALL))
                || (work.getLoadFileDesc() != null && !work.getLoadFileDesc()
                    .getDestinationCreateTable().isEmpty())) {
              StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE);

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
index f9c1997,e1969bb..aa3f186
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
@@@ -109,10 -107,6 +107,8 @@@ public final class TaskFactory 
          MergeFileTask.class));
      taskvec.add(new TaskTuple<DependencyCollectionWork>(DependencyCollectionWork.class,
          DependencyCollectionTask.class));
 +    taskvec.add(new TaskTuple<ImportCommitWork>(ImportCommitWork.class,
 +        ImportCommitTask.class));
-     taskvec.add(new TaskTuple<PartialScanWork>(PartialScanWork.class,
-         PartialScanTask.class));
      taskvec.add(new TaskTuple<IndexMetadataChangeWork>(IndexMetadataChangeWork.class,
          IndexMetadataChangeTask.class));
      taskvec.add(new TaskTuple<TezWork>(TezWork.class, TezTask.class));

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index c702a24,c199780..b6b6d6c
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@@ -3208,21 -3108,15 +3224,22 @@@ public final class Utilities 
        LOG.info("Processing alias " + alias);
  
        // The alias may not have any path
 +      Collection<Map.Entry<Path, ArrayList<String>>> pathToAliases =
 +          work.getPathToAliases().entrySet();
 +      if (!skipDummy) {
 +        // ConcurrentModification otherwise if adding dummy.
 +        pathToAliases = new ArrayList<>(pathToAliases);
 +      }
        boolean isEmptyTable = true;
        boolean hasLogged = false;
 -      // Note: this copies the list because createDummyFileForEmptyPartition may modify the map.
 -      for (Path file : new LinkedList<Path>(work.getPathToAliases().keySet())) {
 +      Path path = null;
 +      for (Map.Entry<Path, ArrayList<String>> e : pathToAliases) {
-         if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT)
+         if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT) {
            throw new IOException("Operation is Canceled.");
+         }
  
 -        List<String> aliases = work.getPathToAliases().get(file);
 +        Path file = e.getKey();
 +        List<String> aliases = e.getValue();
          if (aliases.contains(alias)) {
            if (file != null) {
              isEmptyTable = false;

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
index 51c4090,821d7df..0360816
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
@@@ -44,9 -44,8 +44,10 @@@ import org.apache.hadoop.hive.ql.plan.A
  import org.apache.hadoop.hive.ql.plan.DDLWork;
  import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
  import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+ import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType;
  import org.apache.hadoop.hive.ql.plan.MoveWork;
 +import org.apache.hadoop.hive.ql.session.SessionState;
 +import org.mortbay.jetty.servlet.AbstractSessionManager;
  import org.datanucleus.util.StringUtils;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
@@@ -240,8 -239,7 +241,9 @@@ public class LoadPartitions 
        Path tmpPath) {
      LoadTableDesc loadTableWork = new LoadTableDesc(
          tmpPath, Utilities.getTableDesc(table), partSpec.getPartSpec(),
-         event.replicationSpec().isReplace(), SessionState.get().getTxnMgr().getCurrentTxnId()
 -        event.replicationSpec().isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING);
++        event.replicationSpec().isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING,
++        SessionState.get().getTxnMgr().getCurrentTxnId()
 +    );
      loadTableWork.setInheritTableSpecs(false);
      MoveWork work = new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false,
          context.sessionStateLineageState);

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
index 65a3a59,25a2532..766a9a9
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
@@@ -41,8 -41,8 +41,9 @@@ import org.apache.hadoop.hive.ql.parse.
  import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
  import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
  import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+ import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType;
  import org.apache.hadoop.hive.ql.plan.MoveWork;
 +import org.apache.hadoop.hive.ql.session.SessionState;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  
@@@ -225,9 -225,8 +226,10 @@@ public class LoadTable 
          ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, tmpPath, context.hiveConf);
  
      LoadTableDesc loadTableWork = new LoadTableDesc(
-         tmpPath, Utilities.getTableDesc(table), new TreeMap<>(), replicationSpec.isReplace(),
+         tmpPath, Utilities.getTableDesc(table), new TreeMap<>(),
 -        replicationSpec.isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING);
++        replicationSpec.isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING,
 +        SessionState.get().getTxnMgr().getCurrentTxnId()
 +    );
      MoveWork moveWork =
          new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false,
              context.sessionStateLineageState);

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 37ee6b6,26003f4..68c9482
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@@ -1638,13 -1634,12 +1639,13 @@@ public class Hive 
     * @throws HiveException
     */
    public void loadPartition(Path loadPath, String tableName,
-       Map<String, String> partSpec, boolean replace, boolean inheritTableSpecs,
 -      Map<String, String> partSpec, LoadFileType loadFileType,
 -      boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir,
 -      boolean isSrcLocal, boolean isAcid, boolean hasFollowingStatsTask) throws HiveException {
++      Map<String, String> partSpec, LoadFileType loadFileType, boolean inheritTableSpecs,
 +      boolean isSkewedStoreAsSubdir,  boolean isSrcLocal, boolean isAcid,
 +      boolean hasFollowingStatsTask, Long txnId, int stmtId)
 +          throws HiveException {
      Table tbl = getTable(tableName);
-     loadPartition(loadPath, tbl, partSpec, replace, inheritTableSpecs,
+     loadPartition(loadPath, tbl, partSpec, loadFileType, inheritTableSpecs,
 -        isSkewedStoreAsSubdir, isSrcLocal, isAcid, hasFollowingStatsTask);
 +        isSkewedStoreAsSubdir, isSrcLocal, isAcid, hasFollowingStatsTask, txnId, stmtId);
    }
  
    /**
@@@ -1672,12 -1667,12 +1673,12 @@@
     *          true if there is a following task which updates the stats, so, this method need not update.
     * @return Partition object being loaded with data
     */
 -  public Partition loadPartition(Path loadPath, Table tbl,
 -      Map<String, String> partSpec, LoadFileType loadFileType,
 -      boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir,
 -      boolean isSrcLocal, boolean isAcid, boolean hasFollowingStatsTask) throws HiveException {
 -
 +  public Partition loadPartition(Path loadPath, Table tbl, Map<String, String> partSpec,
-       boolean replace, boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir,
++      LoadFileType loadFileType, boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir,
 +      boolean isSrcLocal, boolean isAcid, boolean hasFollowingStatsTask, Long txnId, int stmtId)
 +          throws HiveException {
      Path tblDataLocationPath =  tbl.getDataLocation();
 +    boolean isMmTableWrite = AcidUtils.isInsertOnlyTable(tbl.getParameters());
      try {
        // Get the partition object if it already exists
        Partition oldPart = getPartition(tbl, partSpec, false);
@@@ -1723,46 -1719,15 +1724,48 @@@
        if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && (null != oldPart)) {
          newFiles = Collections.synchronizedList(new ArrayList<Path>());
        }
 -
 -      if ((loadFileType == LoadFileType.REPLACE_ALL) || (oldPart == null && !isAcid)) {
 -        boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
 -        replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(),
 -            isSrcLocal, isAutoPurge, newFiles);
 +      // Note: this assumes both paths are qualified; which they are, currently.
 +      if (isMmTableWrite && loadPath.equals(newPartPath)) {
 +        // MM insert query, move itself is a no-op.
 +        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
 +          Utilities.FILE_OP_LOGGER.trace("not moving " + loadPath + " to " + newPartPath + " (MM)");
 +        }
 +        assert !isAcid;
 +        if (areEventsForDmlNeeded(tbl, oldPart)) {
 +          newFiles = listFilesCreatedByQuery(loadPath, txnId, stmtId);
 +        }
 +        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
 +          Utilities.FILE_OP_LOGGER.trace("maybe deleting stuff from " + oldPartPath
 +              + " (new " + newPartPath + ") for replace");
 +        }
-         if (replace && oldPartPath != null) {
++        if ((loadFileType == LoadFileType.REPLACE_ALL) && oldPartPath != null) {
 +          boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
 +          deleteOldPathForReplace(newPartPath, oldPartPath, getConf(), isAutoPurge,
 +              new JavaUtils.IdPathFilter(txnId, stmtId, false, true), true,
 +              tbl.isStoredAsSubDirectories() ? tbl.getSkewedColNames().size() : 0);
 +        }
        } else {
 -        FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
 -        Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid,
 -                            (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles);
 +        // Either a non-MM query, or a load into MM table from an external source.
 +        PathFilter filter = FileUtils.HIDDEN_FILES_PATH_FILTER;
 +        Path destPath = newPartPath;
 +        if (isMmTableWrite) {
 +          // We will load into MM directory, and delete from the parent if needed.
 +          destPath = new Path(destPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
-           filter = replace ? new JavaUtils.IdPathFilter(txnId, stmtId, false, true) : filter;
++          filter = (loadFileType == LoadFileType.REPLACE_ALL)
++            ? new JavaUtils.IdPathFilter(txnId, stmtId, false, true) : filter;
 +        }
 +        if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
 +          Utilities.FILE_OP_LOGGER.trace("moving " + loadPath + " to " + destPath);
 +        }
-         if (replace || (oldPart == null && !isAcid)) {
++        if ((loadFileType == LoadFileType.REPLACE_ALL) || (oldPart == null && !isAcid)) {
 +          boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
 +          replaceFiles(tbl.getPath(), loadPath, destPath, oldPartPath, getConf(),
 +              isSrcLocal, isAutoPurge, newFiles, filter, isMmTableWrite);
 +        } else {
 +          FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
-           Hive.copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcid, newFiles);
++          copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcid,
++            (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles);
 +        }
        }
        perfLogger.PerfLogEnd("MoveTask", "FileMoves");
        Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath);
@@@ -2086,8 -1961,8 +2089,8 @@@ private void constructOneLBLocationMap(
     * @throws HiveException
     */
    public Map<Map<String, String>, Partition> loadDynamicPartitions(final Path loadPath,
-       final String tableName, final Map<String, String> partSpec, final boolean replace,
+       final String tableName, final Map<String, String> partSpec, final LoadFileType loadFileType,
 -      final int numDP, final boolean listBucketingEnabled, final boolean isAcid, final long txnId,
 +      final int numDP, final int numLB, final boolean isAcid, final long txnId, final int stmtId,
        final boolean hasFollowingStatsTask, final AcidUtils.Operation operation)
        throws HiveException {
  
@@@ -2137,8 -2007,8 +2140,8 @@@
  
                // load the partition
                Partition newPartition = loadPartition(partPath, tbl, fullPartSpec,
-                   replace, true, numLB > 0,
 -                  loadFileType, true, listBucketingEnabled,
 -                  false, isAcid, hasFollowingStatsTask);
++                  loadFileType, true, numLB > 0,
 +                  false, isAcid, hasFollowingStatsTask, txnId, stmtId);
                partitionsMap.put(fullPartSpec, newPartition);
  
                if (inPlaceEligible) {
@@@ -2161,8 -2031,8 +2164,8 @@@
                    + " partPath=" + partPath + ", "
                    + " table=" + tbl.getTableName() + ", "
                    + " partSpec=" + fullPartSpec + ", "
-                   + " replace=" + replace + ", "
+                   + " loadFileType=" + loadFileType.toString() + ", "
 -                  + " listBucketingEnabled=" + listBucketingEnabled + ", "
 +                  + " listBucketingLevel=" + numLB + ", "
                    + " isAcid=" + isAcid + ", "
                    + " hasFollowingStatsTask=" + hasFollowingStatsTask, t);
                throw t;
@@@ -2228,47 -2099,28 +2232,51 @@@
     *          if there is any following stats task
     * @param isAcid true if this is an ACID based write
     */
-   public void loadTable(Path loadPath, String tableName, boolean replace, boolean isSrcLocal,
+   public void loadTable(Path loadPath, String tableName, LoadFileType loadFileType, boolean isSrcLocal,
 -                        boolean isSkewedStoreAsSubdir, boolean isAcid, boolean hasFollowingStatsTask)
 -      throws HiveException {
 +      boolean isSkewedStoreAsSubdir, boolean isAcid, boolean hasFollowingStatsTask,
 +      Long txnId, int stmtId, boolean isMmTable) throws HiveException {
+ 
      List<Path> newFiles = null;
      Table tbl = getTable(tableName);
      HiveConf sessionConf = SessionState.getSessionConf();
      if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) {
        newFiles = Collections.synchronizedList(new ArrayList<Path>());
      }
 -    if (loadFileType == LoadFileType.REPLACE_ALL) {
 -      Path tableDest = tbl.getPath();
 -      boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
 -      replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal, isAutopurge, newFiles);
 +    // Note: this assumes both paths are qualified; which they are, currently.
 +    if (isMmTable && loadPath.equals(tbl.getPath())) {
 +      Utilities.FILE_OP_LOGGER.debug("not moving " + loadPath + " to " + tbl.getPath());
-       if (replace) {
++      if (loadFileType == LoadFileType.REPLACE_ALL) {
 +        Path tableDest = tbl.getPath();
 +        boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
 +        deleteOldPathForReplace(tableDest, tableDest, sessionConf, isAutopurge,
 +            new JavaUtils.IdPathFilter(txnId, stmtId, false, true), true,
 +            tbl.isStoredAsSubDirectories() ? tbl.getSkewedColNames().size() : 0);
 +      }
 +      newFiles = listFilesCreatedByQuery(loadPath, txnId, stmtId);
      } else {
 -      FileSystem fs;
 -      try {
 -        fs = tbl.getDataLocation().getFileSystem(sessionConf);
 -        copyFiles(sessionConf, loadPath, tbl.getPath(), fs, isSrcLocal, isAcid,
 -                          (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles);
 -      } catch (IOException e) {
 -        throw new HiveException("addFiles: filesystem error in check phase", e);
 +      // Either a non-MM query, or a load into MM table from an external source.
 +      Path tblPath = tbl.getPath(), destPath = tblPath;
 +      PathFilter filter = FileUtils.HIDDEN_FILES_PATH_FILTER;
 +      if (isMmTable) {
 +        // We will load into MM directory, and delete from the parent if needed.
 +        destPath = new Path(destPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
-         filter = replace ? new JavaUtils.IdPathFilter(txnId, stmtId, false, true) : filter;
++        filter = loadFileType == LoadFileType.REPLACE_ALL
++            ? new JavaUtils.IdPathFilter(txnId, stmtId, false, true) : filter;
 +      }
-       Utilities.FILE_OP_LOGGER.debug("moving " + loadPath + " to " + tblPath + " (replace = " + replace + ")");
-       if (replace) {
++      Utilities.FILE_OP_LOGGER.debug("moving " + loadPath + " to " + tblPath
++          + " (replace = " + loadFileType + ")");
++      if (loadFileType == LoadFileType.REPLACE_ALL) {
 +        boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge"));
 +        replaceFiles(tblPath, loadPath, destPath, tblPath,
 +            sessionConf, isSrcLocal, isAutopurge, newFiles, filter, isMmTable);
 +      } else {
 +        try {
 +          FileSystem fs = tbl.getDataLocation().getFileSystem(sessionConf);
-           copyFiles(sessionConf, loadPath, destPath, fs, isSrcLocal, isAcid, newFiles);
++          copyFiles(sessionConf, loadPath, destPath, fs, isSrcLocal, isAcid,
++            loadFileType == LoadFileType.OVERWRITE_EXISTING, newFiles);
 +        } catch (IOException e) {
 +          throw new HiveException("addFiles: filesystem error in check phase", e);
 +        }
        }
      }
      if (!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 9afeadb,51b4f36..1318c18
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@@ -361,38 -344,17 +362,39 @@@ public class ImportSemanticAnalyzer ext
      return tblDesc;
    }
  
 +
    private static Task<?> loadTable(URI fromURI, Table table, boolean replace, Path tgtPath,
 -                            ReplicationSpec replicationSpec, EximUtil.SemanticAnalyzerWrapperContext x) {
 +      ReplicationSpec replicationSpec, EximUtil.SemanticAnalyzerWrapperContext x,
 +      Long txnId, int stmtId, boolean isSourceMm) {
      Path dataPath = new Path(fromURI.toString(), EximUtil.DATA_PATH_NAME);
 -    Path tmpPath = x.getCtx().getExternalTmpPath(tgtPath);
 -    Task<?> copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, tmpPath, x.getConf());
 -    LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
 +    Path destPath = !AcidUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtPath)
 +        : new Path(tgtPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
 +    if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
 +      Utilities.FILE_OP_LOGGER.trace("adding import work for table with source location: " +
 +        dataPath + "; table: " + tgtPath + "; copy destination " + destPath + "; mm " + txnId +
 +        " (src " + isSourceMm + ") for " + (table == null ? "a new table" : table.getTableName()));
 +    }
 +
 +    Task<?> copyTask = null;
 +    if (replicationSpec.isInReplicationScope()) {
 +      if (isSourceMm || isAcid(txnId)) {
 +        // Note: this is replication gap, not MM gap... Repl V2 is not ready yet.
 +        throw new RuntimeException("Replicating MM and ACID tables is not supported");
 +      }
 +      copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, destPath, x.getConf());
 +    } else {
 +      CopyWork cw = new CopyWork(dataPath, destPath, false);
 +      cw.setSkipSourceMmDirs(isSourceMm);
 +      copyTask = TaskFactory.get(cw, x.getConf());
 +    }
 +
 +    LoadTableDesc loadTableWork = new LoadTableDesc(destPath,
-         Utilities.getTableDesc(table), new TreeMap<>(), replace, txnId);
+         Utilities.getTableDesc(table), new TreeMap<>(),
 -        replace ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING);
 -    Task<?> loadTableTask = TaskFactory.get(new MoveWork(x.getInputs(),
 -            x.getOutputs(), loadTableWork, null, false, SessionState.get().getLineageState()),
 -        x.getConf());
++        replace ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING, txnId);
 +    loadTableWork.setTxnId(txnId);
 +    loadTableWork.setStmtId(stmtId);
 +    MoveWork mv = new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, null, false, SessionState.get().getLineageState());
 +    Task<?> loadTableTask = TaskFactory.get(mv, x.getConf());
      copyTask.addDependentTask(loadTableTask);
      x.getTasks().add(copyTask);
      return loadTableTask;
@@@ -458,39 -414,20 +460,41 @@@
            + partSpecToString(partSpec.getPartSpec())
            + " with source location: " + srcLocation);
        Path tgtLocation = new Path(partSpec.getLocation());
 -      Path tmpPath = x.getCtx().getExternalTmpPath(tgtLocation);
 -      Task<?> copyTask = ReplCopyTask.getLoadCopyTask(
 -          replicationSpec, new Path(srcLocation), tmpPath, x.getConf());
 +      Path destPath = !AcidUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtLocation)
 +          : new Path(tgtLocation, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
 +      Path moveTaskSrc =  !AcidUtils.isInsertOnlyTable(table.getParameters()) ? destPath : tgtLocation;
 +      if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
 +        Utilities.FILE_OP_LOGGER.trace("adding import work for partition with source location: "
 +          + srcLocation + "; target: " + tgtLocation + "; copy dest " + destPath + "; mm "
 +          + txnId + " (src " + isSourceMm + ") for " + partSpecToString(partSpec.getPartSpec()));
 +      }
 +
 +
 +      Task<?> copyTask = null;
 +      if (replicationSpec.isInReplicationScope()) {
 +        if (isSourceMm || isAcid(txnId)) {
 +          // Note: this is replication gap, not MM gap... Repl V2 is not ready yet.
 +          throw new RuntimeException("Replicating MM and ACID tables is not supported");
 +        }
 +        copyTask = ReplCopyTask.getLoadCopyTask(
 +            replicationSpec, new Path(srcLocation), destPath, x.getConf());
 +      } else {
 +        CopyWork cw = new CopyWork(new Path(srcLocation), destPath, false);
 +        cw.setSkipSourceMmDirs(isSourceMm);
 +        copyTask = TaskFactory.get(cw, x.getConf());
 +      }
 +
        Task<?> addPartTask = TaskFactory.get(new DDLWork(x.getInputs(),
            x.getOutputs(), addPartitionDesc), x.getConf());
 -      LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath,
 -          Utilities.getTableDesc(table),
 +      LoadTableDesc loadTableWork = new LoadTableDesc(moveTaskSrc, Utilities.getTableDesc(table),
-           partSpec.getPartSpec(), replicationSpec.isReplace(), txnId);
+           partSpec.getPartSpec(),
 -          replicationSpec.isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING);
++          replicationSpec.isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING,
++          txnId);
 +      loadTableWork.setTxnId(txnId);
 +      loadTableWork.setStmtId(stmtId);
        loadTableWork.setInheritTableSpecs(false);
        Task<?> loadPartTask = TaskFactory.get(new MoveWork(
 -              x.getInputs(), x.getOutputs(), loadTableWork, null, false,
 -              SessionState.get().getLineageState()),
 -          x.getConf());
 +          x.getInputs(), x.getOutputs(), loadTableWork, null, false, SessionState.get().getLineageState()), x.getConf());
        copyTask.addDependentTask(loadPartTask);
        addPartTask.addDependentTask(loadPartTask);
        x.getTasks().add(copyTask);

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 51efde1,033235b..5f2a34e
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@@ -50,8 -49,8 +50,9 @@@ import org.apache.hadoop.hive.ql.io.Hiv
  import org.apache.hadoop.hive.ql.metadata.Hive;
  import org.apache.hadoop.hive.ql.metadata.HiveException;
  import org.apache.hadoop.hive.ql.metadata.Partition;
 +import org.apache.hadoop.hive.ql.metadata.Table;
  import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
+ import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType;
  import org.apache.hadoop.hive.ql.plan.MoveWork;
  import org.apache.hadoop.hive.ql.plan.StatsWork;
  import org.apache.hadoop.hive.ql.session.SessionState;
@@@ -283,9 -276,8 +284,10 @@@ public class LoadSemanticAnalyzer exten
  
      LoadTableDesc loadTableWork;
      loadTableWork = new LoadTableDesc(new Path(fromURI),
-       Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite, txnId);
+       Utilities.getTableDesc(ts.tableHandle), partSpec,
 -      isOverWrite ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING);
++      isOverWrite ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING, txnId);
 +    loadTableWork.setTxnId(txnId);
 +    loadTableWork.setStmtId(stmtId);
      if (preservePartitionSpecs){
        // Note : preservePartitionSpecs=true implies inheritTableSpecs=false but
        // but preservePartitionSpecs=false(default) here is not sufficient enough

http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
index 9ac60b0,c497419..9309fbd
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
@@@ -32,11 -30,8 +30,9 @@@ import org.apache.hadoop.hive.conf.Hive
  import org.apache.hadoop.hive.ql.exec.TableScanOperator;
  import org.apache.hadoop.hive.ql.exec.Task;
  import org.apache.hadoop.hive.ql.exec.TaskFactory;
 +import org.apache.hadoop.hive.ql.exec.Utilities;
  import org.apache.hadoop.hive.ql.hooks.WriteEntity;
  import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
- import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork;
  import org.apache.hadoop.hive.ql.lib.Node;
  import org.apache.hadoop.hive.ql.lib.NodeProcessor;
  import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;

[15/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/ql/src/test/results/clientpositive/mm_all.q.out
----------------------------------------------------------------------
diff --cc ql/src/test/results/clientpositive/mm_all.q.out
index e08f1a8,0000000..ea60414
mode 100644,000000..100644
--- a/ql/src/test/results/clientpositive/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/mm_all.q.out
@@@ -1,2443 -1,0 +1,2147 @@@
 +PREHOOK: query: drop table intermediate
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table intermediate
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@intermediate
 +POSTHOOK: query: create table intermediate(key int) partitioned by (p int) stored as orc
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@intermediate
 +PREHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=455
 +POSTHOOK: query: insert into table intermediate partition(p='455') select distinct key from src where key >= 0 order by key desc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=455
 +POSTHOOK: Lineage: intermediate PARTITION(p=455).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=456
 +POSTHOOK: query: insert into table intermediate partition(p='456') select distinct key from src where key is not null order by key asc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=456
 +POSTHOOK: Lineage: intermediate PARTITION(p=456).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@src
 +PREHOOK: Output: default@intermediate@p=457
 +POSTHOOK: query: insert into table intermediate partition(p='457') select distinct key from src where key >= 100 order by key asc limit 2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@src
 +POSTHOOK: Output: default@intermediate@p=457
 +POSTHOOK: Lineage: intermediate PARTITION(p=457).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
 +PREHOOK: query: drop table part_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table part_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@part_mm
 +POSTHOOK: query: create table part_mm(key int) partitioned by (key_mm int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@part_mm
 +PREHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
 +PREHOOK: type: QUERY
 +POSTHOOK: query: explain insert into table part_mm partition(key_mm=455) select key from intermediate
 +POSTHOOK: type: QUERY
 +STAGE DEPENDENCIES:
 +  Stage-1 is a root stage
 +  Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
 +  Stage-4
 +  Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
 +  Stage-2 depends on stages: Stage-0
 +  Stage-3
 +  Stage-5
 +  Stage-6 depends on stages: Stage-5
 +
 +STAGE PLANS:
 +  Stage: Stage-1
 +    Map Reduce
 +      Map Operator Tree:
 +          TableScan
 +            alias: intermediate
 +            Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
 +            Select Operator
 +              expressions: key (type: int)
 +              outputColumnNames: _col0
 +              Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
 +              File Output Operator
 +                compressed: false
 +                Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
 +                table:
 +                    input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +                    output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
 +                    serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
 +                    name: default.part_mm
 +                Write Type: INSERT
 +
 +  Stage: Stage-7
 +    Conditional Operator
 +
 +  Stage: Stage-4
 +    Move Operator
 +      files:
 +          hdfs directory: true
 +#### A masked pattern was here ####
 +
 +  Stage: Stage-0
 +    Move Operator
 +      tables:
 +          partition:
 +            key_mm 455
 +          replace: false
 +          table:
 +              input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +              output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
 +              serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
 +              name: default.part_mm
 +          micromanaged table: true
 +
 +  Stage: Stage-2
 +    Stats-Aggr Operator
 +
 +  Stage: Stage-3
 +    Merge File Operator
 +      Map Operator Tree:
 +          ORC File Merge Operator
 +      merge level: stripe
 +      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +
 +  Stage: Stage-5
 +    Merge File Operator
 +      Map Operator Tree:
 +          ORC File Merge Operator
 +      merge level: stripe
 +      input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
 +
 +  Stage: Stage-6
 +    Move Operator
 +      files:
 +          hdfs directory: true
 +#### A masked pattern was here ####
 +
 +PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@part_mm@key_mm=456
 +POSTHOOK: query: insert into table part_mm partition(key_mm=456) select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@part_mm@key_mm=456
 +POSTHOOK: Lineage: part_mm PARTITION(key_mm=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: query: insert into table part_mm partition(key_mm=455) select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: Lineage: part_mm PARTITION(key_mm=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from part_mm order by key, key_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Input: default@part_mm@key_mm=455
 +PREHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from part_mm order by key, key_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Input: default@part_mm@key_mm=455
 +POSTHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +0	455
 +0	455
 +0	456
 +10	455
 +10	455
 +10	456
 +97	455
 +97	455
 +97	456
 +98	455
 +98	455
 +98	456
 +100	455
 +100	455
 +100	456
 +103	455
 +103	455
 +103	456
 +PREHOOK: query: select * from part_mm order by key, key_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Input: default@part_mm@key_mm=455
 +PREHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from part_mm order by key, key_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Input: default@part_mm@key_mm=455
 +POSTHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +0	455
 +0	455
 +0	456
 +10	455
 +10	455
 +10	456
 +97	455
 +97	455
 +97	456
 +98	455
 +98	455
 +98	456
 +100	455
 +100	455
 +100	456
 +103	455
 +103	455
 +103	456
 +PREHOOK: query: truncate table part_mm
 +PREHOOK: type: TRUNCATETABLE
 +PREHOOK: Output: default@part_mm@key_mm=455
 +PREHOOK: Output: default@part_mm@key_mm=456
 +POSTHOOK: query: truncate table part_mm
 +POSTHOOK: type: TRUNCATETABLE
 +POSTHOOK: Output: default@part_mm@key_mm=455
 +POSTHOOK: Output: default@part_mm@key_mm=456
 +PREHOOK: query: select * from part_mm order by key, key_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Input: default@part_mm@key_mm=455
 +PREHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from part_mm order by key, key_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Input: default@part_mm@key_mm=455
 +POSTHOOK: Input: default@part_mm@key_mm=456
 +#### A masked pattern was here ####
 +PREHOOK: query: drop table part_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@part_mm
 +PREHOOK: Output: default@part_mm
 +POSTHOOK: query: drop table part_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@part_mm
 +POSTHOOK: Output: default@part_mm
 +PREHOOK: query: drop table simple_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table simple_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: create table simple_mm(key int) stored as orc tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@simple_mm
 +PREHOOK: query: insert into table simple_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: insert into table simple_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@simple_mm
 +POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from simple_mm order by key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from simple_mm order by key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +0
 +10
 +97
 +98
 +100
 +103
 +PREHOOK: query: insert into table simple_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: insert into table simple_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@simple_mm
 +POSTHOOK: Lineage: simple_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from simple_mm order by key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from simple_mm order by key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +0
 +0
 +10
 +10
 +97
 +97
 +98
 +98
 +100
 +100
 +103
 +103
 +PREHOOK: query: truncate table simple_mm
 +PREHOOK: type: TRUNCATETABLE
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: truncate table simple_mm
 +POSTHOOK: type: TRUNCATETABLE
 +POSTHOOK: Output: default@simple_mm
 +PREHOOK: query: select * from simple_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from simple_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@simple_mm
 +#### A masked pattern was here ####
 +PREHOOK: query: drop table simple_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@simple_mm
 +PREHOOK: Output: default@simple_mm
 +POSTHOOK: query: drop table simple_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@simple_mm
 +POSTHOOK: Output: default@simple_mm
 +PREHOOK: query: drop table dp_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table dp_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
 +  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@dp_mm
 +POSTHOOK: query: create table dp_mm (key int) partitioned by (key1 string, key2 int) stored as orc
 +  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@dp_mm
 +PREHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@dp_mm@key1=123
 +POSTHOOK: query: insert into table dp_mm partition (key1='123', key2) select key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=0
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=10
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=100
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=103
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=97
 +POSTHOOK: Output: default@dp_mm@key1=123/key2=98
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=0).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=100).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=103).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=10).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=97).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: dp_mm PARTITION(key1=123,key2=98).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from dp_mm order by key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@dp_mm
 +PREHOOK: Input: default@dp_mm@key1=123/key2=0
 +PREHOOK: Input: default@dp_mm@key1=123/key2=10
 +PREHOOK: Input: default@dp_mm@key1=123/key2=100
 +PREHOOK: Input: default@dp_mm@key1=123/key2=103
 +PREHOOK: Input: default@dp_mm@key1=123/key2=97
 +PREHOOK: Input: default@dp_mm@key1=123/key2=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from dp_mm order by key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@dp_mm
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=0
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=10
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=100
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=103
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=97
 +POSTHOOK: Input: default@dp_mm@key1=123/key2=98
 +#### A masked pattern was here ####
 +0	123	0
 +10	123	10
 +97	123	97
 +98	123	98
 +100	123	100
 +103	123	103
 +PREHOOK: query: drop table dp_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@dp_mm
 +PREHOOK: Output: default@dp_mm
 +POSTHOOK: query: drop table dp_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@dp_mm
 +POSTHOOK: Output: default@dp_mm
 +PREHOOK: query: create table union_mm(id int)  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: create table union_mm(id int)  tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@union_mm
 +PREHOOK: query: insert into table union_mm 
 +select temps.p from (
 +select key as p from intermediate 
 +union all 
 +select key + 1 as p from intermediate ) temps
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: insert into table union_mm 
 +select temps.p from (
 +select key as p from intermediate 
 +union all 
 +select key + 1 as p from intermediate ) temps
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@union_mm
 +POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from union_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from union_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +0
 +1
 +10
 +11
 +97
 +98
 +98
 +99
 +100
 +101
 +103
 +104
 +PREHOOK: query: insert into table union_mm 
 +select p from
 +(
 +select key + 1 as p from intermediate
 +union all
 +select key from intermediate
 +) tab group by p
 +union all
 +select key + 2 as p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: insert into table union_mm 
 +select p from
 +(
 +select key + 1 as p from intermediate
 +union all
 +select key from intermediate
 +) tab group by p
 +union all
 +select key + 2 as p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@union_mm
 +POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from union_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from union_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +0
 +0
 +1
 +1
 +2
 +10
 +10
 +11
 +11
 +12
 +97
 +97
 +98
 +98
 +98
 +99
 +99
 +99
 +100
 +100
 +100
 +101
 +101
 +102
 +103
 +103
 +104
 +104
 +105
 +PREHOOK: query: insert into table union_mm
 +SELECT p FROM
 +(
 +  SELECT key + 1 as p FROM intermediate
 +  UNION ALL
 +  SELECT key as p FROM ( 
 +    SELECT distinct key FROM (
 +      SELECT key FROM (
 +        SELECT key + 2 as key FROM intermediate
 +        UNION ALL
 +        SELECT key FROM intermediate
 +      )t1 
 +    group by key)t2
 +  )t3
 +)t4
 +group by p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: insert into table union_mm
 +SELECT p FROM
 +(
 +  SELECT key + 1 as p FROM intermediate
 +  UNION ALL
 +  SELECT key as p FROM ( 
 +    SELECT distinct key FROM (
 +      SELECT key FROM (
 +        SELECT key + 2 as key FROM intermediate
 +        UNION ALL
 +        SELECT key FROM intermediate
 +      )t1 
 +    group by key)t2
 +  )t3
 +)t4
 +group by p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@union_mm
 +POSTHOOK: Lineage: union_mm.id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from union_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from union_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@union_mm
 +#### A masked pattern was here ####
 +0
 +0
 +0
 +1
 +1
 +1
 +2
 +2
 +10
 +10
 +10
 +11
 +11
 +11
 +12
 +12
 +97
 +97
 +97
 +98
 +98
 +98
 +98
 +99
 +99
 +99
 +99
 +100
 +100
 +100
 +100
 +101
 +101
 +101
 +102
 +102
 +103
 +103
 +103
 +104
 +104
 +104
 +105
 +105
 +PREHOOK: query: drop table union_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@union_mm
 +PREHOOK: Output: default@union_mm
 +POSTHOOK: query: drop table union_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@union_mm
 +POSTHOOK: Output: default@union_mm
 +PREHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@partunion_mm
 +POSTHOOK: query: create table partunion_mm(id int) partitioned by (key int) tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@partunion_mm
 +PREHOOK: query: insert into table partunion_mm partition(key)
 +select temps.* from (
 +select key as p, key from intermediate 
 +union all 
 +select key + 1 as p, key + 1 from intermediate ) temps
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@partunion_mm
 +POSTHOOK: query: insert into table partunion_mm partition(key)
 +select temps.* from (
 +select key as p, key from intermediate 
 +union all 
 +select key + 1 as p, key + 1 from intermediate ) temps
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@partunion_mm@key=0
 +POSTHOOK: Output: default@partunion_mm@key=1
 +POSTHOOK: Output: default@partunion_mm@key=10
 +POSTHOOK: Output: default@partunion_mm@key=100
 +POSTHOOK: Output: default@partunion_mm@key=101
 +POSTHOOK: Output: default@partunion_mm@key=103
 +POSTHOOK: Output: default@partunion_mm@key=104
 +POSTHOOK: Output: default@partunion_mm@key=11
 +POSTHOOK: Output: default@partunion_mm@key=97
 +POSTHOOK: Output: default@partunion_mm@key=98
 +POSTHOOK: Output: default@partunion_mm@key=99
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=0).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=100).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=101).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=103).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=104).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=10).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=11).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=1).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=97).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=98).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: partunion_mm PARTITION(key=99).id EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from partunion_mm order by id
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@partunion_mm
 +PREHOOK: Input: default@partunion_mm@key=0
 +PREHOOK: Input: default@partunion_mm@key=1
 +PREHOOK: Input: default@partunion_mm@key=10
 +PREHOOK: Input: default@partunion_mm@key=100
 +PREHOOK: Input: default@partunion_mm@key=101
 +PREHOOK: Input: default@partunion_mm@key=103
 +PREHOOK: Input: default@partunion_mm@key=104
 +PREHOOK: Input: default@partunion_mm@key=11
 +PREHOOK: Input: default@partunion_mm@key=97
 +PREHOOK: Input: default@partunion_mm@key=98
 +PREHOOK: Input: default@partunion_mm@key=99
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from partunion_mm order by id
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@partunion_mm
 +POSTHOOK: Input: default@partunion_mm@key=0
 +POSTHOOK: Input: default@partunion_mm@key=1
 +POSTHOOK: Input: default@partunion_mm@key=10
 +POSTHOOK: Input: default@partunion_mm@key=100
 +POSTHOOK: Input: default@partunion_mm@key=101
 +POSTHOOK: Input: default@partunion_mm@key=103
 +POSTHOOK: Input: default@partunion_mm@key=104
 +POSTHOOK: Input: default@partunion_mm@key=11
 +POSTHOOK: Input: default@partunion_mm@key=97
 +POSTHOOK: Input: default@partunion_mm@key=98
 +POSTHOOK: Input: default@partunion_mm@key=99
 +#### A masked pattern was here ####
 +0	0
 +1	1
 +10	10
 +11	11
 +97	97
 +98	98
 +98	98
 +99	99
 +100	100
 +101	101
 +103	103
 +104	104
 +PREHOOK: query: drop table partunion_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@partunion_mm
 +PREHOOK: Output: default@partunion_mm
 +POSTHOOK: query: drop table partunion_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@partunion_mm
 +POSTHOOK: Output: default@partunion_mm
 +PREHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
 + stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@skew_mm
 +POSTHOOK: query: create table skew_mm(k1 int, k2 int, k4 int) skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3))
 + stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@skew_mm
 +PREHOOK: query: insert into table skew_mm 
 +select key, key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@skew_mm
 +POSTHOOK: query: insert into table skew_mm 
 +select key, key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@skew_mm
 +POSTHOOK: Lineage: skew_mm.k1 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_mm.k2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_mm.k4 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from skew_mm order by k2, k1, k4
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@skew_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from skew_mm order by k2, k1, k4
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@skew_mm
 +#### A masked pattern was here ####
 +0	0	0
 +10	10	10
 +97	97	97
 +98	98	98
 +100	100	100
 +103	103	103
 +PREHOOK: query: drop table skew_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@skew_mm
 +PREHOOK: Output: default@skew_mm
 +POSTHOOK: query: drop table skew_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@skew_mm
 +POSTHOOK: Output: default@skew_mm
 +PREHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) 
 +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@skew_dp_union_mm
 +POSTHOOK: query: create table skew_dp_union_mm(k1 int, k2 int, k4 int) partitioned by (k3 int) 
 +skewed by (k1, k4) on ((0,0),(1,1),(2,2),(3,3)) stored as directories tblproperties ("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@skew_dp_union_mm
 +PREHOOK: query: insert into table skew_dp_union_mm partition (k3)
 +select key as i, key as j, key as k, key as l from intermediate
 +union all 
 +select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@skew_dp_union_mm
 +POSTHOOK: query: insert into table skew_dp_union_mm partition (k3)
 +select key as i, key as j, key as k, key as l from intermediate
 +union all 
 +select key +1 as i, key +2 as j, key +3 as k, key +4 as l from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=0
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=10
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=100
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=101
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=102
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=103
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=104
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=107
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=14
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=4
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=97
 +POSTHOOK: Output: default@skew_dp_union_mm@k3=98
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=0).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=100).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=101).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=102).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=103).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=104).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=107).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=10).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=14).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=4).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=97).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k1 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k2 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: skew_dp_union_mm PARTITION(k3=98).k4 EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@skew_dp_union_mm
 +PREHOOK: Input: default@skew_dp_union_mm@k3=0
 +PREHOOK: Input: default@skew_dp_union_mm@k3=10
 +PREHOOK: Input: default@skew_dp_union_mm@k3=100
 +PREHOOK: Input: default@skew_dp_union_mm@k3=101
 +PREHOOK: Input: default@skew_dp_union_mm@k3=102
 +PREHOOK: Input: default@skew_dp_union_mm@k3=103
 +PREHOOK: Input: default@skew_dp_union_mm@k3=104
 +PREHOOK: Input: default@skew_dp_union_mm@k3=107
 +PREHOOK: Input: default@skew_dp_union_mm@k3=14
 +PREHOOK: Input: default@skew_dp_union_mm@k3=4
 +PREHOOK: Input: default@skew_dp_union_mm@k3=97
 +PREHOOK: Input: default@skew_dp_union_mm@k3=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from skew_dp_union_mm order by k2, k1, k4
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@skew_dp_union_mm
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=0
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=10
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=100
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=101
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=102
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=103
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=104
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=107
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=14
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=4
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=97
 +POSTHOOK: Input: default@skew_dp_union_mm@k3=98
 +#### A masked pattern was here ####
 +0	0	0	0
 +1	2	3	4
 +10	10	10	10
 +11	12	13	14
 +97	97	97	97
 +98	98	98	98
 +98	99	100	101
 +99	100	101	102
 +100	100	100	100
 +101	102	103	104
 +103	103	103	103
 +104	105	106	107
 +PREHOOK: query: drop table skew_dp_union_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@skew_dp_union_mm
 +PREHOOK: Output: default@skew_dp_union_mm
 +POSTHOOK: query: drop table skew_dp_union_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@skew_dp_union_mm
 +POSTHOOK: Output: default@skew_dp_union_mm
 +PREHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: create table merge0_mm (id int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@merge0_mm
 +PREHOOK: query: insert into table merge0_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: insert into table merge0_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge0_mm
 +POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: insert into table merge0_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: insert into table merge0_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge0_mm
 +POSTHOOK: Lineage: merge0_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge0_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: drop table merge0_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@merge0_mm
 +PREHOOK: Output: default@merge0_mm
 +POSTHOOK: query: drop table merge0_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@merge0_mm
 +POSTHOOK: Output: default@merge0_mm
 +PREHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: create table merge2_mm (id int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@merge2_mm
 +PREHOOK: query: insert into table merge2_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: insert into table merge2_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge2_mm
 +POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge2_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge2_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: insert into table merge2_mm select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: insert into table merge2_mm select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge2_mm
 +POSTHOOK: Lineage: merge2_mm.id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge2_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge2_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge2_mm
 +#### A masked pattern was here ####
 +98
 +97
 +0
 +10
 +100
 +103
 +98
 +97
 +0
 +10
 +100
 +103
 +PREHOOK: query: drop table merge2_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@merge2_mm
 +PREHOOK: Output: default@merge2_mm
 +POSTHOOK: query: drop table merge2_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@merge2_mm
 +POSTHOOK: Output: default@merge2_mm
 +PREHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: create table merge1_mm (id int) partitioned by (key int) stored as orc tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@merge1_mm
 +PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge1_mm@key=0
 +POSTHOOK: Output: default@merge1_mm@key=10
 +POSTHOOK: Output: default@merge1_mm@key=100
 +POSTHOOK: Output: default@merge1_mm@key=103
 +POSTHOOK: Output: default@merge1_mm@key=97
 +POSTHOOK: Output: default@merge1_mm@key=98
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge1_mm order by id, key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge1_mm
 +PREHOOK: Input: default@merge1_mm@key=0
 +PREHOOK: Input: default@merge1_mm@key=10
 +PREHOOK: Input: default@merge1_mm@key=100
 +PREHOOK: Input: default@merge1_mm@key=103
 +PREHOOK: Input: default@merge1_mm@key=97
 +PREHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge1_mm order by id, key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge1_mm
 +POSTHOOK: Input: default@merge1_mm@key=0
 +POSTHOOK: Input: default@merge1_mm@key=10
 +POSTHOOK: Input: default@merge1_mm@key=100
 +POSTHOOK: Input: default@merge1_mm@key=103
 +POSTHOOK: Input: default@merge1_mm@key=97
 +POSTHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +0	0
 +10	10
 +97	97
 +98	98
 +100	100
 +103	103
 +PREHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: insert into table merge1_mm partition (key) select key, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@merge1_mm@key=0
 +POSTHOOK: Output: default@merge1_mm@key=10
 +POSTHOOK: Output: default@merge1_mm@key=100
 +POSTHOOK: Output: default@merge1_mm@key=103
 +POSTHOOK: Output: default@merge1_mm@key=97
 +POSTHOOK: Output: default@merge1_mm@key=98
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=0).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=100).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=103).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=10).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=97).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: merge1_mm PARTITION(key=98).id SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from merge1_mm order by id, key
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@merge1_mm
 +PREHOOK: Input: default@merge1_mm@key=0
 +PREHOOK: Input: default@merge1_mm@key=10
 +PREHOOK: Input: default@merge1_mm@key=100
 +PREHOOK: Input: default@merge1_mm@key=103
 +PREHOOK: Input: default@merge1_mm@key=97
 +PREHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from merge1_mm order by id, key
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@merge1_mm
 +POSTHOOK: Input: default@merge1_mm@key=0
 +POSTHOOK: Input: default@merge1_mm@key=10
 +POSTHOOK: Input: default@merge1_mm@key=100
 +POSTHOOK: Input: default@merge1_mm@key=103
 +POSTHOOK: Input: default@merge1_mm@key=97
 +POSTHOOK: Input: default@merge1_mm@key=98
 +#### A masked pattern was here ####
 +0	0
 +0	0
 +10	10
 +10	10
 +97	97
 +97	97
 +98	98
 +98	98
 +100	100
 +100	100
 +103	103
 +103	103
 +PREHOOK: query: drop table merge1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@merge1_mm
 +PREHOOK: Output: default@merge1_mm
 +POSTHOOK: query: drop table merge1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@merge1_mm
 +POSTHOOK: Output: default@merge1_mm
 +PREHOOK: query: drop table ctas0_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table ctas0_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate
 +PREHOOK: type: CREATETABLE_AS_SELECT
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@ctas0_mm
 +POSTHOOK: query: create table ctas0_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as select * from intermediate
 +POSTHOOK: type: CREATETABLE_AS_SELECT
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@ctas0_mm
 +POSTHOOK: Lineage: ctas0_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: ctas0_mm.p SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from ctas0_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@ctas0_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from ctas0_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@ctas0_mm
 +#### A masked pattern was here ####
 +98	455
 +97	455
 +0	456
 +10	456
 +100	457
 +103	457
 +PREHOOK: query: drop table ctas0_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@ctas0_mm
 +PREHOOK: Output: default@ctas0_mm
 +POSTHOOK: query: drop table ctas0_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@ctas0_mm
 +POSTHOOK: Output: default@ctas0_mm
 +PREHOOK: query: drop table ctas1_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table ctas1_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
 +  select * from intermediate union all select * from intermediate
 +PREHOOK: type: CREATETABLE_AS_SELECT
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@ctas1_mm
 +POSTHOOK: query: create table ctas1_mm tblproperties ("transactional"="true", "transactional_properties"="insert_only") as
 +  select * from intermediate union all select * from intermediate
 +POSTHOOK: type: CREATETABLE_AS_SELECT
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@ctas1_mm
 +POSTHOOK: Lineage: ctas1_mm.key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: ctas1_mm.p EXPRESSION [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from ctas1_mm
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@ctas1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from ctas1_mm
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@ctas1_mm
 +#### A masked pattern was here ####
 +98	455
 +98	455
 +97	455
 +97	455
 +0	456
 +0	456
 +10	456
 +10	456
 +100	457
 +100	457
 +103	457
 +103	457
 +PREHOOK: query: drop table ctas1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@ctas1_mm
 +PREHOOK: Output: default@ctas1_mm
 +POSTHOOK: query: drop table ctas1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@ctas1_mm
 +POSTHOOK: Output: default@ctas1_mm
- PREHOOK: query: drop table load0_mm
- PREHOOK: type: DROPTABLE
- POSTHOOK: query: drop table load0_mm
- POSTHOOK: type: DROPTABLE
- PREHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- PREHOOK: type: CREATETABLE
- PREHOOK: Output: database:default
- PREHOOK: Output: default@load0_mm
- POSTHOOK: query: create table load0_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- POSTHOOK: type: CREATETABLE
- POSTHOOK: Output: database:default
- POSTHOOK: Output: default@load0_mm
- PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load0_mm
- POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load0_mm
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load0_mm
- PREHOOK: query: select count(1) from load0_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load0_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- 500
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load0_mm
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table load0_mm
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load0_mm
- PREHOOK: query: select count(1) from load0_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load0_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- 1000
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load0_mm
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' overwrite into table load0_mm
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load0_mm
- PREHOOK: query: select count(1) from load0_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load0_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load0_mm
- #### A masked pattern was here ####
- 500
- PREHOOK: query: drop table load0_mm
- PREHOOK: type: DROPTABLE
- PREHOOK: Input: default@load0_mm
- PREHOOK: Output: default@load0_mm
- POSTHOOK: query: drop table load0_mm
- POSTHOOK: type: DROPTABLE
- POSTHOOK: Input: default@load0_mm
- POSTHOOK: Output: default@load0_mm
- PREHOOK: query: drop table intermediate2
- PREHOOK: type: DROPTABLE
- POSTHOOK: query: drop table intermediate2
- POSTHOOK: type: DROPTABLE
- PREHOOK: query: create table intermediate2 (key string, value string) stored as textfile
- #### A masked pattern was here ####
- PREHOOK: type: CREATETABLE
- #### A masked pattern was here ####
- PREHOOK: Output: database:default
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: create table intermediate2 (key string, value string) stored as textfile
- #### A masked pattern was here ####
- POSTHOOK: type: CREATETABLE
- #### A masked pattern was here ####
- POSTHOOK: Output: database:default
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: drop table load1_mm
- PREHOOK: type: DROPTABLE
- POSTHOOK: query: drop table load1_mm
- POSTHOOK: type: DROPTABLE
- PREHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- PREHOOK: type: CREATETABLE
- PREHOOK: Output: database:default
- PREHOOK: Output: default@load1_mm
- POSTHOOK: query: create table load1_mm (key string, value string) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- POSTHOOK: type: CREATETABLE
- POSTHOOK: Output: database:default
- POSTHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load1_mm
- PREHOOK: query: select count(1) from load1_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load1_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- 1000
- PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- #### A masked pattern was here ####
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load1_mm
- PREHOOK: query: select count(1) from load1_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load1_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- 1050
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- #### A masked pattern was here ####
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load1_mm
- PREHOOK: query: select count(1) from load1_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load1_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load1_mm
- #### A masked pattern was here ####
- 500
- PREHOOK: query: drop table load1_mm
- PREHOOK: type: DROPTABLE
- PREHOOK: Input: default@load1_mm
- PREHOOK: Output: default@load1_mm
- POSTHOOK: query: drop table load1_mm
- POSTHOOK: type: DROPTABLE
- POSTHOOK: Input: default@load1_mm
- POSTHOOK: Output: default@load1_mm
- PREHOOK: query: drop table load2_mm
- PREHOOK: type: DROPTABLE
- POSTHOOK: query: drop table load2_mm
- POSTHOOK: type: DROPTABLE
- PREHOOK: query: create table load2_mm (key string, value string)
-   partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- PREHOOK: type: CREATETABLE
- PREHOOK: Output: database:default
- PREHOOK: Output: default@load2_mm
- POSTHOOK: query: create table load2_mm (key string, value string)
-   partitioned by (k int, l int) stored as textfile tblproperties("transactional"="true", "transactional_properties"="insert_only")
- POSTHOOK: type: CREATETABLE
- POSTHOOK: Output: database:default
- POSTHOOK: Output: default@load2_mm
- PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv2.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- PREHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: load data local inpath '../../data/files/kv3.txt' into table intermediate2
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@intermediate2
- #### A masked pattern was here ####
- PREHOOK: type: LOAD
- #### A masked pattern was here ####
- PREHOOK: Output: default@load2_mm
- #### A masked pattern was here ####
- POSTHOOK: type: LOAD
- #### A masked pattern was here ####
- POSTHOOK: Output: default@load2_mm
- POSTHOOK: Output: default@load2_mm@k=5/l=5
- PREHOOK: query: select count(1) from load2_mm
- PREHOOK: type: QUERY
- PREHOOK: Input: default@load2_mm
- PREHOOK: Input: default@load2_mm@k=5/l=5
- #### A masked pattern was here ####
- POSTHOOK: query: select count(1) from load2_mm
- POSTHOOK: type: QUERY
- POSTHOOK: Input: default@load2_mm
- POSTHOOK: Input: default@load2_mm@k=5/l=5
- #### A masked pattern was here ####
- 1025
- PREHOOK: query: drop table load2_mm
- PREHOOK: type: DROPTABLE
- PREHOOK: Input: default@load2_mm
- PREHOOK: Output: default@load2_mm
- POSTHOOK: query: drop table load2_mm
- POSTHOOK: type: DROPTABLE
- POSTHOOK: Input: default@load2_mm
- POSTHOOK: Output: default@load2_mm
- PREHOOK: query: drop table intermediate2
- PREHOOK: type: DROPTABLE
- PREHOOK: Input: default@intermediate2
- PREHOOK: Output: default@intermediate2
- POSTHOOK: query: drop table intermediate2
- POSTHOOK: type: DROPTABLE
- POSTHOOK: Input: default@intermediate2
- POSTHOOK: Output: default@intermediate2
 +PREHOOK: query: drop table multi0_1_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table multi0_1_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: drop table multi0_2_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table multi0_2_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table multi0_1_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: create table multi0_1_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@multi0_1_mm
 +PREHOOK: query: create table multi0_2_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: create table multi0_2_mm (key int, key2 int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@multi0_2_mm
 +PREHOOK: query: insert into table multi0_1_mm select key, p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: insert into table multi0_1_mm select key, p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_1_mm
 +POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: insert into table multi0_2_mm select p, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: insert into table multi0_2_mm select p, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_2_mm
 +POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select * from multi0_1_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_1_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +0	456
 +10	456
 +97	455
 +98	455
 +100	457
 +103	457
 +PREHOOK: query: select * from multi0_2_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_2_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +455	97
 +455	98
 +456	0
 +456	10
 +457	100
 +457	103
 +PREHOOK: query: insert into table multi0_1_mm select p, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: insert into table multi0_1_mm select p, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_1_mm
 +POSTHOOK: Lineage: multi0_1_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_1_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table multi0_2_mm select key, p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: insert into table multi0_2_mm select key, p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi0_2_mm
 +POSTHOOK: Lineage: multi0_2_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi0_2_mm.key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from multi0_1_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_1_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_1_mm
 +#### A masked pattern was here ####
 +0	456
 +10	456
 +97	455
 +98	455
 +100	457
 +103	457
 +455	97
 +455	98
 +456	0
 +456	10
 +457	100
 +457	103
 +PREHOOK: query: select * from multi0_2_mm order by key, key2
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi0_2_mm order by key, key2
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi0_2_mm
 +#### A masked pattern was here ####
 +0	456
 +10	456
 +97	455
 +98	455
 +100	457
 +103	457
 +455	97
 +455	98
 +456	0
 +456	10
 +457	100
 +457	103
 +PREHOOK: query: drop table multi0_1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@multi0_1_mm
 +PREHOOK: Output: default@multi0_1_mm
 +POSTHOOK: query: drop table multi0_1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@multi0_1_mm
 +POSTHOOK: Output: default@multi0_1_mm
 +PREHOOK: query: drop table multi0_2_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@multi0_2_mm
 +PREHOOK: Output: default@multi0_2_mm
 +POSTHOOK: query: drop table multi0_2_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@multi0_2_mm
 +POSTHOOK: Output: default@multi0_2_mm
 +PREHOOK: query: drop table multi1_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table multi1_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@multi1_mm
 +POSTHOOK: query: create table multi1_mm (key int, key2 int) partitioned by (p int) tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@multi1_mm
 +PREHOOK: query: from intermediate
 +insert into table multi1_mm partition(p=1) select p, key
 +insert into table multi1_mm partition(p=2) select key, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm@p=1
 +PREHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: query: from intermediate
 +insert into table multi1_mm partition(p=1) select p, key
 +insert into table multi1_mm partition(p=2) select key, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +0	456	2
 +10	456	2
 +97	455	2
 +98	455	2
 +100	457	2
 +103	457	2
 +455	97	1
 +455	98	1
 +456	0	1
 +456	10	1
 +457	100	1
 +457	103	1
 +PREHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: query: insert into table multi1_mm partition(p=2) select p, key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=2
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=2).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: query: insert into table multi1_mm partition(p=1) select key, p from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +PREHOOK: query: select * from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +POSTHOOK: query: select * from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
 +#### A masked pattern was here ####
 +0	456	1
 +0	456	2
 +10	456	1
 +10	456	2
 +97	455	1
 +97	455	2
 +98	455	1
 +98	455	2
 +100	457	1
 +100	457	2
 +103	457	1
 +103	457	2
 +455	97	1
 +455	97	2
 +455	98	1
 +455	98	2
 +456	0	1
 +456	0	2
 +456	10	1
 +456	10	2
 +457	100	1
 +457	100	2
 +457	103	1
 +457	103	2
 +PREHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, p
 +insert into table multi1_mm partition(p=1) select key, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm
 +PREHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, p
 +insert into table multi1_mm partition(p=1) select key, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: Output: default@multi1_mm@p=455
 +POSTHOOK: Output: default@multi1_mm@p=456
 +POSTHOOK: Output: default@multi1_mm@p=457
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=455).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=456).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=457).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
 +PREHOOK: Input: default@multi1_mm@p=455
 +PREHOOK: Input: default@multi1_mm@p=456
 +PREHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
 +POSTHOOK: Input: default@multi1_mm@p=455
 +POSTHOOK: Input: default@multi1_mm@p=456
 +POSTHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +0	456	1
 +0	456	1
 +0	456	2
 +10	456	1
 +10	456	1
 +10	456	2
 +97	455	1
 +97	455	1
 +97	455	2
 +98	455	1
 +98	455	1
 +98	455	2
 +100	457	1
 +100	457	1
 +100	457	2
 +103	457	1
 +103	457	1
 +103	457	2
 +455	97	1
 +455	97	2
 +455	97	455
 +455	98	1
 +455	98	2
 +455	98	455
 +456	0	1
 +456	0	2
 +456	0	456
 +456	10	1
 +456	10	2
 +456	10	456
 +457	100	1
 +457	100	2
 +457	100	457
 +457	103	1
 +457	103	2
 +457	103	457
 +PREHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, 1
 +insert into table multi1_mm partition(p=1) select key, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@multi1_mm
 +PREHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: query: from intermediate
 +insert into table multi1_mm partition(p) select p, key, 1
 +insert into table multi1_mm partition(p=1) select key, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@multi1_mm@p=1
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key SIMPLE [(intermediate)intermediate.FieldSchema(name:p, type:int, comment:null), ]
 +POSTHOOK: Lineage: multi1_mm PARTITION(p=1).key2 SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Input: default@multi1_mm@p=1
 +PREHOOK: Input: default@multi1_mm@p=2
 +PREHOOK: Input: default@multi1_mm@p=455
 +PREHOOK: Input: default@multi1_mm@p=456
 +PREHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +POSTHOOK: query: select key, key2, p from multi1_mm order by key, key2, p
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Input: default@multi1_mm@p=1
 +POSTHOOK: Input: default@multi1_mm@p=2
 +POSTHOOK: Input: default@multi1_mm@p=455
 +POSTHOOK: Input: default@multi1_mm@p=456
 +POSTHOOK: Input: default@multi1_mm@p=457
 +#### A masked pattern was here ####
 +0	456	1
 +0	456	1
 +0	456	1
 +0	456	2
 +10	456	1
 +10	456	1
 +10	456	1
 +10	456	2
 +97	455	1
 +97	455	1
 +97	455	1
 +97	455	2
 +98	455	1
 +98	455	1
 +98	455	1
 +98	455	2
 +100	457	1
 +100	457	1
 +100	457	1
 +100	457	2
 +103	457	1
 +103	457	1
 +103	457	1
 +103	457	2
 +455	97	1
 +455	97	1
 +455	97	2
 +455	97	455
 +455	98	1
 +455	98	1
 +455	98	2
 +455	98	455
 +456	0	1
 +456	0	1
 +456	0	2
 +456	0	456
 +456	10	1
 +456	10	1
 +456	10	2
 +456	10	456
 +457	100	1
 +457	100	1
 +457	100	2
 +457	100	457
 +457	103	1
 +457	103	1
 +457	103	2
 +457	103	457
 +PREHOOK: query: drop table multi1_mm
 +PREHOOK: type: DROPTABLE
 +PREHOOK: Input: default@multi1_mm
 +PREHOOK: Output: default@multi1_mm
 +POSTHOOK: query: drop table multi1_mm
 +POSTHOOK: type: DROPTABLE
 +POSTHOOK: Input: default@multi1_mm
 +POSTHOOK: Output: default@multi1_mm
 +PREHOOK: query: drop table stats_mm
 +PREHOOK: type: DROPTABLE
 +POSTHOOK: query: drop table stats_mm
 +POSTHOOK: type: DROPTABLE
 +PREHOOK: query: create table stats_mm(key int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +PREHOOK: type: CREATETABLE
 +PREHOOK: Output: database:default
 +PREHOOK: Output: default@stats_mm
 +POSTHOOK: query: create table stats_mm(key int)  tblproperties("transactional"="true", "transactional_properties"="insert_only")
 +POSTHOOK: type: CREATETABLE
 +POSTHOOK: Output: database:default
 +POSTHOOK: Output: default@stats_mm
 +PREHOOK: query: insert into table stats_mm  select key from intermediate
 +PREHOOK: type: QUERY
 +PREHOOK: Input: default@intermediate
 +PREHOOK: Input: default@intermediate@p=455
 +PREHOOK: Input: default@intermediate@p=456
 +PREHOOK: Input: default@intermediate@p=457
 +PREHOOK: Output: default@stats_mm
 +POSTHOOK: query: insert into table stats_mm  select key from intermediate
 +POSTHOOK: type: QUERY
 +POSTHOOK: Input: default@intermediate
 +POSTHOOK: Input: default@intermediate@p=455
 +POSTHOOK: Input: default@intermediate@p=456
 +POSTHOOK: Input: default@intermediate@p=457
 +POSTHOOK: Output: default@stats_mm
 +POSTHOOK: Lineage: stats_mm.key SIMPLE [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
 +PREHOOK: query: desc formatted stats_mm
 +PREHOOK: type: DESCTABLE
 +PREHOOK: Input: default@stats_mm
 +POSTHOOK: query: desc formatted stats_mm
 +POSTHOOK: type: DESCTABLE
 +POSTHOOK: Input: default@stats_mm
 +# col_name            	data_type           	comment             
 +	 	 
 +key                 	int                 	                    
 +	 	 
 +# Detailed Table Information	 	 
 +Database:           	default             	 
 +#### A masked p

<TRUNCATED>

[12/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Wei Zheng)

Posted by se...@apache.org.

http://git-wip-us.apache.org/repos/asf/hive/blob/42a38577/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
----------------------------------------------------------------------
diff --cc standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
index 0000000,d5963f3..cfbc488
mode 000000,100644..100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@@ -1,0 -1,9033 +1,9034 @@@
+ /**
+  * Autogenerated by Thrift Compiler (0.9.3)
+  *
+  * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+  *  @generated
+  */
+ #ifndef hive_metastore_TYPES_H
+ #define hive_metastore_TYPES_H
+ 
+ #include <iosfwd>
+ 
+ #include <thrift/Thrift.h>
+ #include <thrift/TApplicationException.h>
+ #include <thrift/protocol/TProtocol.h>
+ #include <thrift/transport/TTransport.h>
+ 
+ #include <thrift/cxxfunctional.h>
+ #include "fb303_types.h"
+ 
+ 
+ namespace Apache { namespace Hadoop { namespace Hive {
+ 
+ struct HiveObjectType {
+   enum type {
+     GLOBAL = 1,
+     DATABASE = 2,
+     TABLE = 3,
+     PARTITION = 4,
+     COLUMN = 5
+   };
+ };
+ 
+ extern const std::map<int, const char*> _HiveObjectType_VALUES_TO_NAMES;
+ 
+ struct PrincipalType {
+   enum type {
+     USER = 1,
+     ROLE = 2,
+     GROUP = 3
+   };
+ };
+ 
+ extern const std::map<int, const char*> _PrincipalType_VALUES_TO_NAMES;
+ 
+ struct PartitionEventType {
+   enum type {
+     LOAD_DONE = 1
+   };
+ };
+ 
+ extern const std::map<int, const char*> _PartitionEventType_VALUES_TO_NAMES;
+ 
+ struct TxnState {
+   enum type {
+     COMMITTED = 1,
+     ABORTED = 2,
+     OPEN = 3
+   };
+ };
+ 
+ extern const std::map<int, const char*> _TxnState_VALUES_TO_NAMES;
+ 
+ struct LockLevel {
+   enum type {
+     DB = 1,
+     TABLE = 2,
+     PARTITION = 3
+   };
+ };
+ 
+ extern const std::map<int, const char*> _LockLevel_VALUES_TO_NAMES;
+ 
+ struct LockState {
+   enum type {
+     ACQUIRED = 1,
+     WAITING = 2,
+     ABORT = 3,
+     NOT_ACQUIRED = 4
+   };
+ };
+ 
+ extern const std::map<int, const char*> _LockState_VALUES_TO_NAMES;
+ 
+ struct LockType {
+   enum type {
+     SHARED_READ = 1,
+     SHARED_WRITE = 2,
+     EXCLUSIVE = 3
+   };
+ };
+ 
+ extern const std::map<int, const char*> _LockType_VALUES_TO_NAMES;
+ 
+ struct CompactionType {
+   enum type {
+     MINOR = 1,
+     MAJOR = 2
+   };
+ };
+ 
+ extern const std::map<int, const char*> _CompactionType_VALUES_TO_NAMES;
+ 
+ struct GrantRevokeType {
+   enum type {
+     GRANT = 1,
+     REVOKE = 2
+   };
+ };
+ 
+ extern const std::map<int, const char*> _GrantRevokeType_VALUES_TO_NAMES;
+ 
+ struct DataOperationType {
+   enum type {
+     SELECT = 1,
+     INSERT = 2,
+     UPDATE = 3,
+     DELETE = 4,
+     UNSET = 5,
+     NO_TXN = 6
+   };
+ };
+ 
+ extern const std::map<int, const char*> _DataOperationType_VALUES_TO_NAMES;
+ 
+ struct EventRequestType {
+   enum type {
+     INSERT = 1,
+     UPDATE = 2,
+     DELETE = 3
+   };
+ };
+ 
+ extern const std::map<int, const char*> _EventRequestType_VALUES_TO_NAMES;
+ 
+ struct FunctionType {
+   enum type {
+     JAVA = 1
+   };
+ };
+ 
+ extern const std::map<int, const char*> _FunctionType_VALUES_TO_NAMES;
+ 
+ struct ResourceType {
+   enum type {
+     JAR = 1,
+     FILE = 2,
+     ARCHIVE = 3
+   };
+ };
+ 
+ extern const std::map<int, const char*> _ResourceType_VALUES_TO_NAMES;
+ 
+ struct FileMetadataExprType {
+   enum type {
+     ORC_SARG = 1
+   };
+ };
+ 
+ extern const std::map<int, const char*> _FileMetadataExprType_VALUES_TO_NAMES;
+ 
+ struct ClientCapability {
+   enum type {
 -    TEST_CAPABILITY = 1
++    TEST_CAPABILITY = 1,
++    INSERT_ONLY_TABLES = 2
+   };
+ };
+ 
+ extern const std::map<int, const char*> _ClientCapability_VALUES_TO_NAMES;
+ 
+ class Version;
+ 
+ class FieldSchema;
+ 
+ class SQLPrimaryKey;
+ 
+ class SQLForeignKey;
+ 
+ class SQLUniqueConstraint;
+ 
+ class SQLNotNullConstraint;
+ 
+ class Type;
+ 
+ class HiveObjectRef;
+ 
+ class PrivilegeGrantInfo;
+ 
+ class HiveObjectPrivilege;
+ 
+ class PrivilegeBag;
+ 
+ class PrincipalPrivilegeSet;
+ 
+ class GrantRevokePrivilegeRequest;
+ 
+ class GrantRevokePrivilegeResponse;
+ 
+ class Role;
+ 
+ class RolePrincipalGrant;
+ 
+ class GetRoleGrantsForPrincipalRequest;
+ 
+ class GetRoleGrantsForPrincipalResponse;
+ 
+ class GetPrincipalsInRoleRequest;
+ 
+ class GetPrincipalsInRoleResponse;
+ 
+ class GrantRevokeRoleRequest;
+ 
+ class GrantRevokeRoleResponse;
+ 
+ class Database;
+ 
+ class SerDeInfo;
+ 
+ class Order;
+ 
+ class SkewedInfo;
+ 
+ class StorageDescriptor;
+ 
+ class Table;
+ 
+ class Partition;
+ 
+ class PartitionWithoutSD;
+ 
+ class PartitionSpecWithSharedSD;
+ 
+ class PartitionListComposingSpec;
+ 
+ class PartitionSpec;
+ 
+ class Index;
+ 
+ class BooleanColumnStatsData;
+ 
+ class DoubleColumnStatsData;
+ 
+ class LongColumnStatsData;
+ 
+ class StringColumnStatsData;
+ 
+ class BinaryColumnStatsData;
+ 
+ class Decimal;
+ 
+ class DecimalColumnStatsData;
+ 
+ class Date;
+ 
+ class DateColumnStatsData;
+ 
+ class ColumnStatisticsData;
+ 
+ class ColumnStatisticsObj;
+ 
+ class ColumnStatisticsDesc;
+ 
+ class ColumnStatistics;
+ 
+ class AggrStats;
+ 
+ class SetPartitionsStatsRequest;
+ 
+ class Schema;
+ 
+ class EnvironmentContext;
+ 
+ class PrimaryKeysRequest;
+ 
+ class PrimaryKeysResponse;
+ 
+ class ForeignKeysRequest;
+ 
+ class ForeignKeysResponse;
+ 
+ class UniqueConstraintsRequest;
+ 
+ class UniqueConstraintsResponse;
+ 
+ class NotNullConstraintsRequest;
+ 
+ class NotNullConstraintsResponse;
+ 
+ class DropConstraintRequest;
+ 
+ class AddPrimaryKeyRequest;
+ 
+ class AddForeignKeyRequest;
+ 
+ class AddUniqueConstraintRequest;
+ 
+ class AddNotNullConstraintRequest;
+ 
+ class PartitionsByExprResult;
+ 
+ class PartitionsByExprRequest;
+ 
+ class TableStatsResult;
+ 
+ class PartitionsStatsResult;
+ 
+ class TableStatsRequest;
+ 
+ class PartitionsStatsRequest;
+ 
+ class AddPartitionsResult;
+ 
+ class AddPartitionsRequest;
+ 
+ class DropPartitionsResult;
+ 
+ class DropPartitionsExpr;
+ 
+ class RequestPartsSpec;
+ 
+ class DropPartitionsRequest;
+ 
+ class ResourceUri;
+ 
+ class Function;
+ 
+ class TxnInfo;
+ 
+ class GetOpenTxnsInfoResponse;
+ 
+ class GetOpenTxnsResponse;
+ 
+ class OpenTxnRequest;
+ 
+ class OpenTxnsResponse;
+ 
+ class AbortTxnRequest;
+ 
+ class AbortTxnsRequest;
+ 
+ class CommitTxnRequest;
+ 
+ class LockComponent;
+ 
+ class LockRequest;
+ 
+ class LockResponse;
+ 
+ class CheckLockRequest;
+ 
+ class UnlockRequest;
+ 
+ class ShowLocksRequest;
+ 
+ class ShowLocksResponseElement;
+ 
+ class ShowLocksResponse;
+ 
+ class HeartbeatRequest;
+ 
+ class HeartbeatTxnRangeRequest;
+ 
+ class HeartbeatTxnRangeResponse;
+ 
+ class CompactionRequest;
+ 
+ class CompactionResponse;
+ 
+ class ShowCompactRequest;
+ 
+ class ShowCompactResponseElement;
+ 
+ class ShowCompactResponse;
+ 
+ class AddDynamicPartitions;
+ 
+ class NotificationEventRequest;
+ 
+ class NotificationEvent;
+ 
+ class NotificationEventResponse;
+ 
+ class CurrentNotificationEventId;
+ 
+ class NotificationEventsCountRequest;
+ 
+ class NotificationEventsCountResponse;
+ 
+ class InsertEventRequestData;
+ 
+ class FireEventRequestData;
+ 
+ class FireEventRequest;
+ 
+ class FireEventResponse;
+ 
+ class MetadataPpdResult;
+ 
+ class GetFileMetadataByExprResult;
+ 
+ class GetFileMetadataByExprRequest;
+ 
+ class GetFileMetadataResult;
+ 
+ class GetFileMetadataRequest;
+ 
+ class PutFileMetadataResult;
+ 
+ class PutFileMetadataRequest;
+ 
+ class ClearFileMetadataResult;
+ 
+ class ClearFileMetadataRequest;
+ 
+ class CacheFileMetadataResult;
+ 
+ class CacheFileMetadataRequest;
+ 
+ class GetAllFunctionsResponse;
+ 
+ class ClientCapabilities;
+ 
+ class GetTableRequest;
+ 
+ class GetTableResult;
+ 
+ class GetTablesRequest;
+ 
+ class GetTablesResult;
+ 
+ class CmRecycleRequest;
+ 
+ class CmRecycleResponse;
+ 
+ class TableMeta;
+ 
+ class MetaException;
+ 
+ class UnknownTableException;
+ 
+ class UnknownDBException;
+ 
+ class AlreadyExistsException;
+ 
+ class InvalidPartitionException;
+ 
+ class UnknownPartitionException;
+ 
+ class InvalidObjectException;
+ 
+ class NoSuchObjectException;
+ 
+ class IndexAlreadyExistsException;
+ 
+ class InvalidOperationException;
+ 
+ class ConfigValSecurityException;
+ 
+ class InvalidInputException;
+ 
+ class NoSuchTxnException;
+ 
+ class TxnAbortedException;
+ 
+ class TxnOpenException;
+ 
+ class NoSuchLockException;
+ 
+ typedef struct _Version__isset {
+   _Version__isset() : version(false), comments(false) {}
+   bool version :1;
+   bool comments :1;
+ } _Version__isset;
+ 
+ class Version {
+  public:
+ 
+   Version(const Version&);
+   Version& operator=(const Version&);
+   Version() : version(), comments() {
+   }
+ 
+   virtual ~Version() throw();
+   std::string version;
+   std::string comments;
+ 
+   _Version__isset __isset;
+ 
+   void __set_version(const std::string& val);
+ 
+   void __set_comments(const std::string& val);
+ 
+   bool operator == (const Version & rhs) const
+   {
+     if (!(version == rhs.version))
+       return false;
+     if (!(comments == rhs.comments))
+       return false;
+     return true;
+   }
+   bool operator != (const Version &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const Version & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(Version &a, Version &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const Version& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _FieldSchema__isset {
+   _FieldSchema__isset() : name(false), type(false), comment(false) {}
+   bool name :1;
+   bool type :1;
+   bool comment :1;
+ } _FieldSchema__isset;
+ 
+ class FieldSchema {
+  public:
+ 
+   FieldSchema(const FieldSchema&);
+   FieldSchema& operator=(const FieldSchema&);
+   FieldSchema() : name(), type(), comment() {
+   }
+ 
+   virtual ~FieldSchema() throw();
+   std::string name;
+   std::string type;
+   std::string comment;
+ 
+   _FieldSchema__isset __isset;
+ 
+   void __set_name(const std::string& val);
+ 
+   void __set_type(const std::string& val);
+ 
+   void __set_comment(const std::string& val);
+ 
+   bool operator == (const FieldSchema & rhs) const
+   {
+     if (!(name == rhs.name))
+       return false;
+     if (!(type == rhs.type))
+       return false;
+     if (!(comment == rhs.comment))
+       return false;
+     return true;
+   }
+   bool operator != (const FieldSchema &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const FieldSchema & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(FieldSchema &a, FieldSchema &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const FieldSchema& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _SQLPrimaryKey__isset {
+   _SQLPrimaryKey__isset() : table_db(false), table_name(false), column_name(false), key_seq(false), pk_name(false), enable_cstr(false), validate_cstr(false), rely_cstr(false) {}
+   bool table_db :1;
+   bool table_name :1;
+   bool column_name :1;
+   bool key_seq :1;
+   bool pk_name :1;
+   bool enable_cstr :1;
+   bool validate_cstr :1;
+   bool rely_cstr :1;
+ } _SQLPrimaryKey__isset;
+ 
+ class SQLPrimaryKey {
+  public:
+ 
+   SQLPrimaryKey(const SQLPrimaryKey&);
+   SQLPrimaryKey& operator=(const SQLPrimaryKey&);
+   SQLPrimaryKey() : table_db(), table_name(), column_name(), key_seq(0), pk_name(), enable_cstr(0), validate_cstr(0), rely_cstr(0) {
+   }
+ 
+   virtual ~SQLPrimaryKey() throw();
+   std::string table_db;
+   std::string table_name;
+   std::string column_name;
+   int32_t key_seq;
+   std::string pk_name;
+   bool enable_cstr;
+   bool validate_cstr;
+   bool rely_cstr;
+ 
+   _SQLPrimaryKey__isset __isset;
+ 
+   void __set_table_db(const std::string& val);
+ 
+   void __set_table_name(const std::string& val);
+ 
+   void __set_column_name(const std::string& val);
+ 
+   void __set_key_seq(const int32_t val);
+ 
+   void __set_pk_name(const std::string& val);
+ 
+   void __set_enable_cstr(const bool val);
+ 
+   void __set_validate_cstr(const bool val);
+ 
+   void __set_rely_cstr(const bool val);
+ 
+   bool operator == (const SQLPrimaryKey & rhs) const
+   {
+     if (!(table_db == rhs.table_db))
+       return false;
+     if (!(table_name == rhs.table_name))
+       return false;
+     if (!(column_name == rhs.column_name))
+       return false;
+     if (!(key_seq == rhs.key_seq))
+       return false;
+     if (!(pk_name == rhs.pk_name))
+       return false;
+     if (!(enable_cstr == rhs.enable_cstr))
+       return false;
+     if (!(validate_cstr == rhs.validate_cstr))
+       return false;
+     if (!(rely_cstr == rhs.rely_cstr))
+       return false;
+     return true;
+   }
+   bool operator != (const SQLPrimaryKey &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const SQLPrimaryKey & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(SQLPrimaryKey &a, SQLPrimaryKey &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const SQLPrimaryKey& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _SQLForeignKey__isset {
+   _SQLForeignKey__isset() : pktable_db(false), pktable_name(false), pkcolumn_name(false), fktable_db(false), fktable_name(false), fkcolumn_name(false), key_seq(false), update_rule(false), delete_rule(false), fk_name(false), pk_name(false), enable_cstr(false), validate_cstr(false), rely_cstr(false) {}
+   bool pktable_db :1;
+   bool pktable_name :1;
+   bool pkcolumn_name :1;
+   bool fktable_db :1;
+   bool fktable_name :1;
+   bool fkcolumn_name :1;
+   bool key_seq :1;
+   bool update_rule :1;
+   bool delete_rule :1;
+   bool fk_name :1;
+   bool pk_name :1;
+   bool enable_cstr :1;
+   bool validate_cstr :1;
+   bool rely_cstr :1;
+ } _SQLForeignKey__isset;
+ 
+ class SQLForeignKey {
+  public:
+ 
+   SQLForeignKey(const SQLForeignKey&);
+   SQLForeignKey& operator=(const SQLForeignKey&);
+   SQLForeignKey() : pktable_db(), pktable_name(), pkcolumn_name(), fktable_db(), fktable_name(), fkcolumn_name(), key_seq(0), update_rule(0), delete_rule(0), fk_name(), pk_name(), enable_cstr(0), validate_cstr(0), rely_cstr(0) {
+   }
+ 
+   virtual ~SQLForeignKey() throw();
+   std::string pktable_db;
+   std::string pktable_name;
+   std::string pkcolumn_name;
+   std::string fktable_db;
+   std::string fktable_name;
+   std::string fkcolumn_name;
+   int32_t key_seq;
+   int32_t update_rule;
+   int32_t delete_rule;
+   std::string fk_name;
+   std::string pk_name;
+   bool enable_cstr;
+   bool validate_cstr;
+   bool rely_cstr;
+ 
+   _SQLForeignKey__isset __isset;
+ 
+   void __set_pktable_db(const std::string& val);
+ 
+   void __set_pktable_name(const std::string& val);
+ 
+   void __set_pkcolumn_name(const std::string& val);
+ 
+   void __set_fktable_db(const std::string& val);
+ 
+   void __set_fktable_name(const std::string& val);
+ 
+   void __set_fkcolumn_name(const std::string& val);
+ 
+   void __set_key_seq(const int32_t val);
+ 
+   void __set_update_rule(const int32_t val);
+ 
+   void __set_delete_rule(const int32_t val);
+ 
+   void __set_fk_name(const std::string& val);
+ 
+   void __set_pk_name(const std::string& val);
+ 
+   void __set_enable_cstr(const bool val);
+ 
+   void __set_validate_cstr(const bool val);
+ 
+   void __set_rely_cstr(const bool val);
+ 
+   bool operator == (const SQLForeignKey & rhs) const
+   {
+     if (!(pktable_db == rhs.pktable_db))
+       return false;
+     if (!(pktable_name == rhs.pktable_name))
+       return false;
+     if (!(pkcolumn_name == rhs.pkcolumn_name))
+       return false;
+     if (!(fktable_db == rhs.fktable_db))
+       return false;
+     if (!(fktable_name == rhs.fktable_name))
+       return false;
+     if (!(fkcolumn_name == rhs.fkcolumn_name))
+       return false;
+     if (!(key_seq == rhs.key_seq))
+       return false;
+     if (!(update_rule == rhs.update_rule))
+       return false;
+     if (!(delete_rule == rhs.delete_rule))
+       return false;
+     if (!(fk_name == rhs.fk_name))
+       return false;
+     if (!(pk_name == rhs.pk_name))
+       return false;
+     if (!(enable_cstr == rhs.enable_cstr))
+       return false;
+     if (!(validate_cstr == rhs.validate_cstr))
+       return false;
+     if (!(rely_cstr == rhs.rely_cstr))
+       return false;
+     return true;
+   }
+   bool operator != (const SQLForeignKey &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const SQLForeignKey & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(SQLForeignKey &a, SQLForeignKey &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const SQLForeignKey& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _SQLUniqueConstraint__isset {
+   _SQLUniqueConstraint__isset() : table_db(false), table_name(false), column_name(false), key_seq(false), uk_name(false), enable_cstr(false), validate_cstr(false), rely_cstr(false) {}
+   bool table_db :1;
+   bool table_name :1;
+   bool column_name :1;
+   bool key_seq :1;
+   bool uk_name :1;
+   bool enable_cstr :1;
+   bool validate_cstr :1;
+   bool rely_cstr :1;
+ } _SQLUniqueConstraint__isset;
+ 
+ class SQLUniqueConstraint {
+  public:
+ 
+   SQLUniqueConstraint(const SQLUniqueConstraint&);
+   SQLUniqueConstraint& operator=(const SQLUniqueConstraint&);
+   SQLUniqueConstraint() : table_db(), table_name(), column_name(), key_seq(0), uk_name(), enable_cstr(0), validate_cstr(0), rely_cstr(0) {
+   }
+ 
+   virtual ~SQLUniqueConstraint() throw();
+   std::string table_db;
+   std::string table_name;
+   std::string column_name;
+   int32_t key_seq;
+   std::string uk_name;
+   bool enable_cstr;
+   bool validate_cstr;
+   bool rely_cstr;
+ 
+   _SQLUniqueConstraint__isset __isset;
+ 
+   void __set_table_db(const std::string& val);
+ 
+   void __set_table_name(const std::string& val);
+ 
+   void __set_column_name(const std::string& val);
+ 
+   void __set_key_seq(const int32_t val);
+ 
+   void __set_uk_name(const std::string& val);
+ 
+   void __set_enable_cstr(const bool val);
+ 
+   void __set_validate_cstr(const bool val);
+ 
+   void __set_rely_cstr(const bool val);
+ 
+   bool operator == (const SQLUniqueConstraint & rhs) const
+   {
+     if (!(table_db == rhs.table_db))
+       return false;
+     if (!(table_name == rhs.table_name))
+       return false;
+     if (!(column_name == rhs.column_name))
+       return false;
+     if (!(key_seq == rhs.key_seq))
+       return false;
+     if (!(uk_name == rhs.uk_name))
+       return false;
+     if (!(enable_cstr == rhs.enable_cstr))
+       return false;
+     if (!(validate_cstr == rhs.validate_cstr))
+       return false;
+     if (!(rely_cstr == rhs.rely_cstr))
+       return false;
+     return true;
+   }
+   bool operator != (const SQLUniqueConstraint &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const SQLUniqueConstraint & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(SQLUniqueConstraint &a, SQLUniqueConstraint &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const SQLUniqueConstraint& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _SQLNotNullConstraint__isset {
+   _SQLNotNullConstraint__isset() : table_db(false), table_name(false), column_name(false), nn_name(false), enable_cstr(false), validate_cstr(false), rely_cstr(false) {}
+   bool table_db :1;
+   bool table_name :1;
+   bool column_name :1;
+   bool nn_name :1;
+   bool enable_cstr :1;
+   bool validate_cstr :1;
+   bool rely_cstr :1;
+ } _SQLNotNullConstraint__isset;
+ 
+ class SQLNotNullConstraint {
+  public:
+ 
+   SQLNotNullConstraint(const SQLNotNullConstraint&);
+   SQLNotNullConstraint& operator=(const SQLNotNullConstraint&);
+   SQLNotNullConstraint() : table_db(), table_name(), column_name(), nn_name(), enable_cstr(0), validate_cstr(0), rely_cstr(0) {
+   }
+ 
+   virtual ~SQLNotNullConstraint() throw();
+   std::string table_db;
+   std::string table_name;
+   std::string column_name;
+   std::string nn_name;
+   bool enable_cstr;
+   bool validate_cstr;
+   bool rely_cstr;
+ 
+   _SQLNotNullConstraint__isset __isset;
+ 
+   void __set_table_db(const std::string& val);
+ 
+   void __set_table_name(const std::string& val);
+ 
+   void __set_column_name(const std::string& val);
+ 
+   void __set_nn_name(const std::string& val);
+ 
+   void __set_enable_cstr(const bool val);
+ 
+   void __set_validate_cstr(const bool val);
+ 
+   void __set_rely_cstr(const bool val);
+ 
+   bool operator == (const SQLNotNullConstraint & rhs) const
+   {
+     if (!(table_db == rhs.table_db))
+       return false;
+     if (!(table_name == rhs.table_name))
+       return false;
+     if (!(column_name == rhs.column_name))
+       return false;
+     if (!(nn_name == rhs.nn_name))
+       return false;
+     if (!(enable_cstr == rhs.enable_cstr))
+       return false;
+     if (!(validate_cstr == rhs.validate_cstr))
+       return false;
+     if (!(rely_cstr == rhs.rely_cstr))
+       return false;
+     return true;
+   }
+   bool operator != (const SQLNotNullConstraint &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const SQLNotNullConstraint & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(SQLNotNullConstraint &a, SQLNotNullConstraint &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const SQLNotNullConstraint& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _Type__isset {
+   _Type__isset() : name(false), type1(false), type2(false), fields(false) {}
+   bool name :1;
+   bool type1 :1;
+   bool type2 :1;
+   bool fields :1;
+ } _Type__isset;
+ 
+ class Type {
+  public:
+ 
+   Type(const Type&);
+   Type& operator=(const Type&);
+   Type() : name(), type1(), type2() {
+   }
+ 
+   virtual ~Type() throw();
+   std::string name;
+   std::string type1;
+   std::string type2;
+   std::vector<FieldSchema>  fields;
+ 
+   _Type__isset __isset;
+ 
+   void __set_name(const std::string& val);
+ 
+   void __set_type1(const std::string& val);
+ 
+   void __set_type2(const std::string& val);
+ 
+   void __set_fields(const std::vector<FieldSchema> & val);
+ 
+   bool operator == (const Type & rhs) const
+   {
+     if (!(name == rhs.name))
+       return false;
+     if (__isset.type1 != rhs.__isset.type1)
+       return false;
+     else if (__isset.type1 && !(type1 == rhs.type1))
+       return false;
+     if (__isset.type2 != rhs.__isset.type2)
+       return false;
+     else if (__isset.type2 && !(type2 == rhs.type2))
+       return false;
+     if (__isset.fields != rhs.__isset.fields)
+       return false;
+     else if (__isset.fields && !(fields == rhs.fields))
+       return false;
+     return true;
+   }
+   bool operator != (const Type &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const Type & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(Type &a, Type &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const Type& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _HiveObjectRef__isset {
+   _HiveObjectRef__isset() : objectType(false), dbName(false), objectName(false), partValues(false), columnName(false) {}
+   bool objectType :1;
+   bool dbName :1;
+   bool objectName :1;
+   bool partValues :1;
+   bool columnName :1;
+ } _HiveObjectRef__isset;
+ 
+ class HiveObjectRef {
+  public:
+ 
+   HiveObjectRef(const HiveObjectRef&);
+   HiveObjectRef& operator=(const HiveObjectRef&);
+   HiveObjectRef() : objectType((HiveObjectType::type)0), dbName(), objectName(), columnName() {
+   }
+ 
+   virtual ~HiveObjectRef() throw();
+   HiveObjectType::type objectType;
+   std::string dbName;
+   std::string objectName;
+   std::vector<std::string>  partValues;
+   std::string columnName;
+ 
+   _HiveObjectRef__isset __isset;
+ 
+   void __set_objectType(const HiveObjectType::type val);
+ 
+   void __set_dbName(const std::string& val);
+ 
+   void __set_objectName(const std::string& val);
+ 
+   void __set_partValues(const std::vector<std::string> & val);
+ 
+   void __set_columnName(const std::string& val);
+ 
+   bool operator == (const HiveObjectRef & rhs) const
+   {
+     if (!(objectType == rhs.objectType))
+       return false;
+     if (!(dbName == rhs.dbName))
+       return false;
+     if (!(objectName == rhs.objectName))
+       return false;
+     if (!(partValues == rhs.partValues))
+       return false;
+     if (!(columnName == rhs.columnName))
+       return false;
+     return true;
+   }
+   bool operator != (const HiveObjectRef &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const HiveObjectRef & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(HiveObjectRef &a, HiveObjectRef &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const HiveObjectRef& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _PrivilegeGrantInfo__isset {
+   _PrivilegeGrantInfo__isset() : privilege(false), createTime(false), grantor(false), grantorType(false), grantOption(false) {}
+   bool privilege :1;
+   bool createTime :1;
+   bool grantor :1;
+   bool grantorType :1;
+   bool grantOption :1;
+ } _PrivilegeGrantInfo__isset;
+ 
+ class PrivilegeGrantInfo {
+  public:
+ 
+   PrivilegeGrantInfo(const PrivilegeGrantInfo&);
+   PrivilegeGrantInfo& operator=(const PrivilegeGrantInfo&);
+   PrivilegeGrantInfo() : privilege(), createTime(0), grantor(), grantorType((PrincipalType::type)0), grantOption(0) {
+   }
+ 
+   virtual ~PrivilegeGrantInfo() throw();
+   std::string privilege;
+   int32_t createTime;
+   std::string grantor;
+   PrincipalType::type grantorType;
+   bool grantOption;
+ 
+   _PrivilegeGrantInfo__isset __isset;
+ 
+   void __set_privilege(const std::string& val);
+ 
+   void __set_createTime(const int32_t val);
+ 
+   void __set_grantor(const std::string& val);
+ 
+   void __set_grantorType(const PrincipalType::type val);
+ 
+   void __set_grantOption(const bool val);
+ 
+   bool operator == (const PrivilegeGrantInfo & rhs) const
+   {
+     if (!(privilege == rhs.privilege))
+       return false;
+     if (!(createTime == rhs.createTime))
+       return false;
+     if (!(grantor == rhs.grantor))
+       return false;
+     if (!(grantorType == rhs.grantorType))
+       return false;
+     if (!(grantOption == rhs.grantOption))
+       return false;
+     return true;
+   }
+   bool operator != (const PrivilegeGrantInfo &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const PrivilegeGrantInfo & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(PrivilegeGrantInfo &a, PrivilegeGrantInfo &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const PrivilegeGrantInfo& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _HiveObjectPrivilege__isset {
+   _HiveObjectPrivilege__isset() : hiveObject(false), principalName(false), principalType(false), grantInfo(false) {}
+   bool hiveObject :1;
+   bool principalName :1;
+   bool principalType :1;
+   bool grantInfo :1;
+ } _HiveObjectPrivilege__isset;
+ 
+ class HiveObjectPrivilege {
+  public:
+ 
+   HiveObjectPrivilege(const HiveObjectPrivilege&);
+   HiveObjectPrivilege& operator=(const HiveObjectPrivilege&);
+   HiveObjectPrivilege() : principalName(), principalType((PrincipalType::type)0) {
+   }
+ 
+   virtual ~HiveObjectPrivilege() throw();
+   HiveObjectRef hiveObject;
+   std::string principalName;
+   PrincipalType::type principalType;
+   PrivilegeGrantInfo grantInfo;
+ 
+   _HiveObjectPrivilege__isset __isset;
+ 
+   void __set_hiveObject(const HiveObjectRef& val);
+ 
+   void __set_principalName(const std::string& val);
+ 
+   void __set_principalType(const PrincipalType::type val);
+ 
+   void __set_grantInfo(const PrivilegeGrantInfo& val);
+ 
+   bool operator == (const HiveObjectPrivilege & rhs) const
+   {
+     if (!(hiveObject == rhs.hiveObject))
+       return false;
+     if (!(principalName == rhs.principalName))
+       return false;
+     if (!(principalType == rhs.principalType))
+       return false;
+     if (!(grantInfo == rhs.grantInfo))
+       return false;
+     return true;
+   }
+   bool operator != (const HiveObjectPrivilege &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const HiveObjectPrivilege & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(HiveObjectPrivilege &a, HiveObjectPrivilege &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const HiveObjectPrivilege& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _PrivilegeBag__isset {
+   _PrivilegeBag__isset() : privileges(false) {}
+   bool privileges :1;
+ } _PrivilegeBag__isset;
+ 
+ class PrivilegeBag {
+  public:
+ 
+   PrivilegeBag(const PrivilegeBag&);
+   PrivilegeBag& operator=(const PrivilegeBag&);
+   PrivilegeBag() {
+   }
+ 
+   virtual ~PrivilegeBag() throw();
+   std::vector<HiveObjectPrivilege>  privileges;
+ 
+   _PrivilegeBag__isset __isset;
+ 
+   void __set_privileges(const std::vector<HiveObjectPrivilege> & val);
+ 
+   bool operator == (const PrivilegeBag & rhs) const
+   {
+     if (!(privileges == rhs.privileges))
+       return false;
+     return true;
+   }
+   bool operator != (const PrivilegeBag &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const PrivilegeBag & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(PrivilegeBag &a, PrivilegeBag &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const PrivilegeBag& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _PrincipalPrivilegeSet__isset {
+   _PrincipalPrivilegeSet__isset() : userPrivileges(false), groupPrivileges(false), rolePrivileges(false) {}
+   bool userPrivileges :1;
+   bool groupPrivileges :1;
+   bool rolePrivileges :1;
+ } _PrincipalPrivilegeSet__isset;
+ 
+ class PrincipalPrivilegeSet {
+  public:
+ 
+   PrincipalPrivilegeSet(const PrincipalPrivilegeSet&);
+   PrincipalPrivilegeSet& operator=(const PrincipalPrivilegeSet&);
+   PrincipalPrivilegeSet() {
+   }
+ 
+   virtual ~PrincipalPrivilegeSet() throw();
+   std::map<std::string, std::vector<PrivilegeGrantInfo> >  userPrivileges;
+   std::map<std::string, std::vector<PrivilegeGrantInfo> >  groupPrivileges;
+   std::map<std::string, std::vector<PrivilegeGrantInfo> >  rolePrivileges;
+ 
+   _PrincipalPrivilegeSet__isset __isset;
+ 
+   void __set_userPrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val);
+ 
+   void __set_groupPrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val);
+ 
+   void __set_rolePrivileges(const std::map<std::string, std::vector<PrivilegeGrantInfo> > & val);
+ 
+   bool operator == (const PrincipalPrivilegeSet & rhs) const
+   {
+     if (!(userPrivileges == rhs.userPrivileges))
+       return false;
+     if (!(groupPrivileges == rhs.groupPrivileges))
+       return false;
+     if (!(rolePrivileges == rhs.rolePrivileges))
+       return false;
+     return true;
+   }
+   bool operator != (const PrincipalPrivilegeSet &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const PrincipalPrivilegeSet & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(PrincipalPrivilegeSet &a, PrincipalPrivilegeSet &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const PrincipalPrivilegeSet& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _GrantRevokePrivilegeRequest__isset {
+   _GrantRevokePrivilegeRequest__isset() : requestType(false), privileges(false), revokeGrantOption(false) {}
+   bool requestType :1;
+   bool privileges :1;
+   bool revokeGrantOption :1;
+ } _GrantRevokePrivilegeRequest__isset;
+ 
+ class GrantRevokePrivilegeRequest {
+  public:
+ 
+   GrantRevokePrivilegeRequest(const GrantRevokePrivilegeRequest&);
+   GrantRevokePrivilegeRequest& operator=(const GrantRevokePrivilegeRequest&);
+   GrantRevokePrivilegeRequest() : requestType((GrantRevokeType::type)0), revokeGrantOption(0) {
+   }
+ 
+   virtual ~GrantRevokePrivilegeRequest() throw();
+   GrantRevokeType::type requestType;
+   PrivilegeBag privileges;
+   bool revokeGrantOption;
+ 
+   _GrantRevokePrivilegeRequest__isset __isset;
+ 
+   void __set_requestType(const GrantRevokeType::type val);
+ 
+   void __set_privileges(const PrivilegeBag& val);
+ 
+   void __set_revokeGrantOption(const bool val);
+ 
+   bool operator == (const GrantRevokePrivilegeRequest & rhs) const
+   {
+     if (!(requestType == rhs.requestType))
+       return false;
+     if (!(privileges == rhs.privileges))
+       return false;
+     if (__isset.revokeGrantOption != rhs.__isset.revokeGrantOption)
+       return false;
+     else if (__isset.revokeGrantOption && !(revokeGrantOption == rhs.revokeGrantOption))
+       return false;
+     return true;
+   }
+   bool operator != (const GrantRevokePrivilegeRequest &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const GrantRevokePrivilegeRequest & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(GrantRevokePrivilegeRequest &a, GrantRevokePrivilegeRequest &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const GrantRevokePrivilegeRequest& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _GrantRevokePrivilegeResponse__isset {
+   _GrantRevokePrivilegeResponse__isset() : success(false) {}
+   bool success :1;
+ } _GrantRevokePrivilegeResponse__isset;
+ 
+ class GrantRevokePrivilegeResponse {
+  public:
+ 
+   GrantRevokePrivilegeResponse(const GrantRevokePrivilegeResponse&);
+   GrantRevokePrivilegeResponse& operator=(const GrantRevokePrivilegeResponse&);
+   GrantRevokePrivilegeResponse() : success(0) {
+   }
+ 
+   virtual ~GrantRevokePrivilegeResponse() throw();
+   bool success;
+ 
+   _GrantRevokePrivilegeResponse__isset __isset;
+ 
+   void __set_success(const bool val);
+ 
+   bool operator == (const GrantRevokePrivilegeResponse & rhs) const
+   {
+     if (__isset.success != rhs.__isset.success)
+       return false;
+     else if (__isset.success && !(success == rhs.success))
+       return false;
+     return true;
+   }
+   bool operator != (const GrantRevokePrivilegeResponse &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const GrantRevokePrivilegeResponse & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(GrantRevokePrivilegeResponse &a, GrantRevokePrivilegeResponse &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const GrantRevokePrivilegeResponse& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _Role__isset {
+   _Role__isset() : roleName(false), createTime(false), ownerName(false) {}
+   bool roleName :1;
+   bool createTime :1;
+   bool ownerName :1;
+ } _Role__isset;
+ 
+ class Role {
+  public:
+ 
+   Role(const Role&);
+   Role& operator=(const Role&);
+   Role() : roleName(), createTime(0), ownerName() {
+   }
+ 
+   virtual ~Role() throw();
+   std::string roleName;
+   int32_t createTime;
+   std::string ownerName;
+ 
+   _Role__isset __isset;
+ 
+   void __set_roleName(const std::string& val);
+ 
+   void __set_createTime(const int32_t val);
+ 
+   void __set_ownerName(const std::string& val);
+ 
+   bool operator == (const Role & rhs) const
+   {
+     if (!(roleName == rhs.roleName))
+       return false;
+     if (!(createTime == rhs.createTime))
+       return false;
+     if (!(ownerName == rhs.ownerName))
+       return false;
+     return true;
+   }
+   bool operator != (const Role &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const Role & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(Role &a, Role &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const Role& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _RolePrincipalGrant__isset {
+   _RolePrincipalGrant__isset() : roleName(false), principalName(false), principalType(false), grantOption(false), grantTime(false), grantorName(false), grantorPrincipalType(false) {}
+   bool roleName :1;
+   bool principalName :1;
+   bool principalType :1;
+   bool grantOption :1;
+   bool grantTime :1;
+   bool grantorName :1;
+   bool grantorPrincipalType :1;
+ } _RolePrincipalGrant__isset;
+ 
+ class RolePrincipalGrant {
+  public:
+ 
+   RolePrincipalGrant(const RolePrincipalGrant&);
+   RolePrincipalGrant& operator=(const RolePrincipalGrant&);
+   RolePrincipalGrant() : roleName(), principalName(), principalType((PrincipalType::type)0), grantOption(0), grantTime(0), grantorName(), grantorPrincipalType((PrincipalType::type)0) {
+   }
+ 
+   virtual ~RolePrincipalGrant() throw();
+   std::string roleName;
+   std::string principalName;
+   PrincipalType::type principalType;
+   bool grantOption;
+   int32_t grantTime;
+   std::string grantorName;
+   PrincipalType::type grantorPrincipalType;
+ 
+   _RolePrincipalGrant__isset __isset;
+ 
+   void __set_roleName(const std::string& val);
+ 
+   void __set_principalName(const std::string& val);
+ 
+   void __set_principalType(const PrincipalType::type val);
+ 
+   void __set_grantOption(const bool val);
+ 
+   void __set_grantTime(const int32_t val);
+ 
+   void __set_grantorName(const std::string& val);
+ 
+   void __set_grantorPrincipalType(const PrincipalType::type val);
+ 
+   bool operator == (const RolePrincipalGrant & rhs) const
+   {
+     if (!(roleName == rhs.roleName))
+       return false;
+     if (!(principalName == rhs.principalName))
+       return false;
+     if (!(principalType == rhs.principalType))
+       return false;
+     if (!(grantOption == rhs.grantOption))
+       return false;
+     if (!(grantTime == rhs.grantTime))
+       return false;
+     if (!(grantorName == rhs.grantorName))
+       return false;
+     if (!(grantorPrincipalType == rhs.grantorPrincipalType))
+       return false;
+     return true;
+   }
+   bool operator != (const RolePrincipalGrant &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const RolePrincipalGrant & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(RolePrincipalGrant &a, RolePrincipalGrant &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const RolePrincipalGrant& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ 
+ class GetRoleGrantsForPrincipalRequest {
+  public:
+ 
+   GetRoleGrantsForPrincipalRequest(const GetRoleGrantsForPrincipalRequest&);
+   GetRoleGrantsForPrincipalRequest& operator=(const GetRoleGrantsForPrincipalRequest&);
+   GetRoleGrantsForPrincipalRequest() : principal_name(), principal_type((PrincipalType::type)0) {
+   }
+ 
+   virtual ~GetRoleGrantsForPrincipalRequest() throw();
+   std::string principal_name;
+   PrincipalType::type principal_type;
+ 
+   void __set_principal_name(const std::string& val);
+ 
+   void __set_principal_type(const PrincipalType::type val);
+ 
+   bool operator == (const GetRoleGrantsForPrincipalRequest & rhs) const
+   {
+     if (!(principal_name == rhs.principal_name))
+       return false;
+     if (!(principal_type == rhs.principal_type))
+       return false;
+     return true;
+   }
+   bool operator != (const GetRoleGrantsForPrincipalRequest &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const GetRoleGrantsForPrincipalRequest & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(GetRoleGrantsForPrincipalRequest &a, GetRoleGrantsForPrincipalRequest &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const GetRoleGrantsForPrincipalRequest& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ 
+ class GetRoleGrantsForPrincipalResponse {
+  public:
+ 
+   GetRoleGrantsForPrincipalResponse(const GetRoleGrantsForPrincipalResponse&);
+   GetRoleGrantsForPrincipalResponse& operator=(const GetRoleGrantsForPrincipalResponse&);
+   GetRoleGrantsForPrincipalResponse() {
+   }
+ 
+   virtual ~GetRoleGrantsForPrincipalResponse() throw();
+   std::vector<RolePrincipalGrant>  principalGrants;
+ 
+   void __set_principalGrants(const std::vector<RolePrincipalGrant> & val);
+ 
+   bool operator == (const GetRoleGrantsForPrincipalResponse & rhs) const
+   {
+     if (!(principalGrants == rhs.principalGrants))
+       return false;
+     return true;
+   }
+   bool operator != (const GetRoleGrantsForPrincipalResponse &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const GetRoleGrantsForPrincipalResponse & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(GetRoleGrantsForPrincipalResponse &a, GetRoleGrantsForPrincipalResponse &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const GetRoleGrantsForPrincipalResponse& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ 
+ class GetPrincipalsInRoleRequest {
+  public:
+ 
+   GetPrincipalsInRoleRequest(const GetPrincipalsInRoleRequest&);
+   GetPrincipalsInRoleRequest& operator=(const GetPrincipalsInRoleRequest&);
+   GetPrincipalsInRoleRequest() : roleName() {
+   }
+ 
+   virtual ~GetPrincipalsInRoleRequest() throw();
+   std::string roleName;
+ 
+   void __set_roleName(const std::string& val);
+ 
+   bool operator == (const GetPrincipalsInRoleRequest & rhs) const
+   {
+     if (!(roleName == rhs.roleName))
+       return false;
+     return true;
+   }
+   bool operator != (const GetPrincipalsInRoleRequest &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const GetPrincipalsInRoleRequest & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(GetPrincipalsInRoleRequest &a, GetPrincipalsInRoleRequest &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const GetPrincipalsInRoleRequest& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ 
+ class GetPrincipalsInRoleResponse {
+  public:
+ 
+   GetPrincipalsInRoleResponse(const GetPrincipalsInRoleResponse&);
+   GetPrincipalsInRoleResponse& operator=(const GetPrincipalsInRoleResponse&);
+   GetPrincipalsInRoleResponse() {
+   }
+ 
+   virtual ~GetPrincipalsInRoleResponse() throw();
+   std::vector<RolePrincipalGrant>  principalGrants;
+ 
+   void __set_principalGrants(const std::vector<RolePrincipalGrant> & val);
+ 
+   bool operator == (const GetPrincipalsInRoleResponse & rhs) const
+   {
+     if (!(principalGrants == rhs.principalGrants))
+       return false;
+     return true;
+   }
+   bool operator != (const GetPrincipalsInRoleResponse &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const GetPrincipalsInRoleResponse & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(GetPrincipalsInRoleResponse &a, GetPrincipalsInRoleResponse &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const GetPrincipalsInRoleResponse& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _GrantRevokeRoleRequest__isset {
+   _GrantRevokeRoleRequest__isset() : requestType(false), roleName(false), principalName(false), principalType(false), grantor(false), grantorType(false), grantOption(false) {}
+   bool requestType :1;
+   bool roleName :1;
+   bool principalName :1;
+   bool principalType :1;
+   bool grantor :1;
+   bool grantorType :1;
+   bool grantOption :1;
+ } _GrantRevokeRoleRequest__isset;
+ 
+ class GrantRevokeRoleRequest {
+  public:
+ 
+   GrantRevokeRoleRequest(const GrantRevokeRoleRequest&);
+   GrantRevokeRoleRequest& operator=(const GrantRevokeRoleRequest&);
+   GrantRevokeRoleRequest() : requestType((GrantRevokeType::type)0), roleName(), principalName(), principalType((PrincipalType::type)0), grantor(), grantorType((PrincipalType::type)0), grantOption(0) {
+   }
+ 
+   virtual ~GrantRevokeRoleRequest() throw();
+   GrantRevokeType::type requestType;
+   std::string roleName;
+   std::string principalName;
+   PrincipalType::type principalType;
+   std::string grantor;
+   PrincipalType::type grantorType;
+   bool grantOption;
+ 
+   _GrantRevokeRoleRequest__isset __isset;
+ 
+   void __set_requestType(const GrantRevokeType::type val);
+ 
+   void __set_roleName(const std::string& val);
+ 
+   void __set_principalName(const std::string& val);
+ 
+   void __set_principalType(const PrincipalType::type val);
+ 
+   void __set_grantor(const std::string& val);
+ 
+   void __set_grantorType(const PrincipalType::type val);
+ 
+   void __set_grantOption(const bool val);
+ 
+   bool operator == (const GrantRevokeRoleRequest & rhs) const
+   {
+     if (!(requestType == rhs.requestType))
+       return false;
+     if (!(roleName == rhs.roleName))
+       return false;
+     if (!(principalName == rhs.principalName))
+       return false;
+     if (!(principalType == rhs.principalType))
+       return false;
+     if (__isset.grantor != rhs.__isset.grantor)
+       return false;
+     else if (__isset.grantor && !(grantor == rhs.grantor))
+       return false;
+     if (__isset.grantorType != rhs.__isset.grantorType)
+       return false;
+     else if (__isset.grantorType && !(grantorType == rhs.grantorType))
+       return false;
+     if (__isset.grantOption != rhs.__isset.grantOption)
+       return false;
+     else if (__isset.grantOption && !(grantOption == rhs.grantOption))
+       return false;
+     return true;
+   }
+   bool operator != (const GrantRevokeRoleRequest &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const GrantRevokeRoleRequest & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(GrantRevokeRoleRequest &a, GrantRevokeRoleRequest &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const GrantRevokeRoleRequest& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _GrantRevokeRoleResponse__isset {
+   _GrantRevokeRoleResponse__isset() : success(false) {}
+   bool success :1;
+ } _GrantRevokeRoleResponse__isset;
+ 
+ class GrantRevokeRoleResponse {
+  public:
+ 
+   GrantRevokeRoleResponse(const GrantRevokeRoleResponse&);
+   GrantRevokeRoleResponse& operator=(const GrantRevokeRoleResponse&);
+   GrantRevokeRoleResponse() : success(0) {
+   }
+ 
+   virtual ~GrantRevokeRoleResponse() throw();
+   bool success;
+ 
+   _GrantRevokeRoleResponse__isset __isset;
+ 
+   void __set_success(const bool val);
+ 
+   bool operator == (const GrantRevokeRoleResponse & rhs) const
+   {
+     if (__isset.success != rhs.__isset.success)
+       return false;
+     else if (__isset.success && !(success == rhs.success))
+       return false;
+     return true;
+   }
+   bool operator != (const GrantRevokeRoleResponse &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const GrantRevokeRoleResponse & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(GrantRevokeRoleResponse &a, GrantRevokeRoleResponse &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const GrantRevokeRoleResponse& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _Database__isset {
+   _Database__isset() : name(false), description(false), locationUri(false), parameters(false), privileges(false), ownerName(false), ownerType(false) {}
+   bool name :1;
+   bool description :1;
+   bool locationUri :1;
+   bool parameters :1;
+   bool privileges :1;
+   bool ownerName :1;
+   bool ownerType :1;
+ } _Database__isset;
+ 
+ class Database {
+  public:
+ 
+   Database(const Database&);
+   Database& operator=(const Database&);
+   Database() : name(), description(), locationUri(), ownerName(), ownerType((PrincipalType::type)0) {
+   }
+ 
+   virtual ~Database() throw();
+   std::string name;
+   std::string description;
+   std::string locationUri;
+   std::map<std::string, std::string>  parameters;
+   PrincipalPrivilegeSet privileges;
+   std::string ownerName;
+   PrincipalType::type ownerType;
+ 
+   _Database__isset __isset;
+ 
+   void __set_name(const std::string& val);
+ 
+   void __set_description(const std::string& val);
+ 
+   void __set_locationUri(const std::string& val);
+ 
+   void __set_parameters(const std::map<std::string, std::string> & val);
+ 
+   void __set_privileges(const PrincipalPrivilegeSet& val);
+ 
+   void __set_ownerName(const std::string& val);
+ 
+   void __set_ownerType(const PrincipalType::type val);
+ 
+   bool operator == (const Database & rhs) const
+   {
+     if (!(name == rhs.name))
+       return false;
+     if (!(description == rhs.description))
+       return false;
+     if (!(locationUri == rhs.locationUri))
+       return false;
+     if (!(parameters == rhs.parameters))
+       return false;
+     if (__isset.privileges != rhs.__isset.privileges)
+       return false;
+     else if (__isset.privileges && !(privileges == rhs.privileges))
+       return false;
+     if (__isset.ownerName != rhs.__isset.ownerName)
+       return false;
+     else if (__isset.ownerName && !(ownerName == rhs.ownerName))
+       return false;
+     if (__isset.ownerType != rhs.__isset.ownerType)
+       return false;
+     else if (__isset.ownerType && !(ownerType == rhs.ownerType))
+       return false;
+     return true;
+   }
+   bool operator != (const Database &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const Database & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(Database &a, Database &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const Database& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _SerDeInfo__isset {
+   _SerDeInfo__isset() : name(false), serializationLib(false), parameters(false) {}
+   bool name :1;
+   bool serializationLib :1;
+   bool parameters :1;
+ } _SerDeInfo__isset;
+ 
+ class SerDeInfo {
+  public:
+ 
+   SerDeInfo(const SerDeInfo&);
+   SerDeInfo& operator=(const SerDeInfo&);
+   SerDeInfo() : name(), serializationLib() {
+   }
+ 
+   virtual ~SerDeInfo() throw();
+   std::string name;
+   std::string serializationLib;
+   std::map<std::string, std::string>  parameters;
+ 
+   _SerDeInfo__isset __isset;
+ 
+   void __set_name(const std::string& val);
+ 
+   void __set_serializationLib(const std::string& val);
+ 
+   void __set_parameters(const std::map<std::string, std::string> & val);
+ 
+   bool operator == (const SerDeInfo & rhs) const
+   {
+     if (!(name == rhs.name))
+       return false;
+     if (!(serializationLib == rhs.serializationLib))
+       return false;
+     if (!(parameters == rhs.parameters))
+       return false;
+     return true;
+   }
+   bool operator != (const SerDeInfo &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const SerDeInfo & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(SerDeInfo &a, SerDeInfo &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const SerDeInfo& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _Order__isset {
+   _Order__isset() : col(false), order(false) {}
+   bool col :1;
+   bool order :1;
+ } _Order__isset;
+ 
+ class Order {
+  public:
+ 
+   Order(const Order&);
+   Order& operator=(const Order&);
+   Order() : col(), order(0) {
+   }
+ 
+   virtual ~Order() throw();
+   std::string col;
+   int32_t order;
+ 
+   _Order__isset __isset;
+ 
+   void __set_col(const std::string& val);
+ 
+   void __set_order(const int32_t val);
+ 
+   bool operator == (const Order & rhs) const
+   {
+     if (!(col == rhs.col))
+       return false;
+     if (!(order == rhs.order))
+       return false;
+     return true;
+   }
+   bool operator != (const Order &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const Order & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(Order &a, Order &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const Order& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _SkewedInfo__isset {
+   _SkewedInfo__isset() : skewedColNames(false), skewedColValues(false), skewedColValueLocationMaps(false) {}
+   bool skewedColNames :1;
+   bool skewedColValues :1;
+   bool skewedColValueLocationMaps :1;
+ } _SkewedInfo__isset;
+ 
+ class SkewedInfo {
+  public:
+ 
+   SkewedInfo(const SkewedInfo&);
+   SkewedInfo& operator=(const SkewedInfo&);
+   SkewedInfo() {
+   }
+ 
+   virtual ~SkewedInfo() throw();
+   std::vector<std::string>  skewedColNames;
+   std::vector<std::vector<std::string> >  skewedColValues;
+   std::map<std::vector<std::string> , std::string>  skewedColValueLocationMaps;
+ 
+   _SkewedInfo__isset __isset;
+ 
+   void __set_skewedColNames(const std::vector<std::string> & val);
+ 
+   void __set_skewedColValues(const std::vector<std::vector<std::string> > & val);
+ 
+   void __set_skewedColValueLocationMaps(const std::map<std::vector<std::string> , std::string> & val);
+ 
+   bool operator == (const SkewedInfo & rhs) const
+   {
+     if (!(skewedColNames == rhs.skewedColNames))
+       return false;
+     if (!(skewedColValues == rhs.skewedColValues))
+       return false;
+     if (!(skewedColValueLocationMaps == rhs.skewedColValueLocationMaps))
+       return false;
+     return true;
+   }
+   bool operator != (const SkewedInfo &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const SkewedInfo & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(SkewedInfo &a, SkewedInfo &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const SkewedInfo& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _StorageDescriptor__isset {
+   _StorageDescriptor__isset() : cols(false), location(false), inputFormat(false), outputFormat(false), compressed(false), numBuckets(false), serdeInfo(false), bucketCols(false), sortCols(false), parameters(false), skewedInfo(false), storedAsSubDirectories(false) {}
+   bool cols :1;
+   bool location :1;
+   bool inputFormat :1;
+   bool outputFormat :1;
+   bool compressed :1;
+   bool numBuckets :1;
+   bool serdeInfo :1;
+   bool bucketCols :1;
+   bool sortCols :1;
+   bool parameters :1;
+   bool skewedInfo :1;
+   bool storedAsSubDirectories :1;
+ } _StorageDescriptor__isset;
+ 
+ class StorageDescriptor {
+  public:
+ 
+   StorageDescriptor(const StorageDescriptor&);
+   StorageDescriptor& operator=(const StorageDescriptor&);
+   StorageDescriptor() : location(), inputFormat(), outputFormat(), compressed(0), numBuckets(0), storedAsSubDirectories(0) {
+   }
+ 
+   virtual ~StorageDescriptor() throw();
+   std::vector<FieldSchema>  cols;
+   std::string location;
+   std::string inputFormat;
+   std::string outputFormat;
+   bool compressed;
+   int32_t numBuckets;
+   SerDeInfo serdeInfo;
+   std::vector<std::string>  bucketCols;
+   std::vector<Order>  sortCols;
+   std::map<std::string, std::string>  parameters;
+   SkewedInfo skewedInfo;
+   bool storedAsSubDirectories;
+ 
+   _StorageDescriptor__isset __isset;
+ 
+   void __set_cols(const std::vector<FieldSchema> & val);
+ 
+   void __set_location(const std::string& val);
+ 
+   void __set_inputFormat(const std::string& val);
+ 
+   void __set_outputFormat(const std::string& val);
+ 
+   void __set_compressed(const bool val);
+ 
+   void __set_numBuckets(const int32_t val);
+ 
+   void __set_serdeInfo(const SerDeInfo& val);
+ 
+   void __set_bucketCols(const std::vector<std::string> & val);
+ 
+   void __set_sortCols(const std::vector<Order> & val);
+ 
+   void __set_parameters(const std::map<std::string, std::string> & val);
+ 
+   void __set_skewedInfo(const SkewedInfo& val);
+ 
+   void __set_storedAsSubDirectories(const bool val);
+ 
+   bool operator == (const StorageDescriptor & rhs) const
+   {
+     if (!(cols == rhs.cols))
+       return false;
+     if (!(location == rhs.location))
+       return false;
+     if (!(inputFormat == rhs.inputFormat))
+       return false;
+     if (!(outputFormat == rhs.outputFormat))
+       return false;
+     if (!(compressed == rhs.compressed))
+       return false;
+     if (!(numBuckets == rhs.numBuckets))
+       return false;
+     if (!(serdeInfo == rhs.serdeInfo))
+       return false;
+     if (!(bucketCols == rhs.bucketCols))
+       return false;
+     if (!(sortCols == rhs.sortCols))
+       return false;
+     if (!(parameters == rhs.parameters))
+       return false;
+     if (__isset.skewedInfo != rhs.__isset.skewedInfo)
+       return false;
+     else if (__isset.skewedInfo && !(skewedInfo == rhs.skewedInfo))
+       return false;
+     if (__isset.storedAsSubDirectories != rhs.__isset.storedAsSubDirectories)
+       return false;
+     else if (__isset.storedAsSubDirectories && !(storedAsSubDirectories == rhs.storedAsSubDirectories))
+       return false;
+     return true;
+   }
+   bool operator != (const StorageDescriptor &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const StorageDescriptor & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(StorageDescriptor &a, StorageDescriptor &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const StorageDescriptor& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _Table__isset {
+   _Table__isset() : tableName(false), dbName(false), owner(false), createTime(false), lastAccessTime(false), retention(false), sd(false), partitionKeys(false), parameters(false), viewOriginalText(false), viewExpandedText(false), tableType(false), privileges(false), temporary(true), rewriteEnabled(false) {}
+   bool tableName :1;
+   bool dbName :1;
+   bool owner :1;
+   bool createTime :1;
+   bool lastAccessTime :1;
+   bool retention :1;
+   bool sd :1;
+   bool partitionKeys :1;
+   bool parameters :1;
+   bool viewOriginalText :1;
+   bool viewExpandedText :1;
+   bool tableType :1;
+   bool privileges :1;
+   bool temporary :1;
+   bool rewriteEnabled :1;
+ } _Table__isset;
+ 
+ class Table {
+  public:
+ 
+   Table(const Table&);
+   Table& operator=(const Table&);
+   Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), retention(0), viewOriginalText(), viewExpandedText(), tableType(), temporary(false), rewriteEnabled(0) {
+   }
+ 
+   virtual ~Table() throw();
+   std::string tableName;
+   std::string dbName;
+   std::string owner;
+   int32_t createTime;
+   int32_t lastAccessTime;
+   int32_t retention;
+   StorageDescriptor sd;
+   std::vector<FieldSchema>  partitionKeys;
+   std::map<std::string, std::string>  parameters;
+   std::string viewOriginalText;
+   std::string viewExpandedText;
+   std::string tableType;
+   PrincipalPrivilegeSet privileges;
+   bool temporary;
+   bool rewriteEnabled;
+ 
+   _Table__isset __isset;
+ 
+   void __set_tableName(const std::string& val);
+ 
+   void __set_dbName(const std::string& val);
+ 
+   void __set_owner(const std::string& val);
+ 
+   void __set_createTime(const int32_t val);
+ 
+   void __set_lastAccessTime(const int32_t val);
+ 
+   void __set_retention(const int32_t val);
+ 
+   void __set_sd(const StorageDescriptor& val);
+ 
+   void __set_partitionKeys(const std::vector<FieldSchema> & val);
+ 
+   void __set_parameters(const std::map<std::string, std::string> & val);
+ 
+   void __set_viewOriginalText(const std::string& val);
+ 
+   void __set_viewExpandedText(const std::string& val);
+ 
+   void __set_tableType(const std::string& val);
+ 
+   void __set_privileges(const PrincipalPrivilegeSet& val);
+ 
+   void __set_temporary(const bool val);
+ 
+   void __set_rewriteEnabled(const bool val);
+ 
+   bool operator == (const Table & rhs) const
+   {
+     if (!(tableName == rhs.tableName))
+       return false;
+     if (!(dbName == rhs.dbName))
+       return false;
+     if (!(owner == rhs.owner))
+       return false;
+     if (!(createTime == rhs.createTime))
+       return false;
+     if (!(lastAccessTime == rhs.lastAccessTime))
+       return false;
+     if (!(retention == rhs.retention))
+       return false;
+     if (!(sd == rhs.sd))
+       return false;
+     if (!(partitionKeys == rhs.partitionKeys))
+       return false;
+     if (!(parameters == rhs.parameters))
+       return false;
+     if (!(viewOriginalText == rhs.viewOriginalText))
+       return false;
+     if (!(viewExpandedText == rhs.viewExpandedText))
+       return false;
+     if (!(tableType == rhs.tableType))
+       return false;
+     if (__isset.privileges != rhs.__isset.privileges)
+       return false;
+     else if (__isset.privileges && !(privileges == rhs.privileges))
+       return false;
+     if (__isset.temporary != rhs.__isset.temporary)
+       return false;
+     else if (__isset.temporary && !(temporary == rhs.temporary))
+       return false;
+     if (__isset.rewriteEnabled != rhs.__isset.rewriteEnabled)
+       return false;
+     else if (__isset.rewriteEnabled && !(rewriteEnabled == rhs.rewriteEnabled))
+       return false;
+     return true;
+   }
+   bool operator != (const Table &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const Table & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(Table &a, Table &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const Table& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _Partition__isset {
+   _Partition__isset() : values(false), dbName(false), tableName(false), createTime(false), lastAccessTime(false), sd(false), parameters(false), privileges(false) {}
+   bool values :1;
+   bool dbName :1;
+   bool tableName :1;
+   bool createTime :1;
+   bool lastAccessTime :1;
+   bool sd :1;
+   bool parameters :1;
+   bool privileges :1;
+ } _Partition__isset;
+ 
+ class Partition {
+  public:
+ 
+   Partition(const Partition&);
+   Partition& operator=(const Partition&);
+   Partition() : dbName(), tableName(), createTime(0), lastAccessTime(0) {
+   }
+ 
+   virtual ~Partition() throw();
+   std::vector<std::string>  values;
+   std::string dbName;
+   std::string tableName;
+   int32_t createTime;
+   int32_t lastAccessTime;
+   StorageDescriptor sd;
+   std::map<std::string, std::string>  parameters;
+   PrincipalPrivilegeSet privileges;
+ 
+   _Partition__isset __isset;
+ 
+   void __set_values(const std::vector<std::string> & val);
+ 
+   void __set_dbName(const std::string& val);
+ 
+   void __set_tableName(const std::string& val);
+ 
+   void __set_createTime(const int32_t val);
+ 
+   void __set_lastAccessTime(const int32_t val);
+ 
+   void __set_sd(const StorageDescriptor& val);
+ 
+   void __set_parameters(const std::map<std::string, std::string> & val);
+ 
+   void __set_privileges(const PrincipalPrivilegeSet& val);
+ 
+   bool operator == (const Partition & rhs) const
+   {
+     if (!(values == rhs.values))
+       return false;
+     if (!(dbName == rhs.dbName))
+       return false;
+     if (!(tableName == rhs.tableName))
+       return false;
+     if (!(createTime == rhs.createTime))
+       return false;
+     if (!(lastAccessTime == rhs.lastAccessTime))
+       return false;
+     if (!(sd == rhs.sd))
+       return false;
+     if (!(parameters == rhs.parameters))
+       return false;
+     if (__isset.privileges != rhs.__isset.privileges)
+       return false;
+     else if (__isset.privileges && !(privileges == rhs.privileges))
+       return false;
+     return true;
+   }
+   bool operator != (const Partition &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const Partition & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(Partition &a, Partition &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const Partition& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _PartitionWithoutSD__isset {
+   _PartitionWithoutSD__isset() : values(false), createTime(false), lastAccessTime(false), relativePath(false), parameters(false), privileges(false) {}
+   bool values :1;
+   bool createTime :1;
+   bool lastAccessTime :1;
+   bool relativePath :1;
+   bool parameters :1;
+   bool privileges :1;
+ } _PartitionWithoutSD__isset;
+ 
+ class PartitionWithoutSD {
+  public:
+ 
+   PartitionWithoutSD(const PartitionWithoutSD&);
+   PartitionWithoutSD& operator=(const PartitionWithoutSD&);
+   PartitionWithoutSD() : createTime(0), lastAccessTime(0), relativePath() {
+   }
+ 
+   virtual ~PartitionWithoutSD() throw();
+   std::vector<std::string>  values;
+   int32_t createTime;
+   int32_t lastAccessTime;
+   std::string relativePath;
+   std::map<std::string, std::string>  parameters;
+   PrincipalPrivilegeSet privileges;
+ 
+   _PartitionWithoutSD__isset __isset;
+ 
+   void __set_values(const std::vector<std::string> & val);
+ 
+   void __set_createTime(const int32_t val);
+ 
+   void __set_lastAccessTime(const int32_t val);
+ 
+   void __set_relativePath(const std::string& val);
+ 
+   void __set_parameters(const std::map<std::string, std::string> & val);
+ 
+   void __set_privileges(const PrincipalPrivilegeSet& val);
+ 
+   bool operator == (const PartitionWithoutSD & rhs) const
+   {
+     if (!(values == rhs.values))
+       return false;
+     if (!(createTime == rhs.createTime))
+       return false;
+     if (!(lastAccessTime == rhs.lastAccessTime))
+       return false;
+     if (!(relativePath == rhs.relativePath))
+       return false;
+     if (!(parameters == rhs.parameters))
+       return false;
+     if (__isset.privileges != rhs.__isset.privileges)
+       return false;
+     else if (__isset.privileges && !(privileges == rhs.privileges))
+       return false;
+     return true;
+   }
+   bool operator != (const PartitionWithoutSD &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const PartitionWithoutSD & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(PartitionWithoutSD &a, PartitionWithoutSD &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const PartitionWithoutSD& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _PartitionSpecWithSharedSD__isset {
+   _PartitionSpecWithSharedSD__isset() : partitions(false), sd(false) {}
+   bool partitions :1;
+   bool sd :1;
+ } _PartitionSpecWithSharedSD__isset;
+ 
+ class PartitionSpecWithSharedSD {
+  public:
+ 
+   PartitionSpecWithSharedSD(const PartitionSpecWithSharedSD&);
+   PartitionSpecWithSharedSD& operator=(const PartitionSpecWithSharedSD&);
+   PartitionSpecWithSharedSD() {
+   }
+ 
+   virtual ~PartitionSpecWithSharedSD() throw();
+   std::vector<PartitionWithoutSD>  partitions;
+   StorageDescriptor sd;
+ 
+   _PartitionSpecWithSharedSD__isset __isset;
+ 
+   void __set_partitions(const std::vector<PartitionWithoutSD> & val);
+ 
+   void __set_sd(const StorageDescriptor& val);
+ 
+   bool operator == (const PartitionSpecWithSharedSD & rhs) const
+   {
+     if (!(partitions == rhs.partitions))
+       return false;
+     if (!(sd == rhs.sd))
+       return false;
+     return true;
+   }
+   bool operator != (const PartitionSpecWithSharedSD &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const PartitionSpecWithSharedSD & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(PartitionSpecWithSharedSD &a, PartitionSpecWithSharedSD &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const PartitionSpecWithSharedSD& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _PartitionListComposingSpec__isset {
+   _PartitionListComposingSpec__isset() : partitions(false) {}
+   bool partitions :1;
+ } _PartitionListComposingSpec__isset;
+ 
+ class PartitionListComposingSpec {
+  public:
+ 
+   PartitionListComposingSpec(const PartitionListComposingSpec&);
+   PartitionListComposingSpec& operator=(const PartitionListComposingSpec&);
+   PartitionListComposingSpec() {
+   }
+ 
+   virtual ~PartitionListComposingSpec() throw();
+   std::vector<Partition>  partitions;
+ 
+   _PartitionListComposingSpec__isset __isset;
+ 
+   void __set_partitions(const std::vector<Partition> & val);
+ 
+   bool operator == (const PartitionListComposingSpec & rhs) const
+   {
+     if (!(partitions == rhs.partitions))
+       return false;
+     return true;
+   }
+   bool operator != (const PartitionListComposingSpec &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const PartitionListComposingSpec & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(PartitionListComposingSpec &a, PartitionListComposingSpec &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const PartitionListComposingSpec& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _PartitionSpec__isset {
+   _PartitionSpec__isset() : dbName(false), tableName(false), rootPath(false), sharedSDPartitionSpec(false), partitionList(false) {}
+   bool dbName :1;
+   bool tableName :1;
+   bool rootPath :1;
+   bool sharedSDPartitionSpec :1;
+   bool partitionList :1;
+ } _PartitionSpec__isset;
+ 
+ class PartitionSpec {
+  public:
+ 
+   PartitionSpec(const PartitionSpec&);
+   PartitionSpec& operator=(const PartitionSpec&);
+   PartitionSpec() : dbName(), tableName(), rootPath() {
+   }
+ 
+   virtual ~PartitionSpec() throw();
+   std::string dbName;
+   std::string tableName;
+   std::string rootPath;
+   PartitionSpecWithSharedSD sharedSDPartitionSpec;
+   PartitionListComposingSpec partitionList;
+ 
+   _PartitionSpec__isset __isset;
+ 
+   void __set_dbName(const std::string& val);
+ 
+   void __set_tableName(const std::string& val);
+ 
+   void __set_rootPath(const std::string& val);
+ 
+   void __set_sharedSDPartitionSpec(const PartitionSpecWithSharedSD& val);
+ 
+   void __set_partitionList(const PartitionListComposingSpec& val);
+ 
+   bool operator == (const PartitionSpec & rhs) const
+   {
+     if (!(dbName == rhs.dbName))
+       return false;
+     if (!(tableName == rhs.tableName))
+       return false;
+     if (!(rootPath == rhs.rootPath))
+       return false;
+     if (__isset.sharedSDPartitionSpec != rhs.__isset.sharedSDPartitionSpec)
+       return false;
+     else if (__isset.sharedSDPartitionSpec && !(sharedSDPartitionSpec == rhs.sharedSDPartitionSpec))
+       return false;
+     if (__isset.partitionList != rhs.__isset.partitionList)
+       return false;
+     else if (__isset.partitionList && !(partitionList == rhs.partitionList))
+       return false;
+     return true;
+   }
+   bool operator != (const PartitionSpec &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const PartitionSpec & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(PartitionSpec &a, PartitionSpec &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const PartitionSpec& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _Index__isset {
+   _Index__isset() : indexName(false), indexHandlerClass(false), dbName(false), origTableName(false), createTime(false), lastAccessTime(false), indexTableName(false), sd(false), parameters(false), deferredRebuild(false) {}
+   bool indexName :1;
+   bool indexHandlerClass :1;
+   bool dbName :1;
+   bool origTableName :1;
+   bool createTime :1;
+   bool lastAccessTime :1;
+   bool indexTableName :1;
+   bool sd :1;
+   bool parameters :1;
+   bool deferredRebuild :1;
+ } _Index__isset;
+ 
+ class Index {
+  public:
+ 
+   Index(const Index&);
+   Index& operator=(const Index&);
+   Index() : indexName(), indexHandlerClass(), dbName(), origTableName(), createTime(0), lastAccessTime(0), indexTableName(), deferredRebuild(0) {
+   }
+ 
+   virtual ~Index() throw();
+   std::string indexName;
+   std::string indexHandlerClass;
+   std::string dbName;
+   std::string origTableName;
+   int32_t createTime;
+   int32_t lastAccessTime;
+   std::string indexTableName;
+   StorageDescriptor sd;
+   std::map<std::string, std::string>  parameters;
+   bool deferredRebuild;
+ 
+   _Index__isset __isset;
+ 
+   void __set_indexName(const std::string& val);
+ 
+   void __set_indexHandlerClass(const std::string& val);
+ 
+   void __set_dbName(const std::string& val);
+ 
+   void __set_origTableName(const std::string& val);
+ 
+   void __set_createTime(const int32_t val);
+ 
+   void __set_lastAccessTime(const int32_t val);
+ 
+   void __set_indexTableName(const std::string& val);
+ 
+   void __set_sd(const StorageDescriptor& val);
+ 
+   void __set_parameters(const std::map<std::string, std::string> & val);
+ 
+   void __set_deferredRebuild(const bool val);
+ 
+   bool operator == (const Index & rhs) const
+   {
+     if (!(indexName == rhs.indexName))
+       return false;
+     if (!(indexHandlerClass == rhs.indexHandlerClass))
+       return false;
+     if (!(dbName == rhs.dbName))
+       return false;
+     if (!(origTableName == rhs.origTableName))
+       return false;
+     if (!(createTime == rhs.createTime))
+       return false;
+     if (!(lastAccessTime == rhs.lastAccessTime))
+       return false;
+     if (!(indexTableName == rhs.indexTableName))
+       return false;
+     if (!(sd == rhs.sd))
+       return false;
+     if (!(parameters == rhs.parameters))
+       return false;
+     if (!(deferredRebuild == rhs.deferredRebuild))
+       return false;
+     return true;
+   }
+   bool operator != (const Index &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const Index & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(Index &a, Index &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const Index& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _BooleanColumnStatsData__isset {
+   _BooleanColumnStatsData__isset() : bitVectors(false) {}
+   bool bitVectors :1;
+ } _BooleanColumnStatsData__isset;
+ 
+ class BooleanColumnStatsData {
+  public:
+ 
+   BooleanColumnStatsData(const BooleanColumnStatsData&);
+   BooleanColumnStatsData& operator=(const BooleanColumnStatsData&);
+   BooleanColumnStatsData() : numTrues(0), numFalses(0), numNulls(0), bitVectors() {
+   }
+ 
+   virtual ~BooleanColumnStatsData() throw();
+   int64_t numTrues;
+   int64_t numFalses;
+   int64_t numNulls;
+   std::string bitVectors;
+ 
+   _BooleanColumnStatsData__isset __isset;
+ 
+   void __set_numTrues(const int64_t val);
+ 
+   void __set_numFalses(const int64_t val);
+ 
+   void __set_numNulls(const int64_t val);
+ 
+   void __set_bitVectors(const std::string& val);
+ 
+   bool operator == (const BooleanColumnStatsData & rhs) const
+   {
+     if (!(numTrues == rhs.numTrues))
+       return false;
+     if (!(numFalses == rhs.numFalses))
+       return false;
+     if (!(numNulls == rhs.numNulls))
+       return false;
+     if (__isset.bitVectors != rhs.__isset.bitVectors)
+       return false;
+     else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
+       return false;
+     return true;
+   }
+   bool operator != (const BooleanColumnStatsData &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const BooleanColumnStatsData & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(BooleanColumnStatsData &a, BooleanColumnStatsData &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const BooleanColumnStatsData& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _DoubleColumnStatsData__isset {
+   _DoubleColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false) {}
+   bool lowValue :1;
+   bool highValue :1;
+   bool bitVectors :1;
+ } _DoubleColumnStatsData__isset;
+ 
+ class DoubleColumnStatsData {
+  public:
+ 
+   DoubleColumnStatsData(const DoubleColumnStatsData&);
+   DoubleColumnStatsData& operator=(const DoubleColumnStatsData&);
+   DoubleColumnStatsData() : lowValue(0), highValue(0), numNulls(0), numDVs(0), bitVectors() {
+   }
+ 
+   virtual ~DoubleColumnStatsData() throw();
+   double lowValue;
+   double highValue;
+   int64_t numNulls;
+   int64_t numDVs;
+   std::string bitVectors;
+ 
+   _DoubleColumnStatsData__isset __isset;
+ 
+   void __set_lowValue(const double val);
+ 
+   void __set_highValue(const double val);
+ 
+   void __set_numNulls(const int64_t val);
+ 
+   void __set_numDVs(const int64_t val);
+ 
+   void __set_bitVectors(const std::string& val);
+ 
+   bool operator == (const DoubleColumnStatsData & rhs) const
+   {
+     if (__isset.lowValue != rhs.__isset.lowValue)
+       return false;
+     else if (__isset.lowValue && !(lowValue == rhs.lowValue))
+       return false;
+     if (__isset.highValue != rhs.__isset.highValue)
+       return false;
+     else if (__isset.highValue && !(highValue == rhs.highValue))
+       return false;
+     if (!(numNulls == rhs.numNulls))
+       return false;
+     if (!(numDVs == rhs.numDVs))
+       return false;
+     if (__isset.bitVectors != rhs.__isset.bitVectors)
+       return false;
+     else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
+       return false;
+     return true;
+   }
+   bool operator != (const DoubleColumnStatsData &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const DoubleColumnStatsData & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(DoubleColumnStatsData &a, DoubleColumnStatsData &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const DoubleColumnStatsData& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _LongColumnStatsData__isset {
+   _LongColumnStatsData__isset() : lowValue(false), highValue(false), bitVectors(false) {}
+   bool lowValue :1;
+   bool highValue :1;
+   bool bitVectors :1;
+ } _LongColumnStatsData__isset;
+ 
+ class LongColumnStatsData {
+  public:
+ 
+   LongColumnStatsData(const LongColumnStatsData&);
+   LongColumnStatsData& operator=(const LongColumnStatsData&);
+   LongColumnStatsData() : lowValue(0), highValue(0), numNulls(0), numDVs(0), bitVectors() {
+   }
+ 
+   virtual ~LongColumnStatsData() throw();
+   int64_t lowValue;
+   int64_t highValue;
+   int64_t numNulls;
+   int64_t numDVs;
+   std::string bitVectors;
+ 
+   _LongColumnStatsData__isset __isset;
+ 
+   void __set_lowValue(const int64_t val);
+ 
+   void __set_highValue(const int64_t val);
+ 
+   void __set_numNulls(const int64_t val);
+ 
+   void __set_numDVs(const int64_t val);
+ 
+   void __set_bitVectors(const std::string& val);
+ 
+   bool operator == (const LongColumnStatsData & rhs) const
+   {
+     if (__isset.lowValue != rhs.__isset.lowValue)
+       return false;
+     else if (__isset.lowValue && !(lowValue == rhs.lowValue))
+       return false;
+     if (__isset.highValue != rhs.__isset.highValue)
+       return false;
+     else if (__isset.highValue && !(highValue == rhs.highValue))
+       return false;
+     if (!(numNulls == rhs.numNulls))
+       return false;
+     if (!(numDVs == rhs.numDVs))
+       return false;
+     if (__isset.bitVectors != rhs.__isset.bitVectors)
+       return false;
+     else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
+       return false;
+     return true;
+   }
+   bool operator != (const LongColumnStatsData &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const LongColumnStatsData & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(LongColumnStatsData &a, LongColumnStatsData &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const LongColumnStatsData& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _StringColumnStatsData__isset {
+   _StringColumnStatsData__isset() : bitVectors(false) {}
+   bool bitVectors :1;
+ } _StringColumnStatsData__isset;
+ 
+ class StringColumnStatsData {
+  public:
+ 
+   StringColumnStatsData(const StringColumnStatsData&);
+   StringColumnStatsData& operator=(const StringColumnStatsData&);
+   StringColumnStatsData() : maxColLen(0), avgColLen(0), numNulls(0), numDVs(0), bitVectors() {
+   }
+ 
+   virtual ~StringColumnStatsData() throw();
+   int64_t maxColLen;
+   double avgColLen;
+   int64_t numNulls;
+   int64_t numDVs;
+   std::string bitVectors;
+ 
+   _StringColumnStatsData__isset __isset;
+ 
+   void __set_maxColLen(const int64_t val);
+ 
+   void __set_avgColLen(const double val);
+ 
+   void __set_numNulls(const int64_t val);
+ 
+   void __set_numDVs(const int64_t val);
+ 
+   void __set_bitVectors(const std::string& val);
+ 
+   bool operator == (const StringColumnStatsData & rhs) const
+   {
+     if (!(maxColLen == rhs.maxColLen))
+       return false;
+     if (!(avgColLen == rhs.avgColLen))
+       return false;
+     if (!(numNulls == rhs.numNulls))
+       return false;
+     if (!(numDVs == rhs.numDVs))
+       return false;
+     if (__isset.bitVectors != rhs.__isset.bitVectors)
+       return false;
+     else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
+       return false;
+     return true;
+   }
+   bool operator != (const StringColumnStatsData &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const StringColumnStatsData & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(StringColumnStatsData &a, StringColumnStatsData &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const StringColumnStatsData& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ typedef struct _BinaryColumnStatsData__isset {
+   _BinaryColumnStatsData__isset() : bitVectors(false) {}
+   bool bitVectors :1;
+ } _BinaryColumnStatsData__isset;
+ 
+ class BinaryColumnStatsData {
+  public:
+ 
+   BinaryColumnStatsData(const BinaryColumnStatsData&);
+   BinaryColumnStatsData& operator=(const BinaryColumnStatsData&);
+   BinaryColumnStatsData() : maxColLen(0), avgColLen(0), numNulls(0), bitVectors() {
+   }
+ 
+   virtual ~BinaryColumnStatsData() throw();
+   int64_t maxColLen;
+   double avgColLen;
+   int64_t numNulls;
+   std::string bitVectors;
+ 
+   _BinaryColumnStatsData__isset __isset;
+ 
+   void __set_maxColLen(const int64_t val);
+ 
+   void __set_avgColLen(const double val);
+ 
+   void __set_numNulls(const int64_t val);
+ 
+   void __set_bitVectors(const std::string& val);
+ 
+   bool operator == (const BinaryColumnStatsData & rhs) const
+   {
+     if (!(maxColLen == rhs.maxColLen))
+       return false;
+     if (!(avgColLen == rhs.avgColLen))
+       return false;
+     if (!(numNulls == rhs.numNulls))
+       return false;
+     if (__isset.bitVectors != rhs.__isset.bitVectors)
+       return false;
+     else if (__isset.bitVectors && !(bitVectors == rhs.bitVectors))
+       return false;
+     return true;
+   }
+   bool operator != (const BinaryColumnStatsData &rhs) const {
+     return !(*this == rhs);
+   }
+ 
+   bool operator < (const BinaryColumnStatsData & ) const;
+ 
+   uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
+   uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ 
+   virtual void printTo(std::ostream& out) const;
+ };
+ 
+ void swap(BinaryColumnStatsData &a, BinaryColumnStatsData &b);
+ 
+ inline std::ostream& operator<<(std::ostream& out, const BinaryColumnStatsData& obj)
+ {
+   obj.printTo(out);
+   return out;
+ }
+ 
+ 
+ class Decimal {
+  public:
+ 
+   Decimal(const Decimal&);
+   Decimal& operator=(const Decimal&);
+   Decimal() : unscaled(), scale(0) {
+   }
+ 
+   virtual ~Decimal() throw();
+   std::string unscaled;
+   int16_t scale;
+ 
+   void __set_unscaled(const std::string& val);
+ 
+   void __set_scale(const int16_t val);
+ 
+   bool operator == (const Decimal & rhs) const
+   {
+     if (!(unscaled == rhs.unscaled))
+       return false;
+     if (!(scale =

<TRUNCATED>

[42/50] [abbrv] hive git commit: HIVE-17674 : grep TODO HIVE-15212.17.patch |wc - l = 49 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-17674 : grep TODO HIVE-15212.17.patch |wc - l = 49 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/64015972
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/64015972
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/64015972

Branch: refs/heads/master
Commit: 640159726f4406e752e9555b0b9a54452d8e9157
Parents: 77f3f28
Author: sergey <se...@apache.org>
Authored: Tue Oct 3 18:59:41 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Tue Oct 3 18:59:41 2017 -0700

----------------------------------------------------------------------
 .../hcatalog/mapreduce/HCatOutputFormat.java    |  3 +-
 .../hadoop/hive/metastore/HiveMetaStore.java    |  4 +-
 .../hadoop/hive/metastore/MetaStoreUtils.java   | 58 ++-----------------
 .../TransactionalValidationListener.java        |  4 +-
 .../apache/hadoop/hive/ql/exec/CopyTask.java    |  2 +-
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 24 ++++----
 .../hadoop/hive/ql/exec/FetchOperator.java      |  4 +-
 .../hadoop/hive/ql/exec/FileSinkOperator.java   |  2 +-
 .../hive/ql/exec/HashTableSinkOperator.java     |  3 +-
 .../apache/hadoop/hive/ql/exec/MoveTask.java    |  3 +-
 .../apache/hadoop/hive/ql/exec/Utilities.java   |  3 -
 .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 59 ++++++++++++++++++--
 .../hive/ql/io/CombineHiveInputFormat.java      |  2 +-
 .../hadoop/hive/ql/io/HiveInputFormat.java      | 11 ++--
 .../apache/hadoop/hive/ql/metadata/Hive.java    | 20 +++----
 .../formatting/JsonMetaDataFormatter.java       |  2 +-
 .../formatting/TextMetaDataFormatter.java       |  3 +-
 .../ql/optimizer/AbstractBucketJoinProc.java    |  3 +-
 .../BucketingSortingReduceSinkOptimizer.java    |  4 +-
 .../hive/ql/optimizer/GenMapRedUtils.java       | 11 +---
 .../hadoop/hive/ql/optimizer/SamplePruner.java  |  5 +-
 .../optimizer/physical/SamplingOptimizer.java   |  3 +-
 .../hive/ql/optimizer/physical/Vectorizer.java  |  2 +-
 .../hive/ql/parse/DDLSemanticAnalyzer.java      |  4 +-
 .../hive/ql/parse/ImportSemanticAnalyzer.java   | 25 +++++----
 .../hive/ql/parse/LoadSemanticAnalyzer.java     |  4 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  | 18 +++---
 .../hadoop/hive/ql/plan/FileSinkDesc.java       |  4 +-
 .../hadoop/hive/ql/plan/LoadTableDesc.java      |  4 +-
 .../apache/hadoop/hive/ql/plan/MoveWork.java    |  8 ---
 .../hive/ql/txn/compactor/CompactorMR.java      |  2 +-
 .../hadoop/hive/ql/txn/compactor/Initiator.java |  2 +-
 32 files changed, 147 insertions(+), 159 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java
index bbd594a..2654212 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/HCatOutputFormat.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.common.classification.InterfaceStability;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Index;
@@ -114,7 +115,7 @@ public class HCatOutputFormat extends HCatBaseOutputFormat {
         throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a partition with sorted column definition from Pig/Mapreduce is not supported");
       }
 
-      if (MetaStoreUtils.isInsertOnlyTable(table.getParameters())) {
+      if (AcidUtils.isInsertOnlyTable(table.getParameters())) {
         throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into an insert-only ACID table from Pig/Mapreduce is not supported");
       }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 375b34a..497b5af 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -2378,7 +2378,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
       Exception ex = null;
       try {
         t = get_table_core(dbname, name);
-        if (MetaStoreUtils.isInsertOnlyTable(t.getParameters())) {
+        if (MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) {
           assertClientHasCapability(capabilities, ClientCapability.INSERT_ONLY_TABLES,
               "insert-only tables", "get_table_req");
         }
@@ -2510,7 +2510,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
           startIndex = endIndex;
         }
         for (Table t : tables) {
-          if (MetaStoreUtils.isInsertOnlyTable(t.getParameters())) {
+          if (MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) {
             assertClientHasCapability(capabilities, ClientCapability.INSERT_ONLY_TABLES,
                 "insert-only tables", "get_table_req");
           }

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 235d042..96c8871 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -1821,58 +1821,6 @@ public class MetaStoreUtils {
     return cols;
   }
 
-  // TODO The following two utility methods can be moved to AcidUtils once no class in metastore is relying on them,
-  // right now ObjectStore.getAllMmTablesForCleanup is calling these method
-  /**
-   * Checks if a table is an ACID table that only supports INSERT, but not UPDATE/DELETE
-   * @param params table properties
-   * @return true if table is an INSERT_ONLY table, false otherwise
-   */
-  public static boolean isInsertOnlyTable(Map<String, String> params) {
-    return isInsertOnlyTable(params, false);
-  }
-
-  // TODO: CTAS for MM may currently be broken. It used to work. See the old code and why isCtas isn't used?
-  public static boolean isInsertOnlyTable(Map<String, String> params, boolean isCtas) {
-    String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
-    return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
-  }
-
-   public static boolean isInsertOnlyTable(Properties params) {
-     String transactionalProp = params.getProperty(
-         hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
-     return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
-  }
-
-   /** The method for altering table props; may set the table to MM, non-MM, or not affect MM. */
-  public static Boolean isToInsertOnlyTable(Map<String, String> props) {
-    // TODO: Setting these separately is a very hairy issue in certain combinations, since we
-    //       cannot decide what type of table this becomes without taking both into account, and
-    //       in many cases the conversion might be illegal.
-    //       The only thing we allow is tx = true w/o tx-props, for backward compat.
-    String transactional = props.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
-    String transactionalProp = props.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
-    if (transactional == null && transactionalProp == null) return null; // Not affected.
-    boolean isSetToTxn = "true".equalsIgnoreCase(transactional);
-    if (transactionalProp == null) {
-      if (isSetToTxn) return false; // Assume the full ACID table.
-      throw new RuntimeException("Cannot change '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL
-          + "' without '" + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "'");
-    }
-    if (!"insert_only".equalsIgnoreCase(transactionalProp)) return false; // Not MM.
-    if (!isSetToTxn) {
-      throw new RuntimeException("Cannot set '"
-          + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "' to 'insert_only' without "
-          + "setting '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL + "' to 'true'");
-    }
-    return true;
-  }
-
-  public static boolean isRemovedInsertOnlyTable(Set<String> removedSet) {
-    boolean hasTxn = removedSet.contains(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL),
-        hasProps = removedSet.contains(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
-    return hasTxn || hasProps;
-  }
 
   // given a list of partStats, this function will give you an aggr stats
   public static List<ColumnStatisticsObj> aggrPartitionStats(List<ColumnStatistics> partStats,
@@ -2056,4 +2004,10 @@ public class MetaStoreUtils {
     ipAddress = (ipAddress == null) ? StringUtils.EMPTY : ipAddress;
     return machineList.includes(ipAddress);
   }
+
+  /** Duplicates AcidUtils; used in a couple places in metastore. */
+  public static boolean isInsertOnlyTableParam(Map<String, String> params) {
+    String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+    return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index dabede4..29d8da8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -139,7 +139,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
       hasValidTransactionalValue = true;
     }
 
-    if (!hasValidTransactionalValue && !MetaStoreUtils.isInsertOnlyTable(oldTable.getParameters())) {
+    if (!hasValidTransactionalValue && !MetaStoreUtils.isInsertOnlyTableParam(oldTable.getParameters())) {
       // if here, there is attempt to set transactional to something other than 'true'
       // and NOT the same value it was before
       throw new MetaException("TBLPROPERTIES with 'transactional'='true' cannot be unset");
@@ -158,7 +158,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
         // null and an attempt is made to set it. This behaviour can be changed in the future.
         if ((oldTransactionalPropertiesValue == null
             || !oldTransactionalPropertiesValue.equalsIgnoreCase(transactionalPropertiesValue))
-            && !MetaStoreUtils.isInsertOnlyTable(oldTable.getParameters())) {
+            && !MetaStoreUtils.isInsertOnlyTableParam(oldTable.getParameters())) {
           throw new MetaException("TBLPROPERTIES with 'transactional_properties' cannot be "
               + "altered after the table is created");
         }

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
index 7299ed1..664a11d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
@@ -108,7 +108,7 @@ public class CopyTask extends Task<CopyWork> implements Serializable {
       FileSystem fs, Path path, boolean isSourceMm) throws IOException {
     if (!fs.exists(path)) return null;
     if (!isSourceMm) return matchFilesOneDir(fs, path, null);
-    // TODO: this doesn't handle list bucketing properly. Does the original exim do that?
+    // Note: this doesn't handle list bucketing properly; neither does the original code.
     FileStatus[] mmDirs = fs.listStatus(path, new JavaUtils.AnyIdDirFilter());
     if (mmDirs == null || mmDirs.length == 0) return null;
     List<FileStatus> allFiles = new ArrayList<FileStatus>();

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 4cf0f89..205ca77 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -1911,7 +1911,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
   private int compact(Hive db, AlterTableSimpleDesc desc) throws HiveException {
 
     Table tbl = db.getTable(desc.getTableName());
-    if (!AcidUtils.isFullAcidTable(tbl) && !MetaStoreUtils.isInsertOnlyTable(tbl.getParameters())) {
+    if (!AcidUtils.isFullAcidTable(tbl) && !AcidUtils.isInsertOnlyTable(tbl.getParameters())) {
       throw new HiveException(ErrorMsg.NONACID_COMPACTION_NOT_SUPPORTED, tbl.getDbName(),
           tbl.getTableName());
     }
@@ -4098,8 +4098,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
     List<Task<?>> result = null;
     if (part == null) {
       Set<String> removedSet = alterTbl.getProps().keySet();
-      boolean isFromMmTable = MetaStoreUtils.isInsertOnlyTable(tbl.getParameters()),
-          isRemoved = MetaStoreUtils.isRemovedInsertOnlyTable(removedSet);
+      boolean isFromMmTable = AcidUtils.isInsertOnlyTable(tbl.getParameters()),
+          isRemoved = AcidUtils.isRemovedInsertOnlyTable(removedSet);
       if (isFromMmTable && isRemoved) {
         result = generateRemoveMmTasks(tbl);
       }
@@ -4124,11 +4124,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
     // operation commits. Deleting uncommitted things is safe, but moving stuff before we convert
     // could cause data loss.
     List<Path> allMmDirs = new ArrayList<>();
-    if (tbl.isStoredAsSubDirectories()) {
-      // TODO: support this? we only bail because it's a PITA and hardly anyone seems to care.
-      throw new HiveException("Converting list bucketed tables stored as subdirectories "
-          + " to and from MM is not supported");
-    }
+    checkMmLb(tbl);
     List<String> bucketCols = tbl.getBucketCols();
     if (bucketCols != null && !bucketCols.isEmpty()
         && HiveConf.getBoolVar(conf, ConfVars.HIVE_STRICT_CHECKS_BUCKETING)) {
@@ -4175,16 +4171,16 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
 
   private void checkMmLb(Table tbl) throws HiveException {
     if (!tbl.isStoredAsSubDirectories()) return;
-    // TODO: support this?
+    // TODO [MM gap?]: by design; no-one seems to use LB tables. They will work, but not convert.
+    //                 It's possible to work around this by re-creating and re-inserting the table.
     throw new HiveException("Converting list bucketed tables stored as subdirectories "
-        + " to and from MM is not supported");
+        + " to and from MM is not supported. Please re-create a table in the desired format.");
   }
 
   private void checkMmLb(Partition part) throws HiveException {
     if (!part.isStoredAsSubDirectories()) return;
-    // TODO: support this?
     throw new HiveException("Converting list bucketed tables stored as subdirectories "
-        + " to and from MM is not supported. Please create a table in the desired format.");
+        + " to and from MM is not supported. Please re-create a table in the desired format.");
   }
 
   private void handleRemoveMm(
@@ -4289,8 +4285,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
     if (part != null) {
       part.getTPartition().getParameters().putAll(alterTbl.getProps());
     } else {
-      boolean isFromMmTable = MetaStoreUtils.isInsertOnlyTable(tbl.getParameters());
-      Boolean isToMmTable = MetaStoreUtils.isToInsertOnlyTable(alterTbl.getProps());
+      boolean isFromMmTable = AcidUtils.isInsertOnlyTable(tbl.getParameters());
+      Boolean isToMmTable = AcidUtils.isToInsertOnlyTable(alterTbl.getProps());
       if (isToMmTable != null) {
         if (!isFromMmTable && isToMmTable) {
           result = generateAddMmTasks(tbl);

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
index d09b12b..b85a243 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
@@ -275,7 +275,7 @@ public class FetchOperator implements Serializable {
       FileSystem fs = currPath.getFileSystem(job);
       if (fs.exists(currPath)) {
         if (extractValidTxnList() != null &&
-            MetaStoreUtils.isInsertOnlyTable(currDesc.getTableDesc().getProperties())) {
+            AcidUtils.isInsertOnlyTable(currDesc.getTableDesc().getProperties())) {
           return true;
         }
         for (FileStatus fStat : listStatusUnderPath(fs, currPath)) {
@@ -408,7 +408,7 @@ public class FetchOperator implements Serializable {
       return StringUtils.escapeString(currPath.toString()); // No need to process here.
     }
     ValidTxnList validTxnList;
-    if (MetaStoreUtils.isInsertOnlyTable(currDesc.getTableDesc().getProperties())) {
+    if (AcidUtils.isInsertOnlyTable(currDesc.getTableDesc().getProperties())) {
       validTxnList = extractValidTxnList();
     } else {
       validTxnList = null;  // non-MM case

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 280d3cf..93b967f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -1303,7 +1303,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
         } else {
           int dpLevels = dpCtx == null ? 0 : dpCtx.getNumDPCols(),
               lbLevels = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel();
-          // TODO: why is it stored in both?
+          // TODO: why is it stored in both table and dpCtx?
           int numBuckets = (conf.getTable() != null) ? conf.getTable().getNumBuckets()
               : (dpCtx != null ? dpCtx.getNumBuckets() : 0);
           MissingBucketsContext mbc = new MissingBucketsContext(

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
index 0b8045e..9690acb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
@@ -301,7 +301,8 @@ public class HashTableSinkOperator extends TerminalOperator<HashTableSinkDesc> i
       String bigBucketFileName = getExecContext().getCurrentBigBucketFile();
       String fileName = getExecContext().getLocalWork().getBucketFileName(bigBucketFileName);
       // get the tmp URI path; it will be a hdfs path if not local mode
-      // TODO: this doesn't work... the path for writer and reader mismatch
+      // TODO [MM gap?]: this doesn't work, however this is MR only.
+      //      The path for writer and reader mismatch:
       //      Dump the side-table for tag ... -local-10004/HashTable-Stage-1/MapJoin-a-00-(ds%3D2008-04-08)mm_2.hashtable
       //      Load back 1 hashtable file      -local-10004/HashTable-Stage-1/MapJoin-a-00-srcsortbucket3outof4.txt.hashtable
       //      Hive3 probably won't support MR so do we really care? 

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index e644845..7cc5c30 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -266,7 +266,6 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
 
   @Override
   public int execute(DriverContext driverContext) {
-    if (work.isNoop()) return 0;
     if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
       Utilities.FILE_OP_LOGGER.trace("Executing MoveWork " + System.identityHashCode(work)
         + " with " + work.getLoadFileWork() + "; " + work.getLoadTableWork() + "; "
@@ -713,7 +712,7 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
 
     boolean updateBucketCols = false;
     if (bucketCols != null) {
-      // TODO: this particular bit will not work for MM tables, as there can be multiple
+      // Note: this particular bit will not work for MM tables, as there can be multiple
       //       directories for different MM IDs. We could put the path here that would account
       //       for the current MM ID being written, but it will not guarantee that other MM IDs
       //       have the correct buckets. The existing code discards the inferred data when the

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 99adf7f..c702a24 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -1642,7 +1642,6 @@ public final class Utilities {
           addFilesToPathSet(taskIDToFile.values(), filesKept);
         }
 
-        // TODO: not clear why two if conditions are different. Preserve the existing logic for now.
         addBucketFileToResults(taskIDToFile, numBuckets, hconf, result);
       }
     } else {
@@ -1672,7 +1671,6 @@ public final class Utilities {
           addFilesToPathSet(taskIDToFile.values(), filesKept);
         }
       }
-      // TODO: not clear why two if conditions are different. Preserve the existing logic for now.
       addBucketFileToResults2(taskIDToFile, numBuckets, hconf, result);
     }
 
@@ -4179,7 +4177,6 @@ public final class Utilities {
     Utilities.FILE_OP_LOGGER.debug("Looking for files in: " + specPath);
     JavaUtils.IdPathFilter filter = new JavaUtils.IdPathFilter(txnId, stmtId, true);
     if (isMmCtas && !fs.exists(specPath)) {
-      // TODO: do we also need to do this when creating an empty partition from select?
       Utilities.FILE_OP_LOGGER.info("Creating table directory for CTAS with no output at " + specPath);
       FileUtils.mkdir(fs, specPath, hconf);
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index a461bb9..4c0b71f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -26,6 +26,7 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 import org.apache.hadoop.conf.Configuration;
@@ -447,8 +448,6 @@ public class AcidUtils {
           case HASH_BASED_MERGE_STRING:
             obj.setHashBasedMerge(true);
             break;
-          case INSERT_ONLY_STRING:
-            obj.setInsertOnly(true);
           default:
             throw new IllegalArgumentException(
                 "Unexpected value " + option + " for ACID operational properties!");
@@ -1052,7 +1051,7 @@ public class AcidUtils {
       String deltaPrefix =
               (fn.startsWith(DELTA_PREFIX)) ? DELTA_PREFIX : DELETE_DELTA_PREFIX;
       ParsedDelta delta = parseDelta(child, deltaPrefix);
-      if (tblproperties != null && MetaStoreUtils.isInsertOnlyTable(tblproperties) &&
+      if (tblproperties != null && AcidUtils.isInsertOnlyTable(tblproperties) &&
           ValidTxnList.RangeResponse.ALL == txnList.isTxnRangeAborted(delta.minTransaction, delta.maxTransaction)) {
         aborted.add(child);
       }
@@ -1210,7 +1209,7 @@ public class AcidUtils {
   }
 
   public static boolean isFullAcidTable(Table table) {
-    return isAcidTable(table) && !MetaStoreUtils.isInsertOnlyTable(table.getParameters());
+    return isAcidTable(table) && !AcidUtils.isInsertOnlyTable(table.getParameters());
   }
 
   /**
@@ -1327,4 +1326,56 @@ public class AcidUtils {
      */
     throw new IOException(lengths + " found but is not readable.  Consider waiting or orcfiledump --recover");
   }
+
+
+  /**
+   * Checks if a table is an ACID table that only supports INSERT, but not UPDATE/DELETE
+   * @param params table properties
+   * @return true if table is an INSERT_ONLY table, false otherwise
+   */
+  public static boolean isInsertOnlyTable(Map<String, String> params) {
+    return isInsertOnlyTable(params, false);
+  }
+
+  // TODO [MM gap]: CTAS may currently be broken. It used to work. See the old code, and why isCtas isn't used?
+  public static boolean isInsertOnlyTable(Map<String, String> params, boolean isCtas) {
+    String transactionalProp = params.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+    return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
+  }
+
+  public static boolean isInsertOnlyTable(Properties params) {
+    String transactionalProp = params.getProperty(
+        hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+    return (transactionalProp != null && "insert_only".equalsIgnoreCase(transactionalProp));
+  }
+
+   /** The method for altering table props; may set the table to MM, non-MM, or not affect MM. */
+  public static Boolean isToInsertOnlyTable(Map<String, String> props) {
+    // Note: Setting these separately is a very hairy issue in certain combinations, since we
+    //       cannot decide what type of table this becomes without taking both into account, and
+    //       in many cases the conversion might be illegal.
+    //       The only thing we allow is tx = true w/o tx-props, for backward compat.
+    String transactional = props.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+    String transactionalProp = props.get(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+    if (transactional == null && transactionalProp == null) return null; // Not affected.
+    boolean isSetToTxn = "true".equalsIgnoreCase(transactional);
+    if (transactionalProp == null) {
+      if (isSetToTxn) return false; // Assume the full ACID table.
+      throw new RuntimeException("Cannot change '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL
+          + "' without '" + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "'");
+    }
+    if (!"insert_only".equalsIgnoreCase(transactionalProp)) return false; // Not MM.
+    if (!isSetToTxn) {
+      throw new RuntimeException("Cannot set '"
+          + hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES + "' to 'insert_only' without "
+          + "setting '" + hive_metastoreConstants.TABLE_IS_TRANSACTIONAL + "' to 'true'");
+    }
+    return true;
+  }
+
+  public static boolean isRemovedInsertOnlyTable(Set<String> removedSet) {
+    boolean hasTxn = removedSet.contains(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL),
+        hasProps = removedSet.contains(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+    return hasTxn || hasProps;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
index 93de69f..6a188ac 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
@@ -118,7 +118,7 @@ public class CombineHiveInputFormat<K extends WritableComparable, V extends Writ
         TableDesc tbl = part.getTableDesc();
         boolean isMmNonMerge = false;
         if (tbl != null) {
-          isMmNonMerge = !isMerge && MetaStoreUtils.isInsertOnlyTable(tbl.getProperties());
+          isMmNonMerge = !isMerge && AcidUtils.isInsertOnlyTable(tbl.getProperties());
         } else {
           // This would be the case for obscure tasks like truncate column (unsupported for MM).
           Utilities.FILE_OP_LOGGER.warn("Assuming not insert-only; no table in partition spec " + part);

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index fba5b7e..6a1dc72 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -459,7 +459,7 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
       TableDesc table, List<InputSplit> result)
           throws IOException {
     ValidTxnList validTxnList;
-    if (MetaStoreUtils.isInsertOnlyTable(table.getProperties())) {
+    if (AcidUtils.isInsertOnlyTable(table.getProperties())) {
       String txnString = conf.get(ValidTxnList.VALID_TXNS_KEY);
       validTxnList = txnString == null ? new ValidReadTxnList() : new ValidReadTxnList(txnString);
     } else {
@@ -549,7 +549,7 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
         } else if (!hadAcidState) {
           AcidUtils.Directory dirInfo = AcidUtils.getAcidState(currDir, conf, validTxnList, Ref.from(false), true, null);
           hadAcidState = true;
-          // TODO: for IOW, we also need to count in base dir, if any
+          // TODO [MM gap]: for IOW, we also need to count in base dir, if any
           for (AcidUtils.ParsedDelta delta : dirInfo.getCurrentDirectories()) {
             Utilities.FILE_OP_LOGGER.debug("Adding input " + delta.getPath());
             finalPaths.add(delta.getPath());
@@ -706,9 +706,10 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
       throws IOException {
     PartitionDesc partDesc = pathToPartitionInfo.get(dir);
     if (partDesc == null) {
-      // TODO: HiveFileFormatUtils.getPartitionDescFromPathRecursively for MM tables?
-      //       So far, the only case when this is called for a MM directory was in error.
-      //       Keep it like this for now; may need replacement if we find a valid usage like this.
+      // Note: we could call HiveFileFormatUtils.getPartitionDescFromPathRecursively for MM tables.
+      //       The recursive call is usually needed for non-MM tables, because the path management
+      //       is not strict and the code does whatever. That should not happen for MM tables.
+      //       Keep it like this for now; may need replacement if we find a valid use case.
       partDesc = pathToPartitionInfo.get(Path.getPathWithoutSchemeAndAuthority(dir));
     }
     if (partDesc == null) {

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 5e52485..7464a29 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1676,7 +1676,7 @@ public class Hive {
       boolean isSrcLocal, boolean isAcid, boolean hasFollowingStatsTask, Long txnId, int stmtId)
           throws HiveException {
     Path tblDataLocationPath =  tbl.getDataLocation();
-    boolean isMmTableWrite = MetaStoreUtils.isInsertOnlyTable(tbl.getParameters());
+    boolean isMmTableWrite = AcidUtils.isInsertOnlyTable(tbl.getParameters());
     try {
       // Get the partition object if it already exists
       Partition oldPart = getPartition(tbl, partSpec, false);
@@ -1722,7 +1722,7 @@ public class Hive {
       if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && (null != oldPart)) {
         newFiles = Collections.synchronizedList(new ArrayList<Path>());
       }
-      // TODO: this assumes both paths are qualified; which they are, currently.
+      // Note: this assumes both paths are qualified; which they are, currently.
       if (isMmTableWrite && loadPath.equals(newPartPath)) {
         // MM insert query, move itself is a no-op.
         if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
@@ -1870,7 +1870,7 @@ public class Hive {
     }
     PathFilter subdirFilter = null;
 
-    // TODO: just like the move path, we only do one level of recursion.
+    // Note: just like the move path, we only do one level of recursion.
     for (FileStatus src : srcs) {
       if (src.isDirectory()) {
         if (subdirFilter == null) {
@@ -1920,8 +1920,6 @@ public class Hive {
 private void walkDirTree(FileStatus fSta, FileSystem fSys,
     Map<List<String>, String> skewedColValueLocationMaps, Path newPartPath, SkewedInfo skewedInfo)
     throws IOException {
-  // TODO: may be broken? no LB bugs for now but if any are found.
-
   /* Base Case. It's leaf. */
   if (!fSta.isDir()) {
     if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
@@ -1958,7 +1956,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
   List<String> skewedValue = new ArrayList<String>();
   String lbDirName = FileUtils.unescapePathName(lbdPath.toString());
   String partDirName = FileUtils.unescapePathName(newPartPath.toString());
-  String lbDirSuffix = lbDirName.replace(partDirName, ""); // TODO: wtf?
+  String lbDirSuffix = lbDirName.replace(partDirName, ""); // TODO: should it rather do a prefix?
   if (lbDirSuffix.startsWith(Path.SEPARATOR)) {
     lbDirSuffix = lbDirSuffix.substring(1);
   }
@@ -2105,7 +2103,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
     // Get all valid partition paths and existing partitions for them (if any)
     final Table tbl = getTable(tableName);
     final Set<Path> validPartitions = getValidPartitionsInPath(numDP, numLB, loadPath, txnId, stmtId,
-        MetaStoreUtils.isInsertOnlyTable(tbl.getParameters()));
+        AcidUtils.isInsertOnlyTable(tbl.getParameters()));
 
     final int partsToLoad = validPartitions.size();
     final AtomicInteger partitionsLoaded = new AtomicInteger(0);
@@ -2238,7 +2236,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
     if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) {
       newFiles = Collections.synchronizedList(new ArrayList<Path>());
     }
-    // TODO: this assumes both paths are qualified; which they are, currently.
+    // Note: this assumes both paths are qualified; which they are, currently.
     if (isMmTable && loadPath.equals(tbl.getPath())) {
       Utilities.FILE_OP_LOGGER.debug("not moving " + loadPath + " to " + tbl.getPath());
       if (replace) {
@@ -3797,7 +3795,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
       }
 
       if (oldPath != null) {
-        // TODO: we assume lbLevels is 0 here. Same as old code for non-MM.
+        // Note: we assume lbLevels is 0 here. Same as old code for non-MM.
         //       For MM tables, this can only be a LOAD command. Does LOAD even support LB?
         deleteOldPathForReplace(destf, oldPath, conf, purge, deletePathFilter, isMmTable, 0);
       }
@@ -3863,8 +3861,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
         } else {
           // We need to clean up different MM IDs from each LB directory separately.
           // Avoid temporary directories in the immediate table/part dir.
-          // TODO: we could just find directories with any MM directories inside?
-          //       the rest doesn't have to be cleaned up.
+          // Note: we could just find directories with any MM directories inside?
+          //       the rest doesn't have to be cleaned up. Play it safe.
           String mask = "[^._]*";
           for (int i = 0; i < lbLevels - 1; ++i) {
             mask += Path.SEPARATOR + "*";

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java
index ee8c249..d795a19 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java
@@ -248,7 +248,7 @@ public class JsonMetaDataFormatter implements MetaDataFormatter {
    * @param tblPath not NULL
    * @throws IOException
    */
-  // Duplicates logic in TextMetaDataFormatter TODO: wtf?!!
+  // Duplicates logic in TextMetaDataFormatter
   private void putFileSystemsStats(MapBuilder builder, List<Path> locations,
       HiveConf conf, Path tblPath)
           throws IOException {

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
index 3c25896..5a18e33 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
@@ -318,7 +318,8 @@ class TextMetaDataFormatter implements MetaDataFormatter {
     public int numOfFiles = 0;
   }
 
-  // TODO: why is this in text formatter? grrr
+  // TODO: why is this in text formatter?!!
+  //       This computes stats and should be in stats (de-duplicated too).
   private void writeFileSystemStats(DataOutputStream outStream,
       HiveConf conf,
       List<Path> locations,

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
index 66ea4e2..c62d98f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.lib.Node;
 import org.apache.hadoop.hive.ql.lib.NodeProcessor;
 import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
@@ -264,7 +265,7 @@ abstract public class AbstractBucketJoinProc implements NodeProcessor {
       }
 
       Table tbl = tso.getConf().getTableMetadata();
-      if (MetaStoreUtils.isInsertOnlyTable(tbl.getParameters())) {
+      if (AcidUtils.isInsertOnlyTable(tbl.getParameters())) {
         Utilities.FILE_OP_LOGGER.debug("No bucketed join on MM table " + tbl.getTableName());
         return false;
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
index 85dcdd7..fc1d4f9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
@@ -415,7 +415,7 @@ public class BucketingSortingReduceSinkOptimizer extends Transform {
           /*ACID tables have complex directory layout and require merging of delta files
           * on read thus we should not try to read bucket files directly*/
           return null;
-        } else if (MetaStoreUtils.isInsertOnlyTable(tab.getParameters())) {
+        } else if (AcidUtils.isInsertOnlyTable(tab.getParameters())) {
           // Do not support MM tables either at this point. We could do it with some extra logic.
           return null;
         }
@@ -510,7 +510,7 @@ public class BucketingSortingReduceSinkOptimizer extends Transform {
             TableScanOperator ts = (TableScanOperator) op;
             Table srcTable = ts.getConf().getTableMetadata();
             // Not supported for MM tables for now.
-            if (MetaStoreUtils.isInsertOnlyTable(destTable.getParameters())) {
+            if (AcidUtils.isInsertOnlyTable(destTable.getParameters())) {
               return null;
             }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
index 9ffda7e..0d13a51 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
@@ -1372,12 +1372,6 @@ public final class GenMapRedUtils {
       dummyMv = new MoveWork(null, null, null,
           new LoadFileDesc(inputDirName, finalName, true, null, null, false), false,
           SessionState.get().getLineageState());
-    } else {
-      // TODO# noop MoveWork to avoid q file changes in HIVE-14990. Remove (w/the flag) after merge.
-      dummyMv = new MoveWork(null, null, null,
-          new LoadFileDesc(inputDirName, finalName, true, null, null, false), false,
-          SessionState.get().getLineageState());
-      dummyMv.setNoop(true);
     }
     // Use the original fsOp path here in case of MM - while the new FSOP merges files inside the
     // MM directory, the original MoveTask still commits based on the parent. Note that this path
@@ -1756,7 +1750,7 @@ public final class GenMapRedUtils {
     // Create a dummy task if no move is needed.
     Serializable moveWork = mvWork != null ? mvWork : new DependencyCollectionWork();
 
-    // TODO: this should never happen for mm tables.
+    // Note: this should never happen for mm tables.
     boolean shouldMergeMovePaths = (moveTaskToLink != null && dependencyTask == null
         && shouldMergeMovePaths(conf, condInputPath, condOutputPath, moveTaskToLink.getWork()));
 
@@ -1840,7 +1834,6 @@ public final class GenMapRedUtils {
     // find the move task
     for (Task<MoveWork> mvTsk : mvTasks) {
       MoveWork mvWork = mvTsk.getWork();
-      if (mvWork.isNoop()) continue;
       Path srcDir = null;
       boolean isLfd = false;
       if (mvWork.getLoadFileWork() != null) {
@@ -1878,7 +1871,7 @@ public final class GenMapRedUtils {
     MoveTask mvTask = (MoveTask) GenMapRedUtils.findMoveTaskForFsopOutput(
         mvTasks, fsOp.getConf().getFinalDirName(), fsOp.getConf().isMmTable());
 
-    // TODO: wtf? wtf?!! why is this in this method?
+    // TODO: wtf?!! why is this in this method? This has nothing to do with anything.
     if (mvTask != null && isInsertTable && hconf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER)
         && !fsOp.getConf().isMaterialization()) {
       // mark the MapredWork and FileSinkOperator for gathering stats

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java
index 9d2e031..b5891ab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
 import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
 import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -191,8 +192,8 @@ public class SamplePruner extends Transform {
     String fullScanMsg = "";
 
     // check if input pruning is possible
-    // TODO: this relies a lot on having one file per bucket. No support for MM tables for now.
-    boolean isMmTable = MetaStoreUtils.isInsertOnlyTable(part.getTable().getParameters());
+    // TODO: this code is buggy - it relies on having one file per bucket; no MM support (by design).
+    boolean isMmTable = AcidUtils.isInsertOnlyTable(part.getTable().getParameters());
     if (sampleDescr.getInputPruning() && !isMmTable) {
       LOG.trace("numerator = " + num);
       LOG.trace("denominator = " + den);

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SamplingOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SamplingOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SamplingOptimizer.java
index 5b89059..b28315a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SamplingOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SamplingOptimizer.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.MapWork;
@@ -66,7 +67,7 @@ public class SamplingOptimizer implements PhysicalPlanResolver {
       if (tbl == null) {
         continue;
       }
-      if (MetaStoreUtils.isInsertOnlyTable(tbl.getParameters())) {
+      if (AcidUtils.isInsertOnlyTable(tbl.getParameters())) {
         // Not supported for MM tables - sampler breaks separate MM dirs into splits, resulting in
         // mismatch when the downstream task looks at them again assuming they are MM table roots.
         // We could somehow unset the MM flag for the main job when the sampler succeeds, since the

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 87e4ee8..68f7b47 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -1005,7 +1005,7 @@ public class Vectorizer implements PhysicalPlanResolver {
           setOperatorIssue("Alias " + alias + " not present in aliases " + aliases);
           return new ImmutablePair<Boolean,Boolean>(false, false);
         }
-        // TODO: should this use getPartitionDescFromPathRecursively?
+        // TODO: should this use getPartitionDescFromPathRecursively? That's what other code uses.
         PartitionDesc partDesc = pathToPartitionInfo.get(path);
         if (partDesc.getVectorPartitionDesc() != null) {
           // We've seen this already.

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 36bb89f..cbb422f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -994,7 +994,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
           throw new SemanticException(ErrorMsg.TRUNCATE_COLUMN_INDEXED_TABLE.getMsg());
         }
         // It would be possible to support this, but this is such a pointless command.
-        if (MetaStoreUtils.isInsertOnlyTable(table.getParameters())) {
+        if (AcidUtils.isInsertOnlyTable(table.getParameters())) {
           throw new SemanticException("Truncating MM table columns not presently supported");
         }
 
@@ -1639,7 +1639,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
     try {
       tblObj = getTable(tableName);
       // TODO: we should probably block all ACID tables here.
-      if (MetaStoreUtils.isInsertOnlyTable(tblObj.getParameters())) {
+      if (AcidUtils.isInsertOnlyTable(tblObj.getParameters())) {
         throw new SemanticException("Merge is not supported for MM tables");
       }
       mergeDesc.setTableDesc(Utilities.getTableDesc(tblObj));

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 41b5156..9afeadb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -134,7 +134,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
             parsedTableName = dbTablePair.getValue();
             // get partition metadata if partition specified
             if (child.getChildCount() == 2) {
-              @SuppressWarnings("unused") // TODO: wtf?
+              @SuppressWarnings("unused")
               ASTNode partspec = (ASTNode) child.getChild(1);
               isPartSpecSet = true;
               parsePartitionSpec(child, parsedPartSpec);
@@ -233,7 +233,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
     } catch (Exception e) {
       throw new HiveException(e);
     }
-    boolean isSourceMm = MetaStoreUtils.isInsertOnlyTable(tblDesc.getTblProps());
+    boolean isSourceMm = AcidUtils.isInsertOnlyTable(tblDesc.getTblProps());
 
     if ((replicationSpec != null) && replicationSpec.isInReplicationScope()){
       tblDesc.setReplicationSpec(replicationSpec);
@@ -313,7 +313,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
 
     Long txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
     int stmtId = 0;
-    // TODO: due to the master merge, tblDesc is no longer CreateTableDesc, but ImportTableDesc
+    // TODO [MM gap?]: bad merge; tblDesc is no longer CreateTableDesc, but ImportTableDesc.
+    //                 We need to verify the tests to see if this works correctly.
     /*
     if (isAcid(txnId)) {
       tblDesc.setInitialMmWriteId(txnId);
@@ -365,7 +366,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       ReplicationSpec replicationSpec, EximUtil.SemanticAnalyzerWrapperContext x,
       Long txnId, int stmtId, boolean isSourceMm) {
     Path dataPath = new Path(fromURI.toString(), EximUtil.DATA_PATH_NAME);
-    Path destPath = !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtPath)
+    Path destPath = !AcidUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtPath)
         : new Path(tgtPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
     if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
       Utilities.FILE_OP_LOGGER.trace("adding import work for table with source location: " +
@@ -376,7 +377,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
     Task<?> copyTask = null;
     if (replicationSpec.isInReplicationScope()) {
       if (isSourceMm || isAcid(txnId)) {
-        // TODO: ReplCopyTask is completely screwed. Need to support when it's not as screwed.
+        // Note: this is replication gap, not MM gap... Repl V2 is not ready yet.
         throw new RuntimeException("Replicating MM and ACID tables is not supported");
       }
       copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, destPath, x.getConf());
@@ -457,9 +458,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
           + partSpecToString(partSpec.getPartSpec())
           + " with source location: " + srcLocation);
       Path tgtLocation = new Path(partSpec.getLocation());
-      Path destPath = !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtLocation)
+      Path destPath = !AcidUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtLocation)
           : new Path(tgtLocation, AcidUtils.deltaSubdir(txnId, txnId, stmtId));
-      Path moveTaskSrc =  !MetaStoreUtils.isInsertOnlyTable(table.getParameters()) ? destPath : tgtLocation;
+      Path moveTaskSrc =  !AcidUtils.isInsertOnlyTable(table.getParameters()) ? destPath : tgtLocation;
       if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
         Utilities.FILE_OP_LOGGER.trace("adding import work for partition with source location: "
           + srcLocation + "; target: " + tgtLocation + "; copy dest " + destPath + "; mm "
@@ -470,7 +471,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       Task<?> copyTask = null;
       if (replicationSpec.isInReplicationScope()) {
         if (isSourceMm || isAcid(txnId)) {
-          // TODO: ReplCopyTask is completely screwed. Need to support when it's not as screwed.
+          // Note: this is replication gap, not MM gap... Repl V2 is not ready yet.
           throw new RuntimeException("Replicating MM and ACID tables is not supported");
         }
         copyTask = ReplCopyTask.getLoadCopyTask(
@@ -792,7 +793,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
         x.getLOG().debug("table partitioned");
         Task<?> ict = createImportCommitTask(
             table.getDbName(), table.getTableName(), txnId, stmtId, x.getConf(),
-            MetaStoreUtils.isInsertOnlyTable(table.getParameters()));
+            AcidUtils.isInsertOnlyTable(table.getParameters()));
 
         for (AddPartitionDesc addPartitionDesc : partitionDescs) {
           Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
@@ -829,7 +830,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       if (isPartitioned(tblDesc)) {
         Task<?> ict = createImportCommitTask(
             tblDesc.getDatabaseName(), tblDesc.getTableName(), txnId, stmtId, x.getConf(),
-            MetaStoreUtils.isInsertOnlyTable(tblDesc.getTblProps()));
+            AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
         for (AddPartitionDesc addPartitionDesc : partitionDescs) {
           t.addDependentTask(addSinglePartition(fromURI, fs, tblDesc, table, wh, addPartitionDesc,
             replicationSpec, x, txnId, stmtId, isSourceMm, ict));
@@ -956,7 +957,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
         if (isPartitioned(tblDesc)) {
           Task<?> ict = createImportCommitTask(
               tblDesc.getDatabaseName(), tblDesc.getTableName(), txnId, stmtId, x.getConf(),
-              MetaStoreUtils.isInsertOnlyTable(tblDesc.getTblProps()));
+              AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
           for (AddPartitionDesc addPartitionDesc : partitionDescs) {
             addPartitionDesc.setReplicationSpec(replicationSpec);
             t.addDependentTask(
@@ -982,7 +983,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
           org.apache.hadoop.hive.ql.metadata.Partition ptn = null;
           Task<?> ict = replicationSpec.isMetadataOnly() ? null : createImportCommitTask(
               tblDesc.getDatabaseName(), tblDesc.getTableName(), txnId, stmtId, x.getConf(),
-              MetaStoreUtils.isInsertOnlyTable(tblDesc.getTblProps()));
+              AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
           if ((ptn = x.getHive().getPartition(table, partSpec, false)) == null) {
             if (!replicationSpec.isMetadataOnly()){
               x.getTasks().add(addSinglePartition(

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index ab2b65f..51efde1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -228,7 +228,7 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
           + " and use 'insert... select' to allow Hive to enforce bucketing. " + error);
     }
 
-    if(AcidUtils.isAcidTable(ts.tableHandle) && !MetaStoreUtils.isInsertOnlyTable(ts.tableHandle.getParameters())) {
+    if(AcidUtils.isAcidTable(ts.tableHandle) && !AcidUtils.isInsertOnlyTable(ts.tableHandle.getParameters())) {
       throw new SemanticException(ErrorMsg.LOAD_DATA_ON_ACID_TABLE, ts.tableHandle.getCompleteName());
     }
     // make sure the arguments make sense
@@ -277,7 +277,7 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
     Long txnId = null;
     int stmtId = 0;
     Table tbl = ts.tableHandle;
-    if (MetaStoreUtils.isInsertOnlyTable(tbl.getParameters())) {
+    if (AcidUtils.isInsertOnlyTable(tbl.getParameters())) {
       txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index f3f932b..99b8712 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6837,7 +6837,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
       }
 
       boolean isNonNativeTable = dest_tab.isNonNative();
-      isMmTable = MetaStoreUtils.isInsertOnlyTable(dest_tab.getParameters());
+      isMmTable = AcidUtils.isInsertOnlyTable(dest_tab.getParameters());
       if (isNonNativeTable || isMmTable) {
         queryTmpdir = dest_path;
       } else {
@@ -6873,7 +6873,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
           acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
           checkAcidConstraints(qb, table_desc, dest_tab);
         }
-        if (MetaStoreUtils.isInsertOnlyTable(table_desc.getProperties())) {
+        if (AcidUtils.isInsertOnlyTable(table_desc.getProperties())) {
           acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
         }
         if (isMmTable) {
@@ -6924,7 +6924,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
       dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri()
           .getAuthority(), partPath.toUri().getPath());
 
-      isMmTable = MetaStoreUtils.isInsertOnlyTable(dest_tab.getParameters());
+      isMmTable = AcidUtils.isInsertOnlyTable(dest_tab.getParameters());
       queryTmpdir = isMmTable ? dest_path : ctx.getTempDirForPath(dest_path, true);
       if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
         Utilities.FILE_OP_LOGGER.trace("create filesink w/DEST_PARTITION specifying "
@@ -6947,7 +6947,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
         checkAcidConstraints(qb, table_desc, dest_tab);
       }
-      if (MetaStoreUtils.isInsertOnlyTable(dest_part.getTable().getParameters())) {
+      if (AcidUtils.isInsertOnlyTable(dest_part.getTable().getParameters())) {
         acidOp = getAcidType(table_desc.getOutputFileFormatClass(), dest);
       }
       if (isMmTable) {
@@ -6990,7 +6990,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         field_schemas = new ArrayList<FieldSchema>();
         destTableIsTemporary = tblDesc.isTemporary();
         destTableIsMaterialization = tblDesc.isMaterialization();
-        if (!destTableIsTemporary && MetaStoreUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) {
+        if (!destTableIsTemporary && AcidUtils.isInsertOnlyTable(tblDesc.getTblProps(), true)) {
           isMmTable = isMmCtas = true;
           txnId = SessionState.get().getTxnMgr().getCurrentTxnId();
           tblDesc.setInitialMmWriteId(txnId);
@@ -7253,8 +7253,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     // If this is an insert, update, or delete on an ACID table then mark that so the
     // FileSinkOperator knows how to properly write to it.
     boolean isDestInsertOnly = (dest_part != null && dest_part.getTable() != null &&
-        MetaStoreUtils.isInsertOnlyTable(dest_part.getTable().getParameters()))
-        || (table_desc != null && MetaStoreUtils.isInsertOnlyTable(table_desc.getProperties()));
+        AcidUtils.isInsertOnlyTable(dest_part.getTable().getParameters()))
+        || (table_desc != null && AcidUtils.isInsertOnlyTable(table_desc.getProperties()));
 
     if (isDestInsertOnly) {
       fileSinkDesc.setWriteType(Operation.INSERT);
@@ -12056,7 +12056,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
       if (p != null) {
         tbl = p.getTable();
       }
-      if (tbl != null && (AcidUtils.isFullAcidTable(tbl) || MetaStoreUtils.isInsertOnlyTable(tbl.getParameters()))) {
+      if (tbl != null && (AcidUtils.isFullAcidTable(tbl) || AcidUtils.isInsertOnlyTable(tbl.getParameters()))) {
         acidInQuery = true;
         checkAcidTxnManager(tbl);
       }
@@ -12119,7 +12119,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         tbl = writeEntity.getTable();
       }
 
-      if (tbl != null && (AcidUtils.isFullAcidTable(tbl) || MetaStoreUtils.isInsertOnlyTable(tbl.getParameters()))) {
+      if (tbl != null && (AcidUtils.isFullAcidTable(tbl) || AcidUtils.isInsertOnlyTable(tbl.getParameters()))) {
         acidInQuery = true;
         checkAcidTxnManager(tbl);
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index 4732f0a..ea8fc19 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -283,9 +283,9 @@ public class FileSinkDesc extends AbstractOperatorDesc {
 
   public boolean isMmTable() {
     if (getTable() != null) {
-      return MetaStoreUtils.isInsertOnlyTable(table.getParameters());
+      return AcidUtils.isInsertOnlyTable(table.getParameters());
     } else { // Dynamic Partition Insert case
-      return MetaStoreUtils.isInsertOnlyTable(getTableInfo().getProperties());
+      return AcidUtils.isInsertOnlyTable(getTableInfo().getProperties());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
index 0b7fe01..f246115 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
@@ -42,7 +42,7 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
   private int stmtId;
   private Long currentTransactionId;
 
-  // TODO: the below seems like they should just be combined into partitionDesc
+  // TODO: the below seem like they should just be combined into partitionDesc
   private org.apache.hadoop.hive.ql.plan.TableDesc table;
   private Map<String, String> partitionSpec; // NOTE: this partitionSpec has to be ordered map
 
@@ -162,7 +162,7 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
   }
 
   public boolean isMmTable() {
-    return MetaStoreUtils.isInsertOnlyTable(table.getProperties());
+    return AcidUtils.isInsertOnlyTable(table.getProperties());
   }
 
   public void setReplace(boolean replace) {

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
index 17d2f31..28a3374 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
@@ -165,14 +165,6 @@ public class MoveWork implements Serializable {
   public void setSrcLocal(boolean srcLocal) {
     this.srcLocal = srcLocal;
   }
-
-  public void setNoop(boolean b) {
-    this.isNoop = true;
-  }
-
-  public boolean isNoop() {
-    return this.isNoop;
-  }
   
   public LineageState getLineagState() {
     return sessionStateLineageState;

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 8b3ad75..7d4d379 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -210,7 +210,7 @@ public class CompactorMR {
 
     // For MM tables we don't need to launch MR jobs as there is no compaction needed.
     // We just need to delete the directories for aborted transactions.
-    if (MetaStoreUtils.isInsertOnlyTable(t.getParameters())) {
+    if (AcidUtils.isInsertOnlyTable(t.getParameters())) {
       LOG.debug("Going to delete directories for aborted transactions for MM table "
           + t.getDbName() + "." + t.getTableName());
       removeFiles(conf, sd.getLocation(), txns, t);

http://git-wip-us.apache.org/repos/asf/hive/blob/64015972/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
index c52bd3e..8daa5c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
@@ -253,7 +253,7 @@ public class Initiator extends CompactorThread {
                                                  StorageDescriptor sd, Map<String, String> tblproperties)
       throws IOException, InterruptedException {
 
-    if (MetaStoreUtils.isInsertOnlyTable(tblproperties)) {
+    if (AcidUtils.isInsertOnlyTable(tblproperties)) {
       return CompactionType.MINOR;
     }

[41/50] [abbrv] hive git commit: HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)

Posted by se...@apache.org.

HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/77f3f28e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/77f3f28e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/77f3f28e

Branch: refs/heads/master
Commit: 77f3f28e26fa069c3e3c49d0e2f45073b59b2b60
Parents: 4717849 8ffd8ea
Author: sergey <se...@apache.org>
Authored: Tue Oct 3 15:54:51 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Tue Oct 3 15:54:51 2017 -0700

----------------------------------------------------------------------
 .../metrics2/JsonFileMetricsReporter.java       |  192 +-
 .../org/apache/hadoop/hive/conf/HiveConf.java   |    5 +
 .../metrics/metrics2/TestCodahaleMetrics.java   |   86 +-
 data/conf/perf-reg/hive-site.xml                |  285 --
 data/conf/perf-reg/tez-site.xml                 |   10 -
 data/conf/perf-reg/tez/hive-site.xml            |  285 ++
 data/conf/perf-reg/tez/tez-site.xml             |   10 +
 .../security/TestHadoopAuthBridge23.java        |    2 +-
 .../test/resources/testconfiguration.properties |    7 +
 ...SQLStdHiveAuthorizationValidatorForTest.java |   12 +
 .../hadoop/hive/metastore/HiveMetaStore.java    |    3 +-
 pom.xml                                         |    2 +-
 .../java/org/apache/hadoop/hive/ql/Context.java |   15 +
 .../apache/hadoop/hive/ql/exec/ExplainTask.java |    7 +-
 .../apache/hadoop/hive/ql/exec/ExportTask.java  |   69 +
 .../hadoop/hive/ql/exec/ReplCopyTask.java       |   33 +-
 .../apache/hadoop/hive/ql/exec/TaskFactory.java |    2 +
 .../hadoop/hive/ql/exec/repl/ReplDumpTask.java  |    2 +-
 .../hive/ql/parse/ExportSemanticAnalyzer.java   |   20 +-
 .../hadoop/hive/ql/parse/ParseDriver.java       |   10 +-
 .../apache/hadoop/hive/ql/parse/ParseUtils.java |    6 +-
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |   50 +-
 .../apache/hadoop/hive/ql/parse/TableMask.java  |    6 +-
 .../hadoop/hive/ql/parse/repl/CopyUtils.java    |  161 +-
 .../ql/parse/repl/dump/PartitionExport.java     |    5 +-
 .../hive/ql/parse/repl/dump/TableExport.java    |   77 +-
 .../apache/hadoop/hive/ql/plan/ExportWork.java  |   73 +
 ql/src/test/queries/clientpositive/masking_12.q |   34 +
 .../authorization_uri_export.q.out              |    1 -
 .../exim_12_nonnative_export.q.out              |    6 +-
 .../results/clientpositive/masking_12.q.out     |  437 ++
 .../results/clientpositive/perf/query1.q.out    |  169 -
 .../results/clientpositive/perf/query10.q.out   |  288 --
 .../results/clientpositive/perf/query11.q.out   |  351 --
 .../results/clientpositive/perf/query12.q.out   |  138 -
 .../results/clientpositive/perf/query13.q.out   |  205 -
 .../results/clientpositive/perf/query14.q.out   | 1286 ------
 .../results/clientpositive/perf/query15.q.out   |  119 -
 .../results/clientpositive/perf/query16.q.out   |  192 -
 .../results/clientpositive/perf/query17.q.out   |  225 -
 .../results/clientpositive/perf/query18.q.out   |  194 -
 .../results/clientpositive/perf/query19.q.out   |  161 -
 .../results/clientpositive/perf/query2.q.out    |  255 --
 .../results/clientpositive/perf/query20.q.out   |  130 -
 .../results/clientpositive/perf/query21.q.out   |  139 -
 .../results/clientpositive/perf/query22.q.out   |  121 -
 .../results/clientpositive/perf/query23.q.out   |  556 ---
 .../results/clientpositive/perf/query24.q.out   |  290 --
 .../results/clientpositive/perf/query25.q.out   |  229 --
 .../results/clientpositive/perf/query26.q.out   |  132 -
 .../results/clientpositive/perf/query27.q.out   |  140 -
 .../results/clientpositive/perf/query28.q.out   |  256 --
 .../results/clientpositive/perf/query29.q.out   |  230 --
 .../results/clientpositive/perf/query3.q.out    |  102 -
 .../results/clientpositive/perf/query30.q.out   |  211 -
 .../results/clientpositive/perf/query31.q.out   |  358 --
 .../results/clientpositive/perf/query32.q.out   |  144 -
 .../results/clientpositive/perf/query33.q.out   |  393 --
 .../results/clientpositive/perf/query34.q.out   |  156 -
 .../results/clientpositive/perf/query35.q.out   |  284 --
 .../results/clientpositive/perf/query36.q.out   |  149 -
 .../results/clientpositive/perf/query37.q.out   |  103 -
 .../results/clientpositive/perf/query38.q.out   |  212 -
 .../results/clientpositive/perf/query39.q.out   |  255 --
 .../results/clientpositive/perf/query4.q.out    |  500 ---
 .../results/clientpositive/perf/query40.q.out   |  148 -
 .../results/clientpositive/perf/query42.q.out   |  108 -
 .../results/clientpositive/perf/query43.q.out   |  100 -
 .../results/clientpositive/perf/query44.q.out   |  258 --
 .../results/clientpositive/perf/query45.q.out   |  180 -
 .../results/clientpositive/perf/query46.q.out   |  191 -
 .../results/clientpositive/perf/query47.q.out   |  325 --
 .../results/clientpositive/perf/query48.q.out   |  218 -
 .../results/clientpositive/perf/query49.q.out   |  496 ---
 .../results/clientpositive/perf/query5.q.out    |  473 ---
 .../results/clientpositive/perf/query50.q.out   |  210 -
 .../results/clientpositive/perf/query51.q.out   |  188 -
 .../results/clientpositive/perf/query52.q.out   |  106 -
 .../results/clientpositive/perf/query53.q.out   |  141 -
 .../results/clientpositive/perf/query54.q.out   |  386 --
 .../results/clientpositive/perf/query55.q.out   |   88 -
 .../results/clientpositive/perf/query56.q.out   |  379 --
 .../results/clientpositive/perf/query57.q.out   |  319 --
 .../results/clientpositive/perf/query58.q.out   |  440 --
 .../results/clientpositive/perf/query59.q.out   |  230 --
 .../results/clientpositive/perf/query6.q.out    |  226 -
 .../results/clientpositive/perf/query60.q.out   |  399 --
 .../results/clientpositive/perf/query61.q.out   |  280 --
 .../results/clientpositive/perf/query63.q.out   |  143 -
 .../results/clientpositive/perf/query64.q.out   |  772 ----
 .../results/clientpositive/perf/query65.q.out   |  169 -
 .../results/clientpositive/perf/query66.q.out   |  612 ---
 .../results/clientpositive/perf/query67.q.out   |  179 -
 .../results/clientpositive/perf/query68.q.out   |  205 -
 .../results/clientpositive/perf/query69.q.out   |  268 --
 .../results/clientpositive/perf/query7.q.out    |  132 -
 .../results/clientpositive/perf/query70.q.out   |  205 -
 .../results/clientpositive/perf/query71.q.out   |  208 -
 .../results/clientpositive/perf/query72.q.out   |  250 --
 .../results/clientpositive/perf/query73.q.out   |  150 -
 .../results/clientpositive/perf/query74.q.out   |  313 --
 .../results/clientpositive/perf/query75.q.out   |  481 ---
 .../results/clientpositive/perf/query76.q.out   |  187 -
 .../results/clientpositive/perf/query77.q.out   |  481 ---
 .../results/clientpositive/perf/query78.q.out   |  292 --
 .../results/clientpositive/perf/query79.q.out   |  140 -
 .../results/clientpositive/perf/query8.q.out    |  358 --
 .../results/clientpositive/perf/query80.q.out   |  478 ---
 .../results/clientpositive/perf/query81.q.out   |  214 -
 .../results/clientpositive/perf/query82.q.out   |  103 -
 .../results/clientpositive/perf/query83.q.out   |  372 --
 .../results/clientpositive/perf/query84.q.out   |  135 -
 .../results/clientpositive/perf/query85.q.out   |  304 --
 .../results/clientpositive/perf/query86.q.out   |  126 -
 .../results/clientpositive/perf/query87.q.out   |  239 --
 .../results/clientpositive/perf/query88.q.out   |  569 ---
 .../results/clientpositive/perf/query89.q.out   |  147 -
 .../results/clientpositive/perf/query9.q.out    |  414 --
 .../results/clientpositive/perf/query90.q.out   |  170 -
 .../results/clientpositive/perf/query91.q.out   |  184 -
 .../results/clientpositive/perf/query92.q.out   |  155 -
 .../results/clientpositive/perf/query93.q.out   |   98 -
 .../results/clientpositive/perf/query94.q.out   |  188 -
 .../results/clientpositive/perf/query95.q.out   |  231 --
 .../results/clientpositive/perf/query96.q.out   |  108 -
 .../results/clientpositive/perf/query97.q.out   |  132 -
 .../results/clientpositive/perf/query98.q.out   |  134 -
 .../results/clientpositive/perf/query99.q.out   |  164 -
 .../clientpositive/perf/tez/query1.q.out        |  169 +
 .../clientpositive/perf/tez/query10.q.out       |  288 ++
 .../clientpositive/perf/tez/query11.q.out       |  351 ++
 .../clientpositive/perf/tez/query12.q.out       |  138 +
 .../clientpositive/perf/tez/query13.q.out       |  205 +
 .../clientpositive/perf/tez/query14.q.out       | 1286 ++++++
 .../clientpositive/perf/tez/query15.q.out       |  119 +
 .../clientpositive/perf/tez/query16.q.out       |  192 +
 .../clientpositive/perf/tez/query17.q.out       |  225 +
 .../clientpositive/perf/tez/query18.q.out       |  194 +
 .../clientpositive/perf/tez/query19.q.out       |  161 +
 .../clientpositive/perf/tez/query2.q.out        |  255 ++
 .../clientpositive/perf/tez/query20.q.out       |  130 +
 .../clientpositive/perf/tez/query21.q.out       |  139 +
 .../clientpositive/perf/tez/query22.q.out       |  121 +
 .../clientpositive/perf/tez/query23.q.out       |  556 +++
 .../clientpositive/perf/tez/query24.q.out       |  290 ++
 .../clientpositive/perf/tez/query25.q.out       |  229 ++
 .../clientpositive/perf/tez/query26.q.out       |  132 +
 .../clientpositive/perf/tez/query27.q.out       |  140 +
 .../clientpositive/perf/tez/query28.q.out       |  256 ++
 .../clientpositive/perf/tez/query29.q.out       |  230 ++
 .../clientpositive/perf/tez/query3.q.out        |  102 +
 .../clientpositive/perf/tez/query30.q.out       |  211 +
 .../clientpositive/perf/tez/query31.q.out       |  358 ++
 .../clientpositive/perf/tez/query32.q.out       |  144 +
 .../clientpositive/perf/tez/query33.q.out       |  393 ++
 .../clientpositive/perf/tez/query34.q.out       |  156 +
 .../clientpositive/perf/tez/query35.q.out       |  284 ++
 .../clientpositive/perf/tez/query36.q.out       |  149 +
 .../clientpositive/perf/tez/query37.q.out       |  103 +
 .../clientpositive/perf/tez/query38.q.out       |  212 +
 .../clientpositive/perf/tez/query39.q.out       |  255 ++
 .../clientpositive/perf/tez/query4.q.out        |  500 +++
 .../clientpositive/perf/tez/query40.q.out       |  148 +
 .../clientpositive/perf/tez/query42.q.out       |  108 +
 .../clientpositive/perf/tez/query43.q.out       |  100 +
 .../clientpositive/perf/tez/query44.q.out       |  258 ++
 .../clientpositive/perf/tez/query45.q.out       |  180 +
 .../clientpositive/perf/tez/query46.q.out       |  191 +
 .../clientpositive/perf/tez/query47.q.out       |  325 ++
 .../clientpositive/perf/tez/query48.q.out       |  218 +
 .../clientpositive/perf/tez/query49.q.out       |  496 +++
 .../clientpositive/perf/tez/query5.q.out        |  473 +++
 .../clientpositive/perf/tez/query50.q.out       |  210 +
 .../clientpositive/perf/tez/query51.q.out       |  188 +
 .../clientpositive/perf/tez/query52.q.out       |  106 +
 .../clientpositive/perf/tez/query53.q.out       |  141 +
 .../clientpositive/perf/tez/query54.q.out       |  386 ++
 .../clientpositive/perf/tez/query55.q.out       |   88 +
 .../clientpositive/perf/tez/query56.q.out       |  379 ++
 .../clientpositive/perf/tez/query57.q.out       |  319 ++
 .../clientpositive/perf/tez/query58.q.out       |  440 ++
 .../clientpositive/perf/tez/query59.q.out       |  230 ++
 .../clientpositive/perf/tez/query6.q.out        |  226 +
 .../clientpositive/perf/tez/query60.q.out       |  399 ++
 .../clientpositive/perf/tez/query61.q.out       |  280 ++
 .../clientpositive/perf/tez/query63.q.out       |  143 +
 .../clientpositive/perf/tez/query64.q.out       |  772 ++++
 .../clientpositive/perf/tez/query65.q.out       |  169 +
 .../clientpositive/perf/tez/query66.q.out       |  612 +++
 .../clientpositive/perf/tez/query67.q.out       |  179 +
 .../clientpositive/perf/tez/query68.q.out       |  205 +
 .../clientpositive/perf/tez/query69.q.out       |  268 ++
 .../clientpositive/perf/tez/query7.q.out        |  132 +
 .../clientpositive/perf/tez/query70.q.out       |  205 +
 .../clientpositive/perf/tez/query71.q.out       |  208 +
 .../clientpositive/perf/tez/query72.q.out       |  250 ++
 .../clientpositive/perf/tez/query73.q.out       |  150 +
 .../clientpositive/perf/tez/query74.q.out       |  313 ++
 .../clientpositive/perf/tez/query75.q.out       |  481 +++
 .../clientpositive/perf/tez/query76.q.out       |  187 +
 .../clientpositive/perf/tez/query77.q.out       |  481 +++
 .../clientpositive/perf/tez/query78.q.out       |  292 ++
 .../clientpositive/perf/tez/query79.q.out       |  140 +
 .../clientpositive/perf/tez/query8.q.out        |  358 ++
 .../clientpositive/perf/tez/query80.q.out       |  478 +++
 .../clientpositive/perf/tez/query81.q.out       |  214 +
 .../clientpositive/perf/tez/query82.q.out       |  103 +
 .../clientpositive/perf/tez/query83.q.out       |  372 ++
 .../clientpositive/perf/tez/query84.q.out       |  135 +
 .../clientpositive/perf/tez/query85.q.out       |  304 ++
 .../clientpositive/perf/tez/query86.q.out       |  126 +
 .../clientpositive/perf/tez/query87.q.out       |  239 ++
 .../clientpositive/perf/tez/query88.q.out       |  569 +++
 .../clientpositive/perf/tez/query89.q.out       |  147 +
 .../clientpositive/perf/tez/query9.q.out        |  414 ++
 .../clientpositive/perf/tez/query90.q.out       |  170 +
 .../clientpositive/perf/tez/query91.q.out       |  184 +
 .../clientpositive/perf/tez/query92.q.out       |  155 +
 .../clientpositive/perf/tez/query93.q.out       |   98 +
 .../clientpositive/perf/tez/query94.q.out       |  188 +
 .../clientpositive/perf/tez/query95.q.out       |  231 ++
 .../clientpositive/perf/tez/query96.q.out       |  108 +
 .../clientpositive/perf/tez/query97.q.out       |  132 +
 .../clientpositive/perf/tez/query98.q.out       |  134 +
 .../clientpositive/perf/tez/query99.q.out       |  164 +
 .../clientpositive/spark/auto_join32.q.out      |  574 +++
 .../spark/auto_sortmerge_join_2.q.out           |  594 +++
 .../results/clientpositive/spark/insert1.q.out  |  412 ++
 .../results/clientpositive/spark/join40.q.out   | 3845 ++++++++++++++++++
 .../clientpositive/spark/join_empty.q.out       |   56 +
 .../clientpositive/spark/ppd_join4.q.out        |  128 +
 .../clientpositive/spark/transform1.q.out       |  138 +
 .../hive/service/auth/HiveAuthFactory.java      |    3 +-
 .../hive/thrift/HadoopThriftAuthBridge.java     |   27 +-
 .../hive/metastore/ReplChangeManager.java       |   73 +-
 .../hive/metastore/metrics/JsonReporter.java    |  131 +-
 .../security/HadoopThriftAuthBridge.java        |   26 +-
 .../hive/metastore/metrics/TestMetrics.java     |   75 +-
 238 files changed, 32142 insertions(+), 25397 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/77f3f28e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/77f3f28e/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/77f3f28e/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/77f3f28e/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/77f3f28e/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/77f3f28e/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hive/blob/77f3f28e/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
index 54ee7ae,f094805..2400f9c
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
@@@ -51,13 -21,12 +51,16 @@@ import org.apache.hadoop.hive.ql.metada
  import org.antlr.runtime.tree.Tree;
  import org.apache.hadoop.hive.ql.ErrorMsg;
  import org.apache.hadoop.hive.ql.QueryState;
+ import org.apache.hadoop.hive.ql.exec.Task;
+ import org.apache.hadoop.hive.ql.exec.TaskFactory;
  import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
 +import org.apache.hadoop.hive.ql.metadata.Partition;
 +import org.apache.hadoop.hive.ql.metadata.PartitionIterable;
  import org.apache.hadoop.hive.ql.metadata.Table;
 +import org.apache.hadoop.hive.ql.plan.CopyWork;
 +import org.slf4j.Logger;
  import org.apache.hadoop.hive.ql.parse.repl.dump.TableExport;
+ import org.apache.hadoop.hive.ql.plan.ExportWork;
  
  /**
   * ExportSemanticAnalyzer.

http://git-wip-us.apache.org/repos/asf/hive/blob/77f3f28e/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------