You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2016/01/31 21:16:58 UTC
[1/2] hive git commit: HIVE-12893: Sorted dynamic partition does not
work if subset of partition columns are constant folded (Prasanth
Jayachandran reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 208ab3523 -> 0bd8ff6fa
http://git-wip-us.apache.org/repos/asf/hive/blob/0bd8ff6f/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
index d05b9bb..a28f0a4 100644
--- a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out
@@ -2383,3 +2383,835 @@ POSTHOOK: Input: default@over1k_part_buck_sort2@t=27
POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__
#### A masked pattern was here ####
19
+PREHOOK: query: create table over1k_part3(
+ si smallint,
+ b bigint,
+ f float)
+ partitioned by (s string, t tinyint, i int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: create table over1k_part3(
+ si smallint,
+ b bigint,
+ f float)
+ partitioned by (s string, t tinyint, i int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@over1k_part3
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (s = 'foo') (type: boolean)
+ Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), t (type: tinyint), i (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5
+ Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 'foo' (type: string), _col4 (type: tinyint), _col5 (type: int)
+ sort order: +++
+ Map-reduce partition columns: 'foo' (type: string), _col4 (type: tinyint), _col5 (type: int)
+ Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), 'foo' (type: string), _col4 (type: tinyint), _col5 (type: int)
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (t = 27) (type: boolean)
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), i (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col5
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: string), 27 (type: tinyint), _col5 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col3 (type: string), 27 (type: tinyint), _col5 (type: int)
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), _col3 (type: string), 27 (type: tinyint), _col5 (type: int)
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (i = 100) (type: boolean)
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), t (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: string), _col4 (type: tinyint), 100 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), 100 (type: int)
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), _col3 (type: string), _col4 (type: tinyint), 100 (type: int)
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((i = 100) and (t = 27)) (type: boolean)
+ Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: string), 27 (type: tinyint), 100 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col3 (type: string), 27 (type: tinyint), 100 (type: int)
+ Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), _col3 (type: string), 27 (type: tinyint), 100 (type: int)
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((i = 100) and (s = 'foo')) (type: boolean)
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), t (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col4
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 'foo' (type: string), _col4 (type: tinyint), 100 (type: int)
+ sort order: +++
+ Map-reduce partition columns: 'foo' (type: string), _col4 (type: tinyint), 100 (type: int)
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), 'foo' (type: string), _col4 (type: tinyint), 100 (type: int)
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((t = 27) and (s = 'foo')) (type: boolean)
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), i (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col5
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 'foo' (type: string), 27 (type: tinyint), _col5 (type: int)
+ sort order: +++
+ Map-reduce partition columns: 'foo' (type: string), 27 (type: tinyint), _col5 (type: int)
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), 'foo' (type: string), 27 (type: tinyint), _col5 (type: int)
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((i = 100) and (t = 27)) and (s = 'foo')) (type: boolean)
+ Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27 (type: tinyint), 100 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636
+POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619
+POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617
+POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628
+POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787
+POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778
+POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779
+POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624
+POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536
+POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708
+POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677
+POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675
+POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680
+POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: select sum(hash(*)) from over1k_part3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k_part3
+PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636
+PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619
+PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617
+PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628
+PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787
+PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778
+PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779
+PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624
+PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536
+PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708
+PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677
+PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675
+PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680
+PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from over1k_part3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k_part3
+POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636
+POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619
+POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617
+POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628
+POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787
+POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778
+POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779
+POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624
+POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536
+POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708
+POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677
+POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675
+POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680
+POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+#### A masked pattern was here ####
+17814641134
+PREHOOK: query: -- cross verify results with SDPO disabled
+drop table over1k_part3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@over1k_part3
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: -- cross verify results with SDPO disabled
+drop table over1k_part3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@over1k_part3
+POSTHOOK: Output: default@over1k_part3
+PREHOOK: query: create table over1k_part3(
+ si smallint,
+ b bigint,
+ f float)
+ partitioned by (s string, t tinyint, i int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: create table over1k_part3(
+ si smallint,
+ b bigint,
+ f float)
+ partitioned by (s string, t tinyint, i int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@over1k_part3
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636
+POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619
+POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617
+POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628
+POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787
+POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778
+POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779
+POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624
+POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536
+POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708
+POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677
+POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675
+POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680
+POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: select sum(hash(*)) from over1k_part3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k_part3
+PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636
+PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619
+PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617
+PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628
+PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787
+PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778
+PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779
+PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624
+PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536
+PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708
+PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677
+PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675
+PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680
+PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from over1k_part3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k_part3
+POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636
+POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619
+POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617
+POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628
+POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787
+POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778
+POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779
+POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624
+POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536
+POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708
+POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677
+POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675
+POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680
+POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+#### A masked pattern was here ####
+17814641134
http://git-wip-us.apache.org/repos/asf/hive/blob/0bd8ff6f/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out
index a377e25..97f59d9 100644
--- a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization2.q.out
@@ -1732,7 +1732,6 @@ STAGE PLANS:
#### A masked pattern was here ####
Edges:
Reducer 2 <- Map 1 (SIMPLE_EDGE)
- Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
#### A masked pattern was here ####
Vertices:
Map 1
@@ -1769,27 +1768,14 @@ STAGE PLANS:
expressions: UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int), 'day' (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col2 (type: string)
- sort order: +
- Map-reduce partition columns: _col2 (type: string)
+ File Output Operator
+ compressed: false
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
- Reducer 3
- Execution mode: vectorized
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
- serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
- name: default.hive13_dp1
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.hive13_dp1
Stage: Stage-2
Dependency Collection
[2/2] hive git commit: HIVE-12893: Sorted dynamic partition does not
work if subset of partition columns are constant folded (Prasanth
Jayachandran reviewed by Ashutosh Chauhan)
Posted by pr...@apache.org.
HIVE-12893: Sorted dynamic partition does not work if subset of partition columns are constant folded (Prasanth Jayachandran reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0bd8ff6f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0bd8ff6f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0bd8ff6f
Branch: refs/heads/master
Commit: 0bd8ff6fae20eb2b017888e11781cb7e751f3cba
Parents: 208ab35
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Sun Jan 31 14:15:51 2016 -0600
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Sun Jan 31 14:16:31 2016 -0600
----------------------------------------------------------------------
.../BucketingSortingReduceSinkOptimizer.java | 11 +
.../hadoop/hive/ql/optimizer/Optimizer.java | 14 +-
.../optimizer/SortedDynPartitionOptimizer.java | 40 +-
.../hadoop/hive/ql/parse/ParseContext.java | 9 +
.../clientpositive/dynpart_sort_optimization.q | 43 +
.../dynpart_sort_optimization.q.out | 803 ++++++++++++++++++
.../dynpart_sort_optimization2.q.out | 39 +-
.../llap/dynpart_sort_optimization2.q.out | 48 +-
.../tez/dynpart_sort_optimization.q.out | 832 +++++++++++++++++++
.../tez/dynpart_sort_optimization2.q.out | 28 +-
10 files changed, 1784 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0bd8ff6f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
index 0d7bf77..d5f3057 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
import org.apache.hadoop.hive.shims.ShimLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This transformation does optimization for enforcing bucketing and sorting.
@@ -120,6 +122,7 @@ public class BucketingSortingReduceSinkOptimizer extends Transform {
*
*/
public class BucketSortReduceSinkProcessor implements NodeProcessor {
+ private final Logger LOG = LoggerFactory.getLogger(BucketSortReduceSinkProcessor.class);
protected ParseContext pGraphContext;
public BucketSortReduceSinkProcessor(ParseContext pGraphContext) {
@@ -364,6 +367,14 @@ public class BucketingSortingReduceSinkOptimizer extends Transform {
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
+ // We should not use this optimization if sorted dynamic partition optimizer is used,
+ // as RS will be required.
+ if (pGraphContext.isReduceSinkAddedBySortedDynPartition()) {
+ LOG.info("Reduce Sink is added by Sorted Dynamic Partition Optimizer. Bailing out of" +
+ " Bucketing Sorting Reduce Sink Optimizer");
+ return null;
+ }
+
// If the reduce sink has not been introduced due to bucketing/sorting, ignore it
FileSinkOperator fsOp = (FileSinkOperator) nd;
ReduceSinkOperator rsOp = (ReduceSinkOperator) fsOp.getParentOperators().get(0).getParentOperators().get(0);
http://git-wip-us.apache.org/repos/asf/hive/blob/0bd8ff6f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index 033fc8d..60a2e02 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -108,12 +108,20 @@ public class Optimizer {
transformations.add(new SyntheticJoinPredicate());
transformations.add(new SimplePredicatePushDown());
}
+
if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTCONSTANTPROPAGATION)) {
// We run constant propagation twice because after predicate pushdown, filter expressions
// are combined and may become eligible for reduction (like is not null filter).
transformations.add(new ConstantPropagate());
}
+ if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONING) &&
+ HiveConf.getVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") &&
+ HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITION) &&
+ !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) {
+ transformations.add(new SortedDynPartitionOptimizer());
+ }
+
if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTPPD)) {
transformations.add(new PartitionPruner());
transformations.add(new PartitionConditionRemover());
@@ -182,12 +190,6 @@ public class Optimizer {
transformations.add(new FixedBucketPruningOptimizer());
}
- if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONING) &&
- HiveConf.getVar(hiveConf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equals("nonstrict") &&
- HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTSORTDYNAMICPARTITION) &&
- !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTLISTBUCKETING)) {
- transformations.add(new SortedDynPartitionOptimizer());
- }
if(HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION)) {
transformations.add(new ReduceSinkDeDuplication());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0bd8ff6f/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
index d4be5cf..27b0457 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
@@ -143,6 +143,15 @@ public class SortedDynPartitionOptimizer extends Transform {
return null;
}
+ // unlink connection between FS and its parent
+ Operator<? extends OperatorDesc> fsParent = fsOp.getParentOperators().get(0);
+ // if all dp columns got constant folded then disable this optimization
+ if (allStaticPartitions(fsParent, fsOp.getConf().getDynPartCtx())) {
+ LOG.debug("Bailing out of sorted dynamic partition optimizer as all dynamic partition" +
+ " columns got constant folded (static partitioning)");
+ return null;
+ }
+
// if RS is inserted by enforce bucketing or sorting, we need to remove it
// since ReduceSinkDeDuplication will not merge them to single RS.
// RS inserted by enforce bucketing/sorting will have bucketing column in
@@ -158,7 +167,7 @@ public class SortedDynPartitionOptimizer extends Transform {
}
// unlink connection between FS and its parent
- Operator<? extends OperatorDesc> fsParent = fsOp.getParentOperators().get(0);
+ fsParent = fsOp.getParentOperators().get(0);
fsParent.getChildOperators().clear();
DynamicPartitionCtx dpCtx = fsOp.getConf().getDynPartCtx();
@@ -261,9 +270,38 @@ public class SortedDynPartitionOptimizer extends Transform {
LOG.info("Inserted " + rsOp.getOperatorId() + " and " + selOp.getOperatorId()
+ " as parent of " + fsOp.getOperatorId() + " and child of " + fsParent.getOperatorId());
+
+ parseCtx.setReduceSinkAddedBySortedDynPartition(true);
return null;
}
+ private boolean allStaticPartitions(Operator<? extends OperatorDesc> op,
+ final DynamicPartitionCtx dynPartCtx) {
+ int numDpCols = dynPartCtx.getNumDPCols();
+ int numCols = op.getSchema().getColumnNames().size();
+ List<String> dpCols = op.getSchema().getColumnNames().subList(numCols - numDpCols, numCols);
+ if (op.getColumnExprMap() == null) {
+ // find first operator upstream with valid (non-null) column expression map
+ for(Operator<? extends OperatorDesc> parent : op.getParentOperators()) {
+ if (parent.getColumnExprMap() != null) {
+ op = parent;
+ break;
+ }
+ }
+ }
+ if (op.getColumnExprMap() != null) {
+ for(String dpCol : dpCols) {
+ ExprNodeDesc end = op.getColumnExprMap().get(dpCol);
+ if (!(end instanceof ExprNodeConstantDesc)) {
+ return false;
+ }
+ }
+ } else {
+ return false;
+ }
+ return true;
+ }
+
// Remove RS and SEL introduced by enforce bucketing/sorting config
// Convert PARENT -> RS -> SEL -> FS to PARENT -> FS
private boolean removeRSInsertedByEnforceBucketing(FileSinkOperator fsOp) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0bd8ff6f/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
index bee0175..642c227 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
@@ -105,6 +105,7 @@ public class ParseContext {
private AnalyzeRewriteContext analyzeRewrite;
private CreateTableDesc createTableDesc;
+ private boolean reduceSinkAddedBySortedDynPartition;
public ParseContext() {
@@ -530,4 +531,12 @@ public class ParseContext {
this.createTableDesc = createTableDesc;
}
+ public void setReduceSinkAddedBySortedDynPartition(
+ final boolean reduceSinkAddedBySortedDynPartition) {
+ this.reduceSinkAddedBySortedDynPartition = reduceSinkAddedBySortedDynPartition;
+ }
+
+ public boolean isReduceSinkAddedBySortedDynPartition() {
+ return reduceSinkAddedBySortedDynPartition;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0bd8ff6f/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
index 8b7f4eb..b4ff0e8 100644
--- a/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
+++ b/ql/src/test/queries/clientpositive/dynpart_sort_optimization.q
@@ -164,3 +164,46 @@ desc formatted over1k_part_buck_sort2 partition(t="__HIVE_DEFAULT_PARTITION__");
select * from over1k_part_buck_sort2;
select count(*) from over1k_part_buck_sort2;
+
+create table over1k_part3(
+ si smallint,
+ b bigint,
+ f float)
+ partitioned by (s string, t tinyint, i int);
+
+set hive.optimize.sort.dynamic.partition=true;
+explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo";
+explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27;
+explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100;
+explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27;
+explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo";
+explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo";
+explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo";
+
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo";
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27;
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100;
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27;
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo";
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo";
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo";
+
+select sum(hash(*)) from over1k_part3;
+
+-- cross verify results with SDPO disabled
+drop table over1k_part3;
+create table over1k_part3(
+ si smallint,
+ b bigint,
+ f float)
+ partitioned by (s string, t tinyint, i int);
+set hive.optimize.sort.dynamic.partition=false;
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo";
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27;
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100;
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27;
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo";
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo";
+insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo";
+
+select sum(hash(*)) from over1k_part3;
http://git-wip-us.apache.org/repos/asf/hive/blob/0bd8ff6f/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out
index 18c5399..2105d8b 100644
--- a/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out
+++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out
@@ -2258,3 +2258,806 @@ POSTHOOK: Input: default@over1k_part_buck_sort2@t=27
POSTHOOK: Input: default@over1k_part_buck_sort2@t=__HIVE_DEFAULT_PARTITION__
#### A masked pattern was here ####
19
+PREHOOK: query: create table over1k_part3(
+ si smallint,
+ b bigint,
+ f float)
+ partitioned by (s string, t tinyint, i int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: create table over1k_part3(
+ si smallint,
+ b bigint,
+ f float)
+ partitioned by (s string, t tinyint, i int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@over1k_part3
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (s = 'foo') (type: boolean)
+ Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), t (type: tinyint), i (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col4, _col5
+ Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 'foo' (type: string), _col4 (type: tinyint), _col5 (type: int)
+ sort order: +++
+ Map-reduce partition columns: 'foo' (type: string), _col4 (type: tinyint), _col5 (type: int)
+ Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), 'foo' (type: string), _col4 (type: tinyint), _col5 (type: int)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1974 Data size: 53304 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (t = 27) (type: boolean)
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), i (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col5
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: string), 27 (type: tinyint), _col5 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col3 (type: string), 27 (type: tinyint), _col5 (type: int)
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), _col3 (type: string), 27 (type: tinyint), _col5 (type: int)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (i = 100) (type: boolean)
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string), t (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: string), _col4 (type: tinyint), 100 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col3 (type: string), _col4 (type: tinyint), 100 (type: int)
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), _col3 (type: string), _col4 (type: tinyint), 100 (type: int)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 429 Data size: 53255 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 859 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((i = 100) and (t = 27)) (type: boolean)
+ Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), s (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: string), 27 (type: tinyint), 100 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col3 (type: string), 27 (type: tinyint), 100 (type: int)
+ Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), _col3 (type: string), 27 (type: tinyint), 100 (type: int)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 214 Data size: 26565 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((i = 100) and (s = 'foo')) (type: boolean)
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), t (type: tinyint)
+ outputColumnNames: _col0, _col1, _col2, _col4
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 'foo' (type: string), _col4 (type: tinyint), 100 (type: int)
+ sort order: +++
+ Map-reduce partition columns: 'foo' (type: string), _col4 (type: tinyint), 100 (type: int)
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), 'foo' (type: string), _col4 (type: tinyint), 100 (type: int)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((t = 27) and (s = 'foo')) (type: boolean)
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), i (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col5
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 'foo' (type: string), 27 (type: tinyint), _col5 (type: int)
+ sort order: +++
+ Map-reduce partition columns: 'foo' (type: string), 27 (type: tinyint), _col5 (type: int)
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: smallint), _col1 (type: bigint), _col2 (type: float), 'foo' (type: string), 27 (type: tinyint), _col5 (type: int)
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: bigint), VALUE._col2 (type: float), VALUE._col3 (type: string), VALUE._col4 (type: tinyint), VALUE._col5 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 987 Data size: 26652 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+PREHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
+ Stage-4
+ Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
+ Stage-2 depends on stages: Stage-0
+ Stage-3
+ Stage-5
+ Stage-6 depends on stages: Stage-5
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: over1k
+ Statistics: Num rows: 3949 Data size: 106636 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((i = 100) and (t = 27)) and (s = 'foo')) (type: boolean)
+ Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: si (type: smallint), b (type: bigint), f (type: float), 'foo' (type: string), 27 (type: tinyint), 100 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 493 Data size: 13312 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-4
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ i
+ s
+ t
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.over1k_part3
+
+ Stage: Stage-6
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636
+POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619
+POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617
+POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628
+POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787
+POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778
+POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779
+POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624
+POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536
+POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708
+POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677
+POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675
+POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680
+POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: select sum(hash(*)) from over1k_part3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k_part3
+PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636
+PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619
+PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617
+PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628
+PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787
+PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778
+PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779
+PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624
+PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536
+PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708
+PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677
+PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675
+PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680
+PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from over1k_part3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k_part3
+POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636
+POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619
+POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617
+POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628
+POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787
+POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778
+POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779
+POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624
+POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536
+POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708
+POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677
+POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675
+POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680
+POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+#### A masked pattern was here ####
+17814641134
+PREHOOK: query: -- cross verify results with SDPO disabled
+drop table over1k_part3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@over1k_part3
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: -- cross verify results with SDPO disabled
+drop table over1k_part3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@over1k_part3
+POSTHOOK: Output: default@over1k_part3
+PREHOOK: query: create table over1k_part3(
+ si smallint,
+ b bigint,
+ f float)
+ partitioned by (s string, t tinyint, i int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: create table over1k_part3(
+ si smallint,
+ b bigint,
+ f float)
+ partitioned by (s string, t tinyint, i int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@over1k_part3
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+POSTHOOK: Output: default@over1k_part3@s=alice quirinius/t=27/i=65636
+POSTHOOK: Output: default@over1k_part3@s=bob ovid/t=27/i=65619
+POSTHOOK: Output: default@over1k_part3@s=david allen/t=27/i=65617
+POSTHOOK: Output: default@over1k_part3@s=ethan laertes/t=27/i=65628
+POSTHOOK: Output: default@over1k_part3@s=irene underhill/t=27/i=65787
+POSTHOOK: Output: default@over1k_part3@s=jessica zipper/t=27/i=65778
+POSTHOOK: Output: default@over1k_part3@s=mike zipper/t=27/i=65779
+POSTHOOK: Output: default@over1k_part3@s=oscar carson/t=27/i=65624
+POSTHOOK: Output: default@over1k_part3@s=oscar ovid/t=27/i=65536
+POSTHOOK: Output: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+POSTHOOK: Output: default@over1k_part3@s=quinn allen/t=27/i=65708
+POSTHOOK: Output: default@over1k_part3@s=rachel carson/t=27/i=65677
+POSTHOOK: Output: default@over1k_part3@s=tom brown/t=27/i=65675
+POSTHOOK: Output: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+POSTHOOK: Output: default@over1k_part3@s=wendy van buren/t=27/i=65680
+POSTHOOK: Output: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=alice quirinius,t=27,i=65636).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=bob ovid,t=27,i=65619).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=david allen,t=27,i=65617).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ethan laertes,t=27,i=65628).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=irene underhill,t=27,i=65787).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=jessica zipper,t=27,i=65778).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=mike zipper,t=27,i=65779).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar carson,t=27,i=65624).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=oscar ovid,t=27,i=65536).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=priscilla zipper,t=27,i=65622).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=quinn allen,t=27,i=65708).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=rachel carson,t=27,i=65677).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=tom brown,t=27,i=65675).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=ulysses underhill,t=27,i=65570).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=wendy van buren,t=27,i=65680).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).b SIMPLE [(over1k)over1k.FieldSchema(name:b, type:bigint, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).f SIMPLE [(over1k)over1k.FieldSchema(name:f, type:float, comment:null), ]
+POSTHOOK: Lineage: over1k_part3 PARTITION(s=xavier quirinius,t=27,i=65599).si SIMPLE [(over1k)over1k.FieldSchema(name:si, type:smallint, comment:null), ]
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where t=27 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k
+PREHOOK: Output: default@over1k_part3
+POSTHOOK: query: insert overwrite table over1k_part3 partition(s,t,i) select si,b,f,s,t,i from over1k where i=100 and t=27 and s="foo"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k
+PREHOOK: query: select sum(hash(*)) from over1k_part3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over1k_part3
+PREHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636
+PREHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619
+PREHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617
+PREHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628
+PREHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787
+PREHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778
+PREHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779
+PREHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624
+PREHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536
+PREHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+PREHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708
+PREHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677
+PREHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675
+PREHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+PREHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680
+PREHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+#### A masked pattern was here ####
+POSTHOOK: query: select sum(hash(*)) from over1k_part3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over1k_part3
+POSTHOOK: Input: default@over1k_part3@s=alice quirinius/t=27/i=65636
+POSTHOOK: Input: default@over1k_part3@s=bob ovid/t=27/i=65619
+POSTHOOK: Input: default@over1k_part3@s=david allen/t=27/i=65617
+POSTHOOK: Input: default@over1k_part3@s=ethan laertes/t=27/i=65628
+POSTHOOK: Input: default@over1k_part3@s=irene underhill/t=27/i=65787
+POSTHOOK: Input: default@over1k_part3@s=jessica zipper/t=27/i=65778
+POSTHOOK: Input: default@over1k_part3@s=mike zipper/t=27/i=65779
+POSTHOOK: Input: default@over1k_part3@s=oscar carson/t=27/i=65624
+POSTHOOK: Input: default@over1k_part3@s=oscar ovid/t=27/i=65536
+POSTHOOK: Input: default@over1k_part3@s=priscilla zipper/t=27/i=65622
+POSTHOOK: Input: default@over1k_part3@s=quinn allen/t=27/i=65708
+POSTHOOK: Input: default@over1k_part3@s=rachel carson/t=27/i=65677
+POSTHOOK: Input: default@over1k_part3@s=tom brown/t=27/i=65675
+POSTHOOK: Input: default@over1k_part3@s=ulysses underhill/t=27/i=65570
+POSTHOOK: Input: default@over1k_part3@s=wendy van buren/t=27/i=65680
+POSTHOOK: Input: default@over1k_part3@s=xavier quirinius/t=27/i=65599
+#### A masked pattern was here ####
+17814641134
http://git-wip-us.apache.org/repos/asf/hive/blob/0bd8ff6f/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out
index 0100eea..3b24a2e 100644
--- a/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out
+++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization2.q.out
@@ -1642,9 +1642,8 @@ group by "day", key
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-0 depends on stages: Stage-2
- Stage-3 depends on stages: Stage-0
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
STAGE PLANS:
Stage: Stage-1
@@ -1682,34 +1681,12 @@ STAGE PLANS:
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- Reduce Output Operator
- key expressions: _col2 (type: string)
- sort order: +
- Map-reduce partition columns: _col2 (type: string)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
- Reduce Operator Tree:
- Select Operator
- expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
- serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
- name: default.hive13_dp1
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.hive13_dp1
Stage: Stage-0
Move Operator
@@ -1723,7 +1700,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
name: default.hive13_dp1
- Stage: Stage-3
+ Stage: Stage-2
Stats-Aggr Operator
PREHOOK: query: insert overwrite table `hive13_dp1` partition(`day`)
http://git-wip-us.apache.org/repos/asf/hive/blob/0bd8ff6f/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
index 34c2307..06e94c3 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
@@ -200,7 +200,7 @@ Database: default
Table: ss_part
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 11
rawDataSize 151
@@ -260,7 +260,7 @@ Database: default
Table: ss_part
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 13
rawDataSize 186
@@ -429,7 +429,7 @@ Database: default
Table: ss_part
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 11
rawDataSize 151
@@ -489,7 +489,7 @@ Database: default
Table: ss_part
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 13
rawDataSize 186
@@ -679,7 +679,7 @@ Database: default
Table: ss_part
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 11
rawDataSize 151
@@ -739,7 +739,7 @@ Database: default
Table: ss_part
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 13
rawDataSize 186
@@ -907,7 +907,7 @@ Database: default
Table: ss_part
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 11
rawDataSize 151
@@ -967,7 +967,7 @@ Database: default
Table: ss_part
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 13
rawDataSize 186
@@ -1212,11 +1212,11 @@ Database: default
Table: ss_part_orc
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 11
rawDataSize 88
- totalSize 433
+ totalSize 454
#### A masked pattern was here ####
# Storage Information
@@ -1272,11 +1272,11 @@ Database: default
Table: ss_part_orc
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 13
rawDataSize 104
- totalSize 456
+ totalSize 477
#### A masked pattern was here ####
# Storage Information
@@ -1440,11 +1440,11 @@ Database: default
Table: ss_part_orc
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 11
rawDataSize 88
- totalSize 433
+ totalSize 454
#### A masked pattern was here ####
# Storage Information
@@ -1500,11 +1500,11 @@ Database: default
Table: ss_part_orc
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 13
rawDataSize 104
- totalSize 456
+ totalSize 477
#### A masked pattern was here ####
# Storage Information
@@ -1623,9 +1623,9 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
+ key expressions: 'day' (type: string), _col1 (type: string)
sort order: ++
- Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
+ Map-reduce partition columns: 'day' (type: string), _col1 (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint)
Execution mode: llap
@@ -1634,12 +1634,12 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ keys: 'day' (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int), _col0 (type: string)
+ expressions: UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int), 'day' (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -1751,9 +1751,9 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: _col0 (type: string), _col1 (type: string)
+ key expressions: 'day' (type: string), _col1 (type: string)
sort order: ++
- Map-reduce partition columns: _col0 (type: string)
+ Map-reduce partition columns: 'day' (type: string), _col1 (type: string)
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
value expressions: _col2 (type: bigint)
Execution mode: llap
@@ -1762,12 +1762,12 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: count(VALUE._col0)
- keys: KEY._col0 (type: string), KEY._col1 (type: string)
+ keys: 'day' (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int), _col0 (type: string)
+ expressions: UDFToInteger(_col1) (type: int), UDFToInteger(_col2) (type: int), 'day' (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
File Output Operator