You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dj...@apache.org on 2018/04/16 18:09:20 UTC
[11/11] hive git commit: HIVE-19195,
HIVE-17055 : Fix flaky tests and cleanup testconfiguration to run
llap specific tests in llap only.
HIVE-19195,HIVE-17055 : Fix flaky tests and cleanup testconfiguration to run llap specific tests in llap only.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/77da6997
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/77da6997
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/77da6997
Branch: refs/heads/master
Commit: 77da6997d4357c1d88f3a966fe549f1f41efeab5
Parents: 1e9f3d7
Author: Deepak Jaiswal <dj...@apache.org>
Authored: Mon Apr 16 11:08:30 2018 -0700
Committer: Deepak Jaiswal <dj...@apache.org>
Committed: Mon Apr 16 11:08:58 2018 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 60 +-
.../clientpositive/auto_sortmerge_join_1.q.out | 1324 ----------
.../clientpositive/auto_sortmerge_join_10.q.out | 620 -----
.../clientpositive/auto_sortmerge_join_11.q.out | 1625 -------------
.../clientpositive/auto_sortmerge_join_12.q.out | 2295 ------------------
.../clientpositive/auto_sortmerge_join_13.q.out | 775 ------
.../clientpositive/auto_sortmerge_join_14.q.out | 298 ---
.../clientpositive/auto_sortmerge_join_15.q.out | 266 --
.../clientpositive/auto_sortmerge_join_2.q.out | 1109 ---------
.../clientpositive/auto_sortmerge_join_3.q.out | 1153 ---------
.../clientpositive/auto_sortmerge_join_4.q.out | 1169 ---------
.../clientpositive/auto_sortmerge_join_5.q.out | 856 -------
.../clientpositive/auto_sortmerge_join_7.q.out | 1489 ------------
.../results/clientpositive/llap/llap_smb.q.out | 10 +-
.../clientpositive/llap/tez_smb_main.q.out | 142 +-
.../test/results/clientpositive/llap_acid.q.out | 401 ---
.../results/clientpositive/llap_acid_fast.q.out | 395 ---
.../results/clientpositive/tez_join_hash.q.out | 921 -------
18 files changed, 85 insertions(+), 14823 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/77da6997/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index f99d6a1..2845ab6 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -99,30 +99,12 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
auto_join30.q,\
auto_join_filters.q,\
auto_join_nulls.q,\
- auto_sortmerge_join_1.q,\
- auto_sortmerge_join_10.q,\
- auto_sortmerge_join_11.q,\
- auto_sortmerge_join_12.q,\
- auto_sortmerge_join_13.q,\
- auto_sortmerge_join_14.q,\
- auto_sortmerge_join_15.q,\
- auto_sortmerge_join_16.q,\
- auto_sortmerge_join_2.q,\
- auto_sortmerge_join_3.q,\
- auto_sortmerge_join_4.q,\
- auto_sortmerge_join_5.q,\
- auto_sortmerge_join_6.q,\
- auto_sortmerge_join_7.q,\
- auto_sortmerge_join_8.q,\
- auto_sortmerge_join_9.q,\
autoColumnStats_1.q,\
autoColumnStats_10.q,\
autoColumnStats_2.q,\
bucket2.q,\
bucket3.q,\
bucket4.q,\
- bucket_map_join_tez1.q,\
- bucket_map_join_tez2.q,\
cbo_gby.q,\
cbo_gby_empty.q,\
cbo_join.q,\
@@ -169,8 +151,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
delete_where_partitioned.q,\
delete_whole_partition.q,\
disable_merge_for_bucketing.q,\
- dynamic_partition_pruning.q,\
- dynamic_semijoin_reduction.q,\
dynpart_sort_opt_vectorization.q,\
dynpart_sort_optimization.q,\
dynpart_sort_optimization2.q,\
@@ -200,8 +180,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
kill_query.q,\
leftsemijoin.q,\
limit_pushdown.q,\
- llap_acid.q,\
- llap_acid_fast.q,\
load_dyn_part1.q,\
load_dyn_part2.q,\
load_dyn_part3.q,\
@@ -253,29 +231,6 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
subquery_exists.q,\
subquery_in.q,\
temp_table.q,\
- tez_bmj_schema_evolution.q,\
- tez_dml.q,\
- tez_dynpart_hashjoin_1.q,\
- tez_dynpart_hashjoin_2.q,\
- tez_fsstat.q,\
- tez_insert_overwrite_local_directory_1.q,\
- tez_join.q,\
- tez_join_hash.q,\
- tez_join_result_complex.q,\
- tez_join_tests.q,\
- tez_joins_explain.q,\
- tez_multi_union.q,\
- tez_schema_evolution.q,\
- tez_self_join.q,\
- tez_smb_1.q,\
- tez_smb_main.q,\
- tez_union.q,\
- tez_union2.q,\
- tez_union_decimal.q,\
- tez_union_group_by.q,\
- tez_union_multiinsert.q,\
- tez_vector_dynpart_hashjoin_1.q,\
- tez_vector_dynpart_hashjoin_2.q,\
union2.q,\
union3.q,\
union4.q,\
@@ -448,7 +403,6 @@ minillap.query.files=acid_bucket_pruning.q,\
intersect_all.q,\
intersect_distinct.q,\
intersect_merge.q,\
- llap_smb.q,\
llap_udf.q,\
llapdecider.q,\
reduce_deduplicate.q,\
@@ -494,8 +448,20 @@ minillaplocal.query.files=\
auto_join30.q,\
auto_join_filters.q,\
auto_join_nulls.q,\
+ auto_sortmerge_join_1.q,\
+ auto_sortmerge_join_10.q,\
+ auto_sortmerge_join_11.q,\
+ auto_sortmerge_join_12.q,\
+ auto_sortmerge_join_13.q,\
+ auto_sortmerge_join_14.q,\
+ auto_sortmerge_join_15.q,\
auto_sortmerge_join_16.q,\
+ auto_sortmerge_join_2.q,\
+ auto_sortmerge_join_3.q,\
+ auto_sortmerge_join_4.q,\
+ auto_sortmerge_join_5.q,\
auto_sortmerge_join_6.q,\
+ auto_sortmerge_join_7.q,\
auto_sortmerge_join_8.q,\
auto_sortmerge_join_9.q,\
bucket4.q,\
@@ -578,6 +544,8 @@ minillaplocal.query.files=\
lineage2.q,\
lineage3.q,\
list_bucket_dml_10.q,\
+ llap_acid.q,\
+ llap_acid_fast.q,\
llap_acid2.q,\
llap_partitioned.q,\
llap_vector_nohybridgrace.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/77da6997/ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out b/ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out
deleted file mode 100644
index 3ccdb19..0000000
--- a/ql/src/test/results/clientpositive/auto_sortmerge_join_1.q.out
+++ /dev/null
@@ -1,1324 +0,0 @@
-PREHOOK: query: CREATE TABLE bucket_small (key string, value string) partitioned by (ds string)
-CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@bucket_small
-POSTHOOK: query: CREATE TABLE bucket_small (key string, value string) partitioned by (ds string)
-CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@bucket_small
-PREHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/small/000000_0' INTO TABLE bucket_small partition(ds='2008-04-08')
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@bucket_small
-POSTHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/small/000000_0' INTO TABLE bucket_small partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@bucket_small
-POSTHOOK: Output: default@bucket_small@ds=2008-04-08
-PREHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/small/000001_0' INTO TABLE bucket_small partition(ds='2008-04-08')
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@bucket_small@ds=2008-04-08
-POSTHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/small/000001_0' INTO TABLE bucket_small partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@bucket_small@ds=2008-04-08
-PREHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@bucket_big
-POSTHOOK: query: CREATE TABLE bucket_big (key string, value string) partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@bucket_big
-PREHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000000_0' INTO TABLE bucket_big partition(ds='2008-04-08')
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@bucket_big
-POSTHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000000_0' INTO TABLE bucket_big partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@bucket_big
-POSTHOOK: Output: default@bucket_big@ds=2008-04-08
-PREHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000001_0' INTO TABLE bucket_big partition(ds='2008-04-08')
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@bucket_big@ds=2008-04-08
-POSTHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000001_0' INTO TABLE bucket_big partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@bucket_big@ds=2008-04-08
-PREHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000002_0' INTO TABLE bucket_big partition(ds='2008-04-08')
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@bucket_big@ds=2008-04-08
-POSTHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000002_0' INTO TABLE bucket_big partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@bucket_big@ds=2008-04-08
-PREHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000003_0' INTO TABLE bucket_big partition(ds='2008-04-08')
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@bucket_big@ds=2008-04-08
-POSTHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000003_0' INTO TABLE bucket_big partition(ds='2008-04-08')
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@bucket_big@ds=2008-04-08
-PREHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000000_0' INTO TABLE bucket_big partition(ds='2008-04-09')
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@bucket_big
-POSTHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000000_0' INTO TABLE bucket_big partition(ds='2008-04-09')
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@bucket_big
-POSTHOOK: Output: default@bucket_big@ds=2008-04-09
-PREHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000001_0' INTO TABLE bucket_big partition(ds='2008-04-09')
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@bucket_big@ds=2008-04-09
-POSTHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000001_0' INTO TABLE bucket_big partition(ds='2008-04-09')
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@bucket_big@ds=2008-04-09
-PREHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000002_0' INTO TABLE bucket_big partition(ds='2008-04-09')
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@bucket_big@ds=2008-04-09
-POSTHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000002_0' INTO TABLE bucket_big partition(ds='2008-04-09')
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@bucket_big@ds=2008-04-09
-PREHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000003_0' INTO TABLE bucket_big partition(ds='2008-04-09')
-PREHOOK: type: LOAD
-#### A masked pattern was here ####
-PREHOOK: Output: default@bucket_big@ds=2008-04-09
-POSTHOOK: query: load data local inpath '../../data/files/auto_sortmerge_join/big/000003_0' INTO TABLE bucket_big partition(ds='2008-04-09')
-POSTHOOK: type: LOAD
-#### A masked pattern was here ####
-POSTHOOK: Output: default@bucket_big@ds=2008-04-09
-PREHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key
-PREHOOK: type: QUERY
-POSTHOOK: query: explain extended select count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- Position of Big Table: 1
- BucketMapJoin: true
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- null sort order:
- sort order:
- tag: -1
- value expressions: _col0 (type: bigint)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: ds=2008-04-08
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
-#### A masked pattern was here ####
- Partition
- base file name: ds=2008-04-09
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
- Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [$hdt$_1:b]
- /bucket_big/ds=2008-04-09 [$hdt$_1:b]
- Needs Tagging: false
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- File Output Operator
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
-#### A masked pattern was here ####
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- properties:
- columns _col0
- columns.types bigint
- escape.delim \
- hive.serialization.extend.additional.nesting.levels true
- serialization.escape.crlf true
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@bucket_big
-PREHOOK: Input: default@bucket_big@ds=2008-04-08
-PREHOOK: Input: default@bucket_big@ds=2008-04-09
-PREHOOK: Input: default@bucket_small
-PREHOOK: Input: default@bucket_small@ds=2008-04-08
-#### A masked pattern was here ####
-POSTHOOK: query: select count(*) FROM bucket_small a JOIN bucket_big b ON a.key = b.key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@bucket_big
-POSTHOOK: Input: default@bucket_big@ds=2008-04-08
-POSTHOOK: Input: default@bucket_big@ds=2008-04-09
-POSTHOOK: Input: default@bucket_small
-POSTHOOK: Input: default@bucket_small@ds=2008-04-08
-#### A masked pattern was here ####
-38
-PREHOOK: query: explain extended select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key
-PREHOOK: type: QUERY
-POSTHOOK: query: explain extended select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: a
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- Position of Big Table: 0
- BucketMapJoin: true
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- null sort order:
- sort order:
- tag: -1
- value expressions: _col0 (type: bigint)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: ds=2008-04-08
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
-#### A masked pattern was here ####
- Partition
- base file name: ds=2008-04-09
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
- Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [$hdt$_0:a]
- /bucket_big/ds=2008-04-09 [$hdt$_0:a]
- Needs Tagging: false
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- File Output Operator
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
-#### A masked pattern was here ####
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- properties:
- columns _col0
- columns.types bigint
- escape.delim \
- hive.serialization.extend.additional.nesting.levels true
- serialization.escape.crlf true
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@bucket_big
-PREHOOK: Input: default@bucket_big@ds=2008-04-08
-PREHOOK: Input: default@bucket_big@ds=2008-04-09
-PREHOOK: Input: default@bucket_small
-PREHOOK: Input: default@bucket_small@ds=2008-04-08
-#### A masked pattern was here ####
-POSTHOOK: query: select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@bucket_big
-POSTHOOK: Input: default@bucket_big@ds=2008-04-08
-POSTHOOK: Input: default@bucket_big@ds=2008-04-09
-POSTHOOK: Input: default@bucket_small
-POSTHOOK: Input: default@bucket_small@ds=2008-04-08
-#### A masked pattern was here ####
-38
-PREHOOK: query: explain extended select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key
-PREHOOK: type: QUERY
-POSTHOOK: query: explain extended select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-5 is a root stage , consists of Stage-6, Stage-7, Stage-1
- Stage-6 has a backup stage: Stage-1
- Stage-3 depends on stages: Stage-6
- Stage-7 has a backup stage: Stage-1
- Stage-4 depends on stages: Stage-7
- Stage-1
- Stage-0 depends on stages: Stage-3, Stage-4, Stage-1
-
-STAGE PLANS:
- Stage: Stage-5
- Conditional Operator
-
- Stage: Stage-6
- Map Reduce Local Work
- Alias -> Map Local Tables:
- $hdt$_1:b
- Fetch Operator
- limit: -1
- Partition Description:
- Partition
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- properties:
- bucket_count 2
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_small
- numFiles 2
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_small { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 114
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 2
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_small
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_small { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_small
- name: default.bucket_small
- Alias -> Map Local Operator Tree:
- $hdt$_1:b
- TableScan
- alias: b
- Statistics: Num rows: 2 Data size: 1140 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 1140 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 1140 Basic stats: COMPLETE Column stats: NONE
- HashTable Sink Operator
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- Position of Big Table: 0
-
- Stage: Stage-3
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: a
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- Position of Big Table: 0
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- null sort order:
- sort order:
- tag: -1
- value expressions: _col0 (type: bigint)
- auto parallelism: false
- Local Work:
- Map Reduce Local Work
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: ds=2008-04-08
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
-#### A masked pattern was here ####
- Partition
- base file name: ds=2008-04-09
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
-#### A masked pattern was here ####
- Partition
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- properties:
- bucket_count 2
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_small
- numFiles 2
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_small { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 114
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 2
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_small
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_small { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_small
- name: default.bucket_small
- Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [$hdt$_0:a]
- /bucket_big/ds=2008-04-09 [$hdt$_0:a]
- Needs Tagging: false
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- File Output Operator
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
-#### A masked pattern was here ####
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- properties:
- columns _col0
- columns.types bigint
- escape.delim \
- hive.serialization.extend.additional.nesting.levels true
- serialization.escape.crlf true
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
-
- Stage: Stage-7
- Map Reduce Local Work
- Alias -> Map Local Tables:
- $hdt$_0:a
- Fetch Operator
- limit: -1
- Partition Description:
- Partition
- base file name: ds=2008-04-08
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
- Partition
- base file name: ds=2008-04-09
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
- Alias -> Map Local Operator Tree:
- $hdt$_0:a
- TableScan
- alias: a
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- HashTable Sink Operator
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- Position of Big Table: 1
-
- Stage: Stage-4
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: b
- Statistics: Num rows: 2 Data size: 1140 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 1140 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 1140 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- Position of Big Table: 1
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- null sort order:
- sort order:
- tag: -1
- value expressions: _col0 (type: bigint)
- auto parallelism: false
- Local Work:
- Map Reduce Local Work
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: ds=2008-04-08
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
-#### A masked pattern was here ####
- Partition
- base file name: ds=2008-04-09
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
-#### A masked pattern was here ####
- Partition
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- properties:
- bucket_count 2
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_small
- numFiles 2
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_small { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 114
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 2
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_small
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_small { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_small
- name: default.bucket_small
- Truncated Path -> Alias:
- /bucket_small/ds=2008-04-08 [$hdt$_1:b]
- Needs Tagging: false
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- File Output Operator
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
-#### A masked pattern was here ####
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- properties:
- columns _col0
- columns.types bigint
- escape.delim \
- hive.serialization.extend.additional.nesting.levels true
- serialization.escape.crlf true
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
-
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: a
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- GatherStats: false
- Filter Operator
- isSamplingPred: false
- predicate: key is not null (type: boolean)
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
- Statistics: Num rows: 240 Data size: 116240 Basic stats: COMPLETE Column stats: NONE
- Sorted Merge Bucket Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- Position of Big Table: 0
- BucketMapJoin: true
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Reduce Output Operator
- null sort order:
- sort order:
- tag: -1
- value expressions: _col0 (type: bigint)
- auto parallelism: false
- Path -> Alias:
-#### A masked pattern was here ####
- Path -> Partition:
-#### A masked pattern was here ####
- Partition
- base file name: ds=2008-04-08
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-08
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
-#### A masked pattern was here ####
- Partition
- base file name: ds=2008-04-09
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- partition values:
- ds 2008-04-09
- properties:
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- numFiles 4
- numRows 0
- partition_columns ds
- partition_columns.types string
- rawDataSize 0
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- totalSize 5812
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- properties:
- SORTBUCKETCOLSPREFIX TRUE
- bucket_count 4
- bucket_field_name key
- column.name.delimiter ,
- columns key,value
- columns.comments
- columns.types string:string
-#### A masked pattern was here ####
- name default.bucket_big
- partition_columns ds
- partition_columns.types string
- serialization.ddl struct bucket_big { string key, string value}
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-#### A masked pattern was here ####
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.bucket_big
- name: default.bucket_big
- Truncated Path -> Alias:
- /bucket_big/ds=2008-04-08 [$hdt$_0:a]
- /bucket_big/ds=2008-04-09 [$hdt$_0:a]
- Needs Tagging: false
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- File Output Operator
- compressed: false
- GlobalTableId: 0
-#### A masked pattern was here ####
- NumFilesPerFileSink: 1
-#### A masked pattern was here ####
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- properties:
- columns _col0
- columns.types bigint
- escape.delim \
- hive.serialization.extend.additional.nesting.levels true
- serialization.escape.crlf true
- serialization.format 1
- serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- TotalFiles: 1
- GatherStats: false
- MultiFileSpray: false
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key
-PREHOOK: type: QUERY
-PREHOOK: Input: default@bucket_big
-PREHOOK: Input: default@bucket_big@ds=2008-04-08
-PREHOOK: Input: default@bucket_big@ds=2008-04-09
-PREHOOK: Input: default@bucket_small
-PREHOOK: Input: default@bucket_small@ds=2008-04-08
-#### A masked pattern was here ####
-POSTHOOK: query: select count(*) FROM bucket_big a JOIN bucket_small b ON a.key = b.key
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@bucket_big
-POSTHOOK: Input: default@bucket_big@ds=2008-04-08
-POSTHOOK: Input: default@bucket_big@ds=2008-04-09
-POSTHOOK: Input: default@bucket_small
-POSTHOOK: Input: default@bucket_small@ds=2008-04-08
-#### A masked pattern was here ####
-38