You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2015/11/20 23:33:18 UTC
[51/51] [partial] hive git commit: HIVE-12017: Do not disable CBO by
default when number of joins in a query is equal or less than 1 (Jesus
Camacho Rodriguez, reviewed by Ashutosh Chauhan)
HIVE-12017: Do not disable CBO by default when number of joins in a query is equal or less than 1 (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7dab21ac
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7dab21ac
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7dab21ac
Branch: refs/heads/master
Commit: 7dab21acffdd43e80e6fefb5011139bbf30fa541
Parents: 55b589e
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Sat Nov 14 09:12:49 2015 +0100
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Fri Nov 20 23:31:11 2015 +0100
----------------------------------------------------------------------
.../test/results/positive/hbase_queries.q.out | 12 +-
.../ql/optimizer/calcite/HiveRelOptUtil.java | 40 +
.../rules/HiveAggregateProjectMergeRule.java | 13 +-
.../translator/SqlFunctionConverter.java | 8 +
.../hadoop/hive/ql/parse/CalcitePlanner.java | 169 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 16 -
ql/src/test/queries/clientpositive/mergejoin.q | 9 +
.../bucket_mapjoin_mismatch1.q.out | 36 +-
.../clientnegative/join_nonexistent_part.q.out | 1 -
.../annotate_stats_join_pkfk.q.out | 230 +-
.../archive_excludeHadoop20.q.out | 1 +
.../results/clientpositive/archive_multi.q.out | 1 +
.../results/clientpositive/auto_join1.q.out | 12 +-
.../results/clientpositive/auto_join10.q.out | 12 +-
.../results/clientpositive/auto_join11.q.out | 12 +-
.../results/clientpositive/auto_join14.q.out | 8 +-
.../results/clientpositive/auto_join24.q.out | 54 +-
.../results/clientpositive/auto_join26.q.out | 30 +-
.../results/clientpositive/auto_join32.q.out | 153 +-
.../clientpositive/auto_join_filters.q.out | 12 +-
.../clientpositive/auto_join_nulls.q.out | 2 +-
.../auto_join_reordering_values.q.out | 152 +-
.../clientpositive/auto_join_stats.q.out | 392 ++--
.../clientpositive/auto_join_stats2.q.out | 214 +-
.../clientpositive/auto_smb_mapjoin_14.q.out | 145 +-
.../clientpositive/auto_sortmerge_join_1.q.out | 240 +-
.../clientpositive/auto_sortmerge_join_10.q.out | 103 +-
.../clientpositive/auto_sortmerge_join_11.q.out | 124 +-
.../clientpositive/auto_sortmerge_join_12.q.out | 177 +-
.../clientpositive/auto_sortmerge_join_14.q.out | 152 +-
.../clientpositive/auto_sortmerge_join_15.q.out | 152 +-
.../clientpositive/auto_sortmerge_join_2.q.out | 198 +-
.../clientpositive/auto_sortmerge_join_3.q.out | 234 +-
.../clientpositive/auto_sortmerge_join_4.q.out | 234 +-
.../clientpositive/auto_sortmerge_join_5.q.out | 232 +-
.../clientpositive/auto_sortmerge_join_6.q.out | 1140 ++++------
.../clientpositive/auto_sortmerge_join_7.q.out | 242 +-
.../clientpositive/auto_sortmerge_join_8.q.out | 242 +-
.../clientpositive/auto_sortmerge_join_9.q.out | 1240 +++++-----
.../clientpositive/bucket_map_join_spark1.q.out | 206 +-
.../clientpositive/bucket_map_join_spark2.q.out | 206 +-
.../clientpositive/bucket_map_join_spark3.q.out | 206 +-
.../clientpositive/bucket_map_join_spark4.q.out | 236 +-
.../bucketsortoptimize_insert_2.q.out | 218 +-
.../bucketsortoptimize_insert_4.q.out | 112 +-
.../bucketsortoptimize_insert_5.q.out | 142 +-
.../bucketsortoptimize_insert_6.q.out | 554 +++--
.../bucketsortoptimize_insert_7.q.out | 48 +-
.../bucketsortoptimize_insert_8.q.out | 76 +-
.../cbo_rp_cross_product_check_2.q.out | 468 ++--
.../results/clientpositive/cbo_rp_join1.q.out | 60 +-
.../clientpositive/cbo_rp_lineage2.q.out | 18 +-
.../clientpositive/column_access_stats.q.out | 84 +-
.../results/clientpositive/constprog2.q.out | 20 +-
.../clientpositive/constprog_partitioner.q.out | 10 +-
.../clientpositive/correlationoptimizer1.q.out | 886 ++++----
.../clientpositive/correlationoptimizer11.q.out | 120 +-
.../clientpositive/correlationoptimizer13.q.out | 26 +-
.../clientpositive/correlationoptimizer4.q.out | 972 ++++----
.../clientpositive/correlationoptimizer5.q.out | 248 +-
.../clientpositive/correlationoptimizer9.q.out | 250 +-
.../results/clientpositive/create_view.q.out | 2 +-
.../clientpositive/cross_product_check_1.q.out | 252 ++-
.../clientpositive/cross_product_check_2.q.out | 332 +--
.../results/clientpositive/decimal_join2.q.out | 90 +-
.../clientpositive/dynamic_rdd_cache.q.out | 265 ++-
.../encryption_join_unencrypted_tbl.q.out | 98 +-
...on_join_with_different_encryption_keys.q.out | 102 +-
.../clientpositive/explain_logical.q.out | 142 +-
.../clientpositive/explain_rearrange.q.out | 288 +--
.../clientpositive/filter_join_breaktask.q.out | 86 +-
.../clientpositive/groupby_grouping_sets4.q.out | 8 +-
.../clientpositive/groupby_sort_1_23.q.out | 124 +-
.../clientpositive/groupby_sort_skew_1_23.q.out | 124 +-
.../test/results/clientpositive/having2.q.out | 190 +-
.../clientpositive/index_auto_mult_tables.q.out | 110 +-
.../index_auto_mult_tables_compact.q.out | 110 +-
.../results/clientpositive/index_bitmap3.q.out | 2 +-
.../clientpositive/index_bitmap_auto.q.out | 2 +-
.../infer_bucket_sort_map_operators.q.out | 8 +-
.../test/results/clientpositive/innerjoin.q.out | 14 +-
ql/src/test/results/clientpositive/join1.q.out | 14 +-
ql/src/test/results/clientpositive/join10.q.out | 28 +-
ql/src/test/results/clientpositive/join11.q.out | 14 +-
ql/src/test/results/clientpositive/join14.q.out | 34 +-
ql/src/test/results/clientpositive/join19.q.out | 10 +-
ql/src/test/results/clientpositive/join29.q.out | 44 +-
ql/src/test/results/clientpositive/join31.q.out | 112 +-
ql/src/test/results/clientpositive/join40.q.out | 14 +-
ql/src/test/results/clientpositive/join41.q.out | 108 +-
ql/src/test/results/clientpositive/join42.q.out | 157 +-
ql/src/test/results/clientpositive/join43.q.out | 40 +-
.../join_cond_pushdown_unqual1.q.out | 252 ++-
.../join_cond_pushdown_unqual2.q.out | 186 +-
.../join_cond_pushdown_unqual3.q.out | 292 +--
.../join_cond_pushdown_unqual4.q.out | 196 +-
.../results/clientpositive/join_filters.q.out | 12 +-
.../clientpositive/join_filters_overlap.q.out | 604 ++---
.../clientpositive/join_grp_diff_keys.q.out | 58 +-
.../results/clientpositive/join_hive_626.q.out | 62 +-
.../test/results/clientpositive/join_rc.q.out | 34 +-
.../results/clientpositive/join_reorder.q.out | 104 +-
.../test/results/clientpositive/join_star.q.out | 486 ++--
.../test/results/clientpositive/join_vc.q.out | 76 +-
.../test/results/clientpositive/lineage2.q.out | 10 +-
.../llap/bucket_map_join_tez1.q.out | 832 ++++---
.../llap/bucket_map_join_tez2.q.out | 212 +-
.../clientpositive/llap/constprog_dpp.q.out | 63 +-
.../llap/dynamic_partition_pruning.q.out | 2114 +++++++++--------
.../llap/dynamic_partition_pruning_2.q.out | 503 +++--
.../clientpositive/llap/explainuser_1.q.out | 610 ++---
.../clientpositive/llap/explainuser_2.q.out | 68 +-
.../llap/hybridgrace_hashjoin_1.q.out | 333 +--
.../clientpositive/llap/mapjoin_decimal.q.out | 40 +-
.../test/results/clientpositive/llap/mrr.q.out | 27 +-
.../llap/tez_bmj_schema_evolution.q.out | 44 +-
.../results/clientpositive/llap/tez_join.q.out | 21 +-
.../clientpositive/llap/tez_join_hash.q.out | 52 +-
.../llap/tez_join_result_complex.q.out | 618 ++++-
.../clientpositive/llap/tez_self_join.q.out | 46 +-
.../results/clientpositive/llap/tez_smb_1.q.out | 192 +-
.../clientpositive/llap/tez_smb_main.q.out | 548 +++--
.../vectorized_dynamic_partition_pruning.q.out | 2130 ++++++++++--------
.../clientpositive/louter_join_ppr.q.out | 76 +-
.../clientpositive/mapjoin_memcheck.q.out | 38 +-
.../clientpositive/mapjoin_subquery2.q.out | 82 +-
.../test/results/clientpositive/mergejoin.q.out | 926 ++++++--
.../results/clientpositive/mergejoins.q.out | 102 +-
.../clientpositive/mergejoins_mixed.q.out | 1110 +++++----
.../results/clientpositive/multiMapJoin1.q.out | 1316 ++++++-----
.../results/clientpositive/multiMapJoin2.q.out | 50 +-
.../clientpositive/multi_join_union.q.out | 66 +-
.../test/results/clientpositive/orc_llap.q.out | 128 +-
.../results/clientpositive/parallel_join1.q.out | 14 +-
.../results/clientpositive/parquet_join.q.out | 124 +-
ql/src/test/results/clientpositive/pcr.q.out | 166 +-
.../results/clientpositive/pointlookup2.q.out | 364 +--
.../results/clientpositive/pointlookup3.q.out | 268 ++-
.../test/results/clientpositive/ppd_join5.q.out | 112 +-
.../clientpositive/ppd_outer_join5.q.out | 280 ++-
.../clientpositive/ppd_repeated_alias.q.out | 256 ++-
.../results/clientpositive/ppd_union_view.q.out | 48 +-
.../results/clientpositive/ptfgroupbyjoin.q.out | 150 +-
.../results/clientpositive/quotedid_skew.q.out | 96 +-
.../runtime_skewjoin_mapjoin_spark.q.out | 174 +-
.../test/results/clientpositive/skewjoin.q.out | 36 +-
.../clientpositive/skewjoin_mapjoin1.q.out | 344 +--
.../clientpositive/skewjoin_mapjoin10.q.out | 332 +--
.../clientpositive/skewjoin_mapjoin11.q.out | 68 +-
.../clientpositive/skewjoin_mapjoin2.q.out | 164 +-
.../clientpositive/skewjoin_mapjoin3.q.out | 68 +-
.../clientpositive/skewjoin_mapjoin4.q.out | 112 +-
.../clientpositive/skewjoin_mapjoin5.q.out | 94 +-
.../clientpositive/skewjoin_mapjoin7.q.out | 256 ++-
.../clientpositive/skewjoin_mapjoin8.q.out | 60 +-
.../clientpositive/skewjoin_mapjoin9.q.out | 124 +-
.../clientpositive/skewjoin_onesideskew.q.out | 30 +-
.../skewjoin_union_remove_1.q.out | 392 ++--
.../skewjoin_union_remove_2.q.out | 126 +-
.../results/clientpositive/skewjoinopt1.q.out | 296 ++-
.../results/clientpositive/skewjoinopt11.q.out | 168 +-
.../results/clientpositive/skewjoinopt12.q.out | 88 +-
.../results/clientpositive/skewjoinopt13.q.out | 70 +-
.../results/clientpositive/skewjoinopt14.q.out | 102 +-
.../results/clientpositive/skewjoinopt15.q.out | 296 ++-
.../results/clientpositive/skewjoinopt16.q.out | 88 +-
.../results/clientpositive/skewjoinopt17.q.out | 184 +-
.../results/clientpositive/skewjoinopt18.q.out | 50 +-
.../results/clientpositive/skewjoinopt19.q.out | 96 +-
.../results/clientpositive/skewjoinopt2.q.out | 280 ++-
.../results/clientpositive/skewjoinopt20.q.out | 96 +-
.../results/clientpositive/skewjoinopt21.q.out | 32 +-
.../results/clientpositive/skewjoinopt3.q.out | 192 +-
.../results/clientpositive/skewjoinopt4.q.out | 192 +-
.../results/clientpositive/skewjoinopt5.q.out | 96 +-
.../results/clientpositive/skewjoinopt6.q.out | 96 +-
.../results/clientpositive/skewjoinopt7.q.out | 126 +-
.../results/clientpositive/skewjoinopt8.q.out | 126 +-
.../results/clientpositive/skewjoinopt9.q.out | 72 +-
.../results/clientpositive/smb_mapjoin_17.q.out | 248 +-
.../results/clientpositive/smb_mapjoin_25.q.out | 385 +++-
.../clientpositive/spark/auto_join1.q.out | 12 +-
.../clientpositive/spark/auto_join10.q.out | 12 +-
.../clientpositive/spark/auto_join11.q.out | 12 +-
.../clientpositive/spark/auto_join14.q.out | 10 +-
.../clientpositive/spark/auto_join24.q.out | 54 +-
.../clientpositive/spark/auto_join26.q.out | 36 +-
.../clientpositive/spark/auto_join32.q.out | 231 +-
.../spark/auto_join_filters.q.out | 12 +-
.../clientpositive/spark/auto_join_nulls.q.out | 2 +-
.../spark/auto_join_reordering_values.q.out | 132 +-
.../clientpositive/spark/auto_join_stats.q.out | 156 +-
.../clientpositive/spark/auto_join_stats2.q.out | 190 +-
.../spark/auto_smb_mapjoin_14.q.out | 131 +-
.../spark/auto_sortmerge_join_1.q.out | 152 +-
.../spark/auto_sortmerge_join_10.q.out | 27 +-
.../spark/auto_sortmerge_join_12.q.out | 124 +-
.../spark/auto_sortmerge_join_14.q.out | 100 +-
.../spark/auto_sortmerge_join_15.q.out | 100 +-
.../spark/auto_sortmerge_join_2.q.out | 106 +-
.../spark/auto_sortmerge_join_3.q.out | 148 +-
.../spark/auto_sortmerge_join_4.q.out | 148 +-
.../spark/auto_sortmerge_join_5.q.out | 148 +-
.../spark/auto_sortmerge_join_6.q.out | 450 ++--
.../spark/auto_sortmerge_join_7.q.out | 152 +-
.../spark/auto_sortmerge_join_8.q.out | 152 +-
.../spark/auto_sortmerge_join_9.q.out | 736 +++---
.../spark/bucket_map_join_spark1.q.out | 204 +-
.../spark/bucket_map_join_spark2.q.out | 206 +-
.../spark/bucket_map_join_spark3.q.out | 204 +-
.../spark/bucket_map_join_spark4.q.out | 230 +-
.../spark/bucket_map_join_tez1.q.out | 888 +++++---
.../spark/bucket_map_join_tez2.q.out | 248 +-
.../spark/bucketsortoptimize_insert_2.q.out | 156 +-
.../spark/bucketsortoptimize_insert_4.q.out | 46 +-
.../spark/bucketsortoptimize_insert_6.q.out | 364 +--
.../spark/bucketsortoptimize_insert_7.q.out | 80 +-
.../spark/bucketsortoptimize_insert_8.q.out | 104 +-
.../spark/column_access_stats.q.out | 84 +-
.../spark/constprog_partitioner.q.out | 10 +-
.../spark/cross_product_check_1.q.out | 296 +--
.../spark/cross_product_check_2.q.out | 294 +--
.../spark/dynamic_rdd_cache.q.out | 286 ++-
.../spark/filter_join_breaktask.q.out | 80 +-
.../spark/groupby_sort_1_23.q.out | 118 +-
.../spark/groupby_sort_skew_1_23.q.out | 118 +-
.../clientpositive/spark/index_bitmap3.q.out | 2 +-
.../spark/index_bitmap_auto.q.out | 2 +-
.../spark/infer_bucket_sort_map_operators.q.out | 8 +-
.../clientpositive/spark/innerjoin.q.out | 14 +-
.../results/clientpositive/spark/join1.q.out | 14 +-
.../results/clientpositive/spark/join10.q.out | 28 +-
.../results/clientpositive/spark/join11.q.out | 14 +-
.../results/clientpositive/spark/join14.q.out | 34 +-
.../results/clientpositive/spark/join19.q.out | 10 +-
.../results/clientpositive/spark/join29.q.out | 18 +-
.../results/clientpositive/spark/join31.q.out | 44 +-
.../results/clientpositive/spark/join40.q.out | 14 +-
.../results/clientpositive/spark/join41.q.out | 108 +-
.../spark/join_cond_pushdown_unqual1.q.out | 252 ++-
.../spark/join_cond_pushdown_unqual2.q.out | 186 +-
.../spark/join_cond_pushdown_unqual3.q.out | 292 +--
.../spark/join_cond_pushdown_unqual4.q.out | 196 +-
.../spark/join_filters_overlap.q.out | 628 +++---
.../clientpositive/spark/join_hive_626.q.out | 62 +-
.../results/clientpositive/spark/join_rc.q.out | 34 +-
.../clientpositive/spark/join_reorder.q.out | 104 +-
.../clientpositive/spark/join_star.q.out | 468 ++--
.../results/clientpositive/spark/join_vc.q.out | 76 +-
.../clientpositive/spark/louter_join_ppr.q.out | 172 +-
.../clientpositive/spark/mapjoin_decimal.q.out | 36 +-
.../clientpositive/spark/mapjoin_memcheck.q.out | 38 +-
.../spark/mapjoin_subquery2.q.out | 82 +-
.../clientpositive/spark/mergejoins.q.out | 102 +-
.../clientpositive/spark/mergejoins_mixed.q.out | 1110 +++++----
.../clientpositive/spark/multi_join_union.q.out | 94 +-
.../clientpositive/spark/parallel_join1.q.out | 14 +-
.../clientpositive/spark/parquet_join.q.out | 126 +-
.../test/results/clientpositive/spark/pcr.q.out | 124 +-
.../clientpositive/spark/ppd_join5.q.out | 112 +-
.../clientpositive/spark/ppd_outer_join5.q.out | 258 ++-
.../spark/runtime_skewjoin_mapjoin_spark.q.out | 74 +-
.../results/clientpositive/spark/skewjoin.q.out | 36 +-
.../spark/skewjoin_union_remove_1.q.out | 392 ++--
.../spark/skewjoin_union_remove_2.q.out | 124 +-
.../clientpositive/spark/skewjoinopt1.q.out | 288 ++-
.../clientpositive/spark/skewjoinopt11.q.out | 136 +-
.../clientpositive/spark/skewjoinopt12.q.out | 88 +-
.../clientpositive/spark/skewjoinopt13.q.out | 70 +-
.../clientpositive/spark/skewjoinopt14.q.out | 98 +-
.../clientpositive/spark/skewjoinopt15.q.out | 288 ++-
.../clientpositive/spark/skewjoinopt16.q.out | 88 +-
.../clientpositive/spark/skewjoinopt17.q.out | 184 +-
.../clientpositive/spark/skewjoinopt18.q.out | 50 +-
.../clientpositive/spark/skewjoinopt19.q.out | 96 +-
.../clientpositive/spark/skewjoinopt2.q.out | 272 ++-
.../clientpositive/spark/skewjoinopt20.q.out | 96 +-
.../clientpositive/spark/skewjoinopt3.q.out | 192 +-
.../clientpositive/spark/skewjoinopt4.q.out | 192 +-
.../clientpositive/spark/skewjoinopt5.q.out | 96 +-
.../clientpositive/spark/skewjoinopt6.q.out | 96 +-
.../clientpositive/spark/skewjoinopt7.q.out | 124 +-
.../clientpositive/spark/skewjoinopt8.q.out | 124 +-
.../clientpositive/spark/skewjoinopt9.q.out | 72 +-
.../clientpositive/spark/smb_mapjoin_17.q.out | 300 +--
.../clientpositive/spark/smb_mapjoin_25.q.out | 245 +-
.../clientpositive/spark/subquery_in.q.out | 4 +-
.../spark/table_access_keys_stats.q.out | 37 +-
.../clientpositive/spark/temp_table_join1.q.out | 106 +-
.../results/clientpositive/spark/union22.q.out | 110 +-
.../results/clientpositive/spark/union24.q.out | 90 +-
.../results/clientpositive/spark/union27.q.out | 54 +-
.../results/clientpositive/spark/union32.q.out | 112 +-
.../clientpositive/spark/union_remove_12.q.out | 68 +-
.../clientpositive/spark/union_remove_13.q.out | 72 +-
.../clientpositive/spark/union_remove_14.q.out | 68 +-
.../clientpositive/spark/union_remove_23.q.out | 40 +-
.../spark/vector_decimal_mapjoin.q.out | 36 +-
.../spark/vector_inner_join.q.out | 422 ++--
.../spark/vector_mapjoin_reduce.q.out | 8 +-
.../spark/vector_outer_join0.q.out | 72 +-
.../results/clientpositive/subquery_in.q.out | 4 +-
.../results/clientpositive/subquery_notin.q.out | 128 +-
.../subquery_notin_having.q.java1.7.out | 168 +-
.../table_access_keys_stats.q.out | 37 +-
.../clientpositive/temp_table_join1.q.out | 106 +-
.../results/clientpositive/tez/auto_join1.q.out | 13 +-
.../clientpositive/tez/auto_join_filters.q.out | 12 +-
.../clientpositive/tez/auto_join_nulls.q.out | 2 +-
.../tez/auto_sortmerge_join_1.q.out | 192 +-
.../tez/auto_sortmerge_join_10.q.out | 123 +-
.../tez/auto_sortmerge_join_11.q.out | 128 +-
.../tez/auto_sortmerge_join_12.q.out | 146 +-
.../tez/auto_sortmerge_join_14.q.out | 104 +-
.../tez/auto_sortmerge_join_15.q.out | 104 +-
.../tez/auto_sortmerge_join_2.q.out | 128 +-
.../tez/auto_sortmerge_join_3.q.out | 192 +-
.../tez/auto_sortmerge_join_4.q.out | 192 +-
.../tez/auto_sortmerge_join_5.q.out | 152 +-
.../tez/auto_sortmerge_join_6.q.out | 500 ++--
.../tez/auto_sortmerge_join_7.q.out | 192 +-
.../tez/auto_sortmerge_join_8.q.out | 192 +-
.../tez/auto_sortmerge_join_9.q.out | 1074 +++++----
.../tez/bucket_map_join_tez1.q.out | 832 ++++---
.../tez/bucket_map_join_tez2.q.out | 212 +-
.../clientpositive/tez/constprog_dpp.q.out | 63 +-
.../tez/correlationoptimizer1.q.out | 848 ++++---
.../tez/cross_product_check_1.q.out | 296 +--
.../tez/cross_product_check_2.q.out | 322 +--
.../tez/dynamic_partition_pruning.q.out | 2085 +++++++++--------
.../tez/dynamic_partition_pruning_2.q.out | 503 +++--
.../clientpositive/tez/explainuser_1.q.out | 618 ++---
.../clientpositive/tez/explainuser_2.q.out | 361 +--
.../clientpositive/tez/explainuser_3.q.out | 46 +-
.../tez/filter_join_breaktask.q.out | 80 +-
.../tez/hybridgrace_hashjoin_1.q.out | 333 +--
.../test/results/clientpositive/tez/join1.q.out | 14 +-
.../clientpositive/tez/mapjoin_decimal.q.out | 40 +-
.../results/clientpositive/tez/mergejoin.q.out | 922 ++++++--
.../test/results/clientpositive/tez/mrr.q.out | 27 +-
.../results/clientpositive/tez/skewjoin.q.out | 28 +-
.../clientpositive/tez/subquery_in.q.out | 4 +-
.../tez/tez_bmj_schema_evolution.q.out | 44 +-
.../results/clientpositive/tez/tez_join.q.out | 21 +-
.../clientpositive/tez/tez_join_hash.q.out | 52 +-
.../tez/tez_join_result_complex.q.out | 618 ++++-
.../clientpositive/tez/tez_self_join.q.out | 46 +-
.../results/clientpositive/tez/tez_smb_1.q.out | 188 +-
.../clientpositive/tez/tez_smb_empty.q.out | 134 +-
.../clientpositive/tez/tez_smb_main.q.out | 548 +++--
.../clientpositive/tez/unionDistinct_1.q.out | 650 +++---
.../tez/vector_auto_smb_mapjoin_14.q.out | 296 +--
.../tez/vector_binary_join_groupby.q.out | 66 +-
.../tez/vector_char_mapjoin1.q.out | 122 +-
.../tez/vector_decimal_mapjoin.q.out | 40 +-
.../clientpositive/tez/vector_inner_join.q.out | 452 ++--
.../tez/vector_join_filters.q.out | 6 +-
.../clientpositive/tez/vector_join_nulls.q.out | 2 +-
.../tez/vector_left_outer_join2.q.out | 378 ++--
.../tez/vector_mapjoin_reduce.q.out | 8 +-
.../tez/vector_mr_diff_schema_alias.q.out | 83 +-
.../clientpositive/tez/vector_outer_join0.q.out | 80 +-
.../tez/vector_varchar_mapjoin1.q.out | 122 +-
.../clientpositive/tez/vectorized_context.q.out | 84 +-
.../vectorized_dynamic_partition_pruning.q.out | 2102 +++++++++--------
.../clientpositive/udf_folder_constants.q.out | 14 +-
.../test/results/clientpositive/union22.q.out | 152 +-
.../test/results/clientpositive/union24.q.out | 110 +-
.../test/results/clientpositive/union27.q.out | 62 +-
.../test/results/clientpositive/union32.q.out | 128 +-
.../test/results/clientpositive/union34.q.out | 4 +-
.../clientpositive/unionDistinct_1.q.out | 616 ++---
.../clientpositive/union_remove_12.q.out | 64 +-
.../clientpositive/union_remove_13.q.out | 68 +-
.../clientpositive/union_remove_14.q.out | 64 +-
.../clientpositive/union_remove_23.q.out | 40 +-
.../vector_auto_smb_mapjoin_14.q.out | 145 +-
.../vector_binary_join_groupby.q.out | 62 +-
.../clientpositive/vector_char_mapjoin1.q.out | 110 +-
.../clientpositive/vector_decimal_mapjoin.q.out | 36 +-
.../clientpositive/vector_inner_join.q.out | 408 ++--
.../vector_interval_mapjoin.q.out | 4 +-
.../clientpositive/vector_join_filters.q.out | 6 +-
.../clientpositive/vector_join_nulls.q.out | 2 +-
.../vector_left_outer_join2.q.out | 396 ++--
.../clientpositive/vector_mapjoin_reduce.q.out | 32 +-
.../vector_mr_diff_schema_alias.q.out | 85 +-
.../clientpositive/vector_outer_join0.q.out | 72 +-
.../vector_varchar_mapjoin1.q.out | 110 +-
.../clientpositive/vectorized_context.q.out | 96 +-
390 files changed, 42412 insertions(+), 32574 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/hbase-handler/src/test/results/positive/hbase_queries.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out
index d044c7e..fa348e3 100644
--- a/hbase-handler/src/test/results/positive/hbase_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -155,10 +155,10 @@ STAGE PLANS:
keys:
0 UDFToDouble(_col0) (type: double)
1 UDFToDouble(_col0) (type: double)
- outputColumnNames: _col2, _col3
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col2 (type: string), _col3 (type: string)
+ expressions: _col1 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
@@ -271,7 +271,7 @@ STAGE PLANS:
alias: hbase_table_1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: (100 < key) (type: boolean)
+ predicate: (((100 < key) and (key < 120)) and key is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int)
@@ -286,7 +286,7 @@ STAGE PLANS:
alias: hbase_table_2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: (key < 120) (type: boolean)
+ predicate: (((key < 120) and (100 < key)) and key is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
@@ -305,10 +305,10 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2, _col3
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
- expressions: _col2 (type: int), _col3 (type: string)
+ expressions: _col1 (type: int), _col2 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
index b4e7d47..ea87719 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
@@ -1,11 +1,13 @@
package org.apache.hadoop.hive.ql.optimizer.calcite;
+import java.util.AbstractList;
import java.util.ArrayList;
import java.util.List;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
@@ -289,5 +291,43 @@ public class HiveRelOptUtil extends RelOptUtil {
}
}
+ /**
+ * Creates a relational expression that projects the given fields of the
+ * input.
+ *
+ * <p>Optimizes if the fields are the identity projection.
+ *
+ * @param factory ProjectFactory
+ * @param child Input relational expression
+ * @param posList Source of each projected field
+ * @return Relational expression that projects given fields
+ */
+ public static RelNode createProject(final RelFactories.ProjectFactory factory,
+ final RelNode child, final List<Integer> posList) {
+ RelDataType rowType = child.getRowType();
+ final List<String> fieldNames = rowType.getFieldNames();
+ final RexBuilder rexBuilder = child.getCluster().getRexBuilder();
+ return createProject(child,
+ new AbstractList<RexNode>() {
+ public int size() {
+ return posList.size();
+ }
+
+ public RexNode get(int index) {
+ final int pos = posList.get(index);
+ return rexBuilder.makeInputRef(child, pos);
+ }
+ },
+ new AbstractList<String>() {
+ public int size() {
+ return posList.size();
+ }
+
+ public String get(int index) {
+ final int pos = posList.get(index);
+ return fieldNames.get(pos);
+ }
+ }, true, factory);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
index 53f04ee..0df24a3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
@@ -16,25 +16,25 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
/**
* Planner rule that recognizes a {@link HiveAggregate}
* on top of a {@link HiveProject} and if possible
@@ -140,8 +140,9 @@ public class HiveAggregateProjectMergeRule extends RelOptRule {
i < newAggregate.getRowType().getFieldCount(); i++) {
posList.add(i);
}
- rel = RelOptUtil.createProject(HiveProject.DEFAULT_PROJECT_FACTORY,
+ rel = HiveRelOptUtil.createProject(HiveProject.DEFAULT_PROJECT_FACTORY,
rel, posList);
+
}
return rel;
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
index a17fb94..bcb9ea7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java
@@ -302,6 +302,7 @@ public class SqlFunctionConverter {
registerFunction("and", SqlStdOperatorTable.AND, hToken(HiveParser.KW_AND, "and"));
registerFunction("or", SqlStdOperatorTable.OR, hToken(HiveParser.KW_OR, "or"));
registerFunction("=", SqlStdOperatorTable.EQUALS, hToken(HiveParser.EQUAL, "="));
+ registerDuplicateFunction("==", SqlStdOperatorTable.EQUALS, hToken(HiveParser.EQUAL, "="));
registerFunction("<", SqlStdOperatorTable.LESS_THAN, hToken(HiveParser.LESSTHAN, "<"));
registerFunction("<=", SqlStdOperatorTable.LESS_THAN_OR_EQUAL,
hToken(HiveParser.LESSTHANOREQUALTO, "<="));
@@ -334,6 +335,13 @@ public class SqlFunctionConverter {
}
}
}
+
+ private void registerDuplicateFunction(String name, SqlOperator calciteFn, HiveToken hiveToken) {
+ hiveToCalcite.put(name, calciteFn);
+ if (hiveToken != null) {
+ calciteToHiveToken.put(calciteFn, hiveToken);
+ }
+ }
}
private static HiveToken hToken(int type, String text) {
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index bfe4633..c005b1a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -24,6 +24,7 @@ import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -201,12 +202,15 @@ public class CalcitePlanner extends SemanticAnalyzer {
private final AtomicInteger noColsMissingStats = new AtomicInteger(0);
private SemanticException semanticException;
- private boolean runCBO = true;
+ private boolean runCBO = true;
+ private boolean disableSemJoinReordering = true;
+ private EnumSet<ExtendedCBOProfile> profilesCBO;
public CalcitePlanner(HiveConf conf) throws SemanticException {
super(conf);
if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED)) {
runCBO = false;
+ disableSemJoinReordering = false;
}
}
@@ -242,6 +246,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query
}
runCBO = canCBOHandleAst(queryForCbo, getQB(), cboCtx);
+ profilesCBO = obtainCBOProfiles(queryProperties);
if (runCBO) {
disableJoinMerge = true;
@@ -320,6 +325,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
} finally {
runCBO = false;
disableJoinMerge = false;
+ disableSemJoinReordering = false;
if (reAnalyzeAST) {
init(true);
prunedPartitions.clear();
@@ -427,11 +433,10 @@ public class CalcitePlanner extends SemanticAnalyzer {
boolean isInTest = conf.getBoolVar(ConfVars.HIVE_IN_TEST);
boolean isStrictTest = isInTest
&& !conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict");
- boolean hasEnoughJoins = !topLevelQB || (queryProperties.getJoinCount() > 1) || isInTest || distinctExprsExists(qb);
- if (!isStrictTest && hasEnoughJoins && !queryProperties.hasClusterBy()
- && !queryProperties.hasDistributeBy() && !queryProperties.hasSortBy()
- && !queryProperties.hasPTF() && !queryProperties.usesScript()
+ if (!isStrictTest
+ && !queryProperties.hasClusterBy() && !queryProperties.hasDistributeBy()
+ && !queryProperties.hasSortBy() && !queryProperties.hasPTF() && !queryProperties.usesScript()
&& !queryProperties.hasMultiDestQuery() && !queryProperties.hasLateralViews()) {
// Ok to run CBO.
return null;
@@ -442,8 +447,6 @@ public class CalcitePlanner extends SemanticAnalyzer {
if (verbose) {
if (isStrictTest)
msg += "is in test running in mode other than nonstrict; ";
- if (!hasEnoughJoins)
- msg += "has too few joins; ";
if (queryProperties.hasClusterBy())
msg += "has cluster by; ";
if (queryProperties.hasDistributeBy())
@@ -465,9 +468,24 @@ public class CalcitePlanner extends SemanticAnalyzer {
return msg;
}
+ /* This method inserts the right profiles into profiles CBO depending
+ * on the query characteristics. */
+ private static EnumSet<ExtendedCBOProfile> obtainCBOProfiles(QueryProperties queryProperties) {
+ EnumSet<ExtendedCBOProfile> profilesCBO = EnumSet.noneOf(ExtendedCBOProfile.class);
+ // If the query contains more than one join
+ if (queryProperties.getJoinCount() > 1) {
+ profilesCBO.add(ExtendedCBOProfile.JOIN_REORDERING);
+ }
+ // If the query contains windowing processing
+ if (queryProperties.hasWindowing()) {
+ profilesCBO.add(ExtendedCBOProfile.WINDOWING_POSTPROCESSING);
+ }
+ return profilesCBO;
+ }
+
@Override
boolean continueJoinMerge() {
- return !runCBO;
+ return !(runCBO && disableSemJoinReordering);
}
@Override
@@ -789,6 +807,11 @@ public class CalcitePlanner extends SemanticAnalyzer {
return rr;
}
+ private enum ExtendedCBOProfile {
+ JOIN_REORDERING,
+ WINDOWING_POSTPROCESSING;
+ }
+
/**
* Code responsible for Calcite plan generation and optimization.
*/
@@ -843,61 +866,109 @@ public class CalcitePlanner extends SemanticAnalyzer {
// Create MD provider
HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf);
- // 2. Apply Pre Join Order optimizations
+ // 2. Apply pre-join order optimizations
calcitePreCboPlan = applyPreJoinOrderingTransforms(calciteGenPlan,
mdProvider.getMetadataProvider());
- // 3. Appy Join Order Optimizations using Hep Planner (MST Algorithm)
- List<RelMetadataProvider> list = Lists.newArrayList();
- list.add(mdProvider.getMetadataProvider());
- RelTraitSet desiredTraits = cluster
- .traitSetOf(HiveRelNode.CONVENTION, RelCollations.EMPTY);
-
- HepProgram hepPgm = null;
- HepProgramBuilder hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP)
- .addRuleInstance(new JoinToMultiJoinRule(HiveJoin.class));
- hepPgmBldr.addRuleInstance(new LoptOptimizeJoinRule(HiveJoin.HIVE_JOIN_FACTORY,
- HiveProject.DEFAULT_PROJECT_FACTORY, HiveFilter.DEFAULT_FILTER_FACTORY));
-
- hepPgmBldr.addRuleInstance(ReduceExpressionsRule.JOIN_INSTANCE);
- hepPgmBldr.addRuleInstance(ReduceExpressionsRule.FILTER_INSTANCE);
- hepPgmBldr.addRuleInstance(ReduceExpressionsRule.PROJECT_INSTANCE);
- hepPgmBldr.addRuleInstance(ProjectRemoveRule.INSTANCE);
- hepPgmBldr.addRuleInstance(UnionMergeRule.INSTANCE);
- hepPgmBldr.addRuleInstance(new ProjectMergeRule(false, HiveProject.DEFAULT_PROJECT_FACTORY));
- hepPgmBldr.addRuleInstance(HiveAggregateProjectMergeRule.INSTANCE);
- if (conf.getBoolVar(ConfVars.AGGR_JOIN_TRANSPOSE)) {
- hepPgmBldr.addRuleInstance(HiveAggregateJoinTransposeRule.INSTANCE);
+ // 3. Apply join order optimizations: reordering MST algorithm
+ // If join optimizations failed because of missing stats, we continue with
+ // the rest of optimizations
+ if (profilesCBO.contains(ExtendedCBOProfile.JOIN_REORDERING)) {
+ try {
+ List<RelMetadataProvider> list = Lists.newArrayList();
+ list.add(mdProvider.getMetadataProvider());
+ RelTraitSet desiredTraits = cluster
+ .traitSetOf(HiveRelNode.CONVENTION, RelCollations.EMPTY);
+
+ HepProgramBuilder hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP);
+ hepPgmBldr.addRuleInstance(new JoinToMultiJoinRule(HiveJoin.class));
+ hepPgmBldr.addRuleInstance(new LoptOptimizeJoinRule(HiveJoin.HIVE_JOIN_FACTORY,
+ HiveProject.DEFAULT_PROJECT_FACTORY, HiveFilter.DEFAULT_FILTER_FACTORY));
+
+ HepProgram hepPgm = hepPgmBldr.build();
+ HepPlanner hepPlanner = new HepPlanner(hepPgm);
+
+ hepPlanner.registerMetadataProviders(list);
+ RelMetadataProvider chainedProvider = ChainedRelMetadataProvider.of(list);
+ cluster.setMetadataProvider(new CachingRelMetadataProvider(chainedProvider, hepPlanner));
+
+ RelNode rootRel = calcitePreCboPlan;
+ hepPlanner.setRoot(rootRel);
+ if (!calcitePreCboPlan.getTraitSet().equals(desiredTraits)) {
+ rootRel = hepPlanner.changeTraits(calcitePreCboPlan, desiredTraits);
+ }
+ hepPlanner.setRoot(rootRel);
+
+ calciteOptimizedPlan = hepPlanner.findBestExp();
+ } catch (Exception e) {
+ boolean isMissingStats = noColsMissingStats.get() > 0;
+ if (isMissingStats) {
+ LOG.warn("Missing column stats (see previous messages), skipping join reordering in CBO");
+ noColsMissingStats.set(0);
+ calciteOptimizedPlan = calcitePreCboPlan;
+ disableSemJoinReordering = false;
+ } else {
+ throw e;
+ }
+ }
+ } else {
+ calciteOptimizedPlan = calcitePreCboPlan;
+ disableSemJoinReordering = false;
}
- hepPgm = hepPgmBldr.build();
- HepPlanner hepPlanner = new HepPlanner(hepPgm);
+ // 4. Run other optimizations that do not need stats
+ calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
+ HepMatchOrder.BOTTOM_UP, ReduceExpressionsRule.JOIN_INSTANCE,
+ ReduceExpressionsRule.FILTER_INSTANCE, ReduceExpressionsRule.PROJECT_INSTANCE,
+ ProjectRemoveRule.INSTANCE, UnionMergeRule.INSTANCE,
+ new ProjectMergeRule(false, HiveProject.DEFAULT_PROJECT_FACTORY),
+ HiveAggregateProjectMergeRule.INSTANCE);
+
+ // 5. Run aggregate-join transpose (cost based)
+ // If it failed because of missing stats, we continue with
+ // the rest of optimizations
+ if (conf.getBoolVar(ConfVars.AGGR_JOIN_TRANSPOSE)) {
+ try {
+ HepProgramBuilder hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP);
+ hepPgmBldr.addRuleInstance(HiveAggregateJoinTransposeRule.INSTANCE);
- hepPlanner.registerMetadataProviders(list);
- RelMetadataProvider chainedProvider = ChainedRelMetadataProvider.of(list);
- cluster.setMetadataProvider(new CachingRelMetadataProvider(chainedProvider, hepPlanner));
+ HepProgram hepPgm = hepPgmBldr.build();
+ HepPlanner hepPlanner = new HepPlanner(hepPgm);
- RelNode rootRel = calcitePreCboPlan;
- hepPlanner.setRoot(rootRel);
- if (!calcitePreCboPlan.getTraitSet().equals(desiredTraits)) {
- rootRel = hepPlanner.changeTraits(calcitePreCboPlan, desiredTraits);
- }
- hepPlanner.setRoot(rootRel);
+ List<RelMetadataProvider> list = Lists.newArrayList();
+ list.add(mdProvider.getMetadataProvider());
+ hepPlanner.registerMetadataProviders(list);
+ RelMetadataProvider chainedProvider = ChainedRelMetadataProvider.of(list);
+ cluster.setMetadataProvider(new CachingRelMetadataProvider(chainedProvider, hepPlanner));
+
+ hepPlanner.setRoot(calciteOptimizedPlan);
- calciteOptimizedPlan = hepPlanner.findBestExp();
+ calciteOptimizedPlan = hepPlanner.findBestExp();
+ } catch (Exception e) {
+ boolean isMissingStats = noColsMissingStats.get() > 0;
+ if (isMissingStats) {
+ LOG.warn("Missing column stats (see previous messages), skipping aggregate-join transpose in CBO");
+ noColsMissingStats.set(0);
+ } else {
+ throw e;
+ }
+ }
+ }
- // 4. Run rule to try to remove projects on top of join operators
+ // 6. Run rule to try to remove projects on top of join operators
calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
HepMatchOrder.BOTTOM_UP, HiveJoinCommuteRule.INSTANCE);
- // 5. Run rule to fix windowing issue when it is done over
+ // 7. Run rule to fix windowing issue when it is done over
// aggregation columns (HIVE-10627)
- calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
- HepMatchOrder.BOTTOM_UP, HiveWindowingFixRule.INSTANCE);
+ if (profilesCBO.contains(ExtendedCBOProfile.WINDOWING_POSTPROCESSING)) {
+ calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
+ HepMatchOrder.BOTTOM_UP, HiveWindowingFixRule.INSTANCE);
+ }
- // 6. Run rules to aid in translation from Calcite tree to Hive tree
+ // 8. Run rules to aid in translation from Calcite tree to Hive tree
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
- // 6.1. Merge join into multijoin operators (if possible)
+ // 8.1. Merge join into multijoin operators (if possible)
calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(),
HepMatchOrder.BOTTOM_UP, HiveJoinProjectTransposeRule.BOTH_PROJECT_INCLUDE_OUTER,
HiveJoinProjectTransposeRule.LEFT_PROJECT_INCLUDE_OUTER,
@@ -915,7 +986,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE,
new ProjectMergeRule(false, HiveProject.DEFAULT_PROJECT_FACTORY));
- // 6.2. Introduce exchange operators below join/multijoin operators
+ // 8.2. Introduce exchange operators below join/multijoin operators
calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(),
HepMatchOrder.BOTTOM_UP, HiveInsertExchange4JoinRule.EXCHANGE_BELOW_JOIN,
HiveInsertExchange4JoinRule.EXCHANGE_BELOW_MULTIJOIN);
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 5323a7d..69bb9d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -8708,22 +8708,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
return distinctExprs;
}
- // see if there are any distinct expressions
- protected static boolean distinctExprsExists(QB qb) {
- QBParseInfo qbp = qb.getParseInfo();
-
- TreeSet<String> ks = new TreeSet<String>();
- ks.addAll(qbp.getClauseNames());
-
- for (String dest : ks) {
- List<ASTNode> list = qbp.getDistinctFuncExprsForClause(dest);
- if (!list.isEmpty()) {
- return true;
- }
- }
- return false;
- }
-
@SuppressWarnings("nls")
private Operator genBodyPlan(QB qb, Operator input, Map<String, Operator> aliasToOpInfo)
throws SemanticException {
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/queries/clientpositive/mergejoin.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mergejoin.q b/ql/src/test/queries/clientpositive/mergejoin.q
index 7550e09..95f3d01 100644
--- a/ql/src/test/queries/clientpositive/mergejoin.q
+++ b/ql/src/test/queries/clientpositive/mergejoin.q
@@ -120,6 +120,15 @@ full outer join
select * from
(select * from tab where tab.key = 0)a
+full outer join
+(select * from tab_part where tab_part.key = 98)b on a.key = b.key join tab_part c on b.key = c.key;
+
+select * from
+(select * from tab where tab.key = 0)a
join
(select * from tab_part where tab_part.key = 98)b full outer join tab_part c on a.key = b.key and b.key = c.key;
+select * from
+(select * from tab where tab.key = 0)a
+join
+(select * from tab_part where tab_part.key = 98)b on a.key = b.key full outer join tab_part c on b.key = c.key;
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out b/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out
index 623c2e8..cdecad1 100644
--- a/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out
+++ b/ql/src/test/results/clientnegative/bucket_mapjoin_mismatch1.q.out
@@ -108,35 +108,43 @@ STAGE PLANS:
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
+ Select Operator
+ expressions: key (type: int), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 20 Data size: 2100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
TableScan
alias: b
Statistics: Num rows: 29 Data size: 3062 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: key (type: int)
- sort order: +
- Map-reduce partition columns: key (type: int)
+ Select Operator
+ expressions: key (type: int), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 15 Data size: 1583 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: int)
- 1 key (type: int)
- outputColumnNames: _col0, _col1, _col7
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col4
Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: _col0 (type: int), _col1 (type: string), _col7 (type: string)
+ expressions: _col0 (type: int), _col1 (type: string), _col4 (type: string)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 22 Data size: 2310 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/results/clientnegative/join_nonexistent_part.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/join_nonexistent_part.q.out b/ql/src/test/results/clientnegative/join_nonexistent_part.q.out
index 391dd05..a924895 100644
--- a/ql/src/test/results/clientnegative/join_nonexistent_part.q.out
+++ b/ql/src/test/results/clientnegative/join_nonexistent_part.q.out
@@ -1,2 +1 @@
-Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
Authorization failed:No privilege 'Select' found for inputs { database:default, table:srcpart, columnName:key}. Use SHOW GRANT to get more details.
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
index aa380b2..14e7e60 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
@@ -273,35 +273,35 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: ss
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ alias: s
+ Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ss_store_sk is not null (type: boolean)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: s_store_sk is not null (type: boolean)
+ Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: ss_store_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
- alias: s
- Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ alias: ss
+ Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: s_store_sk is not null (type: boolean)
- Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: ss_store_sk is not null (type: boolean)
+ Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -309,19 +309,15 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col0
Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: int)
- outputColumnNames: _col0
+ File Output Operator
+ compressed: false
Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -342,35 +338,35 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: ss
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ alias: s
+ Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (ss_store_sk > 0) (type: boolean)
- Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (s_store_sk > 0) (type: boolean)
+ Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: ss_store_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
- alias: s
- Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ alias: ss
+ Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (s_store_sk > 0) (type: boolean)
- Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (ss_store_sk > 0) (type: boolean)
+ Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -378,19 +374,15 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col0
Statistics: Num rows: 136 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: int)
- outputColumnNames: _col0
+ File Output Operator
+ compressed: false
Statistics: Num rows: 136 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 136 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -411,35 +403,35 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: ss
- Statistics: Num rows: 1000 Data size: 7668 Basic stats: COMPLETE Column stats: COMPLETE
+ alias: s
+ Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: PARTIAL
Filter Operator
- predicate: ((ss_quantity > 10) and ss_store_sk is not null) (type: boolean)
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: ((s_company_id > 0) and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
Select Operator
- expressions: ss_store_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
TableScan
- alias: s
- Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: PARTIAL
+ alias: ss
+ Statistics: Num rows: 1000 Data size: 7668 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((s_company_id > 0) and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
+ predicate: ((ss_quantity > 10) and ss_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -447,19 +439,15 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2
+ outputColumnNames: _col0
Statistics: Num rows: 131 Data size: 524 Basic stats: COMPLETE Column stats: PARTIAL
- Select Operator
- expressions: _col2 (type: int)
- outputColumnNames: _col0
+ File Output Operator
+ compressed: false
Statistics: Num rows: 131 Data size: 524 Basic stats: COMPLETE Column stats: PARTIAL
- File Output Operator
- compressed: false
- Statistics: Num rows: 131 Data size: 524 Basic stats: COMPLETE Column stats: PARTIAL
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -480,35 +468,35 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: ss
- Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
+ alias: s
+ Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ss_store_sk is not null (type: boolean)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: ((s_floor_space > 0) and s_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: ss_store_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
- alias: s
- Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE
+ alias: ss
+ Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((s_floor_space > 0) and s_store_sk is not null) (type: boolean)
- Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: ss_store_sk is not null (type: boolean)
+ Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 964 Data size: 3716 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -516,19 +504,15 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col1
+ outputColumnNames: _col0
Statistics: Num rows: 393 Data size: 1572 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: int)
- outputColumnNames: _col0
+ File Output Operator
+ compressed: false
Statistics: Num rows: 393 Data size: 1572 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 393 Data size: 1572 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -549,35 +533,35 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: ss
- Statistics: Num rows: 1000 Data size: 7668 Basic stats: COMPLETE Column stats: COMPLETE
+ alias: s
+ Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((ss_quantity > 10) and ss_store_sk is not null) (type: boolean)
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: s_store_sk is not null (type: boolean)
+ Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: ss_store_sk (type: int)
+ expressions: s_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
- alias: s
- Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ alias: ss
+ Statistics: Num rows: 1000 Data size: 7668 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: s_store_sk is not null (type: boolean)
- Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: ((ss_quantity > 10) and ss_store_sk is not null) (type: boolean)
+ Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: s_store_sk (type: int)
+ expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 321 Data size: 2460 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -585,19 +569,15 @@ STAGE PLANS:
keys:
0 _col0 (type: int)
1 _col0 (type: int)
- outputColumnNames: _col2
+ outputColumnNames: _col0
Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col2 (type: int)
- outputColumnNames: _col0
+ File Output Operator
+ compressed: false
Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/results/clientpositive/archive_excludeHadoop20.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/archive_excludeHadoop20.q.out b/ql/src/test/results/clientpositive/archive_excludeHadoop20.q.out
index c2b9872..9b52beb 100644
--- a/ql/src/test/results/clientpositive/archive_excludeHadoop20.q.out
+++ b/ql/src/test/results/clientpositive/archive_excludeHadoop20.q.out
@@ -137,6 +137,7 @@ POSTHOOK: Input: default@tstsrcpart
POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
#### A masked pattern was here ####
0 3
+Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key
WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0'
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/results/clientpositive/archive_multi.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/archive_multi.q.out b/ql/src/test/results/clientpositive/archive_multi.q.out
index 0ad29d1..5f66176 100644
--- a/ql/src/test/results/clientpositive/archive_multi.q.out
+++ b/ql/src/test/results/clientpositive/archive_multi.q.out
@@ -141,6 +141,7 @@ POSTHOOK: Input: ac_test@tstsrcpart
POSTHOOK: Input: ac_test@tstsrcpart@ds=2008-04-08/hr=12
#### A masked pattern was here ####
0 3
+Warning: Shuffle Join JOIN[9][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
PREHOOK: query: SELECT * FROM ac_test.tstsrcpart a JOIN ac_test.tstsrc b ON a.key=b.key
WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0'
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/results/clientpositive/auto_join1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/auto_join1.q.out b/ql/src/test/results/clientpositive/auto_join1.q.out
index 48ad641..ad7681b 100644
--- a/ql/src/test/results/clientpositive/auto_join1.q.out
+++ b/ql/src/test/results/clientpositive/auto_join1.q.out
@@ -40,8 +40,8 @@ STAGE PLANS:
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
@@ -58,8 +58,8 @@ STAGE PLANS:
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -67,10 +67,10 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col1, _col2
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col2) (type: int), _col1 (type: string)
+ expressions: UDFToInteger(_col0) (type: int), _col2 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/results/clientpositive/auto_join10.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/auto_join10.q.out b/ql/src/test/results/clientpositive/auto_join10.q.out
index fa6f62d..3c38de3 100644
--- a/ql/src/test/results/clientpositive/auto_join10.q.out
+++ b/ql/src/test/results/clientpositive/auto_join10.q.out
@@ -35,8 +35,8 @@ STAGE PLANS:
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
@@ -53,8 +53,8 @@ STAGE PLANS:
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -62,10 +62,10 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col1, _col2
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hash(_col0,_col1) (type: int)
+ expressions: hash(_col1,_col2) (type: int)
outputColumnNames: _col0
Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/results/clientpositive/auto_join11.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/auto_join11.q.out b/ql/src/test/results/clientpositive/auto_join11.q.out
index 851920b..7dbfb1c 100644
--- a/ql/src/test/results/clientpositive/auto_join11.q.out
+++ b/ql/src/test/results/clientpositive/auto_join11.q.out
@@ -35,8 +35,8 @@ STAGE PLANS:
predicate: (UDFToDouble(key) < 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ expressions: key (type: string)
+ outputColumnNames: _col0
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
HashTable Sink Operator
keys:
@@ -53,8 +53,8 @@ STAGE PLANS:
predicate: (UDFToDouble(key) < 100.0) (type: boolean)
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string)
- outputColumnNames: _col0
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
@@ -62,10 +62,10 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col1, _col2
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: hash(_col2,_col1) (type: int)
+ expressions: hash(_col0,_col2) (type: int)
outputColumnNames: _col0
Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/results/clientpositive/auto_join14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/auto_join14.q.out b/ql/src/test/results/clientpositive/auto_join14.q.out
index 47e1724..4a0f2f9 100644
--- a/ql/src/test/results/clientpositive/auto_join14.q.out
+++ b/ql/src/test/results/clientpositive/auto_join14.q.out
@@ -28,11 +28,11 @@ STAGE PLANS:
Stage: Stage-5
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_1:src
+ $hdt$_0:src
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_1:src
+ $hdt$_0:src
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -67,10 +67,10 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- outputColumnNames: _col1, _col3
+ outputColumnNames: _col0, _col2
Statistics: Num rows: 366 Data size: 3890 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: UDFToInteger(_col3) (type: int), _col1 (type: string)
+ expressions: UDFToInteger(_col0) (type: int), _col2 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 366 Data size: 3890 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/7dab21ac/ql/src/test/results/clientpositive/auto_join24.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/auto_join24.q.out b/ql/src/test/results/clientpositive/auto_join24.q.out
index 5b57303..8af6302 100644
--- a/ql/src/test/results/clientpositive/auto_join24.q.out
+++ b/ql/src/test/results/clientpositive/auto_join24.q.out
@@ -33,48 +33,56 @@ STAGE PLANS:
Stage: Stage-5
Map Reduce Local Work
Alias -> Map Local Tables:
- a
+ $hdt$_0:a
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- a
+ $hdt$_0:a
TableScan
alias: a
Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 155 Data size: 743 Basic stats: COMPLETE Column stats: NONE
- HashTable Sink Operator
- keys:
- 0 key (type: string)
- 1 key (type: string)
+ Select Operator
+ expressions: key (type: string), cnt (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 155 Data size: 743 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
Stage: Stage-2
Map Reduce
Map Operator Tree:
TableScan
- alias: b
+ alias: a
Statistics: Num rows: 309 Data size: 1482 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
Statistics: Num rows: 155 Data size: 743 Basic stats: COMPLETE Column stats: NONE
- Map Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col1
- Statistics: Num rows: 170 Data size: 817 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(_col1)
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- sort order:
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 155 Data size: 743 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1
+ Statistics: Num rows: 170 Data size: 817 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col1)
+ mode: hash
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: bigint)
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
Local Work:
Map Reduce Local Work
Reduce Operator Tree: