You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jp...@apache.org on 2015/12/15 01:39:59 UTC
[51/51] [partial] hive git commit: HIVE-11110: Reorder
applyPreJoinOrderingTransforms, add NotNULL/FilterMerge rules,
improve Filter selectivity estimation (Laljo John Pullokkaran reviewed by
Jesus Camacho Rodriguez, Ashutosh Chauhan
HIVE-11110: Reorder applyPreJoinOrderingTransforms, add NotNULL/FilterMerge rules, improve Filter selectivity estimation (Laljo John Pullokkaran reviewed by Jesus Camacho Rodriguez, Ashutosh Chauhan
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d8ee05ae
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d8ee05ae
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d8ee05ae
Branch: refs/heads/branch-2.0
Commit: d8ee05aeb3eae9540c4aeb685f86eb9a9b9c1d3a
Parents: 74fe203
Author: jpullokk <jp...@apache.org>
Authored: Mon Dec 14 16:29:44 2015 -0800
Committer: jpullokk <jp...@apache.org>
Committed: Mon Dec 14 16:29:44 2015 -0800
----------------------------------------------------------------------
.../test/results/positive/hbase_queries.q.out | 38 +-
.../ql/optimizer/calcite/HiveCalciteUtil.java | 98 +
.../calcite/HiveDefaultRelMetadataProvider.java | 2 +
.../ql/optimizer/calcite/RelOptHiveTable.java | 9 +-
.../hive/ql/optimizer/calcite/TraitsUtil.java | 4 +
.../calcite/reloperators/HiveProject.java | 25 +-
.../calcite/reloperators/HiveTableScan.java | 46 +-
.../rules/HiveFilterAggregateTransposeRule.java | 45 +
.../rules/HiveFilterProjectTSTransposeRule.java | 113 +
.../HiveJoinPushTransitivePredicatesRule.java | 44 +-
.../calcite/rules/HivePartitionPruneRule.java | 2 +-
.../calcite/rules/HivePreFilteringRule.java | 170 +-
.../stats/FilterSelectivityEstimator.java | 48 +
.../calcite/stats/HiveRelMdPredicates.java | 645 ++++
.../translator/SqlFunctionConverter.java | 3 +-
.../stats/annotation/StatsRulesProcFactory.java | 62 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 117 +-
.../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 8 +
.../special_character_in_tabnames_1.q | 26 +-
.../bucket_mapjoin_mismatch1.q.out | 28 +-
.../sortmerge_mapjoin_mismatch_1.q.out | 10 +-
.../clientpositive/allcolref_in_udf.q.out | 14 +-
.../results/clientpositive/ambiguous_col.q.out | 54 +-
.../clientpositive/annotate_stats_join.q.out | 8 +-
.../annotate_stats_join_pkfk.q.out | 61 +-
.../archive_excludeHadoop20.q.out | 1 -
.../results/clientpositive/archive_multi.q.out | 1 -
.../results/clientpositive/auto_join1.q.out | 14 +-
.../results/clientpositive/auto_join10.q.out | 12 +-
.../results/clientpositive/auto_join12.q.out | 66 +-
.../results/clientpositive/auto_join13.q.out | 10 +-
.../results/clientpositive/auto_join15.q.out | 12 +-
.../results/clientpositive/auto_join16.q.out | 18 +-
.../results/clientpositive/auto_join17.q.out | 14 +-
.../results/clientpositive/auto_join19.q.out | 14 +-
.../results/clientpositive/auto_join2.q.out | 24 +-
.../results/clientpositive/auto_join22.q.out | 18 +-
.../results/clientpositive/auto_join24.q.out | 10 +-
.../results/clientpositive/auto_join26.q.out | 20 +-
.../results/clientpositive/auto_join3.q.out | 18 +-
.../results/clientpositive/auto_join30.q.out | 64 +-
.../results/clientpositive/auto_join33.q.out | 16 +-
.../results/clientpositive/auto_join4.q.out | 4 +-
.../results/clientpositive/auto_join5.q.out | 4 +-
.../results/clientpositive/auto_join8.q.out | 18 +-
.../results/clientpositive/auto_join9.q.out | 14 +-
.../auto_join_reordering_values.q.out | 10 +-
.../clientpositive/auto_join_stats.q.out | 106 +-
.../clientpositive/auto_join_stats2.q.out | 46 +-
.../auto_join_without_localtask.q.out | 148 +-
.../clientpositive/auto_smb_mapjoin_14.q.out | 72 +-
.../clientpositive/auto_sortmerge_join_1.q.out | 20 +-
.../clientpositive/auto_sortmerge_join_11.q.out | 22 +-
.../clientpositive/auto_sortmerge_join_12.q.out | 12 +-
.../clientpositive/auto_sortmerge_join_13.q.out | 6 +-
.../clientpositive/auto_sortmerge_join_2.q.out | 24 +-
.../clientpositive/auto_sortmerge_join_3.q.out | 28 +-
.../clientpositive/auto_sortmerge_join_4.q.out | 28 +-
.../clientpositive/auto_sortmerge_join_5.q.out | 28 +-
.../clientpositive/auto_sortmerge_join_6.q.out | 128 +-
.../clientpositive/auto_sortmerge_join_7.q.out | 28 +-
.../clientpositive/auto_sortmerge_join_8.q.out | 28 +-
.../clientpositive/auto_sortmerge_join_9.q.out | 204 +-
.../clientpositive/bucket_map_join_spark1.q.out | 28 +-
.../clientpositive/bucket_map_join_spark2.q.out | 28 +-
.../clientpositive/bucket_map_join_spark3.q.out | 28 +-
.../clientpositive/bucket_map_join_spark4.q.out | 40 +-
.../clientpositive/bucketcontext_1.q.out | 8 +-
.../clientpositive/bucketcontext_2.q.out | 8 +-
.../clientpositive/bucketcontext_3.q.out | 8 +-
.../clientpositive/bucketcontext_4.q.out | 8 +-
.../clientpositive/bucketcontext_5.q.out | 8 +-
.../clientpositive/bucketcontext_6.q.out | 8 +-
.../clientpositive/bucketcontext_7.q.out | 8 +-
.../clientpositive/bucketcontext_8.q.out | 8 +-
.../bucketizedhiveinputformat.q.out | 2 -
.../results/clientpositive/bucketmapjoin1.q.out | 20 +-
.../clientpositive/bucketmapjoin10.q.out | 6 +-
.../clientpositive/bucketmapjoin11.q.out | 12 +-
.../clientpositive/bucketmapjoin12.q.out | 12 +-
.../clientpositive/bucketmapjoin13.q.out | 24 +-
.../results/clientpositive/bucketmapjoin2.q.out | 30 +-
.../results/clientpositive/bucketmapjoin3.q.out | 20 +-
.../results/clientpositive/bucketmapjoin4.q.out | 20 +-
.../results/clientpositive/bucketmapjoin5.q.out | 20 +-
.../results/clientpositive/bucketmapjoin7.q.out | 12 +-
.../results/clientpositive/bucketmapjoin8.q.out | 12 +-
.../results/clientpositive/bucketmapjoin9.q.out | 12 +-
.../clientpositive/bucketmapjoin_negative.q.out | 10 +-
.../bucketmapjoin_negative2.q.out | 10 +-
.../bucketmapjoin_negative3.q.out | 90 +-
.../bucketsortoptimize_insert_2.q.out | 56 +-
.../bucketsortoptimize_insert_4.q.out | 30 +-
.../bucketsortoptimize_insert_5.q.out | 40 +-
.../bucketsortoptimize_insert_6.q.out | 114 +-
.../bucketsortoptimize_insert_7.q.out | 12 +-
.../bucketsortoptimize_insert_8.q.out | 8 +-
.../clientpositive/cbo_rp_auto_join1.q.out | 396 ++-
.../clientpositive/cbo_rp_auto_join17.q.out | 14 +-
.../cbo_rp_cross_product_check_2.q.out | 56 +-
.../clientpositive/cbo_rp_lineage2.q.out | 8 +-
.../results/clientpositive/cbo_udf_max.q.out | 4 +-
.../clientpositive/column_access_stats.q.out | 52 +-
.../clientpositive/constprog_partitioner.q.out | 44 +-
.../clientpositive/correlationoptimizer1.q.out | 190 +-
.../clientpositive/correlationoptimizer10.q.out | 176 +-
.../clientpositive/correlationoptimizer11.q.out | 72 +-
.../clientpositive/correlationoptimizer12.q.out | 16 +-
.../clientpositive/correlationoptimizer13.q.out | 42 +-
.../clientpositive/correlationoptimizer14.q.out | 234 +-
.../clientpositive/correlationoptimizer15.q.out | 60 +-
.../clientpositive/correlationoptimizer2.q.out | 104 +-
.../clientpositive/correlationoptimizer3.q.out | 220 +-
.../clientpositive/correlationoptimizer4.q.out | 74 +-
.../clientpositive/correlationoptimizer5.q.out | 110 +-
.../clientpositive/correlationoptimizer6.q.out | 616 ++--
.../clientpositive/correlationoptimizer7.q.out | 104 +-
.../clientpositive/correlationoptimizer8.q.out | 34 +-
.../clientpositive/correlationoptimizer9.q.out | 120 +-
.../results/clientpositive/cross_join.q.out | 28 +-
.../clientpositive/cross_product_check_1.q.out | 66 +-
.../clientpositive/cross_product_check_2.q.out | 54 +-
.../results/clientpositive/decimal_join2.q.out | 36 +-
.../results/clientpositive/decimal_udf.q.out | 12 +-
.../clientpositive/dynamic_rdd_cache.q.out | 34 +-
.../encryption_join_unencrypted_tbl.q.out | 20 +-
...on_join_with_different_encryption_keys.q.out | 16 +-
.../clientpositive/explain_logical.q.out | 198 +-
.../clientpositive/filter_cond_pushdown.q.out | 241 +-
.../clientpositive/filter_join_breaktask.q.out | 32 +-
.../clientpositive/groupby_join_pushdown.q.out | 320 +-
.../groupby_multi_single_reducer3.q.out | 16 +-
.../clientpositive/groupby_position.q.out | 30 +-
.../clientpositive/groupby_sort_1_23.q.out | 46 +-
.../clientpositive/groupby_sort_skew_1_23.q.out | 50 +-
.../test/results/clientpositive/having2.q.out | 66 +-
.../identity_project_remove_skip.q.out | 10 +-
.../clientpositive/index_auto_mult_tables.q.out | 54 +-
.../index_auto_mult_tables_compact.q.out | 42 +-
.../clientpositive/index_auto_self_join.q.out | 58 +-
.../results/clientpositive/index_bitmap3.q.out | 30 +-
.../clientpositive/index_bitmap_auto.q.out | 30 +-
.../index_bitmap_auto_partitioned.q.out | 6 +-
.../index_bitmap_compression.q.out | 6 +-
.../infer_bucket_sort_map_operators.q.out | 22 +-
.../test/results/clientpositive/innerjoin.q.out | 18 +-
.../test/results/clientpositive/input39.q.out | 10 +-
.../results/clientpositive/input_part9.q.out | 4 +-
.../clientpositive/input_testxpath2.q.out | 6 +-
.../clientpositive/input_testxpath4.q.out | 12 +-
ql/src/test/results/clientpositive/join1.q.out | 18 +-
ql/src/test/results/clientpositive/join10.q.out | 18 +-
ql/src/test/results/clientpositive/join12.q.out | 65 +-
ql/src/test/results/clientpositive/join13.q.out | 16 +-
ql/src/test/results/clientpositive/join15.q.out | 18 +-
ql/src/test/results/clientpositive/join16.q.out | 20 +-
ql/src/test/results/clientpositive/join17.q.out | 18 +-
ql/src/test/results/clientpositive/join2.q.out | 32 +-
ql/src/test/results/clientpositive/join22.q.out | 28 +-
ql/src/test/results/clientpositive/join25.q.out | 10 +-
ql/src/test/results/clientpositive/join26.q.out | 12 +-
ql/src/test/results/clientpositive/join27.q.out | 10 +-
ql/src/test/results/clientpositive/join28.q.out | 22 +-
ql/src/test/results/clientpositive/join29.q.out | 42 +-
ql/src/test/results/clientpositive/join3.q.out | 24 +-
ql/src/test/results/clientpositive/join30.q.out | 16 +-
ql/src/test/results/clientpositive/join31.q.out | 40 +-
ql/src/test/results/clientpositive/join32.q.out | 22 +-
.../clientpositive/join32_lessSize.q.out | 124 +-
ql/src/test/results/clientpositive/join33.q.out | 22 +-
ql/src/test/results/clientpositive/join34.q.out | 4 +-
ql/src/test/results/clientpositive/join35.q.out | 4 +-
ql/src/test/results/clientpositive/join36.q.out | 10 +-
ql/src/test/results/clientpositive/join37.q.out | 10 +-
ql/src/test/results/clientpositive/join40.q.out | 32 +-
ql/src/test/results/clientpositive/join42.q.out | 77 +-
ql/src/test/results/clientpositive/join43.q.out | 66 +-
ql/src/test/results/clientpositive/join8.q.out | 16 +-
ql/src/test/results/clientpositive/join9.q.out | 18 +-
.../clientpositive/join_alt_syntax.q.out | 144 +-
.../clientpositive/join_cond_pushdown_1.q.out | 86 +-
.../clientpositive/join_cond_pushdown_2.q.out | 74 +-
.../clientpositive/join_cond_pushdown_3.q.out | 86 +-
.../clientpositive/join_cond_pushdown_4.q.out | 74 +-
.../join_cond_pushdown_unqual1.q.out | 24 +-
.../join_cond_pushdown_unqual2.q.out | 44 +-
.../join_cond_pushdown_unqual3.q.out | 24 +-
.../join_cond_pushdown_unqual4.q.out | 44 +-
.../clientpositive/join_grp_diff_keys.q.out | 8 +-
.../results/clientpositive/join_map_ppr.q.out | 24 +-
.../join_merge_multi_expressions.q.out | 20 +-
.../results/clientpositive/join_nullsafe.q.out | 36 +-
.../clientpositive/join_on_varchar.q.out | 14 +-
.../results/clientpositive/join_parse.q.out | 130 +-
.../test/results/clientpositive/join_rc.q.out | 18 +-
.../results/clientpositive/join_reorder.q.out | 24 +-
.../test/results/clientpositive/join_star.q.out | 54 +-
.../results/clientpositive/join_thrift.q.out | 14 +-
.../test/results/clientpositive/join_vc.q.out | 30 +-
.../clientpositive/lateral_view_cp.q.out | 24 +-
.../results/clientpositive/limit_pushdown.q.out | 14 +-
.../limit_pushdown_negative.q.out | 14 +-
.../test/results/clientpositive/lineage2.q.out | 18 +-
.../test/results/clientpositive/lineage3.q.out | 20 +-
.../list_bucket_query_oneskew_2.q.out | 2 +-
.../llap/bucket_map_join_tez1.q.out | 341 +-
.../llap/bucket_map_join_tez2.q.out | 135 +-
.../llap/dynamic_partition_pruning.q.out | 269 +-
.../llap/dynamic_partition_pruning_2.q.out | 21 +-
.../llap/hybridgrace_hashjoin_1.q.out | 97 +-
.../llap/hybridgrace_hashjoin_2.q.out | 128 +-
.../clientpositive/llap/lvj_mapjoin.q.out | 56 +-
.../clientpositive/llap/mapjoin_decimal.q.out | 18 +-
.../results/clientpositive/llap/mergejoin.q.out | 1614 +++++++--
.../test/results/clientpositive/llap/mrr.q.out | 130 +-
.../llap/tez_bmj_schema_evolution.q.out | 18 +-
.../llap/tez_dynpart_hashjoin_1.q.out | 120 +-
.../llap/tez_dynpart_hashjoin_2.q.out | 118 +-
.../clientpositive/llap/tez_join_hash.q.out | 145 +-
.../llap/tez_join_result_complex.q.out | 64 +-
.../clientpositive/llap/tez_self_join.q.out | 28 +-
.../results/clientpositive/llap/tez_smb_1.q.out | 64 +-
.../clientpositive/llap/tez_smb_main.q.out | 207 +-
.../results/clientpositive/llap/tez_union.q.out | 172 +-
.../llap/tez_vector_dynpart_hashjoin_1.q.out | 120 +-
.../llap/tez_vector_dynpart_hashjoin_2.q.out | 122 +-
.../vectorized_dynamic_partition_pruning.q.out | 269 +-
.../llap/vectorized_nested_mapjoin.q.out | 66 +-
.../clientpositive/llap_uncompressed.q.out | 6 +-
.../clientpositive/louter_join_ppr.q.out | 28 +-
.../clientpositive/mapjoin_distinct.q.out | 60 +-
.../clientpositive/mapjoin_mapjoin.q.out | 60 +-
.../clientpositive/mapjoin_memcheck.q.out | 12 +-
.../clientpositive/mapjoin_subquery.q.out | 44 +-
.../test/results/clientpositive/mergejoin.q.out | 210 +-
.../results/clientpositive/mergejoins.q.out | 38 +-
.../clientpositive/mergejoins_mixed.q.out | 266 +-
.../results/clientpositive/metadataonly1.q.out | 8 +-
.../results/clientpositive/multiMapJoin1.q.out | 384 +--
.../results/clientpositive/multiMapJoin2.q.out | 324 +-
.../clientpositive/multi_join_union.q.out | 26 +-
.../results/clientpositive/nonmr_fetch.q.out | 16 +-
.../test/results/clientpositive/orc_llap.q.out | 52 +-
.../clientpositive/orc_predicate_pushdown.q.out | 32 +-
.../results/clientpositive/parallel_join1.q.out | 18 +-
.../results/clientpositive/parquet_join.q.out | 36 +-
.../parquet_predicate_pushdown.q.out | 32 +-
ql/src/test/results/clientpositive/pcr.q.out | 40 +-
.../results/clientpositive/pointlookup2.q.out | 40 +-
.../results/clientpositive/pointlookup3.q.out | 40 +-
.../test/results/clientpositive/ppd_gby.q.out | 49 +-
.../test/results/clientpositive/ppd_gby2.q.out | 55 +-
.../results/clientpositive/ppd_gby_join.q.out | 96 +-
.../test/results/clientpositive/ppd_join.q.out | 86 +-
.../test/results/clientpositive/ppd_join2.q.out | 133 +-
.../test/results/clientpositive/ppd_join3.q.out | 155 +-
.../test/results/clientpositive/ppd_join5.q.out | 10 +-
.../clientpositive/ppd_join_filter.q.out | 130 +-
.../clientpositive/ppd_multi_insert.q.out | 68 +-
.../clientpositive/ppd_outer_join2.q.out | 66 +-
.../clientpositive/ppd_outer_join3.q.out | 66 +-
.../clientpositive/ppd_outer_join4.q.out | 223 +-
.../clientpositive/ppd_outer_join5.q.out | 231 +-
.../results/clientpositive/ppd_random.q.out | 16 +-
.../clientpositive/ppd_repeated_alias.q.out | 2 +-
.../results/clientpositive/ppd_udf_case.q.out | 59 +-
.../results/clientpositive/ppd_udf_col.q.out | 178 +-
.../test/results/clientpositive/ppd_union.q.out | 42 +-
.../results/clientpositive/ppd_union_view.q.out | 34 +-
ql/src/test/results/clientpositive/ppd_vc.q.out | 30 +-
ql/src/test/results/clientpositive/ptf.q.out | 72 +-
.../results/clientpositive/ptf_streaming.q.out | 58 +-
.../results/clientpositive/ptfgroupbyjoin.q.out | 44 +-
.../results/clientpositive/quotedid_basic.q.out | 12 +-
.../clientpositive/quotedid_partition.q.out | 4 +-
.../reduce_deduplicate_exclude_join.q.out | 12 +-
.../reduce_deduplicate_extended.q.out | 86 +-
.../test/results/clientpositive/regex_col.q.out | 22 +-
.../clientpositive/router_join_ppr.q.out | 30 +-
.../runtime_skewjoin_mapjoin_spark.q.out | 58 +-
.../test/results/clientpositive/sample8.q.out | 46 +-
.../clientpositive/select_transform_hint.q.out | 24 +-
.../test/results/clientpositive/semijoin.q.out | 272 +-
.../test/results/clientpositive/skewjoin.q.out | 259 +-
.../clientpositive/skewjoin_mapjoin10.q.out | 32 +-
.../clientpositive/skewjoin_mapjoin6.q.out | 48 +-
.../clientpositive/skewjoin_noskew.q.out | 18 +-
.../clientpositive/skewjoin_onesideskew.q.out | 18 +-
.../results/clientpositive/skewjoinopt10.q.out | 52 +-
.../results/clientpositive/skewjoinopt12.q.out | 8 +-
.../results/clientpositive/skewjoinopt15.q.out | 40 +-
.../results/clientpositive/skewjoinopt16.q.out | 8 +-
.../results/clientpositive/skewjoinopt17.q.out | 8 +-
.../results/clientpositive/skewjoinopt18.q.out | 14 +-
.../results/clientpositive/skewjoinopt2.q.out | 16 +-
.../results/clientpositive/smb_mapjoin9.q.out | 2 +-
.../results/clientpositive/smb_mapjoin_1.q.out | 2 +-
.../results/clientpositive/smb_mapjoin_10.q.out | 2 +-
.../results/clientpositive/smb_mapjoin_11.q.out | 2 +-
.../results/clientpositive/smb_mapjoin_12.q.out | 4 +-
.../results/clientpositive/smb_mapjoin_13.q.out | 14 +-
.../results/clientpositive/smb_mapjoin_14.q.out | 18 +-
.../results/clientpositive/smb_mapjoin_15.q.out | 18 +-
.../results/clientpositive/smb_mapjoin_16.q.out | 2 +-
.../results/clientpositive/smb_mapjoin_17.q.out | 2 +-
.../results/clientpositive/smb_mapjoin_2.q.out | 4 +-
.../results/clientpositive/smb_mapjoin_25.q.out | 105 +-
.../results/clientpositive/smb_mapjoin_3.q.out | 2 +-
.../results/clientpositive/smb_mapjoin_4.q.out | 2 +-
.../results/clientpositive/smb_mapjoin_6.q.out | 4 +-
.../clientpositive/sort_merge_join_desc_2.q.out | 2 +-
.../clientpositive/sort_merge_join_desc_3.q.out | 2 +-
.../clientpositive/sort_merge_join_desc_4.q.out | 6 +-
.../clientpositive/sort_merge_join_desc_5.q.out | 2 +-
.../clientpositive/sort_merge_join_desc_6.q.out | 6 +-
.../clientpositive/sort_merge_join_desc_7.q.out | 6 +-
.../clientpositive/sort_merge_join_desc_8.q.out | 6 +-
.../spark/annotate_stats_join.q.out | 8 +-
.../clientpositive/spark/auto_join1.q.out | 14 +-
.../clientpositive/spark/auto_join10.q.out | 12 +-
.../clientpositive/spark/auto_join12.q.out | 73 +-
.../clientpositive/spark/auto_join13.q.out | 10 +-
.../clientpositive/spark/auto_join15.q.out | 12 +-
.../clientpositive/spark/auto_join16.q.out | 14 +-
.../clientpositive/spark/auto_join17.q.out | 14 +-
.../clientpositive/spark/auto_join19.q.out | 14 +-
.../clientpositive/spark/auto_join2.q.out | 98 +-
.../clientpositive/spark/auto_join22.q.out | 18 +-
.../clientpositive/spark/auto_join24.q.out | 10 +-
.../clientpositive/spark/auto_join26.q.out | 20 +-
.../clientpositive/spark/auto_join3.q.out | 18 +-
.../clientpositive/spark/auto_join30.q.out | 38 +-
.../clientpositive/spark/auto_join8.q.out | 14 +-
.../clientpositive/spark/auto_join9.q.out | 14 +-
.../spark/auto_join_reordering_values.q.out | 10 +-
.../clientpositive/spark/auto_join_stats.q.out | 58 +-
.../clientpositive/spark/auto_join_stats2.q.out | 46 +-
.../spark/auto_join_without_localtask.q.out | 68 +-
.../spark/auto_smb_mapjoin_14.q.out | 126 +-
.../spark/auto_sortmerge_join_1.q.out | 18 +-
.../spark/auto_sortmerge_join_12.q.out | 12 +-
.../spark/auto_sortmerge_join_13.q.out | 44 +-
.../spark/auto_sortmerge_join_2.q.out | 16 +-
.../spark/auto_sortmerge_join_3.q.out | 22 +-
.../spark/auto_sortmerge_join_4.q.out | 22 +-
.../spark/auto_sortmerge_join_5.q.out | 22 +-
.../spark/auto_sortmerge_join_6.q.out | 144 +-
.../spark/auto_sortmerge_join_7.q.out | 22 +-
.../spark/auto_sortmerge_join_8.q.out | 22 +-
.../spark/auto_sortmerge_join_9.q.out | 220 +-
.../spark/bucket_map_join_spark1.q.out | 28 +-
.../spark/bucket_map_join_spark2.q.out | 28 +-
.../spark/bucket_map_join_spark3.q.out | 28 +-
.../spark/bucket_map_join_spark4.q.out | 140 +-
.../spark/bucket_map_join_tez1.q.out | 361 +-
.../spark/bucket_map_join_tez2.q.out | 185 +-
.../spark/bucketizedhiveinputformat.q.out | 2 -
.../clientpositive/spark/bucketmapjoin1.q.out | 20 +-
.../clientpositive/spark/bucketmapjoin10.q.out | 6 +-
.../clientpositive/spark/bucketmapjoin11.q.out | 12 +-
.../clientpositive/spark/bucketmapjoin12.q.out | 12 +-
.../clientpositive/spark/bucketmapjoin13.q.out | 24 +-
.../clientpositive/spark/bucketmapjoin2.q.out | 30 +-
.../clientpositive/spark/bucketmapjoin3.q.out | 20 +-
.../clientpositive/spark/bucketmapjoin4.q.out | 20 +-
.../clientpositive/spark/bucketmapjoin5.q.out | 20 +-
.../clientpositive/spark/bucketmapjoin7.q.out | 12 +-
.../clientpositive/spark/bucketmapjoin8.q.out | 12 +-
.../clientpositive/spark/bucketmapjoin9.q.out | 12 +-
.../spark/bucketmapjoin_negative.q.out | 10 +-
.../spark/bucketmapjoin_negative2.q.out | 10 +-
.../spark/bucketmapjoin_negative3.q.out | 90 +-
.../spark/bucketsortoptimize_insert_2.q.out | 108 +-
.../spark/bucketsortoptimize_insert_4.q.out | 30 +-
.../spark/bucketsortoptimize_insert_6.q.out | 154 +-
.../spark/bucketsortoptimize_insert_7.q.out | 54 +-
.../spark/bucketsortoptimize_insert_8.q.out | 36 +-
.../spark/column_access_stats.q.out | 52 +-
.../spark/constprog_partitioner.q.out | 44 +-
.../clientpositive/spark/cross_join.q.out | 28 +-
.../spark/cross_product_check_1.q.out | 66 +-
.../spark/cross_product_check_2.q.out | 48 +-
.../spark/dynamic_rdd_cache.q.out | 34 +-
.../spark/filter_join_breaktask.q.out | 28 +-
.../spark/groupby_multi_single_reducer3.q.out | 16 +-
.../clientpositive/spark/groupby_position.q.out | 32 +-
.../spark/groupby_sort_1_23.q.out | 46 +-
.../spark/groupby_sort_skew_1_23.q.out | 50 +-
.../spark/identity_project_remove_skip.q.out | 8 +-
.../spark/index_auto_self_join.q.out | 42 +-
.../clientpositive/spark/index_bitmap3.q.out | 30 +-
.../spark/index_bitmap_auto.q.out | 30 +-
.../spark/infer_bucket_sort_map_operators.q.out | 38 +-
.../infer_bucket_sort_reducers_power_two.q.out | 8 +-
.../clientpositive/spark/innerjoin.q.out | 18 +-
.../results/clientpositive/spark/join1.q.out | 18 +-
.../results/clientpositive/spark/join10.q.out | 18 +-
.../results/clientpositive/spark/join12.q.out | 51 +-
.../results/clientpositive/spark/join13.q.out | 16 +-
.../results/clientpositive/spark/join15.q.out | 18 +-
.../results/clientpositive/spark/join16.q.out | 20 +-
.../results/clientpositive/spark/join17.q.out | 18 +-
.../results/clientpositive/spark/join2.q.out | 32 +-
.../results/clientpositive/spark/join22.q.out | 22 +-
.../results/clientpositive/spark/join25.q.out | 10 +-
.../results/clientpositive/spark/join26.q.out | 12 +-
.../results/clientpositive/spark/join27.q.out | 10 +-
.../results/clientpositive/spark/join28.q.out | 22 +-
.../results/clientpositive/spark/join29.q.out | 26 +-
.../results/clientpositive/spark/join3.q.out | 24 +-
.../results/clientpositive/spark/join30.q.out | 16 +-
.../results/clientpositive/spark/join31.q.out | 28 +-
.../results/clientpositive/spark/join32.q.out | 22 +-
.../clientpositive/spark/join32_lessSize.q.out | 138 +-
.../results/clientpositive/spark/join33.q.out | 22 +-
.../results/clientpositive/spark/join34.q.out | 6 +-
.../results/clientpositive/spark/join35.q.out | 6 +-
.../results/clientpositive/spark/join36.q.out | 10 +-
.../results/clientpositive/spark/join37.q.out | 10 +-
.../results/clientpositive/spark/join40.q.out | 32 +-
.../results/clientpositive/spark/join8.q.out | 16 +-
.../results/clientpositive/spark/join9.q.out | 18 +-
.../clientpositive/spark/join_alt_syntax.q.out | 144 +-
.../spark/join_cond_pushdown_1.q.out | 86 +-
.../spark/join_cond_pushdown_2.q.out | 74 +-
.../spark/join_cond_pushdown_3.q.out | 86 +-
.../spark/join_cond_pushdown_4.q.out | 74 +-
.../spark/join_cond_pushdown_unqual1.q.out | 24 +-
.../spark/join_cond_pushdown_unqual2.q.out | 44 +-
.../spark/join_cond_pushdown_unqual3.q.out | 24 +-
.../spark/join_cond_pushdown_unqual4.q.out | 44 +-
.../clientpositive/spark/join_map_ppr.q.out | 24 +-
.../spark/join_merge_multi_expressions.q.out | 20 +-
.../clientpositive/spark/join_nullsafe.q.out | 36 +-
.../results/clientpositive/spark/join_rc.q.out | 18 +-
.../clientpositive/spark/join_reorder.q.out | 24 +-
.../clientpositive/spark/join_star.q.out | 54 +-
.../clientpositive/spark/join_thrift.q.out | 14 +-
.../results/clientpositive/spark/join_vc.q.out | 30 +-
.../clientpositive/spark/limit_pushdown.q.out | 14 +-
.../clientpositive/spark/louter_join_ppr.q.out | 18 +-
.../clientpositive/spark/mapjoin_decimal.q.out | 16 +-
.../clientpositive/spark/mapjoin_distinct.q.out | 60 +-
.../clientpositive/spark/mapjoin_mapjoin.q.out | 158 +-
.../clientpositive/spark/mapjoin_memcheck.q.out | 12 +-
.../clientpositive/spark/mapjoin_subquery.q.out | 44 +-
.../clientpositive/spark/mergejoins.q.out | 38 +-
.../clientpositive/spark/mergejoins_mixed.q.out | 266 +-
.../clientpositive/spark/multi_join_union.q.out | 28 +-
.../clientpositive/spark/parallel_join1.q.out | 18 +-
.../clientpositive/spark/parquet_join.q.out | 42 +-
.../test/results/clientpositive/spark/pcr.q.out | 40 +-
.../clientpositive/spark/ppd_gby_join.q.out | 96 +-
.../results/clientpositive/spark/ppd_join.q.out | 86 +-
.../clientpositive/spark/ppd_join2.q.out | 133 +-
.../clientpositive/spark/ppd_join3.q.out | 153 +-
.../clientpositive/spark/ppd_join5.q.out | 10 +-
.../clientpositive/spark/ppd_join_filter.q.out | 130 +-
.../clientpositive/spark/ppd_multi_insert.q.out | 68 +-
.../clientpositive/spark/ppd_outer_join2.q.out | 66 +-
.../clientpositive/spark/ppd_outer_join3.q.out | 66 +-
.../clientpositive/spark/ppd_outer_join4.q.out | 211 +-
.../clientpositive/spark/ppd_outer_join5.q.out | 207 +-
.../test/results/clientpositive/spark/ptf.q.out | 72 +-
.../clientpositive/spark/ptf_streaming.q.out | 58 +-
.../spark/reduce_deduplicate_exclude_join.q.out | 12 +-
.../clientpositive/spark/router_join_ppr.q.out | 20 +-
.../spark/runtime_skewjoin_mapjoin_spark.q.out | 26 +-
.../results/clientpositive/spark/sample8.q.out | 46 +-
.../results/clientpositive/spark/semijoin.q.out | 272 +-
.../results/clientpositive/spark/skewjoin.q.out | 283 +-
.../clientpositive/spark/skewjoin_noskew.q.out | 18 +-
.../clientpositive/spark/skewjoinopt10.q.out | 48 +-
.../clientpositive/spark/skewjoinopt12.q.out | 8 +-
.../clientpositive/spark/skewjoinopt15.q.out | 32 +-
.../clientpositive/spark/skewjoinopt16.q.out | 8 +-
.../clientpositive/spark/skewjoinopt17.q.out | 8 +-
.../clientpositive/spark/skewjoinopt18.q.out | 14 +-
.../clientpositive/spark/skewjoinopt2.q.out | 16 +-
.../clientpositive/spark/smb_mapjoin_1.q.out | 14 +-
.../clientpositive/spark/smb_mapjoin_10.q.out | 8 +-
.../clientpositive/spark/smb_mapjoin_11.q.out | 12 +-
.../clientpositive/spark/smb_mapjoin_12.q.out | 24 +-
.../clientpositive/spark/smb_mapjoin_13.q.out | 22 +-
.../clientpositive/spark/smb_mapjoin_14.q.out | 52 +-
.../clientpositive/spark/smb_mapjoin_15.q.out | 42 +-
.../clientpositive/spark/smb_mapjoin_16.q.out | 4 +-
.../clientpositive/spark/smb_mapjoin_17.q.out | 4 +-
.../clientpositive/spark/smb_mapjoin_2.q.out | 16 +-
.../clientpositive/spark/smb_mapjoin_25.q.out | 83 +-
.../clientpositive/spark/smb_mapjoin_3.q.out | 14 +-
.../clientpositive/spark/smb_mapjoin_4.q.out | 8 +-
.../clientpositive/spark/smb_mapjoin_5.q.out | 6 +-
.../clientpositive/spark/smb_mapjoin_6.q.out | 16 +-
.../clientpositive/spark/subquery_exists.q.out | 22 +-
.../clientpositive/spark/subquery_in.q.out | 145 +-
.../spark/subquery_multiinsert.q.java1.7.out | 14 +-
.../spark/table_access_keys_stats.q.out | 37 +-
.../clientpositive/spark/temp_table_join1.q.out | 54 +-
.../results/clientpositive/spark/union26.q.out | 28 +-
.../results/clientpositive/spark/union32.q.out | 80 +-
.../results/clientpositive/spark/union34.q.out | 44 +-
.../clientpositive/spark/union_top_level.q.out | 16 +-
.../spark/vector_decimal_mapjoin.q.out | 12 +-
.../spark/vector_mapjoin_reduce.q.out | 166 +-
.../spark/vectorization_short_regress.q.out | 18 +-
.../spark/vectorized_mapjoin.q.out | 12 +-
.../spark/vectorized_nested_mapjoin.q.out | 32 +-
.../clientpositive/spark/vectorized_ptf.q.out | 72 +-
.../spark/vectorized_shufflejoin.q.out | 16 +-
.../test/results/clientpositive/stats11.q.out | 20 +-
.../subq_where_serialization.q.out | 30 +-
.../clientpositive/subquery_exists.q.out | 22 +-
.../clientpositive/subquery_exists_having.q.out | 44 +-
.../results/clientpositive/subquery_in.q.out | 145 +-
.../clientpositive/subquery_in_having.q.out | 118 +-
.../subquery_multiinsert.q.java1.7.out | 14 +-
.../results/clientpositive/subquery_notin.q.out | 16 +-
.../subquery_notin_having.q.java1.7.out | 6 +-
.../subquery_unqual_corr_expr.q.out | 16 +-
.../subquery_unqualcolumnrefs.q.out | 142 +-
.../results/clientpositive/subquery_views.q.out | 26 +-
.../table_access_keys_stats.q.out | 37 +-
.../clientpositive/temp_table_join1.q.out | 54 +-
.../results/clientpositive/tez/auto_join1.q.out | 16 +-
.../clientpositive/tez/auto_join30.q.out | 50 +-
.../tez/auto_sortmerge_join_1.q.out | 18 +-
.../tez/auto_sortmerge_join_11.q.out | 26 +-
.../tez/auto_sortmerge_join_12.q.out | 12 +-
.../tez/auto_sortmerge_join_13.q.out | 54 +-
.../tez/auto_sortmerge_join_2.q.out | 28 +-
.../tez/auto_sortmerge_join_3.q.out | 42 +-
.../tez/auto_sortmerge_join_4.q.out | 42 +-
.../tez/auto_sortmerge_join_5.q.out | 34 +-
.../tez/auto_sortmerge_join_6.q.out | 192 +-
.../tez/auto_sortmerge_join_7.q.out | 42 +-
.../tez/auto_sortmerge_join_8.q.out | 42 +-
.../tez/auto_sortmerge_join_9.q.out | 268 +-
.../tez/bucket_map_join_tez1.q.out | 339 +-
.../tez/bucket_map_join_tez2.q.out | 133 +-
.../tez/correlationoptimizer1.q.out | 208 +-
.../results/clientpositive/tez/cross_join.q.out | 30 +-
.../tez/cross_product_check_1.q.out | 66 +-
.../tez/cross_product_check_2.q.out | 54 +-
.../tez/dynamic_partition_pruning.q.out | 228 +-
.../tez/dynamic_partition_pruning_2.q.out | 21 +-
.../clientpositive/tez/explainuser_1.q.out | 1502 ++++----
.../clientpositive/tez/explainuser_2.q.out | 3195 +++++++++---------
.../clientpositive/tez/explainuser_3.q.out | 34 +-
.../tez/filter_join_breaktask.q.out | 28 +-
.../tez/hybridgrace_hashjoin_1.q.out | 97 +-
.../tez/hybridgrace_hashjoin_2.q.out | 128 +-
.../test/results/clientpositive/tez/join1.q.out | 18 +-
.../clientpositive/tez/join_nullsafe.q.out | 36 +-
.../clientpositive/tez/limit_pushdown.q.out | 14 +-
.../clientpositive/tez/lvj_mapjoin.q.out | 56 +-
.../clientpositive/tez/mapjoin_decimal.q.out | 18 +-
.../clientpositive/tez/mapjoin_mapjoin.q.out | 74 +-
.../results/clientpositive/tez/mergejoin.q.out | 204 +-
.../test/results/clientpositive/tez/mrr.q.out | 130 +-
.../test/results/clientpositive/tez/ptf.q.out | 72 +-
.../clientpositive/tez/ptf_streaming.q.out | 58 +-
.../results/clientpositive/tez/skewjoin.q.out | 109 +-
.../clientpositive/tez/subquery_exists.q.out | 22 +-
.../clientpositive/tez/subquery_in.q.out | 145 +-
.../tez/tez_bmj_schema_evolution.q.out | 18 +-
.../tez/tez_dynpart_hashjoin_1.q.out | 120 +-
.../tez/tez_dynpart_hashjoin_2.q.out | 118 +-
.../clientpositive/tez/tez_join_hash.q.out | 144 +-
.../tez/tez_join_result_complex.q.out | 64 +-
.../clientpositive/tez/tez_self_join.q.out | 28 +-
.../results/clientpositive/tez/tez_smb_1.q.out | 64 +-
.../clientpositive/tez/tez_smb_empty.q.out | 116 +-
.../clientpositive/tez/tez_smb_main.q.out | 206 +-
.../results/clientpositive/tez/tez_union.q.out | 172 +-
.../tez/tez_vector_dynpart_hashjoin_1.q.out | 120 +-
.../tez/tez_vector_dynpart_hashjoin_2.q.out | 120 +-
.../clientpositive/tez/unionDistinct_1.q.out | 190 +-
.../tez/vector_auto_smb_mapjoin_14.q.out | 300 +-
.../tez/vector_binary_join_groupby.q.out | 14 +-
.../tez/vector_char_mapjoin1.q.out | 56 +-
.../tez/vector_decimal_expressions.q.out | 8 +-
.../tez/vector_decimal_mapjoin.q.out | 14 +-
.../clientpositive/tez/vector_decimal_udf.q.out | 18 +-
.../clientpositive/tez/vector_if_expr.q.out | 10 +-
.../tez/vector_interval_mapjoin.q.out | 24 +-
.../clientpositive/tez/vector_join30.q.out | 50 +-
.../tez/vector_join_part_col_char.q.out | 22 +-
.../tez/vector_leftsemi_mapjoin.q.out | 1500 ++++----
.../tez/vector_mapjoin_reduce.q.out | 133 +-
.../tez/vector_nullsafe_join.q.out | 64 +-
.../tez/vector_reduce_groupby_decimal.q.out | 12 +-
.../tez/vector_varchar_mapjoin1.q.out | 56 +-
.../clientpositive/tez/vectorization_8.q.out | 16 +-
.../tez/vectorization_limit.q.out | 22 +-
.../tez/vectorization_short_regress.q.out | 18 +-
.../tez/vectorized_bucketmapjoin1.q.out | 42 +-
.../clientpositive/tez/vectorized_context.q.out | 26 +-
.../vectorized_dynamic_partition_pruning.q.out | 228 +-
.../clientpositive/tez/vectorized_mapjoin.q.out | 14 +-
.../tez/vectorized_nested_mapjoin.q.out | 66 +-
.../clientpositive/tez/vectorized_ptf.q.out | 72 +-
.../tez/vectorized_shufflejoin.q.out | 16 +-
.../udf_case_column_pruning.q.out | 20 +-
.../clientpositive/udf_isnull_isnotnull.q.out | 4 +-
.../test/results/clientpositive/udf_size.q.out | 4 +-
.../clientpositive/udtf_json_tuple.q.out | 26 +-
.../clientpositive/udtf_parse_url_tuple.q.out | 22 +-
.../test/results/clientpositive/union22.q.out | 6 +-
.../test/results/clientpositive/union26.q.out | 34 +-
.../test/results/clientpositive/union32.q.out | 96 +-
.../test/results/clientpositive/union34.q.out | 56 +-
.../clientpositive/unionDistinct_1.q.out | 214 +-
.../clientpositive/union_top_level.q.out | 32 +-
.../vector_auto_smb_mapjoin_14.q.out | 72 +-
.../vector_binary_join_groupby.q.out | 12 +-
.../clientpositive/vector_char_mapjoin1.q.out | 50 +-
.../clientpositive/vector_decimal_cast.q.out | 4 +-
.../vector_decimal_expressions.q.out | 8 +-
.../clientpositive/vector_decimal_mapjoin.q.out | 12 +-
.../clientpositive/vector_decimal_udf.q.out | 18 +-
.../results/clientpositive/vector_if_expr.q.out | 10 +-
.../vector_interval_mapjoin.q.out | 22 +-
.../results/clientpositive/vector_join30.q.out | 64 +-
.../vector_leftsemi_mapjoin.q.out | 1356 ++++----
.../clientpositive/vector_mapjoin_reduce.q.out | 108 +-
.../clientpositive/vector_nullsafe_join.q.out | 48 +-
.../vector_reduce_groupby_decimal.q.out | 12 +-
.../vector_varchar_mapjoin1.q.out | 50 +-
.../clientpositive/vectorization_8.q.out | 16 +-
.../vectorization_decimal_date.q.out | 4 +-
.../clientpositive/vectorization_limit.q.out | 22 +-
.../vectorization_short_regress.q.out | 18 +-
.../vectorized_bucketmapjoin1.q.out | 6 +-
.../clientpositive/vectorized_context.q.out | 22 +-
.../clientpositive/vectorized_mapjoin.q.out | 12 +-
.../vectorized_nested_mapjoin.q.out | 36 +-
.../results/clientpositive/vectorized_ptf.q.out | 72 +-
.../clientpositive/vectorized_shufflejoin.q.out | 16 +-
639 files changed, 22891 insertions(+), 20623 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/hbase-handler/src/test/results/positive/hbase_queries.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out
index 1ab9877..3907bc9 100644
--- a/hbase-handler/src/test/results/positive/hbase_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -121,7 +121,7 @@ STAGE PLANS:
alias: hbase_table_1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: UDFToDouble(key) is not null (type: boolean)
+ predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int)
@@ -136,17 +136,17 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: UDFToDouble(key) is not null (type: boolean)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: UDFToDouble(_col0) (type: double)
sort order: +
Map-reduce partition columns: UDFToDouble(_col0) (type: double)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: string)
Reduce Operator Tree:
Join Operator
@@ -156,11 +156,11 @@ STAGE PLANS:
0 UDFToDouble(_col0) (type: double)
1 UDFToDouble(_col0) (type: double)
outputColumnNames: _col1, _col2
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -175,13 +175,13 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 20
Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
@@ -494,19 +494,19 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: UDFToDouble(key) is not null (type: boolean)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(key)
keys: key (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Group By Operator
@@ -514,7 +514,7 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -529,7 +529,7 @@ STAGE PLANS:
alias: hbase_table_1
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
- predicate: UDFToDouble(key) is not null (type: boolean)
+ predicate: (key is not null and UDFToDouble(key) is not null) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: int), value (type: string)
@@ -546,7 +546,7 @@ STAGE PLANS:
key expressions: UDFToDouble(_col0) (type: double)
sort order: +
Map-reduce partition columns: UDFToDouble(_col0) (type: double)
- Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Join Operator
@@ -556,14 +556,14 @@ STAGE PLANS:
0 UDFToDouble(_col0) (type: double)
1 UDFToDouble(_col0) (type: double)
outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), UDFToInteger(_col3) (type: int)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat
output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
index e1b60b0..4825a61 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
@@ -25,6 +25,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.RelOptUtil.InputFinder;
import org.apache.calcite.plan.RelOptUtil.InputReferencedVisitor;
@@ -32,6 +34,8 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
@@ -44,6 +48,7 @@ import org.apache.calcite.rex.RexLocalRef;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexOver;
import org.apache.calcite.rex.RexRangeRef;
+import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.rex.RexVisitor;
import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.sql.SqlKind;
@@ -613,6 +618,46 @@ public class HiveCalciteUtil {
return (new Pair<RelNode, RelNode>(parentOforiginalProjRel, originalProjRel));
}
+ public static boolean isComparisonOp(RexCall call) {
+ return call.getKind().belongsTo(SqlKind.COMPARISON);
+ }
+
+ private static final Function<RexNode, String> REX_STR_FN = new Function<RexNode, String>() {
+ public String apply(RexNode r) {
+ return r.toString();
+ }
+ };
+
+ public static ImmutableList<RexNode> getPredsNotPushedAlready(RelNode inp, List<RexNode> predsToPushDown) {
+ final RelOptPredicateList predicates = RelMetadataQuery.getPulledUpPredicates(inp);
+ final ImmutableSet<String> alreadyPushedPreds = ImmutableSet.copyOf(Lists.transform(
+ predicates.pulledUpPredicates, REX_STR_FN));
+ final ImmutableList.Builder<RexNode> newConjuncts = ImmutableList.builder();
+ for (RexNode r : predsToPushDown) {
+ if (!alreadyPushedPreds.contains(r.toString())) {
+ newConjuncts.add(r);
+ }
+ }
+ return newConjuncts.build();
+ }
+
+ public static RexNode getTypeSafePred(RelOptCluster cluster, RexNode rex, RelDataType rType) {
+ RexNode typeSafeRex = rex;
+ if ((typeSafeRex instanceof RexCall) && HiveCalciteUtil.isComparisonOp((RexCall) typeSafeRex)) {
+ RexBuilder rb = cluster.getRexBuilder();
+ List<RexNode> fixedPredElems = new ArrayList<RexNode>();
+ RelDataType commonType = cluster.getTypeFactory().leastRestrictive(
+ RexUtil.types(((RexCall) rex).getOperands()));
+ for (RexNode rn : ((RexCall) rex).getOperands()) {
+ fixedPredElems.add(rb.ensureType(commonType, rn, true));
+ }
+
+ typeSafeRex = rb.makeCall(((RexCall) typeSafeRex).getOperator(), fixedPredElems);
+ }
+
+ return typeSafeRex;
+ }
+
public static boolean isDeterministic(RexNode expr) {
boolean deterministic = true;
@@ -635,6 +680,58 @@ public class HiveCalciteUtil {
return deterministic;
}
+ public static boolean isDeterministicFuncOnLiterals(RexNode expr) {
+ boolean deterministicFuncOnLiterals = true;
+
+ RexVisitor<Void> visitor = new RexVisitorImpl<Void>(true) {
+ @Override
+ public Void visitCall(org.apache.calcite.rex.RexCall call) {
+ if (!call.getOperator().isDeterministic()) {
+ throw new Util.FoundOne(call);
+ }
+ return super.visitCall(call);
+ }
+
+ @Override
+ public Void visitInputRef(RexInputRef inputRef) {
+ throw new Util.FoundOne(inputRef);
+ }
+
+ @Override
+ public Void visitLocalRef(RexLocalRef localRef) {
+ throw new Util.FoundOne(localRef);
+ }
+
+ @Override
+ public Void visitOver(RexOver over) {
+ throw new Util.FoundOne(over);
+ }
+
+ @Override
+ public Void visitDynamicParam(RexDynamicParam dynamicParam) {
+ throw new Util.FoundOne(dynamicParam);
+ }
+
+ @Override
+ public Void visitRangeRef(RexRangeRef rangeRef) {
+ throw new Util.FoundOne(rangeRef);
+ }
+
+ @Override
+ public Void visitFieldAccess(RexFieldAccess fieldAccess) {
+ throw new Util.FoundOne(fieldAccess);
+ }
+ };
+
+ try {
+ expr.accept(visitor);
+ } catch (Util.FoundOne e) {
+ deterministicFuncOnLiterals = false;
+ }
+
+ return deterministicFuncOnLiterals;
+ }
+
public static <T> ImmutableMap<Integer, T> getColInfoMap(List<T> hiveCols,
int startIndx) {
Builder<Integer, T> bldr = ImmutableMap.<Integer, T> builder();
@@ -803,6 +900,7 @@ public class HiveCalciteUtil {
public static Set<Integer> getInputRefs(RexNode expr) {
InputRefsCollector irefColl = new InputRefsCollector(true);
+ expr.accept(irefColl);
return irefColl.getInputRefSet();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
index 748c86d..c0609d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveDefaultRelMetadataProvider.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdDistinctRowCou
import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdDistribution;
import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdMemory;
import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdParallelism;
+import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdPredicates;
import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdRowCount;
import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdSelectivity;
import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdSize;
@@ -75,6 +76,7 @@ public class HiveDefaultRelMetadataProvider {
new HiveRelMdParallelism(maxSplitSize).getMetadataProvider(),
HiveRelMdDistribution.SOURCE,
HiveRelMdCollation.SOURCE,
+ HiveRelMdPredicates.SOURCE,
new DefaultRelMetadataProvider()));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
index cce3588..02db680 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/RelOptHiveTable.java
@@ -198,7 +198,7 @@ public class RelOptHiveTable extends RelOptAbstractTable {
if (null == partitionList) {
// we are here either unpartitioned table or partitioned table with no
// predicates
- computePartitionList(hiveConf, null);
+ computePartitionList(hiveConf, null, new HashSet<Integer>());
}
if (hiveTblMetadata.isPartitioned()) {
List<Long> rowCounts = StatsUtils.getBasicStatForPartitions(hiveTblMetadata,
@@ -234,8 +234,7 @@ public class RelOptHiveTable extends RelOptAbstractTable {
return sb.toString();
}
- public void computePartitionList(HiveConf conf, RexNode pruneNode) {
-
+ public void computePartitionList(HiveConf conf, RexNode pruneNode, Set<Integer> partOrVirtualCols) {
try {
if (!hiveTblMetadata.isPartitioned() || pruneNode == null
|| InputFinder.bits(pruneNode).length() == 0) {
@@ -248,7 +247,7 @@ public class RelOptHiveTable extends RelOptAbstractTable {
// We have valid pruning expressions, only retrieve qualifying partitions
ExprNodeDesc pruneExpr = pruneNode.accept(new ExprNodeConverter(getName(), getRowType(),
- HiveCalciteUtil.getInputRefs(pruneNode), this.getRelOptSchema().getTypeFactory()));
+ partOrVirtualCols, this.getRelOptSchema().getTypeFactory()));
partitionList = PartitionPruner.prune(hiveTblMetadata, pruneExpr, conf, getName(),
partitionCache);
@@ -287,7 +286,7 @@ public class RelOptHiveTable extends RelOptAbstractTable {
if (null == partitionList) {
// We could be here either because its an unpartitioned table or because
// there are no pruning predicates on a partitioned table.
- computePartitionList(hiveConf, null);
+ computePartitionList(hiveConf, null, new HashSet<Integer>());
}
// 2. Obtain Col Stats for Non Partition Cols
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
index be28828..024ba46 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/TraitsUtil.java
@@ -34,4 +34,8 @@ public class TraitsUtil {
public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster) {
return cluster.traitSetOf(HiveRelNode.CONVENTION, RelCollations.EMPTY);
}
+
+ public static RelTraitSet getDefaultTraitSet(RelOptCluster cluster, RelTraitSet traitsFromInput) {
+ return RelTraitSet.createEmpty().merge(traitsFromInput).merge(getDefaultTraitSet(cluster));
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
index 4b7887a..db625c1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
@@ -50,6 +50,7 @@ public class HiveProject extends Project implements HiveRelNode {
public static final ProjectFactory DEFAULT_PROJECT_FACTORY = new HiveProjectFactoryImpl();
private final List<Integer> virtualCols;
+ private boolean isSysnthetic;
/**
* Creates a HiveProject.
@@ -165,10 +166,14 @@ public class HiveProject extends Project implements HiveRelNode {
}
@Override
- public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> exps,
- RelDataType rowType) {
+ public Project copy(RelTraitSet traitSet, RelNode input, List<RexNode> exps, RelDataType rowType) {
assert traitSet.containsIfApplicable(HiveRelNode.CONVENTION);
- return new HiveProject(getCluster(), traitSet, input, exps, rowType, getFlags());
+ HiveProject hp = new HiveProject(getCluster(), traitSet, input, exps, rowType, getFlags());
+ if (this.isSysnthetic()) {
+ hp.setSynthetic();
+ }
+
+ return hp;
}
@Override
@@ -184,6 +189,16 @@ public class HiveProject extends Project implements HiveRelNode {
return virtualCols;
}
+ // TODO: this should come through RelBuilder to the constructor as opposed to
+ // set method. This requires calcite change
+ public void setSynthetic() {
+ this.isSysnthetic = true;
+ }
+
+ public boolean isSysnthetic() {
+ return isSysnthetic;
+ }
+
/**
* Implementation of {@link ProjectFactory} that returns
* {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject}
@@ -196,9 +211,9 @@ public class HiveProject extends Project implements HiveRelNode {
List<? extends RexNode> childExprs, List<String> fieldNames) {
RelOptCluster cluster = child.getCluster();
RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames);
+ RelTraitSet trait = TraitsUtil.getDefaultTraitSet(cluster, child.getTraitSet());
RelNode project = HiveProject.create(cluster, child,
- childExprs, rowType,
- child.getTraitSet(), Collections.<RelCollation> emptyList());
+ childExprs, rowType, trait, Collections.<RelCollation> emptyList());
return project;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
index 446dc73..5788805 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
@@ -37,6 +37,7 @@ import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Pair;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
@@ -44,6 +45,7 @@ import org.apache.hadoop.hive.ql.plan.ColStatistics;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
+import com.google.common.collect.ImmutableSet;
/**
@@ -61,6 +63,7 @@ public class HiveTableScan extends TableScan implements HiveRelNode {
private final String tblAlias;
private final String concatQbIDAlias;
private final boolean useQBIdInDigest;
+ private final ImmutableSet<Integer> viurtualOrPartColIndxsInTS;
public String getTableAlias() {
return tblAlias;
@@ -94,7 +97,9 @@ public class HiveTableScan extends TableScan implements HiveRelNode {
this.tblAlias = alias;
this.concatQbIDAlias = concatQbIDAlias;
this.hiveTableScanRowType = newRowtype;
- this.neededColIndxsFrmReloptHT = buildNeededColIndxsFrmReloptHT(table.getRowType(), newRowtype);
+ Pair<ImmutableList<Integer>, ImmutableSet<Integer>> colIndxPair = buildColIndxsFrmReloptHT(table, newRowtype);
+ this.neededColIndxsFrmReloptHT = colIndxPair.getKey();
+ this.viurtualOrPartColIndxsInTS = colIndxPair.getValue();
this.useQBIdInDigest = useQBIdInDigest;
}
@@ -183,7 +188,13 @@ public class HiveTableScan extends TableScan implements HiveRelNode {
fieldNames));
// 5. Add Proj on top of TS
- return relBuilder.push(newHT).project(exprList, new ArrayList<String>(fieldNames)).build();
+ HiveProject hp = (HiveProject) relBuilder.push(newHT)
+ .project(exprList, new ArrayList<String>(fieldNames)).build();
+
+ // 6. Set synthetic flag, so that we would push filter below this one
+ hp.setSynthetic();
+
+ return hp;
}
public List<Integer> getNeededColIndxsFrmReloptHT() {
@@ -194,17 +205,36 @@ public class HiveTableScan extends TableScan implements HiveRelNode {
return hiveTableScanRowType;
}
- private static ImmutableList<Integer> buildNeededColIndxsFrmReloptHT(RelDataType htRowtype,
- RelDataType scanRowType) {
+ public Set<Integer> getPartOrVirtualCols() {
+ return viurtualOrPartColIndxsInTS;
+ }
+
+ private static Pair<ImmutableList<Integer>, ImmutableSet<Integer>> buildColIndxsFrmReloptHT(
+ RelOptHiveTable relOptHTable, RelDataType scanRowType) {
+ RelDataType relOptHtRowtype = relOptHTable.getRowType();
+ ImmutableList<Integer> neededColIndxsFrmReloptHT;
Builder<Integer> neededColIndxsFrmReloptHTBldr = new ImmutableList.Builder<Integer>();
- Map<String, Integer> colNameToPosInReloptHT = HiveCalciteUtil.getRowColNameIndxMap(htRowtype
- .getFieldList());
+ ImmutableSet<Integer> viurtualOrPartColIndxsInTS;
+ ImmutableSet.Builder<Integer> viurtualOrPartColIndxsInTSBldr = new ImmutableSet.Builder<Integer>();
+
+ Map<String, Integer> colNameToPosInReloptHT = HiveCalciteUtil
+ .getRowColNameIndxMap(relOptHtRowtype.getFieldList());
List<String> colNamesInScanRowType = scanRowType.getFieldNames();
+ int partOrVirtualColStartPosInrelOptHtRowtype = relOptHTable.getNonPartColumns().size();
+ int tmp;
for (int i = 0; i < colNamesInScanRowType.size(); i++) {
- neededColIndxsFrmReloptHTBldr.add(colNameToPosInReloptHT.get(colNamesInScanRowType.get(i)));
+ tmp = colNameToPosInReloptHT.get(colNamesInScanRowType.get(i));
+ neededColIndxsFrmReloptHTBldr.add(tmp);
+ if (tmp >= partOrVirtualColStartPosInrelOptHtRowtype) {
+ viurtualOrPartColIndxsInTSBldr.add(i);
+ }
}
- return neededColIndxsFrmReloptHTBldr.build();
+ neededColIndxsFrmReloptHT = neededColIndxsFrmReloptHTBldr.build();
+ viurtualOrPartColIndxsInTS = viurtualOrPartColIndxsInTSBldr.build();
+
+ return new Pair<ImmutableList<Integer>, ImmutableSet<Integer>>(neededColIndxsFrmReloptHT,
+ viurtualOrPartColIndxsInTS);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java
new file mode 100644
index 0000000..0e5c731
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterAggregateTransposeRule.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.RelFactories.FilterFactory;
+import org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
+import org.apache.calcite.rex.RexNode;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+
+public class HiveFilterAggregateTransposeRule extends FilterAggregateTransposeRule {
+
+ public HiveFilterAggregateTransposeRule(Class<? extends Filter> filterClass,
+ FilterFactory filterFactory, Class<? extends Aggregate> aggregateClass) {
+ super(filterClass, filterFactory, aggregateClass);
+ }
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ final Filter filterRel = call.rel(0);
+ RexNode condition = filterRel.getCondition();
+ if (!HiveCalciteUtil.isDeterministic(condition)) {
+ return false;
+ }
+
+ return super.matches(call);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
new file mode 100644
index 0000000..8321504
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.core.RelFactories.FilterFactory;
+import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexOver;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+
+//TODO: Remove this once Calcite FilterProjectTransposeRule can take rule operand
+public class HiveFilterProjectTSTransposeRule extends RelOptRule {
+
+ private final RelFactories.FilterFactory filterFactory;
+ private final RelFactories.ProjectFactory projectFactory;
+
+ public HiveFilterProjectTSTransposeRule(Class<? extends Filter> filterClass,
+ FilterFactory filterFactory, Class<? extends Project> projectClass,
+ ProjectFactory projectFactory, Class<? extends TableScan> tsClass) {
+ super(operand(filterClass, operand(projectClass, operand(tsClass, none()))));
+ this.filterFactory = filterFactory;
+ this.projectFactory = projectFactory;
+ }
+
+ @Override
+ public boolean matches(RelOptRuleCall call) {
+ final HiveProject projRel = call.rel(1);
+
+ // Assumption:
+ // 1. This will be run last after PP, Col Pruning in the PreJoinOrder
+ // optimizations.
+ // 2. If ProjectRel is not synthetic then PPD would have already pushed
+ // relevant pieces down and hence no point in running PPD again.
+ // 3. For synthetic Projects we don't care about non deterministic UDFs
+ if (!projRel.isSysnthetic()) {
+ return false;
+ }
+
+ return super.matches(call);
+ }
+
+ // ~ Methods ----------------------------------------------------------------
+
+ // implement RelOptRule
+ public void onMatch(RelOptRuleCall call) {
+ final Filter filter = call.rel(0);
+ final Project project = call.rel(1);
+
+ if (RexOver.containsOver(project.getProjects(), null)) {
+ // In general a filter cannot be pushed below a windowing calculation.
+ // Applying the filter before the aggregation function changes
+ // the results of the windowing invocation.
+ //
+ // When the filter is on the PARTITION BY expression of the OVER clause
+ // it can be pushed down. For now we don't support this.
+ return;
+ }
+
+ if (RexUtil.containsCorrelation(filter.getCondition())) {
+ // If there is a correlation condition anywhere in the filter, don't
+ // push this filter past project since in some cases it can prevent a
+ // Correlate from being de-correlated.
+ return;
+ }
+
+ // convert the filter to one that references the child of the project
+ RexNode newCondition = RelOptUtil.pushPastProject(filter.getCondition(), project);
+
+ // Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
+ // nullable and not-nullable conditions, but a CAST might get in the way of
+ // other rewrites.
+ final RelDataTypeFactory typeFactory = filter.getCluster().getTypeFactory();
+ if (RexUtil.isNullabilityCast(typeFactory, newCondition)) {
+ newCondition = ((RexCall) newCondition).getOperands().get(0);
+ }
+
+ RelNode newFilterRel = filterFactory == null ? filter.copy(filter.getTraitSet(),
+ project.getInput(), newCondition) : filterFactory.createFilter(project.getInput(),
+ newCondition);
+
+ RelNode newProjRel = projectFactory == null ? project.copy(project.getTraitSet(), newFilterRel,
+ project.getProjects(), project.getRowType()) : projectFactory.createProject(newFilterRel,
+ project.getProjects(), project.getRowType().getFieldNames());
+
+ call.transformTo(newProjRel);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java
index 29deed9..703c8c6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinPushTransitivePredicatesRule.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
import java.util.ArrayList;
import java.util.List;
+import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
@@ -26,6 +27,7 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
@@ -35,6 +37,7 @@ import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.util.Util;
import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
import org.apache.hive.common.util.AnnotationUtils;
@@ -68,14 +71,21 @@ public class HiveJoinPushTransitivePredicatesRule extends RelOptRule {
@Override public void onMatch(RelOptRuleCall call) {
Join join = call.rel(0);
+
+ // Register that we have visited this operator in this rule
+ HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class);
+ if (registry != null) {
+ registry.registerVisited(this, join);
+ }
+
RelOptPredicateList preds = RelMetadataQuery.getPulledUpPredicates(join);
RexBuilder rB = join.getCluster().getRexBuilder();
RelNode lChild = call.rel(1);
RelNode rChild = call.rel(2);
- List<RexNode> leftPreds = getValidPreds(preds.leftInferredPredicates, lChild.getRowType().getFieldList());
- List<RexNode> rightPreds = getValidPreds(preds.rightInferredPredicates, rChild.getRowType().getFieldList());
+ List<RexNode> leftPreds = getValidPreds(join.getCluster(), lChild, preds.leftInferredPredicates, lChild.getRowType());
+ List<RexNode> rightPreds = getValidPreds(join.getCluster(), rChild, preds.rightInferredPredicates, rChild.getRowType());
if (leftPreds.isEmpty() && rightPreds.isEmpty()) {
return;
@@ -97,11 +107,17 @@ public class HiveJoinPushTransitivePredicatesRule extends RelOptRule {
lChild, rChild, join.getJoinType(), join.isSemiJoinDone());
call.getPlanner().onCopy(join, newRel);
+ // We register new Join rel so we do not fire the rule on them again
+ if (registry != null) {
+ registry.registerVisited(this, newRel);
+ }
+
call.transformTo(newRel);
}
- private ImmutableList<RexNode> getValidPreds (List<RexNode> rexs, List<RelDataTypeField> types) {
- InputRefValidator validator = new InputRefValidator(types);
+ private ImmutableList<RexNode> getValidPreds(RelOptCluster cluster, RelNode rn,
+ List<RexNode> rexs, RelDataType rType) {
+ InputRefValidator validator = new InputRefValidator(rType.getFieldList());
List<RexNode> valids = new ArrayList<RexNode>(rexs.size());
for (RexNode rex : rexs) {
try {
@@ -111,7 +127,25 @@ public class HiveJoinPushTransitivePredicatesRule extends RelOptRule {
Util.swallow(e, null);
}
}
- return ImmutableList.copyOf(valids);
+
+ return HiveCalciteUtil.getPredsNotPushedAlready(rn, valids);
+ }
+
+ private RexNode getTypeSafePred(RelOptCluster cluster, RexNode rex, RelDataType rType) {
+ RexNode typeSafeRex = rex;
+ if ((typeSafeRex instanceof RexCall) && HiveCalciteUtil.isComparisonOp((RexCall) typeSafeRex)) {
+ RexBuilder rb = cluster.getRexBuilder();
+ List<RexNode> fixedPredElems = new ArrayList<RexNode>();
+ RelDataType commonType = cluster.getTypeFactory().leastRestrictive(
+ RexUtil.types(((RexCall) rex).getOperands()));
+ for (RexNode rn : ((RexCall) rex).getOperands()) {
+ fixedPredElems.add(rb.ensureType(commonType, rn, true));
+ }
+
+ typeSafeRex = rb.makeCall(((RexCall) typeSafeRex).getOperator(), fixedPredElems);
+ }
+
+ return typeSafeRex;
}
private static class InputRefValidator extends RexVisitorImpl<Void> {
http://git-wip-us.apache.org/repos/asf/hive/blob/d8ee05ae/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java
index ba28055..d8c3a22 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePartitionPruneRule.java
@@ -52,6 +52,6 @@ public class HivePartitionPruneRule extends RelOptRule {
Pair<RexNode, RexNode> predicates = PartitionPrune
.extractPartitionPredicates(filter.getCluster(), hiveTable, predicate);
RexNode partColExpr = predicates.left;
- hiveTable.computePartitionList(conf, partColExpr);
+ hiveTable.computePartitionList(conf, partColExpr, tScan.getPartOrVirtualCols());
}
}