You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by px...@apache.org on 2016/01/17 06:31:45 UTC
[23/23] hive git commit: HIVE-12661:
StatsSetupConst.COLUMN_STATS_ACCURATE is not used correctly (Pengcheng Xiong,
reviewed by Ashutosh Chauhan)
HIVE-12661: StatsSetupConst.COLUMN_STATS_ACCURATE is not used correctly (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d82409d0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d82409d0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d82409d0
Branch: refs/heads/master
Commit: d82409d0d804fa441b7956623ed92719886d2fcd
Parents: aa0f8e0
Author: Pengcheng Xiong <px...@apache.org>
Authored: Sat Jan 16 21:17:07 2016 -0800
Committer: Pengcheng Xiong <px...@apache.org>
Committed: Sat Jan 16 21:17:07 2016 -0800
----------------------------------------------------------------------
.../hadoop/hive/common/StatsSetupConst.java | 211 +++-
.../listener/TestDbNotificationListener.java | 26 +-
.../hadoop/hive/metastore/MetaStoreUtils.java | 14 +-
.../hadoop/hive/metastore/ObjectStore.java | 25 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 7 +-
.../apache/hadoop/hive/ql/exec/StatsTask.java | 12 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 18 +-
.../apache/hadoop/hive/ql/metadata/Table.java | 2 +-
.../hive/ql/optimizer/StatsOptimizer.java | 62 +-
.../apache/hadoop/hive/ql/plan/StatsWork.java | 4 +
.../columnStatsUpdateForStatsOptimizer_1.q | 102 ++
.../columnStatsUpdateForStatsOptimizer_2.q | 58 ++
.../stats_partialscan_autogether.q.out | 3 -
.../clientnegative/unset_table_property.q.out | 3 -
.../clientpositive/alter_file_format.q.out | 33 -
.../clientpositive/alter_merge_stats_orc.q.out | 10 +-
...lter_numbuckets_partitioned_table2_h23.q.out | 18 +-
...alter_numbuckets_partitioned_table_h23.q.out | 8 +-
.../alter_partition_clusterby_sortby.q.out | 9 -
.../alter_partition_coltype.q.out | 16 +-
.../clientpositive/alter_skewed_table.q.out | 9 -
.../clientpositive/alter_table_not_sorted.q.out | 3 -
.../clientpositive/alter_table_serde2.q.out | 4 +-
.../analyze_table_null_partition.q.out | 8 +-
.../auto_join_reordering_values.q.out | 16 +-
.../clientpositive/auto_sortmerge_join_1.q.out | 45 -
.../clientpositive/auto_sortmerge_join_11.q.out | 48 -
.../clientpositive/auto_sortmerge_join_12.q.out | 21 -
.../clientpositive/auto_sortmerge_join_2.q.out | 39 -
.../clientpositive/auto_sortmerge_join_3.q.out | 36 -
.../clientpositive/auto_sortmerge_join_4.q.out | 36 -
.../clientpositive/auto_sortmerge_join_5.q.out | 12 -
.../clientpositive/auto_sortmerge_join_7.q.out | 54 --
.../clientpositive/auto_sortmerge_join_8.q.out | 54 --
.../clientpositive/binary_output_format.q.out | 4 +-
.../test/results/clientpositive/bucket1.q.out | 4 +-
.../test/results/clientpositive/bucket2.q.out | 4 +-
.../test/results/clientpositive/bucket3.q.out | 4 +-
.../test/results/clientpositive/bucket4.q.out | 4 +-
.../test/results/clientpositive/bucket5.q.out | 6 +-
.../results/clientpositive/bucket_many.q.out | 4 +-
.../clientpositive/bucket_map_join_1.q.out | 2 -
.../clientpositive/bucket_map_join_2.q.out | 2 -
.../clientpositive/bucket_map_join_spark1.q.out | 22 +-
.../clientpositive/bucket_map_join_spark2.q.out | 22 +-
.../clientpositive/bucket_map_join_spark3.q.out | 22 +-
.../clientpositive/bucket_map_join_spark4.q.out | 24 +-
.../clientpositive/bucketcontext_1.q.out | 15 -
.../clientpositive/bucketcontext_2.q.out | 15 -
.../clientpositive/bucketcontext_3.q.out | 12 -
.../clientpositive/bucketcontext_4.q.out | 12 -
.../clientpositive/bucketcontext_5.q.out | 4 -
.../clientpositive/bucketcontext_6.q.out | 12 -
.../clientpositive/bucketcontext_7.q.out | 18 -
.../clientpositive/bucketcontext_8.q.out | 18 -
.../results/clientpositive/bucketmapjoin1.q.out | 24 +-
.../clientpositive/bucketmapjoin10.q.out | 12 -
.../clientpositive/bucketmapjoin11.q.out | 24 -
.../clientpositive/bucketmapjoin12.q.out | 12 -
.../clientpositive/bucketmapjoin13.q.out | 18 +-
.../results/clientpositive/bucketmapjoin2.q.out | 53 +-
.../results/clientpositive/bucketmapjoin3.q.out | 28 +-
.../results/clientpositive/bucketmapjoin4.q.out | 20 +-
.../results/clientpositive/bucketmapjoin5.q.out | 28 +-
.../results/clientpositive/bucketmapjoin7.q.out | 6 -
.../results/clientpositive/bucketmapjoin8.q.out | 12 -
.../results/clientpositive/bucketmapjoin9.q.out | 12 -
.../clientpositive/bucketmapjoin_negative.q.out | 5 -
.../bucketmapjoin_negative2.q.out | 8 -
.../bucketmapjoin_negative3.q.out | 18 -
.../cbo_rp_outer_join_ppr.q.java1.7.out | 20 +-
.../columnStatsUpdateForStatsOptimizer_1.q.out | 967 +++++++++++++++++++
.../columnStatsUpdateForStatsOptimizer_2.q.out | 443 +++++++++
.../clientpositive/columnstats_partlvl.q.out | 6 -
.../clientpositive/columnstats_tbllvl.q.out | 4 -
.../test/results/clientpositive/combine2.q.out | 16 +-
.../constantPropagateForSubQuery.q.out | 8 +-
.../create_alter_list_bucketing_table1.q.out | 15 -
.../results/clientpositive/create_like.q.out | 3 -
ql/src/test/results/clientpositive/ctas.q.out | 14 +-
.../results/clientpositive/ctas_colname.q.out | 14 +-
.../ctas_uses_database_location.q.out | 2 +-
.../results/clientpositive/decimal_serde.q.out | 4 +-
.../describe_comment_nonascii.q.out | 3 -
.../results/clientpositive/describe_table.q.out | 4 +-
.../disable_merge_for_bucketing.q.out | 4 +-
.../display_colstats_tbllvl.q.out | 2 -
.../dynamic_partition_skip_default.q.out | 8 +-
.../dynpart_sort_opt_vectorization.q.out | 32 +-
.../dynpart_sort_optimization.q.out | 32 +-
.../dynpart_sort_optimization2.q.out | 24 +-
.../encryption_join_unencrypted_tbl.q.out | 6 +-
...on_join_with_different_encryption_keys.q.out | 8 +-
.../extrapolate_part_stats_full.q.out | 24 +-
.../extrapolate_part_stats_partial.q.out | 76 +-
.../extrapolate_part_stats_partial_ndv.q.out | 38 +-
.../clientpositive/filter_join_breaktask.q.out | 4 +-
.../clientpositive/fouter_join_ppr.q.out | 40 +-
.../clientpositive/groupby_map_ppr.q.out | 4 +-
.../groupby_map_ppr_multi_distinct.q.out | 4 +-
.../results/clientpositive/groupby_ppr.q.out | 4 +-
.../groupby_ppr_multi_distinct.q.out | 4 +-
.../clientpositive/groupby_sort_1_23.q.out | 208 ++--
.../results/clientpositive/groupby_sort_6.q.out | 11 +-
.../clientpositive/groupby_sort_skew_1_23.q.out | 208 ++--
.../clientpositive/infer_bucket_sort.q.out | 50 +-
.../infer_bucket_sort_bucketed_table.q.out | 2 +-
.../infer_bucket_sort_convert_join.q.out | 4 +-
.../infer_bucket_sort_dyn_part.q.out | 16 +-
.../infer_bucket_sort_grouping_operators.q.out | 12 +-
.../infer_bucket_sort_list_bucket.q.out | 4 +-
.../infer_bucket_sort_map_operators.q.out | 8 +-
.../infer_bucket_sort_merge.q.out | 4 +-
.../infer_bucket_sort_multi_insert.q.out | 16 +-
.../infer_bucket_sort_num_buckets.q.out | 4 +-
.../infer_bucket_sort_reducers_power_two.q.out | 12 +-
.../test/results/clientpositive/input23.q.out | 2 +-
.../test/results/clientpositive/input42.q.out | 12 +-
.../results/clientpositive/input_part1.q.out | 2 +-
.../results/clientpositive/input_part2.q.out | 4 +-
.../results/clientpositive/input_part7.q.out | 4 +-
.../results/clientpositive/input_part9.q.out | 4 +-
.../results/clientpositive/insert_into1.q.out | 4 +-
ql/src/test/results/clientpositive/join17.q.out | 4 +-
ql/src/test/results/clientpositive/join26.q.out | 2 +-
ql/src/test/results/clientpositive/join32.q.out | 10 +-
.../clientpositive/join32_lessSize.q.out | 54 +-
ql/src/test/results/clientpositive/join33.q.out | 10 +-
ql/src/test/results/clientpositive/join34.q.out | 8 +-
ql/src/test/results/clientpositive/join35.q.out | 12 +-
ql/src/test/results/clientpositive/join9.q.out | 6 +-
.../clientpositive/join_filters_overlap.q.out | 20 +-
.../results/clientpositive/join_map_ppr.q.out | 20 +-
.../results/clientpositive/lb_fs_stats.q.out | 2 +-
.../clientpositive/list_bucket_dml_1.q.out | 10 +-
.../list_bucket_dml_10.q.java1.7.out | 6 +-
.../list_bucket_dml_11.q.java1.7.out | 8 +-
.../list_bucket_dml_12.q.java1.7.out | 10 +-
.../list_bucket_dml_13.q.java1.7.out | 8 +-
.../clientpositive/list_bucket_dml_14.q.out | 10 +-
.../list_bucket_dml_2.q.java1.7.out | 8 +-
.../clientpositive/list_bucket_dml_3.q.out | 8 +-
.../list_bucket_dml_4.q.java1.7.out | 14 +-
.../list_bucket_dml_5.q.java1.7.out | 12 +-
.../list_bucket_dml_6.q.java1.7.out | 20 +-
.../clientpositive/list_bucket_dml_7.q.out | 20 +-
.../list_bucket_dml_8.q.java1.7.out | 16 +-
.../list_bucket_dml_9.q.java1.7.out | 14 +-
.../list_bucket_query_multiskew_1.q.out | 10 +-
.../list_bucket_query_multiskew_2.q.out | 8 +-
.../list_bucket_query_multiskew_3.q.out | 14 +-
.../list_bucket_query_oneskew_1.q.out | 12 -
.../list_bucket_query_oneskew_2.q.out | 15 -
.../list_bucket_query_oneskew_3.q.out | 6 -
.../clientpositive/llap/tez_fsstat.q.out | 2 +-
.../llap/tez_join_result_complex.q.out | 8 -
.../results/clientpositive/load_dyn_part8.q.out | 8 +-
.../clientpositive/louter_join_ppr.q.out | 36 +-
.../clientpositive/mapjoin_mapjoin.q.out | 16 +-
ql/src/test/results/clientpositive/merge3.q.out | 14 +-
.../results/clientpositive/nullformatCTAS.q.out | 2 +-
.../offset_limit_global_optimizer.q.out | 52 +-
.../clientpositive/optimize_nullscan.q.out | 56 +-
.../results/clientpositive/orc_analyze.q.out | 46 +-
.../clientpositive/orc_int_type_promotion.q.out | 6 +-
.../clientpositive/outer_join_ppr.q.java1.7.out | 20 +-
.../clientpositive/parallel_orderby.q.out | 4 +-
.../parquet_mixed_partition_formats.q.out | 6 -
.../results/clientpositive/parquet_serde.q.out | 6 -
.../partition_coltype_literals.q.out | 17 +-
ql/src/test/results/clientpositive/pcr.q.out | 114 +--
ql/src/test/results/clientpositive/pcs.q.out | 38 +-
.../results/clientpositive/pointlookup2.q.out | 28 +-
.../results/clientpositive/pointlookup3.q.out | 18 +-
.../results/clientpositive/pointlookup4.q.out | 8 +-
.../clientpositive/ppd_join_filter.q.out | 32 +-
.../results/clientpositive/ppd_union_view.q.out | 6 +-
ql/src/test/results/clientpositive/ppd_vc.q.out | 20 +-
.../clientpositive/ppr_allchildsarenull.q.out | 12 +-
.../test/results/clientpositive/push_or.q.out | 4 +-
.../clientpositive/rand_partitionpruner1.q.out | 4 +-
.../clientpositive/rand_partitionpruner2.q.out | 4 +-
.../clientpositive/rand_partitionpruner3.q.out | 4 +-
.../clientpositive/rcfile_default_format.q.out | 10 +-
.../clientpositive/reduce_deduplicate.q.out | 4 +-
.../results/clientpositive/regexp_extract.q.out | 8 +-
.../clientpositive/router_join_ppr.q.out | 36 +-
.../test/results/clientpositive/sample1.q.out | 2 +-
.../test/results/clientpositive/sample10.q.out | 8 +-
.../test/results/clientpositive/sample2.q.out | 4 +-
.../test/results/clientpositive/sample4.q.out | 4 +-
.../test/results/clientpositive/sample5.q.out | 4 +-
.../test/results/clientpositive/sample6.q.out | 32 +-
.../test/results/clientpositive/sample7.q.out | 4 +-
.../test/results/clientpositive/sample8.q.out | 8 +-
.../test/results/clientpositive/sample9.q.out | 4 +-
.../clientpositive/serde_user_properties.q.out | 12 +-
.../show_create_table_alter.q.out | 12 -
.../show_create_table_serde.q.out | 3 -
.../clientpositive/show_tblproperties.q.out | 15 -
.../results/clientpositive/smb_mapjoin_11.q.out | 6 +-
.../results/clientpositive/smb_mapjoin_12.q.out | 4 +-
.../results/clientpositive/smb_mapjoin_13.q.out | 8 +-
.../results/clientpositive/smb_mapjoin_15.q.out | 16 +-
.../clientpositive/sort_merge_join_desc_5.q.out | 2 +-
.../clientpositive/sort_merge_join_desc_6.q.out | 4 +-
.../clientpositive/sort_merge_join_desc_7.q.out | 8 +-
.../spark/alter_merge_stats_orc.q.out | 10 +-
.../spark/auto_join_reordering_values.q.out | 20 +-
.../spark/auto_sortmerge_join_1.q.out | 21 -
.../spark/auto_sortmerge_join_12.q.out | 15 -
.../spark/auto_sortmerge_join_2.q.out | 15 -
.../spark/auto_sortmerge_join_3.q.out | 15 -
.../spark/auto_sortmerge_join_4.q.out | 15 -
.../spark/auto_sortmerge_join_5.q.out | 8 -
.../spark/auto_sortmerge_join_7.q.out | 24 -
.../spark/auto_sortmerge_join_8.q.out | 24 -
.../results/clientpositive/spark/bucket2.q.out | 4 +-
.../results/clientpositive/spark/bucket3.q.out | 4 +-
.../results/clientpositive/spark/bucket4.q.out | 4 +-
.../results/clientpositive/spark/bucket5.q.out | 10 +-
.../spark/bucket_map_join_1.q.out | 4 -
.../spark/bucket_map_join_2.q.out | 4 -
.../spark/bucket_map_join_spark1.q.out | 16 +-
.../spark/bucket_map_join_spark2.q.out | 16 +-
.../spark/bucket_map_join_spark3.q.out | 16 +-
.../spark/bucket_map_join_spark4.q.out | 24 +-
.../clientpositive/spark/bucketmapjoin1.q.out | 14 +-
.../clientpositive/spark/bucketmapjoin10.q.out | 12 -
.../clientpositive/spark/bucketmapjoin11.q.out | 24 -
.../clientpositive/spark/bucketmapjoin12.q.out | 12 -
.../clientpositive/spark/bucketmapjoin13.q.out | 18 +-
.../clientpositive/spark/bucketmapjoin2.q.out | 29 +-
.../clientpositive/spark/bucketmapjoin3.q.out | 16 +-
.../clientpositive/spark/bucketmapjoin4.q.out | 12 +-
.../clientpositive/spark/bucketmapjoin5.q.out | 20 +-
.../clientpositive/spark/bucketmapjoin7.q.out | 6 -
.../clientpositive/spark/bucketmapjoin8.q.out | 12 -
.../clientpositive/spark/bucketmapjoin9.q.out | 12 -
.../spark/bucketmapjoin_negative.q.out | 5 -
.../spark/bucketmapjoin_negative2.q.out | 8 -
.../spark/bucketmapjoin_negative3.q.out | 36 -
.../results/clientpositive/spark/ctas.q.out | 14 +-
.../spark/disable_merge_for_bucketing.q.out | 4 +-
.../spark/filter_join_breaktask.q.out | 6 +-
.../clientpositive/spark/groupby_map_ppr.q.out | 4 +-
.../spark/groupby_map_ppr_multi_distinct.q.out | 4 +-
.../clientpositive/spark/groupby_ppr.q.out | 4 +-
.../spark/groupby_ppr_multi_distinct.q.out | 4 +-
.../spark/groupby_sort_1_23.q.out | 132 +--
.../spark/groupby_sort_skew_1_23.q.out | 132 +--
.../infer_bucket_sort_bucketed_table.q.out | 2 +-
.../spark/infer_bucket_sort_map_operators.q.out | 8 +-
.../spark/infer_bucket_sort_merge.q.out | 4 +-
.../spark/infer_bucket_sort_num_buckets.q.out | 4 +-
.../infer_bucket_sort_reducers_power_two.q.out | 12 +-
.../clientpositive/spark/input_part2.q.out | 4 +-
.../clientpositive/spark/insert_into1.q.out | 4 +-
.../results/clientpositive/spark/join17.q.out | 8 +-
.../results/clientpositive/spark/join26.q.out | 10 +-
.../results/clientpositive/spark/join32.q.out | 10 +-
.../clientpositive/spark/join32_lessSize.q.out | 54 +-
.../results/clientpositive/spark/join33.q.out | 10 +-
.../results/clientpositive/spark/join34.q.out | 12 +-
.../results/clientpositive/spark/join35.q.out | 12 +-
.../results/clientpositive/spark/join9.q.out | 6 +-
.../spark/join_filters_overlap.q.out | 68 +-
.../clientpositive/spark/join_map_ppr.q.out | 24 +-
.../spark/list_bucket_dml_10.q.java1.7.out | 6 +-
.../spark/list_bucket_dml_2.q.java1.7.out | 8 +-
.../clientpositive/spark/load_dyn_part8.q.out | 8 +-
.../clientpositive/spark/louter_join_ppr.q.out | 36 +-
.../clientpositive/spark/mapjoin_mapjoin.q.out | 16 +-
.../spark/optimize_nullscan.q.out | 56 +-
.../spark/outer_join_ppr.q.java1.7.out | 20 +-
.../clientpositive/spark/parallel_orderby.q.out | 4 +-
.../test/results/clientpositive/spark/pcr.q.out | 92 +-
.../clientpositive/spark/ppd_join_filter.q.out | 32 +-
.../spark/reduce_deduplicate.q.out | 4 +-
.../clientpositive/spark/router_join_ppr.q.out | 36 +-
.../results/clientpositive/spark/sample1.q.out | 2 +-
.../results/clientpositive/spark/sample10.q.out | 8 +-
.../results/clientpositive/spark/sample2.q.out | 4 +-
.../results/clientpositive/spark/sample4.q.out | 4 +-
.../results/clientpositive/spark/sample5.q.out | 4 +-
.../results/clientpositive/spark/sample6.q.out | 32 +-
.../results/clientpositive/spark/sample7.q.out | 4 +-
.../results/clientpositive/spark/sample8.q.out | 10 +-
.../clientpositive/spark/smb_mapjoin_11.q.out | 6 +-
.../clientpositive/spark/smb_mapjoin_12.q.out | 4 +-
.../clientpositive/spark/smb_mapjoin_13.q.out | 12 +-
.../clientpositive/spark/smb_mapjoin_15.q.out | 20 +-
.../results/clientpositive/spark/stats0.q.out | 8 +-
.../results/clientpositive/spark/stats1.q.out | 5 +-
.../results/clientpositive/spark/stats10.q.out | 4 +-
.../results/clientpositive/spark/stats12.q.out | 16 +-
.../results/clientpositive/spark/stats13.q.out | 14 +-
.../results/clientpositive/spark/stats14.q.out | 6 +-
.../results/clientpositive/spark/stats15.q.out | 6 +-
.../results/clientpositive/spark/stats16.q.out | 2 +-
.../results/clientpositive/spark/stats18.q.out | 5 +-
.../results/clientpositive/spark/stats3.q.out | 1 -
.../results/clientpositive/spark/stats5.q.out | 4 +-
.../results/clientpositive/spark/stats6.q.out | 10 +-
.../results/clientpositive/spark/stats7.q.out | 4 +-
.../results/clientpositive/spark/stats8.q.out | 16 +-
.../results/clientpositive/spark/stats9.q.out | 4 +-
.../clientpositive/spark/stats_noscan_1.q.out | 28 +-
.../clientpositive/spark/stats_noscan_2.q.out | 12 +-
.../clientpositive/spark/stats_only_null.q.out | 4 +-
.../spark/stats_partscan_1_23.q.out | 9 -
.../results/clientpositive/spark/statsfs.q.out | 16 +-
.../clientpositive/spark/transform_ppr1.q.out | 8 +-
.../clientpositive/spark/transform_ppr2.q.out | 4 +-
.../results/clientpositive/spark/union22.q.out | 6 +-
.../results/clientpositive/spark/union24.q.out | 48 +-
.../clientpositive/spark/union_ppr.q.out | 8 +-
.../clientpositive/spark/union_remove_1.q.out | 3 -
.../clientpositive/spark/union_remove_10.q.out | 3 -
.../clientpositive/spark/union_remove_11.q.out | 3 -
.../clientpositive/spark/union_remove_12.q.out | 3 -
.../clientpositive/spark/union_remove_13.q.out | 3 -
.../clientpositive/spark/union_remove_14.q.out | 3 -
.../clientpositive/spark/union_remove_19.q.out | 3 -
.../clientpositive/spark/union_remove_2.q.out | 3 -
.../clientpositive/spark/union_remove_20.q.out | 3 -
.../clientpositive/spark/union_remove_21.q.out | 3 -
.../clientpositive/spark/union_remove_22.q.out | 3 -
.../clientpositive/spark/union_remove_23.q.out | 3 -
.../clientpositive/spark/union_remove_24.q.out | 3 -
.../clientpositive/spark/union_remove_25.q.out | 9 -
.../clientpositive/spark/union_remove_3.q.out | 3 -
.../clientpositive/spark/union_remove_4.q.out | 3 -
.../clientpositive/spark/union_remove_5.q.out | 3 -
.../clientpositive/spark/union_remove_7.q.out | 3 -
.../clientpositive/spark/union_remove_8.q.out | 3 -
.../clientpositive/spark/union_remove_9.q.out | 3 -
.../clientpositive/spark/vectorized_ptf.q.out | 108 +--
ql/src/test/results/clientpositive/stats0.q.out | 8 +-
ql/src/test/results/clientpositive/stats1.q.out | 5 +-
.../test/results/clientpositive/stats10.q.out | 4 +-
.../test/results/clientpositive/stats11.q.out | 36 +-
.../test/results/clientpositive/stats12.q.out | 16 +-
.../test/results/clientpositive/stats13.q.out | 14 +-
.../test/results/clientpositive/stats14.q.out | 6 +-
.../test/results/clientpositive/stats15.q.out | 6 +-
.../test/results/clientpositive/stats16.q.out | 2 +-
.../test/results/clientpositive/stats18.q.out | 5 +-
ql/src/test/results/clientpositive/stats3.q.out | 1 -
ql/src/test/results/clientpositive/stats4.q.out | 8 +-
ql/src/test/results/clientpositive/stats5.q.out | 4 +-
ql/src/test/results/clientpositive/stats6.q.out | 10 +-
ql/src/test/results/clientpositive/stats7.q.out | 4 +-
ql/src/test/results/clientpositive/stats8.q.out | 16 +-
ql/src/test/results/clientpositive/stats9.q.out | 4 +-
.../clientpositive/stats_empty_partition.q.out | 2 +-
.../clientpositive/stats_invalidation.q.out | 7 +-
.../stats_list_bucket.q.java1.7.out | 4 +-
.../results/clientpositive/stats_noscan_1.q.out | 28 +-
.../results/clientpositive/stats_noscan_2.q.out | 12 +-
.../clientpositive/stats_only_null.q.out | 4 +-
.../clientpositive/stats_partscan_1_23.q.out | 8 +-
.../test/results/clientpositive/statsfs.q.out | 16 +-
.../temp_table_display_colstats_tbllvl.q.out | 2 -
.../tez/alter_merge_stats_orc.q.out | 10 +-
.../tez/auto_sortmerge_join_1.q.out | 27 -
.../tez/auto_sortmerge_join_11.q.out | 42 -
.../tez/auto_sortmerge_join_12.q.out | 15 -
.../tez/auto_sortmerge_join_2.q.out | 18 -
.../tez/auto_sortmerge_join_3.q.out | 27 -
.../tez/auto_sortmerge_join_4.q.out | 27 -
.../tez/auto_sortmerge_join_5.q.out | 12 -
.../tez/auto_sortmerge_join_7.q.out | 36 -
.../tez/auto_sortmerge_join_8.q.out | 36 -
.../results/clientpositive/tez/bucket2.q.out | 4 +-
.../results/clientpositive/tez/bucket3.q.out | 4 +-
.../results/clientpositive/tez/bucket4.q.out | 4 +-
.../test/results/clientpositive/tez/ctas.q.out | 14 +-
.../tez/disable_merge_for_bucketing.q.out | 4 +-
.../tez/dynpart_sort_opt_vectorization.q.out | 32 +-
.../tez/dynpart_sort_optimization.q.out | 32 +-
.../tez/dynpart_sort_optimization2.q.out | 24 +-
.../tez/filter_join_breaktask.q.out | 6 +-
.../clientpositive/tez/insert_into1.q.out | 4 +-
.../clientpositive/tez/mapjoin_mapjoin.q.out | 16 +-
.../tez/metadata_only_queries.q.out | 294 +++++-
.../metadata_only_queries_with_filters.q.out | 92 +-
.../clientpositive/tez/optimize_nullscan.q.out | 56 +-
.../clientpositive/tez/orc_analyze.q.out | 46 +-
.../results/clientpositive/tez/sample1.q.out | 2 +-
.../clientpositive/tez/stats_noscan_1.q.out | 28 +-
.../clientpositive/tez/stats_only_null.q.out | 96 +-
.../results/clientpositive/tez/tez_fsstat.q.out | 2 +-
.../tez/tez_join_result_complex.q.out | 8 -
.../clientpositive/tez/transform_ppr1.q.out | 8 +-
.../clientpositive/tez/transform_ppr2.q.out | 4 +-
.../clientpositive/tez/unionDistinct_1.q.out | 54 +-
.../clientpositive/tez/union_fast_stats.q.out | 12 +-
.../clientpositive/tez/vectorized_ptf.q.out | 108 +--
.../results/clientpositive/transform_ppr1.q.out | 8 +-
.../results/clientpositive/transform_ppr2.q.out | 4 +-
.../clientpositive/truncate_column.q.out | 24 +-
.../truncate_column_list_bucket.q.out | 6 -
.../results/clientpositive/truncate_table.q.out | 14 -
.../results/clientpositive/udf_explode.q.out | 16 +-
.../results/clientpositive/udtf_explode.q.out | 12 +-
.../test/results/clientpositive/union22.q.out | 12 +-
.../test/results/clientpositive/union24.q.out | 48 +-
.../clientpositive/unionDistinct_1.q.out | 60 +-
.../clientpositive/union_fast_stats.q.out | 12 +-
.../test/results/clientpositive/union_ppr.q.out | 4 +-
.../results/clientpositive/union_remove_1.q.out | 3 -
.../clientpositive/union_remove_10.q.out | 3 -
.../clientpositive/union_remove_11.q.out | 3 -
.../clientpositive/union_remove_12.q.out | 3 -
.../clientpositive/union_remove_13.q.out | 3 -
.../clientpositive/union_remove_14.q.out | 3 -
.../clientpositive/union_remove_19.q.out | 3 -
.../results/clientpositive/union_remove_2.q.out | 3 -
.../clientpositive/union_remove_20.q.out | 3 -
.../clientpositive/union_remove_21.q.out | 3 -
.../clientpositive/union_remove_22.q.out | 3 -
.../clientpositive/union_remove_23.q.out | 3 -
.../clientpositive/union_remove_24.q.out | 3 -
.../clientpositive/union_remove_25.q.out | 9 -
.../results/clientpositive/union_remove_3.q.out | 3 -
.../results/clientpositive/union_remove_4.q.out | 3 -
.../results/clientpositive/union_remove_5.q.out | 3 -
.../results/clientpositive/union_remove_7.q.out | 3 -
.../results/clientpositive/union_remove_8.q.out | 3 -
.../results/clientpositive/union_remove_9.q.out | 3 -
.../unset_table_view_property.q.out | 24 -
.../results/clientpositive/vectorized_ptf.q.out | 104 +-
433 files changed, 4647 insertions(+), 4264 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java b/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
index 2ff76ee..029d415 100644
--- a/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
+++ b/common/src/java/org/apache/hadoop/hive/common/StatsSetupConst.java
@@ -19,9 +19,15 @@ package org.apache.hadoop.hive.common;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.json.JSONException;
+import org.json.JSONObject;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -30,6 +36,8 @@ import java.util.Map;
public class StatsSetupConst {
+ protected final static Logger LOG = LoggerFactory.getLogger(StatsSetupConst.class.getName());
+
public enum StatDB {
fs {
@Override
@@ -111,16 +119,211 @@ public class StatsSetupConst {
// update should take place, such as with replication.
public static final String DO_NOT_UPDATE_STATS = "DO_NOT_UPDATE_STATS";
- // This string constant will be persisted in metastore to indicate whether corresponding
- // table or partition's statistics are accurate or not.
+ //This string constant will be persisted in metastore to indicate whether corresponding
+ //table or partition's statistics and table or partition's column statistics are accurate or not.
public static final String COLUMN_STATS_ACCURATE = "COLUMN_STATS_ACCURATE";
+ public static final String COLUMN_STATS = "COLUMN_STATS";
+
+ public static final String BASIC_STATS = "BASIC_STATS";
+
public static final String TRUE = "true";
public static final String FALSE = "false";
- public static boolean areStatsUptoDate(Map<String, String> params) {
+ public static boolean areBasicStatsUptoDate(Map<String, String> params) {
String statsAcc = params.get(COLUMN_STATS_ACCURATE);
- return statsAcc == null ? false : statsAcc.equals(TRUE);
+ if (statsAcc == null) {
+ return false;
+ } else {
+ JSONObject jsonObj;
+ try {
+ jsonObj = new JSONObject(statsAcc);
+ if (jsonObj != null && jsonObj.has(BASIC_STATS)) {
+ return true;
+ } else {
+ return false;
+ }
+ } catch (JSONException e) {
+ // For backward compatibility, if previous value can not be parsed to a
+ // json object, it will come here.
+ LOG.debug("In StatsSetupConst, JsonParser can not parse " + BASIC_STATS + ".");
+ // We try to parse it as TRUE or FALSE
+ if (statsAcc.equals(TRUE)) {
+ setBasicStatsState(params, TRUE);
+ return true;
+ } else {
+ setBasicStatsState(params, FALSE);
+ return false;
+ }
+ }
+ }
+ }
+
+ public static boolean areColumnStatsUptoDate(Map<String, String> params, String colName) {
+ String statsAcc = params.get(COLUMN_STATS_ACCURATE);
+ if (statsAcc == null) {
+ return false;
+ } else {
+ JSONObject jsonObj;
+ try {
+ jsonObj = new JSONObject(statsAcc);
+ if (jsonObj == null || !jsonObj.has(COLUMN_STATS)) {
+ return false;
+ } else {
+ JSONObject columns = jsonObj.getJSONObject(COLUMN_STATS);
+ if (columns != null && columns.has(colName)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ } catch (JSONException e) {
+ // For backward compatibility, if previous value can not be parsed to a
+ // json object, it will come here.
+ LOG.debug("In StatsSetupConst, JsonParser can not parse COLUMN_STATS.");
+ return false;
+ }
+ }
+ }
+
+ // It will only throw JSONException when stats.put(BASIC_STATS, TRUE)
+ // has duplicate key, which is not possible
+ // note that set basic stats false will wipe out column stats too.
+ public static void setBasicStatsState(Map<String, String> params, String setting) {
+ if (setting.equals(FALSE)) {
+ if (params.containsKey(COLUMN_STATS_ACCURATE)) {
+ params.remove(COLUMN_STATS_ACCURATE);
+ }
+ } else {
+ String statsAcc = params.get(COLUMN_STATS_ACCURATE);
+ if (statsAcc == null) {
+ JSONObject stats = new JSONObject(new LinkedHashMap());
+ // duplicate key is not possible
+ try {
+ stats.put(BASIC_STATS, TRUE);
+ } catch (JSONException e) {
+ // impossible to throw any json exceptions.
+ LOG.trace(e.getMessage());
+ }
+ params.put(COLUMN_STATS_ACCURATE, stats.toString());
+ } else {
+ // statsAcc may not be jason format, which will throw exception
+ JSONObject stats;
+ try {
+ stats = new JSONObject(statsAcc);
+ } catch (JSONException e) {
+ // old format of statsAcc, e.g., TRUE or FALSE
+ LOG.debug("In StatsSetupConst, JsonParser can not parse statsAcc.");
+ stats = new JSONObject(new LinkedHashMap());
+ try {
+ if (statsAcc.equals(TRUE)) {
+ stats.put(BASIC_STATS, TRUE);
+ } else {
+ stats.put(BASIC_STATS, FALSE);
+ }
+ } catch (JSONException e1) {
+ // impossible to throw any json exceptions.
+ LOG.trace(e1.getMessage());
+ }
+ }
+ if (!stats.has(BASIC_STATS)) {
+ // duplicate key is not possible
+ try {
+ stats.put(BASIC_STATS, TRUE);
+ } catch (JSONException e) {
+ // impossible to throw any json exceptions.
+ LOG.trace(e.getMessage());
+ }
+ }
+ params.put(COLUMN_STATS_ACCURATE, stats.toString());
+ }
+ }
+ }
+
+ public static void setColumnStatsState(Map<String, String> params, List<String> colNames) {
+ try {
+ String statsAcc = params.get(COLUMN_STATS_ACCURATE);
+ JSONObject colStats = new JSONObject(new LinkedHashMap());
+ // duplicate key is not possible
+ for (String colName : colNames) {
+ colStats.put(colName.toLowerCase(), TRUE);
+ }
+ if (statsAcc == null) {
+ JSONObject stats = new JSONObject(new LinkedHashMap());
+ // duplicate key is not possible
+ stats.put(COLUMN_STATS, colStats);
+ params.put(COLUMN_STATS_ACCURATE, stats.toString());
+ } else {
+ // statsAcc may not be jason format, which will throw exception
+ JSONObject stats;
+ try {
+ stats = new JSONObject(statsAcc);
+ } catch (JSONException e) {
+ // old format of statsAcc, e.g., TRUE or FALSE
+ LOG.debug("In StatsSetupConst, JsonParser can not parse statsAcc.");
+ stats = new JSONObject(new LinkedHashMap());
+ try {
+ if (statsAcc.equals(TRUE)) {
+ stats.put(BASIC_STATS, TRUE);
+ } else {
+ stats.put(BASIC_STATS, FALSE);
+ }
+ } catch (JSONException e1) {
+ // impossible to throw any json exceptions.
+ LOG.trace(e1.getMessage());
+ }
+ }
+ if (!stats.has(COLUMN_STATS)) {
+ // duplicate key is not possible
+ stats.put(COLUMN_STATS, colStats);
+ } else {
+ // getJSONObject(COLUMN_STATS) should be found.
+ JSONObject allColumnStats = stats.getJSONObject(COLUMN_STATS);
+ for (String colName : colNames) {
+ if (!allColumnStats.has(colName)) {
+ // duplicate key is not possible
+ allColumnStats.put(colName, TRUE);
+ }
+ }
+ stats.remove(COLUMN_STATS);
+ // duplicate key is not possible
+ stats.put(COLUMN_STATS, allColumnStats);
+ }
+ params.put(COLUMN_STATS_ACCURATE, stats.toString());
+ }
+ } catch (JSONException e) {
+ //impossible to throw any json exceptions.
+ LOG.trace(e.getMessage());
+ }
+ }
+
+ public static void clearColumnStatsState(Map<String, String> params) {
+ String statsAcc = params.get(COLUMN_STATS_ACCURATE);
+ if (statsAcc != null) {
+ // statsAcc may not be jason format, which will throw exception
+ JSONObject stats;
+ try {
+ stats = new JSONObject(statsAcc);
+ } catch (JSONException e) {
+ // old format of statsAcc, e.g., TRUE or FALSE
+ LOG.debug("In StatsSetupConst, JsonParser can not parse statsAcc.");
+ stats = new JSONObject(new LinkedHashMap());
+ try {
+ if (statsAcc.equals(TRUE)) {
+ stats.put(BASIC_STATS, TRUE);
+ } else {
+ stats.put(BASIC_STATS, FALSE);
+ }
+ } catch (JSONException e1) {
+ // impossible to throw any json exceptions.
+ LOG.trace(e1.getMessage());
+ }
+ }
+ if (stats.has(COLUMN_STATS)) {
+ stats.remove(COLUMN_STATS);
+ }
+ params.put(COLUMN_STATS_ACCURATE, stats.toString());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
----------------------------------------------------------------------
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
index da84c52..56f5c8b 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java
@@ -534,35 +534,35 @@ public class TestDbNotificationListener {
for (NotificationEvent ne : rsp.getEvents()) LOG.debug("EVENT: " + ne.getMessage());
// For reasons not clear to me there's one or more alter partitions after add partition and
// insert.
- assertEquals(19, rsp.getEventsSize());
+ assertEquals(25, rsp.getEventsSize());
NotificationEvent event = rsp.getEvents().get(1);
assertEquals(firstEventId + 2, event.getEventId());
assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT, event.getEventType());
- event = rsp.getEvents().get(4);
- assertEquals(firstEventId + 5, event.getEventId());
+ event = rsp.getEvents().get(5);
+ assertEquals(firstEventId + 6, event.getEventId());
assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
// Make sure the files are listed in the insert
assertTrue(event.getMessage().matches(".*\"files\":\\[\"pfile.*"));
- event = rsp.getEvents().get(7);
- assertEquals(firstEventId + 8, event.getEventId());
- assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
- assertTrue(event.getMessage().matches(".*\"files\":\\[\"pfile.*"));
event = rsp.getEvents().get(9);
assertEquals(firstEventId + 10, event.getEventId());
- assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT, event.getEventType());
- event = rsp.getEvents().get(11);
- assertEquals(firstEventId + 12, event.getEventId());
assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
assertTrue(event.getMessage().matches(".*\"files\":\\[\"pfile.*"));
+ event = rsp.getEvents().get(12);
+ assertEquals(firstEventId + 13, event.getEventId());
+ assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT, event.getEventType());
event = rsp.getEvents().get(14);
assertEquals(firstEventId + 15, event.getEventId());
assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
assertTrue(event.getMessage().matches(".*\"files\":\\[\"pfile.*"));
- event = rsp.getEvents().get(16);
- assertEquals(firstEventId + 17, event.getEventId());
- assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT, event.getEventType());
event = rsp.getEvents().get(18);
assertEquals(firstEventId + 19, event.getEventId());
+ assertEquals(HCatConstants.HCAT_INSERT_EVENT, event.getEventType());
+ assertTrue(event.getMessage().matches(".*\"files\":\\[\"pfile.*"));
+ event = rsp.getEvents().get(21);
+ assertEquals(firstEventId + 22, event.getEventId());
+ assertEquals(HCatConstants.HCAT_ADD_PARTITION_EVENT, event.getEventType());
+ event = rsp.getEvents().get(24);
+ assertEquals(firstEventId + 25, event.getEventId());
assertEquals(HCatConstants.HCAT_DROP_PARTITION_EVENT, event.getEventType());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 432f7d0..eee7f1b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -233,13 +233,10 @@ public class MetaStoreUtils {
LOG.info("Updated size of table " + tbl.getTableName() +" to "+ params.get(StatsSetupConst.TOTAL_SIZE));
if(!params.containsKey(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK)) {
// invalidate stats requiring scan since this is a regular ddl alter case
- for (String stat : StatsSetupConst.statsRequireCompute) {
- params.put(stat, "-1");
- }
- params.put(StatsSetupConst.COLUMN_STATS_ACCURATE, StatsSetupConst.FALSE);
+ StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE);
} else {
params.remove(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK);
- params.put(StatsSetupConst.COLUMN_STATS_ACCURATE, StatsSetupConst.TRUE);
+ StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE);
}
}
tbl.setParameters(params);
@@ -354,13 +351,10 @@ public class MetaStoreUtils {
LOG.warn("Updated size to " + params.get(StatsSetupConst.TOTAL_SIZE));
if(!params.containsKey(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK)) {
// invalidate stats requiring scan since this is a regular ddl alter case
- for (String stat : StatsSetupConst.statsRequireCompute) {
- params.put(stat, "-1");
- }
- params.put(StatsSetupConst.COLUMN_STATS_ACCURATE, StatsSetupConst.FALSE);
+ StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE);
} else {
params.remove(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK);
- params.put(StatsSetupConst.COLUMN_STATS_ACCURATE, StatsSetupConst.TRUE);
+ StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE);
}
}
part.setParameters(params);
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index de30de4..e044c73 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -58,6 +58,7 @@ import javax.jdo.datastore.DataStoreCache;
import javax.jdo.identity.IntIdentity;
import com.google.common.annotations.VisibleForTesting;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configurable;
@@ -65,6 +66,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.ObjectPair;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -6498,12 +6500,24 @@ public class ObjectStore implements RawStore, Configurable {
// DataNucleus objects get detached all over the place for no (real) reason.
// So let's not use them anywhere unless absolutely necessary.
Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName());
+ List<String> colNames = new ArrayList<>();
for (ColumnStatisticsObj statsObj:statsObjs) {
// We have to get mtable again because DataNucleus.
MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(
ensureGetMTable(statsDesc.getDbName(), statsDesc.getTableName()), statsDesc, statsObj);
writeMTableColumnStatistics(table, mStatsObj);
+ colNames.add(statsObj.getColName());
}
+
+ // Set the table properties
+ // No need to check again if it exists.
+ String dbname = table.getDbName();
+ String name = table.getTableName();
+ MTable oldt = getMTable(dbname, name);
+ Map<String, String> parameters = table.getParameters();
+ StatsSetupConst.setColumnStatsState(parameters, colNames);
+ oldt.setParameters(parameters);
+
committed = commitTransaction();
return committed;
} finally {
@@ -6525,6 +6539,7 @@ public class ObjectStore implements RawStore, Configurable {
Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName());
Partition partition = convertToPart(getMPartition(
statsDesc.getDbName(), statsDesc.getTableName(), partVals));
+ List<String> colNames = new ArrayList<>();
for (ColumnStatisticsObj statsObj:statsObjs) {
// We have to get partition again because DataNucleus
MPartition mPartition = getMPartition(
@@ -6535,7 +6550,15 @@ public class ObjectStore implements RawStore, Configurable {
MPartitionColumnStatistics mStatsObj =
StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, statsDesc, statsObj);
writeMPartitionColumnStatistics(table, partition, mStatsObj);
- }
+ colNames.add(statsObj.getColName());
+ }
+ // Set the partition properties
+ // No need to check again if it exists.
+ MPartition mPartition = getMPartition(
+ statsDesc.getDbName(), statsDesc.getTableName(), partVals);
+ Map<String, String> parameters = mPartition.getParameters();
+ StatsSetupConst.setColumnStatsState(parameters, colNames);
+ mPartition.setParameters(parameters);
committed = commitTransaction();
return committed;
} finally {
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index deec8bb..5f3a9cf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -4361,10 +4361,15 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
String statVal = props.get(stat);
if (statVal != null && Long.parseLong(statVal) > 0) {
statsPresent = true;
+ //In the case of truncate table, we set the stats to be 0.
props.put(stat, "0");
- props.put(StatsSetupConst.COLUMN_STATS_ACCURATE, "false");
}
}
+ //first set basic stats to true
+ StatsSetupConst.setBasicStatsState(props, StatsSetupConst.TRUE);
+ props.put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE);
+ //then invalidate column stats
+ StatsSetupConst.clearColumnStatsState(props);
return statsPresent;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
index edf69fe..44277aa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
@@ -178,7 +178,9 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
updateQuickStats(wh, parameters, tTable.getSd());
// write table stats to metastore
- parameters.put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE);
+ if (!getWork().getNoStatsAggregator()) {
+ parameters.put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE);
+ }
db.alterTable(tableFullName, new Table(tTable));
if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) {
@@ -213,7 +215,9 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
updateQuickStats(wh, parameters, tPart.getSd());
- parameters.put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE);
+ if (!getWork().getNoStatsAggregator()) {
+ parameters.put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE);
+ }
updates.add(new Partition(table, tPart));
if (conf.getBoolVar(ConfVars.TEZ_EXEC_SUMMARY)) {
@@ -313,7 +317,7 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
if (work.getLoadTableDesc() != null &&
!work.getLoadTableDesc().getReplace()) {
String originalValue = parameters.get(statType);
- if (originalValue != null && !originalValue.equals("-1")) {
+ if (originalValue != null) {
longValue += Long.parseLong(originalValue); // todo: invalid + valid = invalid
}
}
@@ -338,7 +342,7 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
private void clearStats(Map<String, String> parameters) {
for (String statType : StatsSetupConst.supportedStats) {
if (parameters.containsKey(statType)) {
- parameters.put(statType, "0");
+ parameters.remove(statType);
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index dca1198..290ee99 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -524,7 +524,7 @@ public class Hive {
tbl.setNumBuckets(bucketCount);
tbl.setBucketCols(bucketCols);
if (parameters != null) {
- tbl.setParamters(parameters);
+ tbl.setParameters(parameters);
}
createTable(tbl);
}
@@ -1419,6 +1419,7 @@ public class Hive {
* @param isSrcLocal
* If the source directory is LOCAL
* @param isAcid true if this is an ACID operation
+ * @throws JSONException
*/
public Partition loadPartition(Path loadPath, Table tbl,
Map<String, String> partSpec, boolean replace,
@@ -1478,6 +1479,8 @@ public class Hive {
newTPart = getPartition(tbl, partSpec, true, newPartPath.toString(),
inheritTableSpecs, newFiles);
+ //column stats will be inaccurate
+ StatsSetupConst.clearColumnStatsState(newTPart.getParameters());
// recreate the partition if it existed before
if (isSkewedStoreAsSubdir) {
@@ -1490,7 +1493,7 @@ public class Hive {
skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps);
newCreatedTpart.getSd().setSkewedInfo(skewedInfo);
if(!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
- newTPart.getParameters().put(StatsSetupConst.COLUMN_STATS_ACCURATE, "false");
+ StatsSetupConst.setBasicStatsState(newTPart.getParameters(), StatsSetupConst.FALSE);
}
alterPartition(tbl.getDbName(), tbl.getTableName(), new Partition(tbl, newCreatedTpart));
newTPart = getPartition(tbl, partSpec, true, newPartPath.toString(), inheritTableSpecs,
@@ -1498,9 +1501,9 @@ public class Hive {
return new Partition(tbl, newCreatedTpart);
}
if(!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
- newTPart.getParameters().put(StatsSetupConst.COLUMN_STATS_ACCURATE, "false");
- alterPartition(tbl.getDbName(), tbl.getTableName(), new Partition(tbl, newTPart.getTPartition()));
+ StatsSetupConst.setBasicStatsState(newTPart.getParameters(), StatsSetupConst.FALSE);
}
+ alterPartition(tbl.getDbName(), tbl.getTableName(), new Partition(tbl, newTPart.getTPartition()));
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
throw new HiveException(e);
@@ -1611,6 +1614,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
* @param txnId txnId, can be 0 unless isAcid == true
* @return partition map details (PartitionSpec and Partition)
* @throws HiveException
+ * @throws JSONException
*/
public Map<Map<String, String>, Partition> loadDynamicPartitions(Path loadPath,
String tableName, Map<String, String> partSpec, boolean replace,
@@ -1730,11 +1734,14 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
}
if(!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
- tbl.getParameters().put(StatsSetupConst.COLUMN_STATS_ACCURATE, "false");
+ StatsSetupConst.setBasicStatsState(tbl.getParameters(), StatsSetupConst.FALSE);
} else {
tbl.getParameters().put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, "true");
}
+ //column stats will be inaccurate
+ StatsSetupConst.clearColumnStatsState(tbl.getParameters());
+
try {
if (isSkewedStoreAsSubdir) {
SkewedInfo skewedInfo = tbl.getSkewedInfo();
@@ -1992,7 +1999,6 @@ private void constructOneLBLocationMap(FileStatus fSta,
throw new HiveException("new partition path should not be null or empty.");
}
tpart.getSd().setLocation(partPath);
- tpart.getParameters().put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK,"true");
String fullName = tbl.getTableName();
if (!org.apache.commons.lang.StringUtils.isEmpty(tbl.getDbName())) {
fullName = tbl.getDbName() + "." + tbl.getTableName();
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index f151871..a5217eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -382,7 +382,7 @@ public class Table implements Serializable {
tTable.getParameters().put(name, value);
}
- public void setParamters(Map<String, String> params) {
+ public void setParameters(Map<String, String> params) {
tTable.setParameters(params);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
index 2f9a831..4091c0d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
@@ -365,13 +365,18 @@ public class StatsOptimizer extends Transform {
String colName = desc.getColumn();
StatType type = getType(desc.getTypeString());
if(!tbl.isPartitioned()) {
- if (!StatsSetupConst.areStatsUptoDate(tbl.getParameters())) {
- Logger.debug("Stats for table : " + tbl.getTableName() + " are not upto date.");
+ if (!StatsSetupConst.areBasicStatsUptoDate(tbl.getParameters())) {
+ Logger.debug("Stats for table : " + tbl.getTableName() + " are not up to date.");
return null;
}
rowCnt = Long.parseLong(tbl.getProperty(StatsSetupConst.ROW_COUNT));
- if (rowCnt < 1) {
- Logger.debug("Table doesn't have upto date stats " + tbl.getTableName());
+ if (rowCnt == null) {
+ Logger.debug("Table doesn't have up to date stats " + tbl.getTableName());
+ return null;
+ }
+ if (!StatsSetupConst.areColumnStatsUptoDate(tbl.getParameters(), colName)) {
+ Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName
+ + " are not up to date.");
return null;
}
List<ColumnStatisticsObj> stats = hive.getMSC().getTableColumnStatistics(
@@ -392,20 +397,20 @@ public class StatsOptimizer extends Transform {
Set<Partition> parts = pctx.getPrunedPartitions(
tsOp.getConf().getAlias(), tsOp).getPartitions();
for (Partition part : parts) {
- if (!StatsSetupConst.areStatsUptoDate(part.getParameters())) {
- Logger.debug("Stats for part : " + part.getSpec() + " are not upto date.");
+ if (!StatsSetupConst.areBasicStatsUptoDate(part.getParameters())) {
+ Logger.debug("Stats for part : " + part.getSpec() + " are not up to date.");
return null;
}
Long partRowCnt = Long.parseLong(part.getParameters()
.get(StatsSetupConst.ROW_COUNT));
- if (partRowCnt < 1) {
- Logger.debug("Partition doesn't have upto date stats " + part.getSpec());
+ if (partRowCnt == null) {
+ Logger.debug("Partition doesn't have up to date stats " + part.getSpec());
return null;
}
rowCnt += partRowCnt;
}
Collection<List<ColumnStatisticsObj>> result =
- verifyAndGetPartStats(hive, tbl, colName, parts);
+ verifyAndGetPartColumnStats(hive, tbl, colName, parts);
if (result == null) {
return null; // logging inside
}
@@ -429,8 +434,9 @@ public class StatsOptimizer extends Transform {
String colName = colDesc.getColumn();
StatType type = getType(colDesc.getTypeString());
if(!tbl.isPartitioned()) {
- if (!StatsSetupConst.areStatsUptoDate(tbl.getParameters())) {
- Logger.debug("Stats for table : " + tbl.getTableName() + " are not upto date.");
+ if (!StatsSetupConst.areColumnStatsUptoDate(tbl.getParameters(), colName)) {
+ Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName
+ + " are not up to date.");
return null;
}
List<ColumnStatisticsObj> stats = hive.getMSC().getTableColumnStatistics(
@@ -478,7 +484,7 @@ public class StatsOptimizer extends Transform {
Long maxVal = null;
Collection<List<ColumnStatisticsObj>> result =
- verifyAndGetPartStats(hive, tbl, colName, parts);
+ verifyAndGetPartColumnStats(hive, tbl, colName, parts);
if (result == null) {
return null; // logging inside
}
@@ -504,7 +510,7 @@ public class StatsOptimizer extends Transform {
Double maxVal = null;
Collection<List<ColumnStatisticsObj>> result =
- verifyAndGetPartStats(hive, tbl, colName, parts);
+ verifyAndGetPartColumnStats(hive, tbl, colName, parts);
if (result == null) {
return null; // logging inside
}
@@ -536,8 +542,9 @@ public class StatsOptimizer extends Transform {
String colName = colDesc.getColumn();
StatType type = getType(colDesc.getTypeString());
if (!tbl.isPartitioned()) {
- if (!StatsSetupConst.areStatsUptoDate(tbl.getParameters())) {
- Logger.debug("Stats for table : " + tbl.getTableName() + " are not upto date.");
+ if (!StatsSetupConst.areColumnStatsUptoDate(tbl.getParameters(), colName)) {
+ Logger.debug("Stats for table : " + tbl.getTableName() + " column " + colName
+ + " are not up to date.");
return null;
}
ColumnStatisticsData statData = hive.getMSC().getTableColumnStatistics(
@@ -579,7 +586,7 @@ public class StatsOptimizer extends Transform {
Long minVal = null;
Collection<List<ColumnStatisticsObj>> result =
- verifyAndGetPartStats(hive, tbl, colName, parts);
+ verifyAndGetPartColumnStats(hive, tbl, colName, parts);
if (result == null) {
return null; // logging inside
}
@@ -605,7 +612,7 @@ public class StatsOptimizer extends Transform {
Double minVal = null;
Collection<List<ColumnStatisticsObj>> result =
- verifyAndGetPartStats(hive, tbl, colName, parts);
+ verifyAndGetPartColumnStats(hive, tbl, colName, parts);
if (result == null) {
return null; // logging inside
}
@@ -707,12 +714,13 @@ public class StatsOptimizer extends Transform {
return statObj.get(0).getStatsData();
}
- private Collection<List<ColumnStatisticsObj>> verifyAndGetPartStats(
+ private Collection<List<ColumnStatisticsObj>> verifyAndGetPartColumnStats(
Hive hive, Table tbl, String colName, Set<Partition> parts) throws TException {
List<String> partNames = new ArrayList<String>(parts.size());
for (Partition part : parts) {
- if (!StatsSetupConst.areStatsUptoDate(part.getParameters())) {
- Logger.debug("Stats for part : " + part.getSpec() + " are not upto date.");
+ if (!StatsSetupConst.areColumnStatsUptoDate(part.getParameters(), colName)) {
+ Logger.debug("Stats for part : " + part.getSpec() + " column " + colName
+ + " are not up to date.");
return null;
}
partNames.add(part.getName());
@@ -732,25 +740,25 @@ public class StatsOptimizer extends Transform {
if (tbl.isPartitioned()) {
for (Partition part : pctx.getPrunedPartitions(
tsOp.getConf().getAlias(), tsOp).getPartitions()) {
- if (!StatsSetupConst.areStatsUptoDate(part.getParameters())) {
+ if (!StatsSetupConst.areBasicStatsUptoDate(part.getParameters())) {
return null;
}
- long partRowCnt = Long.parseLong(part.getParameters().get(StatsSetupConst.ROW_COUNT));
- if (partRowCnt < 1) {
- Logger.debug("Partition doesn't have upto date stats " + part.getSpec());
+ Long partRowCnt = Long.parseLong(part.getParameters().get(StatsSetupConst.ROW_COUNT));
+ if (partRowCnt == null) {
+ Logger.debug("Partition doesn't have up to date stats " + part.getSpec());
return null;
}
rowCnt += partRowCnt;
}
} else { // unpartitioned table
- if (!StatsSetupConst.areStatsUptoDate(tbl.getParameters())) {
+ if (!StatsSetupConst.areBasicStatsUptoDate(tbl.getParameters())) {
return null;
}
rowCnt = Long.parseLong(tbl.getProperty(StatsSetupConst.ROW_COUNT));
- if (rowCnt < 1) {
+ if (rowCnt == null) {
// if rowCnt < 1 than its either empty table or table on which stats are not
// computed We assume the worse and don't attempt to optimize.
- Logger.debug("Table doesn't have upto date stats " + tbl.getTableName());
+ Logger.debug("Table doesn't have up to date stats " + tbl.getTableName());
rowCnt = null;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
index d87022d..a5050c5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
@@ -44,6 +44,10 @@ public class StatsWork implements Serializable {
// are still valid. However, if a load file is being performed, the old stats collected by
// aggregator are not valid. It might be a good idea to clear them instead of leaving wrong
// and old stats.
+ // Since HIVE-12661, we maintain the old stats (although may be wrong) for CBO
+ // purpose. We use a flag COLUMN_STATS_ACCURATE to
+ // show the accuracy of the stats.
+
private boolean clearAggregatorStats = false;
private boolean noStatsAggregator = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/test/queries/clientpositive/columnStatsUpdateForStatsOptimizer_1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/columnStatsUpdateForStatsOptimizer_1.q b/ql/src/test/queries/clientpositive/columnStatsUpdateForStatsOptimizer_1.q
new file mode 100644
index 0000000..199c74c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/columnStatsUpdateForStatsOptimizer_1.q
@@ -0,0 +1,102 @@
+set hive.stats.fetch.column.stats=true;
+set hive.stats.fetch.partition.stats=true;
+set hive.compute.query.using.stats=true;
+set hive.mapred.mode=nonstrict;
+
+drop table calendar;
+
+CREATE TABLE calendar (year int, month int);
+
+insert into calendar values (2010, 10), (2011, 11), (2012, 12);
+
+desc formatted calendar;
+
+analyze table calendar compute statistics;
+
+desc formatted calendar;
+
+explain select count(1) from calendar;
+
+explain select max(year) from calendar;
+
+select max(year) from calendar;
+
+select max(month) from calendar;
+
+analyze table calendar compute statistics for columns;
+
+desc formatted calendar;
+
+explain select max(year) from calendar;
+
+select max(year) from calendar;
+
+insert into calendar values (2015, 15);
+
+desc formatted calendar;
+
+explain select max(year) from calendar;
+
+select max(year) from calendar;
+
+explain select max(month) from calendar;
+
+select max(month) from calendar;
+
+analyze table calendar compute statistics for columns year;
+
+desc formatted calendar;
+
+explain select max(year) from calendar;
+
+select max(year) from calendar;
+
+explain select max(month) from calendar;
+
+select max(month) from calendar;
+
+analyze table calendar compute statistics for columns month;
+
+desc formatted calendar;
+
+explain select max(month) from calendar;
+
+select max(month) from calendar;
+
+CREATE TABLE calendarp (`year` int) partitioned by (p int);
+
+insert into table calendarp partition (p=1) values (2010), (2011), (2012);
+
+desc formatted calendarp partition (p=1);
+
+explain select max(year) from calendarp where p=1;
+
+select max(year) from calendarp where p=1;
+
+analyze table calendarp partition (p=1) compute statistics for columns;
+
+desc formatted calendarp partition (p=1);
+
+explain select max(year) from calendarp where p=1;
+
+insert into table calendarp partition (p=1) values (2015);
+
+desc formatted calendarp partition (p=1);
+
+explain select max(year) from calendarp where p=1;
+
+select max(year) from calendarp where p=1;
+
+create table t (key string, value string);
+
+load data local inpath '../../data/files/kv1.txt' into table t;
+
+desc formatted t;
+
+analyze table t compute statistics;
+
+desc formatted t;
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/test/queries/clientpositive/columnStatsUpdateForStatsOptimizer_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/columnStatsUpdateForStatsOptimizer_2.q b/ql/src/test/queries/clientpositive/columnStatsUpdateForStatsOptimizer_2.q
new file mode 100644
index 0000000..222d85f
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/columnStatsUpdateForStatsOptimizer_2.q
@@ -0,0 +1,58 @@
+set hive.stats.fetch.column.stats=true;
+set hive.stats.fetch.partition.stats=true;
+set hive.compute.query.using.stats=true;
+
+
+drop table calendar;
+
+CREATE TABLE calendar (year int, month int) clustered by (month) into 2 buckets stored as orc;
+
+insert into calendar values (2010, 10), (2011, 11), (2012, 12);
+
+desc formatted calendar;
+
+analyze table calendar compute statistics for columns year;
+
+desc formatted calendar;
+
+explain select max(year) from calendar;
+
+select max(year) from calendar;
+
+explain select count(1) from calendar;
+
+select count(1) from calendar;
+
+ALTER TABLE calendar CHANGE year year1 INT;
+
+--after patch, should be old stats rather than -1
+
+desc formatted calendar;
+
+--but basic/column stats can not be used by optimizer
+
+explain select max(month) from calendar;
+
+select max(month) from calendar;
+
+explain select count(1) from calendar;
+
+select count(1) from calendar;
+
+truncate table calendar;
+
+--after patch, should be 0
+
+desc formatted calendar;
+
+--but column stats can not be used by optimizer
+
+explain select max(month) from calendar;
+
+select max(month) from calendar;
+
+--basic stats can be used by optimizer
+
+explain select count(1) from calendar;
+
+select count(1) from calendar;
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out b/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out
index d03c249..4faf327 100644
--- a/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out
+++ b/ql/src/test/results/clientnegative/stats_partialscan_autogether.q.out
@@ -66,10 +66,7 @@ Database: default
Table: analyze_srcpart_partial_scan
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE false
numFiles 1
- numRows -1
- rawDataSize -1
totalSize 5077
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/test/results/clientnegative/unset_table_property.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/unset_table_property.q.out b/ql/src/test/results/clientnegative/unset_table_property.q.out
index 158ed38..0510788 100644
--- a/ql/src/test/results/clientnegative/unset_table_property.q.out
+++ b/ql/src/test/results/clientnegative/unset_table_property.q.out
@@ -18,13 +18,10 @@ PREHOOK: query: SHOW TBLPROPERTIES testTable
PREHOOK: type: SHOW_TBLPROPERTIES
POSTHOOK: query: SHOW TBLPROPERTIES testTable
POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE false
a 1
c 3
#### A masked pattern was here ####
numFiles 0
-numRows -1
-rawDataSize -1
totalSize 0
#### A masked pattern was here ####
FAILED: SemanticException [Error 10215]: Please use the following syntax if not sure whether the property existed or not:
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/test/results/clientpositive/alter_file_format.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_file_format.q.out b/ql/src/test/results/clientpositive/alter_file_format.q.out
index c9e88f8..5d83b23 100644
--- a/ql/src/test/results/clientpositive/alter_file_format.q.out
+++ b/ql/src/test/results/clientpositive/alter_file_format.q.out
@@ -62,11 +62,8 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -106,11 +103,8 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -150,11 +144,8 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -194,11 +185,8 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -238,11 +226,8 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -282,11 +267,8 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -386,11 +368,8 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -435,11 +414,8 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -484,11 +460,8 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -533,11 +506,8 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -582,11 +552,8 @@ Database: default
Table: alter_partition_format_test
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out b/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out
index cefe069..0d5ba01 100644
--- a/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out
+++ b/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out
@@ -89,7 +89,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 3
numRows 1500
rawDataSize 141000
@@ -140,7 +140,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 1500
rawDataSize 141000
@@ -241,7 +241,7 @@ Database: default
Table: src_orc_merge_test_part_stat
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 3
numRows 1500
rawDataSize 141000
@@ -290,7 +290,7 @@ Database: default
Table: src_orc_merge_test_part_stat
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 3
numRows 1500
rawDataSize 141000
@@ -347,7 +347,7 @@ Database: default
Table: src_orc_merge_test_part_stat
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 1500
rawDataSize 141000
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/test/results/clientpositive/alter_numbuckets_partitioned_table2_h23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_numbuckets_partitioned_table2_h23.q.out b/ql/src/test/results/clientpositive/alter_numbuckets_partitioned_table2_h23.q.out
index 29a4c4b..3b71598 100644
--- a/ql/src/test/results/clientpositive/alter_numbuckets_partitioned_table2_h23.q.out
+++ b/ql/src/test/results/clientpositive/alter_numbuckets_partitioned_table2_h23.q.out
@@ -79,7 +79,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 500
rawDataSize 5312
@@ -173,7 +173,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 8
numRows 500
rawDataSize 5312
@@ -267,7 +267,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 8
numRows 500
rawDataSize 5312
@@ -361,7 +361,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 8
numRows 500
rawDataSize 5312
@@ -455,7 +455,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 4
numRows 500
rawDataSize 5312
@@ -549,7 +549,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 4
numRows 500
rawDataSize 5312
@@ -643,7 +643,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 4
numRows 500
rawDataSize 5312
@@ -737,7 +737,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 4
numRows 500
rawDataSize 5312
@@ -831,7 +831,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 1
numRows 500
rawDataSize 5312
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/test/results/clientpositive/alter_numbuckets_partitioned_table_h23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_numbuckets_partitioned_table_h23.q.out b/ql/src/test/results/clientpositive/alter_numbuckets_partitioned_table_h23.q.out
index 4d0f841..cab3de4 100644
--- a/ql/src/test/results/clientpositive/alter_numbuckets_partitioned_table_h23.q.out
+++ b/ql/src/test/results/clientpositive/alter_numbuckets_partitioned_table_h23.q.out
@@ -83,7 +83,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 8
numRows 500
rawDataSize 5312
@@ -144,7 +144,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 12
numRows 500
rawDataSize 5312
@@ -264,7 +264,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
numFiles 12
numRows 500
rawDataSize 5312
@@ -346,7 +346,7 @@ Database: default
Table: tst1
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
#### A masked pattern was here ####
numFiles 12
numRows 500
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/test/results/clientpositive/alter_partition_clusterby_sortby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_partition_clusterby_sortby.q.out b/ql/src/test/results/clientpositive/alter_partition_clusterby_sortby.q.out
index f5e8d1f..184d2e4 100644
--- a/ql/src/test/results/clientpositive/alter_partition_clusterby_sortby.q.out
+++ b/ql/src/test/results/clientpositive/alter_partition_clusterby_sortby.q.out
@@ -48,11 +48,8 @@ Database: default
Table: alter_table_partition_clusterby_sortby
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -101,11 +98,8 @@ Database: default
Table: alter_table_partition_clusterby_sortby
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
@@ -154,11 +148,8 @@ Database: default
Table: alter_table_partition_clusterby_sortby
#### A masked pattern was here ####
Partition Parameters:
- COLUMN_STATS_ACCURATE false
#### A masked pattern was here ####
numFiles 0
- numRows -1
- rawDataSize -1
totalSize 0
#### A masked pattern was here ####
http://git-wip-us.apache.org/repos/asf/hive/blob/d82409d0/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_partition_coltype.q.out b/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
index 08bd2b8..e52f7ac 100644
--- a/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
+++ b/ql/src/test/results/clientpositive/alter_partition_coltype.q.out
@@ -148,7 +148,7 @@ STAGE PLANS:
dt 100
ts 3.0
properties:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
columns key,value
columns.comments
@@ -194,7 +194,7 @@ STAGE PLANS:
dt 100
ts 6.30
properties:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
columns key,value
columns.comments
@@ -374,7 +374,7 @@ STAGE PLANS:
dt 100
ts 6.30
properties:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
columns key,value
columns.comments
@@ -532,7 +532,7 @@ STAGE PLANS:
dt 100
ts 3.0
properties:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
columns key,value
columns.comments
@@ -721,7 +721,7 @@ STAGE PLANS:
dt 100
ts 3.0
properties:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
columns key,value
columns.comments
@@ -765,7 +765,7 @@ STAGE PLANS:
dt 100
ts 6.30
properties:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
columns key,value
columns.comments
@@ -990,7 +990,7 @@ STAGE PLANS:
partcol1 1
partcol2 1
properties:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
columns intcol
columns.comments
@@ -1095,7 +1095,7 @@ STAGE PLANS:
partcol1 2
partcol2 1
properties:
- COLUMN_STATS_ACCURATE true
+ COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"}
bucket_count -1
columns intcol
columns.comments