You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2017/11/07 22:33:59 UTC
[22/22] hive git commit: HIVE-16827 : Merge stats task and column
stats task into a single task (Zoltan Haindrich via Ashutosh Chauhan)
HIVE-16827 : Merge stats task and column stats task into a single task (Zoltan Haindrich via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ec9cc0bc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ec9cc0bc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ec9cc0bc
Branch: refs/heads/master
Commit: ec9cc0bc2967daa42a1061cc0dfc297afe223a5e
Parents: 9454042
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Wed Sep 13 06:02:00 2017 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue Nov 7 14:25:19 2017 -0800
----------------------------------------------------------------------
.../accumulo_single_sourced_multi_insert.q.out | 3 +-
.../hadoop/hive/common/jsonexplain/Vertex.java | 3 +-
.../clientpositive/serde_typedbytes.q.out | 3 +-
.../clientpositive/serde_typedbytes2.q.out | 3 +-
.../clientpositive/serde_typedbytes3.q.out | 3 +-
.../clientpositive/serde_typedbytes4.q.out | 3 +-
.../clientpositive/serde_typedbytes5.q.out | 3 +-
data/scripts/q_test_init_src.sql | 5 +
.../hbase_single_sourced_multi_insert.q.out | 3 +-
.../test/results/clientpositive/explain.q.out | 3 +-
.../insert_into_dynamic_partitions.q.out | 5 +-
.../clientpositive/insert_into_table.q.out | 5 +-
.../insert_overwrite_dynamic_partitions.q.out | 5 +-
.../clientpositive/insert_overwrite_table.q.out | 5 +-
.../write_final_output_blobstore.q.out | 8 +-
.../apache/hadoop/hive/ql/TestMTQueries.java | 1 +
.../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 1 +
.../test/resources/testconfiguration.properties | 5 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 69 +-
.../hadoop/hive/metastore/MetaStoreUtils.java | 25 +-
pom.xml | 1 +
.../apache/hadoop/hive/ql/DriverContext.java | 8 +-
.../hadoop/hive/ql/exec/ColumnStatsTask.java | 451 -----
.../hadoop/hive/ql/exec/FileSinkOperator.java | 5 +-
.../hadoop/hive/ql/exec/StatsNoJobTask.java | 396 -----
.../apache/hadoop/hive/ql/exec/StatsTask.java | 502 +-----
.../org/apache/hadoop/hive/ql/exec/Task.java | 2 +
.../apache/hadoop/hive/ql/exec/TaskFactory.java | 9 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 48 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 12 +-
.../ql/metadata/SessionHiveMetaStoreClient.java | 22 +-
.../hive/ql/optimizer/GenMRTableScan1.java | 49 +-
.../hive/ql/optimizer/GenMapRedUtils.java | 60 +-
.../ql/optimizer/QueryPlanPostProcessor.java | 52 +-
.../ql/optimizer/physical/MemoryDecider.java | 10 +-
.../ql/optimizer/physical/SerializeFilter.java | 2 +-
.../ql/optimizer/physical/SkewJoinResolver.java | 11 +-
.../hive/ql/parse/BaseSemanticAnalyzer.java | 20 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 37 +-
.../hive/ql/parse/ExplainSemanticAnalyzer.java | 6 -
.../hadoop/hive/ql/parse/GenTezUtils.java | 4 -
.../hive/ql/parse/ImportSemanticAnalyzer.java | 4 +-
.../hive/ql/parse/LoadSemanticAnalyzer.java | 21 +-
.../hive/ql/parse/ProcessAnalyzeTable.java | 96 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +-
.../hadoop/hive/ql/parse/TaskCompiler.java | 169 +-
.../parse/spark/SparkProcessAnalyzeTable.java | 38 +-
.../hive/ql/plan/BasicStatsNoJobWork.java | 63 +
.../hadoop/hive/ql/plan/BasicStatsWork.java | 196 +++
.../hadoop/hive/ql/plan/ColumnStatsDesc.java | 27 +-
.../hadoop/hive/ql/plan/ColumnStatsWork.java | 97 --
.../hadoop/hive/ql/plan/FileSinkDesc.java | 8 +-
.../hadoop/hive/ql/plan/IStatsGatherDesc.java | 28 +
.../hadoop/hive/ql/plan/LoadFileDesc.java | 24 +-
.../hadoop/hive/ql/plan/StatsNoJobWork.java | 70 -
.../apache/hadoop/hive/ql/plan/StatsWork.java | 171 +-
.../hadoop/hive/ql/plan/TableScanDesc.java | 5 +-
.../hive/ql/stats/BasicStatsNoJobTask.java | 385 +++++
.../hadoop/hive/ql/stats/BasicStatsTask.java | 499 ++++++
.../hadoop/hive/ql/stats/ColStatsProcessor.java | 188 +++
.../ql/stats/ColumnStatisticsObjTranslator.java | 293 ++++
.../hadoop/hive/ql/stats/IStatsProcessor.java | 36 +
.../apache/hadoop/hive/ql/stats/Partish.java | 186 +++
.../apache/hadoop/hive/ql/TestTxnCommands.java | 2 +-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 1 +
.../hadoop/hive/ql/TxnCommandsBaseForTests.java | 1 +
.../TestHiveReduceExpressionsWithStatsRule.java | 2 +-
.../queries/clientpositive/autoColumnStats_1.q | 4 +
.../queries/clientpositive/autoColumnStats_10.q | 52 +
.../queries/clientpositive/autoColumnStats_5a.q | 30 +
.../queries/clientpositive/basicstat_partval.q | 12 +
.../clientpositive/columnstats_partlvl.q | 2 +
.../clientpositive/columnstats_partlvl_dp.q | 2 +
.../test/queries/clientpositive/deleteAnalyze.q | 2 +
.../clientpositive/exec_parallel_column_stats.q | 6 +-
.../clientpositive/outer_reference_windowed.q | 2 +
.../test/queries/clientpositive/smb_mapjoin_1.q | 3 +
.../temp_table_display_colstats_tbllvl.q | 4 +
.../clientpositive/acid_table_stats.q.out | 10 +
.../clientpositive/alterColumnStatsPart.q.out | 12 +
.../alter_partition_update_status.q.out | 8 +
.../alter_table_column_stats.q.out | 16 +
.../alter_table_update_status.q.out | 8 +-
..._table_update_status_disable_bitvector.q.out | 8 +-
.../clientpositive/analyze_tbl_date.q.out | 2 +
.../clientpositive/analyze_tbl_part.q.out | 20 +
.../annotate_stats_deep_filters.q.out | 2 +
.../clientpositive/annotate_stats_filter.q.out | 2 +
.../clientpositive/annotate_stats_groupby.q.out | 4 +
.../annotate_stats_groupby2.q.out | 2 +
.../clientpositive/annotate_stats_join.q.out | 6 +
.../annotate_stats_join_pkfk.q.out | 8 +
.../clientpositive/annotate_stats_limit.q.out | 2 +
.../clientpositive/annotate_stats_part.q.out | 4 +
.../clientpositive/annotate_stats_select.q.out | 2 +
.../clientpositive/annotate_stats_table.q.out | 4 +
.../clientpositive/annotate_stats_union.q.out | 6 +
.../clientpositive/autoColumnStats_1.q.out | 1433 ++++++++++++++++
.../clientpositive/autoColumnStats_10.q.out | 516 ++++++
.../clientpositive/autoColumnStats_2.q.out | 1557 ++++++++++++++++++
.../clientpositive/autoColumnStats_3.q.out | 10 +-
.../clientpositive/autoColumnStats_4.q.out | 11 +-
.../clientpositive/autoColumnStats_5.q.out | 37 +-
.../clientpositive/autoColumnStats_5a.q.out | 997 +++++++++++
.../clientpositive/autoColumnStats_6.q.out | 7 +-
.../clientpositive/autoColumnStats_7.q.out | 9 +-
.../clientpositive/autoColumnStats_8.q.out | 31 +-
.../clientpositive/autoColumnStats_9.q.out | 9 +-
.../results/clientpositive/auto_join1.q.out | 3 +-
.../results/clientpositive/auto_join14.q.out | 3 +-
.../results/clientpositive/auto_join17.q.out | 3 +-
.../results/clientpositive/auto_join19.q.out | 3 +-
.../clientpositive/auto_join19_inclause.q.out | 3 +-
.../results/clientpositive/auto_join2.q.out | 3 +-
.../results/clientpositive/auto_join26.q.out | 3 +-
.../results/clientpositive/auto_join3.q.out | 3 +-
.../results/clientpositive/auto_join4.q.out | 3 +-
.../results/clientpositive/auto_join5.q.out | 3 +-
.../results/clientpositive/auto_join6.q.out | 3 +-
.../results/clientpositive/auto_join7.q.out | 3 +-
.../results/clientpositive/auto_join8.q.out | 3 +-
.../results/clientpositive/auto_join9.q.out | 3 +-
.../clientpositive/auto_sortmerge_join_13.q.out | 18 +-
.../results/clientpositive/avro_decimal.q.out | 4 +-
.../clientpositive/avro_decimal_native.q.out | 4 +-
.../clientpositive/basicstat_partval.q.out | 132 ++
.../beeline/colstats_all_nulls.q.out | 2 +
.../clientpositive/beeline/smb_mapjoin_1.q.out | 45 +
.../clientpositive/beeline/smb_mapjoin_11.q.out | 3 +-
.../clientpositive/beeline/smb_mapjoin_12.q.out | 6 +-
.../clientpositive/beeline/smb_mapjoin_7.q.out | 3 +-
.../clientpositive/binary_output_format.q.out | 3 +-
.../test/results/clientpositive/bucket1.q.out | 3 +-
.../test/results/clientpositive/bucket2.q.out | 3 +-
.../test/results/clientpositive/bucket3.q.out | 3 +-
.../clientpositive/bucket_map_join_spark1.q.out | 6 +-
.../clientpositive/bucket_map_join_spark2.q.out | 6 +-
.../clientpositive/bucket_map_join_spark3.q.out | 6 +-
.../results/clientpositive/bucketmapjoin5.q.out | 6 +-
.../clientpositive/bucketmapjoin_negative.q.out | 3 +-
.../bucketmapjoin_negative2.q.out | 3 +-
.../bucketsortoptimize_insert_1.q.out | 15 +-
.../bucketsortoptimize_insert_3.q.out | 6 +-
.../bucketsortoptimize_insert_4.q.out | 6 +-
.../bucketsortoptimize_insert_5.q.out | 6 +-
.../bucketsortoptimize_insert_8.q.out | 6 +-
.../clientpositive/case_sensitivity.q.out | 3 +-
ql/src/test/results/clientpositive/cast1.q.out | 3 +-
.../cbo_rp_annotate_stats_groupby.q.out | 4 +
.../clientpositive/cbo_rp_auto_join1.q.out | 4 +
.../clientpositive/cbo_rp_auto_join17.q.out | 3 +-
.../cbo_rp_gby2_map_multi_distinct.q.out | 6 +-
.../cbo_rp_groupby3_noskew_multi_distinct.q.out | 3 +-
.../clientpositive/colstats_all_nulls.q.out | 2 +
.../columnStatsUpdateForStatsOptimizer_2.q.out | 2 +
.../column_pruner_multiple_children.q.out | 9 +-
.../columnarserde_create_shortcut.q.out | 3 +-
.../clientpositive/columnstats_infinity.q.out | 4 +-
.../clientpositive/columnstats_partlvl.q.out | 133 +-
.../clientpositive/columnstats_partlvl_dp.q.out | 145 +-
.../clientpositive/columnstats_quoting.q.out | 10 +-
.../clientpositive/columnstats_tbllvl.q.out | 63 +-
.../results/clientpositive/compustat_avro.q.out | 2 +
.../clientpositive/compute_stats_date.q.out | 9 +-
.../test/results/clientpositive/constGby.q.out | 2 +
.../clientpositive/constant_prop_2.q.out | 3 +-
.../clientpositive/constant_prop_3.q.out | 6 +
.../results/clientpositive/constprog_dp.q.out | 3 +-
.../results/clientpositive/constprog_type.q.out | 3 +-
.../clientpositive/correlated_join_keys.q.out | 2 +
.../clientpositive/correlationoptimizer5.q.out | 9 +-
ql/src/test/results/clientpositive/cp_sel.q.out | 3 +-
ql/src/test/results/clientpositive/ctas.q.out | 15 +-
.../results/clientpositive/ctas_colname.q.out | 21 +-
.../ctas_uses_database_location.q.out | 3 +-
.../results/clientpositive/decimal_stats.q.out | 2 +
.../results/clientpositive/deleteAnalyze.q.out | 56 +-
.../display_colstats_tbllvl.q.out | 35 +-
.../results/clientpositive/distinct_stats.q.out | 2 +
.../clientpositive/drop_table_with_stats.q.out | 12 +
.../dynpart_sort_optimization_acid2.q.out | 3 +-
.../encryption_join_unencrypted_tbl.q.out | 18 +-
.../encrypted/encryption_move_tbl.q.out | 2 +
.../exec_parallel_column_stats.q.out | 33 +-
.../results/clientpositive/explain_ddl.q.out | 15 +-
.../extrapolate_part_stats_date.q.out | 8 +
.../extrapolate_part_stats_full.q.out | 24 +
.../extrapolate_part_stats_partial.q.out | 24 +
.../test/results/clientpositive/fm-sketch.q.out | 16 +-
.../test/results/clientpositive/groupby1.q.out | 3 +-
.../test/results/clientpositive/groupby10.q.out | 18 +-
.../test/results/clientpositive/groupby11.q.out | 6 +-
.../test/results/clientpositive/groupby12.q.out | 3 +-
.../results/clientpositive/groupby1_limit.q.out | 3 +-
.../results/clientpositive/groupby1_map.q.out | 3 +-
.../clientpositive/groupby1_map_nomap.q.out | 3 +-
.../clientpositive/groupby1_map_skew.q.out | 3 +-
.../clientpositive/groupby1_noskew.q.out | 3 +-
.../results/clientpositive/groupby2_map.q.out | 3 +-
.../groupby2_map_multi_distinct.q.out | 6 +-
.../clientpositive/groupby2_map_skew.q.out | 3 +-
.../clientpositive/groupby2_noskew.q.out | 3 +-
.../groupby2_noskew_multi_distinct.q.out | 3 +-
.../test/results/clientpositive/groupby3.q.out | 3 +-
.../results/clientpositive/groupby3_map.q.out | 3 +-
.../groupby3_map_multi_distinct.q.out | 3 +-
.../clientpositive/groupby3_map_skew.q.out | 3 +-
.../clientpositive/groupby3_noskew.q.out | 3 +-
.../groupby3_noskew_multi_distinct.q.out | 3 +-
.../test/results/clientpositive/groupby4.q.out | 3 +-
.../results/clientpositive/groupby4_map.q.out | 3 +-
.../clientpositive/groupby4_map_skew.q.out | 3 +-
.../clientpositive/groupby4_noskew.q.out | 3 +-
.../test/results/clientpositive/groupby5.q.out | 3 +-
.../results/clientpositive/groupby5_map.q.out | 3 +-
.../clientpositive/groupby5_map_skew.q.out | 3 +-
.../clientpositive/groupby5_noskew.q.out | 3 +-
.../test/results/clientpositive/groupby6.q.out | 3 +-
.../results/clientpositive/groupby6_map.q.out | 3 +-
.../clientpositive/groupby6_map_skew.q.out | 3 +-
.../clientpositive/groupby6_noskew.q.out | 3 +-
.../results/clientpositive/groupby7_map.q.out | 6 +-
.../groupby7_map_multi_single_reducer.q.out | 6 +-
.../clientpositive/groupby7_map_skew.q.out | 6 +-
.../clientpositive/groupby7_noskew.q.out | 6 +-
.../groupby7_noskew_multi_single_reducer.q.out | 6 +-
.../test/results/clientpositive/groupby8.q.out | 12 +-
.../results/clientpositive/groupby8_map.q.out | 6 +-
.../clientpositive/groupby8_map_skew.q.out | 6 +-
.../clientpositive/groupby8_noskew.q.out | 6 +-
.../test/results/clientpositive/groupby9.q.out | 30 +-
.../clientpositive/groupby_complex_types.q.out | 9 +-
...pby_complex_types_multi_single_reducer.q.out | 6 +-
.../results/clientpositive/groupby_cube1.q.out | 6 +-
.../clientpositive/groupby_cube_multi_gby.q.out | 6 +-
.../clientpositive/groupby_duplicate_key.q.out | 3 +-
.../clientpositive/groupby_map_ppr.q.out | 3 +-
.../groupby_map_ppr_multi_distinct.q.out | 3 +-
.../groupby_multi_insert_common_distinct.q.out | 6 +-
.../groupby_multi_single_reducer.q.out | 24 +-
.../groupby_multi_single_reducer2.q.out | 6 +-
.../groupby_multi_single_reducer3.q.out | 24 +-
.../clientpositive/groupby_position.q.out | 12 +-
.../results/clientpositive/groupby_ppr.q.out | 3 +-
.../groupby_ppr_multi_distinct.q.out | 6 +-
.../clientpositive/groupby_rollup1.q.out | 6 +-
.../clientpositive/groupby_sort_1_23.q.out | 60 +-
.../results/clientpositive/groupby_sort_2.q.out | 3 +-
.../results/clientpositive/groupby_sort_3.q.out | 6 +-
.../results/clientpositive/groupby_sort_4.q.out | 6 +-
.../results/clientpositive/groupby_sort_5.q.out | 9 +-
.../results/clientpositive/groupby_sort_6.q.out | 9 +-
.../results/clientpositive/groupby_sort_7.q.out | 3 +-
.../clientpositive/groupby_sort_skew_1_23.q.out | 60 +-
.../clientpositive/groupby_sort_test_1.q.out | 3 +-
ql/src/test/results/clientpositive/hll.q.out | 16 +-
.../implicit_cast_during_insert.q.out | 3 +-
.../clientpositive/index_auto_update.q.out | 6 +-
.../infer_bucket_sort_dyn_part.q.out | 3 +-
.../infer_bucket_sort_grouping_operators.q.out | 9 +-
.../infer_bucket_sort_map_operators.q.out | 12 +-
.../infer_bucket_sort_num_buckets.q.out | 3 +-
.../test/results/clientpositive/innerjoin.q.out | 3 +-
.../test/results/clientpositive/input11.q.out | 3 +-
.../results/clientpositive/input11_limit.q.out | 3 +-
.../test/results/clientpositive/input12.q.out | 9 +-
.../test/results/clientpositive/input13.q.out | 9 +-
.../test/results/clientpositive/input14.q.out | 3 +-
.../results/clientpositive/input14_limit.q.out | 3 +-
.../test/results/clientpositive/input17.q.out | 3 +-
.../test/results/clientpositive/input18.q.out | 3 +-
.../results/clientpositive/input1_limit.q.out | 6 +-
.../test/results/clientpositive/input20.q.out | 3 +-
.../test/results/clientpositive/input30.q.out | 6 +-
.../test/results/clientpositive/input31.q.out | 3 +-
.../test/results/clientpositive/input32.q.out | 3 +-
.../test/results/clientpositive/input33.q.out | 3 +-
.../test/results/clientpositive/input34.q.out | 3 +-
.../test/results/clientpositive/input35.q.out | 3 +-
.../test/results/clientpositive/input36.q.out | 3 +-
.../test/results/clientpositive/input38.q.out | 3 +-
.../results/clientpositive/input3_limit.q.out | 3 +-
ql/src/test/results/clientpositive/input4.q.out | 3 +-
ql/src/test/results/clientpositive/input5.q.out | 3 +-
ql/src/test/results/clientpositive/input6.q.out | 3 +-
ql/src/test/results/clientpositive/input7.q.out | 3 +-
ql/src/test/results/clientpositive/input8.q.out | 3 +-
ql/src/test/results/clientpositive/input9.q.out | 3 +-
.../clientpositive/input_columnarserde.q.out | 3 +-
.../clientpositive/input_dynamicserde.q.out | 3 +-
.../clientpositive/input_lazyserde.q.out | 3 +-
.../clientpositive/input_lazyserde2.q.out | 3 +-
.../results/clientpositive/input_part1.q.out | 3 +-
.../results/clientpositive/input_part10.q.out | 3 +-
.../results/clientpositive/input_part2.q.out | 6 +-
.../results/clientpositive/input_part5.q.out | 3 +-
.../clientpositive/input_testsequencefile.q.out | 3 +-
.../clientpositive/input_testxpath.q.out | 3 +-
.../clientpositive/input_testxpath2.q.out | 3 +-
.../test/results/clientpositive/insert1.q.out | 18 +-
.../insert1_overwrite_partitions.q.out | 6 +-
.../insert2_overwrite_partitions.q.out | 6 +-
.../results/clientpositive/insert_into1.q.out | 15 +-
.../results/clientpositive/insert_into2.q.out | 9 +-
.../results/clientpositive/insert_into3.q.out | 12 +-
.../results/clientpositive/insert_into4.q.out | 9 +-
.../results/clientpositive/insert_into5.q.out | 12 +-
.../results/clientpositive/insert_into6.q.out | 6 +-
.../insert_values_orig_table_use_metadata.q.out | 5 +
.../clientpositive/insertoverwrite_bucket.q.out | 6 +-
ql/src/test/results/clientpositive/join14.q.out | 3 +-
ql/src/test/results/clientpositive/join17.q.out | 3 +-
ql/src/test/results/clientpositive/join2.q.out | 3 +-
ql/src/test/results/clientpositive/join25.q.out | 3 +-
ql/src/test/results/clientpositive/join26.q.out | 3 +-
ql/src/test/results/clientpositive/join27.q.out | 3 +-
ql/src/test/results/clientpositive/join28.q.out | 3 +-
ql/src/test/results/clientpositive/join29.q.out | 3 +-
ql/src/test/results/clientpositive/join3.q.out | 3 +-
ql/src/test/results/clientpositive/join30.q.out | 3 +-
ql/src/test/results/clientpositive/join31.q.out | 3 +-
ql/src/test/results/clientpositive/join32.q.out | 3 +-
ql/src/test/results/clientpositive/join33.q.out | 3 +-
ql/src/test/results/clientpositive/join34.q.out | 3 +-
ql/src/test/results/clientpositive/join35.q.out | 3 +-
ql/src/test/results/clientpositive/join36.q.out | 3 +-
ql/src/test/results/clientpositive/join37.q.out | 3 +-
ql/src/test/results/clientpositive/join39.q.out | 3 +-
ql/src/test/results/clientpositive/join4.q.out | 3 +-
ql/src/test/results/clientpositive/join5.q.out | 3 +-
ql/src/test/results/clientpositive/join6.q.out | 3 +-
ql/src/test/results/clientpositive/join7.q.out | 3 +-
ql/src/test/results/clientpositive/join8.q.out | 3 +-
ql/src/test/results/clientpositive/join9.q.out | 3 +-
.../results/clientpositive/join_map_ppr.q.out | 6 +-
.../limit_pushdown_negative.q.out | 6 +-
.../test/results/clientpositive/lineage1.q.out | 3 +-
.../clientpositive/list_bucket_dml_1.q.out | 3 +-
.../clientpositive/list_bucket_dml_11.q.out | 3 +-
.../clientpositive/list_bucket_dml_12.q.out | 3 +-
.../clientpositive/list_bucket_dml_13.q.out | 3 +-
.../clientpositive/list_bucket_dml_14.q.out | 3 +-
.../clientpositive/list_bucket_dml_2.q.out | 3 +-
.../clientpositive/list_bucket_dml_3.q.out | 3 +-
.../clientpositive/list_bucket_dml_4.q.out | 6 +-
.../clientpositive/list_bucket_dml_5.q.out | 3 +-
.../clientpositive/list_bucket_dml_6.q.out | 6 +-
.../clientpositive/list_bucket_dml_7.q.out | 6 +-
.../clientpositive/list_bucket_dml_8.q.out | 3 +-
.../clientpositive/list_bucket_dml_9.q.out | 6 +-
.../llap/acid_bucket_pruning.q.out | 2 +
.../clientpositive/llap/acid_no_buckets.q.out | 12 +-
.../llap/acid_vectorization_original.q.out | 3 +-
.../clientpositive/llap/autoColumnStats_1.q.out | 74 +
.../llap/autoColumnStats_10.q.out | 516 ++++++
.../clientpositive/llap/autoColumnStats_2.q.out | 1 -
.../clientpositive/llap/auto_join1.q.out | 3 +-
.../llap/auto_smb_mapjoin_14.q.out | 12 +-
.../llap/auto_sortmerge_join_13.q.out | 18 +-
.../results/clientpositive/llap/bucket2.q.out | 3 +-
.../results/clientpositive/llap/bucket3.q.out | 3 +-
.../results/clientpositive/llap/bucket4.q.out | 3 +-
.../results/clientpositive/llap/bucket5.q.out | 6 +-
.../results/clientpositive/llap/bucket6.q.out | 3 +-
.../clientpositive/llap/bucket_many.q.out | 3 +-
.../clientpositive/llap/bucketmapjoin1.q.out | 6 +-
.../clientpositive/llap/bucketmapjoin2.q.out | 9 +-
.../clientpositive/llap/bucketmapjoin3.q.out | 6 +-
.../clientpositive/llap/bucketmapjoin4.q.out | 6 +-
.../llap/bucketsortoptimize_insert_2.q.out | 18 +-
.../llap/bucketsortoptimize_insert_6.q.out | 21 +-
.../llap/bucketsortoptimize_insert_7.q.out | 9 +-
.../llap/column_table_stats.q.out | 36 +-
.../llap/column_table_stats_orc.q.out | 21 +-
.../test/results/clientpositive/llap/ctas.q.out | 15 +-
.../clientpositive/llap/deleteAnalyze.q.out | 48 +-
.../llap/disable_merge_for_bucketing.q.out | 3 +-
.../llap/dynamic_partition_pruning.q.out | 3 +-
.../llap/dynamic_semijoin_reduction_3.q.out | 27 +-
.../llap/dynpart_sort_opt_vectorization.q.out | 45 +-
.../llap/dynpart_sort_optimization.q.out | 66 +-
.../llap/dynpart_sort_optimization2.q.out | 24 +-
.../llap/dynpart_sort_optimization_acid.q.out | 36 +-
.../clientpositive/llap/explainuser_1.q.out | 14 +-
.../clientpositive/llap/explainuser_2.q.out | 24 +-
.../results/clientpositive/llap/groupby1.q.out | 3 +-
.../results/clientpositive/llap/groupby2.q.out | 3 +-
.../results/clientpositive/llap/groupby3.q.out | 3 +-
.../results/clientpositive/llap/insert1.q.out | 18 +-
.../clientpositive/llap/insert_into1.q.out | 15 +-
.../clientpositive/llap/insert_into2.q.out | 9 +-
.../results/clientpositive/llap/join1.q.out | 3 +-
.../clientpositive/llap/join32_lessSize.q.out | 18 +-
.../llap/list_bucket_dml_10.q.out | 3 +-
.../clientpositive/llap/llap_stats.q.out | 7 +-
.../clientpositive/llap/load_dyn_part1.q.out | 6 +-
.../clientpositive/llap/load_dyn_part2.q.out | 3 +-
.../clientpositive/llap/load_dyn_part3.q.out | 3 +-
.../clientpositive/llap/load_dyn_part5.q.out | 3 +-
.../clientpositive/llap/mapreduce1.q.out | 3 +-
.../clientpositive/llap/mapreduce2.q.out | 3 +-
.../results/clientpositive/llap/merge1.q.out | 9 +-
.../results/clientpositive/llap/merge2.q.out | 9 +-
.../results/clientpositive/llap/mm_all.q.out | 7 +-
.../clientpositive/llap/multi_insert.q.out | 72 +-
.../llap/multi_insert_lateral_view.q.out | 42 +-
.../clientpositive/llap/orc_merge1.q.out | 9 +-
.../clientpositive/llap/orc_merge10.q.out | 12 +-
.../clientpositive/llap/orc_merge2.q.out | 3 +-
.../clientpositive/llap/orc_merge3.q.out | 3 +-
.../clientpositive/llap/orc_merge4.q.out | 3 +-
.../clientpositive/llap/orc_merge5.q.out | 9 +-
.../clientpositive/llap/orc_merge6.q.out | 9 +-
.../clientpositive/llap/orc_merge7.q.out | 9 +-
.../clientpositive/llap/orc_merge_diff_fs.q.out | 9 +-
.../llap/orc_merge_incompat1.q.out | 3 +-
.../llap/orc_merge_incompat2.q.out | 6 +-
.../results/clientpositive/llap/parallel.q.out | 6 +-
.../clientpositive/llap/parallel_colstats.q.out | 22 +-
.../test/results/clientpositive/llap/ptf.q.out | 6 +-
.../clientpositive/llap/rcfile_createas1.q.out | 3 +-
.../clientpositive/llap/rcfile_merge2.q.out | 3 +-
.../clientpositive/llap/rcfile_merge3.q.out | 3 +-
.../clientpositive/llap/rcfile_merge4.q.out | 3 +-
.../llap/reduce_deduplicate.q.out | 6 +-
.../results/clientpositive/llap/sample1.q.out | 3 +-
.../results/clientpositive/llap/skewjoin.q.out | 3 +-
.../clientpositive/llap/smb_mapjoin_18.q.out | 9 +-
.../clientpositive/llap/smb_mapjoin_19.q.out | 3 +-
.../clientpositive/llap/smb_mapjoin_6.q.out | 12 +-
.../results/clientpositive/llap/sqlmerge.q.out | 15 +-
.../results/clientpositive/llap/stats11.q.out | 9 +-
.../clientpositive/llap/stats_noscan_1.q.out | 6 +-
.../clientpositive/llap/temp_table.q.out | 6 +-
.../results/clientpositive/llap/tez_dml.q.out | 12 +-
.../llap/tez_join_result_complex.q.out | 6 +-
.../clientpositive/llap/tez_nway_join.q.out | 90 +-
.../llap/tez_union_dynamic_partition.q.out | 3 +-
.../llap/tez_union_dynamic_partition_2.q.out | 3 +-
.../llap/tez_union_multiinsert.q.out | 30 +-
.../results/clientpositive/llap/union4.q.out | 3 +-
.../results/clientpositive/llap/union6.q.out | 3 +-
.../clientpositive/llap/unionDistinct_1.q.out | 69 +-
.../clientpositive/llap/union_stats.q.out | 3 +-
.../clientpositive/llap/union_top_level.q.out | 9 +-
.../llap/vector_auto_smb_mapjoin_14.q.out | 12 +-
.../clientpositive/llap/vector_bucket.q.out | 3 +-
.../clientpositive/llap/vector_char_4.q.out | 3 +-
.../llap/vector_complex_all.q.out | 3 +-
.../clientpositive/llap/vector_decimal_6.q.out | 3 +-
.../clientpositive/llap/vector_groupby4.q.out | 3 +-
.../clientpositive/llap/vector_groupby6.q.out | 3 +-
.../llap/vector_groupby_cube1.q.out | 6 +-
.../llap/vector_groupby_rollup1.q.out | 6 +-
.../llap/vector_multi_insert.q.out | 9 +-
.../llap/vector_udf_character_length.q.out | 3 +-
.../llap/vector_udf_octet_length.q.out | 3 +-
.../clientpositive/llap/vector_varchar_4.q.out | 3 +-
.../llap/vector_varchar_simple.q.out | 3 +-
.../clientpositive/llap/vector_windowing.q.out | 9 +-
.../llap/vector_windowing_expressions.q.out | 6 +-
.../llap/vector_windowing_streaming.q.out | 3 +-
.../vectorized_dynamic_partition_pruning.q.out | 3 +-
.../clientpositive/llap/vectorized_ptf.q.out | 6 +-
.../results/clientpositive/load_dyn_part1.q.out | 6 +-
.../clientpositive/load_dyn_part10.q.out | 3 +-
.../clientpositive/load_dyn_part13.q.out | 3 +-
.../clientpositive/load_dyn_part14.q.out | 3 +-
.../results/clientpositive/load_dyn_part2.q.out | 3 +-
.../results/clientpositive/load_dyn_part3.q.out | 3 +-
.../results/clientpositive/load_dyn_part4.q.out | 3 +-
.../results/clientpositive/load_dyn_part8.q.out | 6 +-
.../results/clientpositive/load_dyn_part9.q.out | 3 +-
.../results/clientpositive/mapreduce1.q.out | 3 +-
.../results/clientpositive/mapreduce2.q.out | 3 +-
.../results/clientpositive/mapreduce3.q.out | 3 +-
.../results/clientpositive/mapreduce4.q.out | 3 +-
.../results/clientpositive/mapreduce5.q.out | 3 +-
.../results/clientpositive/mapreduce6.q.out | 3 +-
.../results/clientpositive/mapreduce7.q.out | 3 +-
.../results/clientpositive/mapreduce8.q.out | 3 +-
.../results/clientpositive/masking_11.q.out | 2 +
.../materialized_view_rewrite_ssb.q.out | 10 +
.../materialized_view_rewrite_ssb_2.q.out | 10 +
ql/src/test/results/clientpositive/merge1.q.out | 9 +-
ql/src/test/results/clientpositive/merge2.q.out | 9 +-
ql/src/test/results/clientpositive/merge3.q.out | 9 +-
ql/src/test/results/clientpositive/merge4.q.out | 9 +-
.../merge_dynamic_partition.q.out | 9 +-
.../merge_dynamic_partition2.q.out | 3 +-
.../merge_dynamic_partition3.q.out | 3 +-
.../merge_dynamic_partition4.q.out | 3 +-
.../merge_dynamic_partition5.q.out | 3 +-
.../clientpositive/metadata_only_queries.q.out | 14 +
.../metadata_only_queries_with_filters.q.out | 8 +
ql/src/test/results/clientpositive/mm_all.q.out | 7 +-
.../clientpositive/multi_insert_gby.q.out | 12 +-
.../clientpositive/multi_insert_gby2.q.out | 6 +-
.../clientpositive/multi_insert_gby3.q.out | 27 +-
.../clientpositive/multi_insert_gby4.q.out | 9 +-
.../clientpositive/multi_insert_mixed.q.out | 9 +-
...i_insert_move_tasks_share_dependencies.q.out | 96 +-
.../clientpositive/multi_insert_union_src.q.out | 6 +-
.../multi_insert_with_join2.q.out | 42 +-
.../clientpositive/multigroupby_singlemr.q.out | 33 +-
.../results/clientpositive/nonmr_fetch.q.out | 6 +-
.../nonreserved_keywords_insert_into1.q.out | 9 +-
.../results/clientpositive/notable_alias1.q.out | 3 +-
.../results/clientpositive/notable_alias2.q.out | 3 +-
.../results/clientpositive/nullformatCTAS.q.out | 3 +-
.../optimize_filter_literal.q.out | 8 +
.../results/clientpositive/orc_createas1.q.out | 6 +-
.../results/clientpositive/orc_merge1.q.out | 9 +-
.../results/clientpositive/orc_merge10.q.out | 12 +-
.../results/clientpositive/orc_merge2.q.out | 3 +-
.../results/clientpositive/orc_merge3.q.out | 3 +-
.../results/clientpositive/orc_merge4.q.out | 3 +-
.../results/clientpositive/orc_merge5.q.out | 9 +-
.../results/clientpositive/orc_merge6.q.out | 9 +-
.../clientpositive/orc_merge_diff_fs.q.out | 9 +-
.../clientpositive/orc_merge_incompat1.q.out | 3 +-
.../clientpositive/orc_merge_incompat2.q.out | 6 +-
.../outer_reference_windowed.q.out | 122 +-
.../test/results/clientpositive/parallel.q.out | 6 +-
.../clientpositive/parallel_colstats.q.out | 30 +-
.../results/clientpositive/parallel_join1.q.out | 3 +-
.../clientpositive/parallel_orderby.q.out | 3 +-
.../clientpositive/partial_column_stats.q.out | 5 +-
.../partition_coltype_literals.q.out | 4 +
ql/src/test/results/clientpositive/pcr.q.out | 12 +-
ql/src/test/results/clientpositive/pcs.q.out | 8 +
.../clientpositive/ppd_constant_expr.q.out | 6 +-
.../clientpositive/ppd_multi_insert.q.out | 18 +-
ql/src/test/results/clientpositive/quote1.q.out | 3 +-
.../results/clientpositive/quotedid_stats.q.out | 2 +
.../clientpositive/rand_partitionpruner2.q.out | 3 +-
.../clientpositive/rcfile_null_value.q.out | 3 +-
.../clientpositive/remove_exprs_stats.q.out | 6 +
.../rename_external_partition_location.q.out | 4 +
.../rename_table_update_column_stats.q.out | 4 +
.../test/results/clientpositive/sample1.q.out | 3 +-
.../test/results/clientpositive/sample2.q.out | 3 +-
.../test/results/clientpositive/sample4.q.out | 3 +-
.../test/results/clientpositive/sample5.q.out | 3 +-
.../test/results/clientpositive/sample6.q.out | 3 +-
.../test/results/clientpositive/sample7.q.out | 3 +-
.../test/results/clientpositive/skewjoin.q.out | 3 +-
.../clientpositive/skewjoin_noskew.q.out | 3 +-
.../clientpositive/skewjoin_onesideskew.q.out | 3 +-
.../results/clientpositive/smb_mapjoin9.q.out | 3 +-
.../results/clientpositive/smb_mapjoin_1.q.out | 43 +
.../results/clientpositive/smb_mapjoin_11.q.out | 3 +-
.../results/clientpositive/smb_mapjoin_12.q.out | 6 +-
.../results/clientpositive/smb_mapjoin_20.q.out | 9 +-
.../results/clientpositive/smb_mapjoin_21.q.out | 18 +-
.../results/clientpositive/smb_mapjoin_22.q.out | 6 +-
.../results/clientpositive/smb_mapjoin_7.q.out | 3 +-
.../spark/annotate_stats_join.q.out | 6 +
.../clientpositive/spark/auto_join1.q.out | 3 +-
.../clientpositive/spark/auto_join14.q.out | 3 +-
.../clientpositive/spark/auto_join17.q.out | 3 +-
.../clientpositive/spark/auto_join19.q.out | 3 +-
.../clientpositive/spark/auto_join2.q.out | 3 +-
.../clientpositive/spark/auto_join26.q.out | 3 +-
.../clientpositive/spark/auto_join3.q.out | 3 +-
.../clientpositive/spark/auto_join4.q.out | 3 +-
.../clientpositive/spark/auto_join5.q.out | 3 +-
.../clientpositive/spark/auto_join6.q.out | 3 +-
.../clientpositive/spark/auto_join7.q.out | 3 +-
.../clientpositive/spark/auto_join8.q.out | 3 +-
.../clientpositive/spark/auto_join9.q.out | 3 +-
.../spark/auto_smb_mapjoin_14.q.out | 12 +-
.../spark/auto_sortmerge_join_13.q.out | 18 +-
.../spark/avro_decimal_native.q.out | 4 +-
.../results/clientpositive/spark/bucket2.q.out | 3 +-
.../results/clientpositive/spark/bucket3.q.out | 3 +-
.../results/clientpositive/spark/bucket4.q.out | 3 +-
.../results/clientpositive/spark/bucket5.q.out | 6 +-
.../results/clientpositive/spark/bucket6.q.out | 3 +-
.../spark/bucket_map_join_spark1.q.out | 6 +-
.../spark/bucket_map_join_spark2.q.out | 6 +-
.../spark/bucket_map_join_spark3.q.out | 6 +-
.../spark/bucket_map_join_tez1.q.out | 104 +-
.../spark/bucket_map_join_tez2.q.out | 22 +
.../clientpositive/spark/bucketmapjoin1.q.out | 6 +-
.../clientpositive/spark/bucketmapjoin2.q.out | 9 +-
.../clientpositive/spark/bucketmapjoin3.q.out | 6 +-
.../clientpositive/spark/bucketmapjoin4.q.out | 6 +-
.../clientpositive/spark/bucketmapjoin5.q.out | 6 +-
.../clientpositive/spark/bucketmapjoin7.q.out | 4 -
.../spark/bucketmapjoin_negative.q.out | 3 +-
.../spark/bucketmapjoin_negative2.q.out | 3 +-
.../spark/bucketsortoptimize_insert_2.q.out | 18 +-
.../spark/bucketsortoptimize_insert_4.q.out | 6 +-
.../spark/bucketsortoptimize_insert_6.q.out | 21 +-
.../spark/bucketsortoptimize_insert_7.q.out | 9 +-
.../spark/bucketsortoptimize_insert_8.q.out | 6 +-
.../results/clientpositive/spark/ctas.q.out | 15 +-
.../spark/disable_merge_for_bucketing.q.out | 3 +-
.../spark/dynamic_rdd_cache.q.out | 9 +-
.../results/clientpositive/spark/groupby1.q.out | 3 +-
.../clientpositive/spark/groupby10.q.out | 18 +-
.../clientpositive/spark/groupby11.q.out | 6 +-
.../clientpositive/spark/groupby1_map.q.out | 3 +-
.../spark/groupby1_map_nomap.q.out | 3 +-
.../spark/groupby1_map_skew.q.out | 3 +-
.../clientpositive/spark/groupby1_noskew.q.out | 3 +-
.../results/clientpositive/spark/groupby2.q.out | 3 +-
.../clientpositive/spark/groupby2_map.q.out | 3 +-
.../spark/groupby2_map_multi_distinct.q.out | 6 +-
.../spark/groupby2_map_skew.q.out | 3 +-
.../clientpositive/spark/groupby2_noskew.q.out | 3 +-
.../spark/groupby2_noskew_multi_distinct.q.out | 3 +-
.../results/clientpositive/spark/groupby3.q.out | 3 +-
.../clientpositive/spark/groupby3_map.q.out | 3 +-
.../spark/groupby3_map_multi_distinct.q.out | 3 +-
.../spark/groupby3_map_skew.q.out | 3 +-
.../clientpositive/spark/groupby3_noskew.q.out | 3 +-
.../spark/groupby3_noskew_multi_distinct.q.out | 3 +-
.../results/clientpositive/spark/groupby4.q.out | 3 +-
.../clientpositive/spark/groupby4_map.q.out | 3 +-
.../spark/groupby4_map_skew.q.out | 3 +-
.../clientpositive/spark/groupby4_noskew.q.out | 3 +-
.../results/clientpositive/spark/groupby5.q.out | 3 +-
.../clientpositive/spark/groupby5_map.q.out | 3 +-
.../spark/groupby5_map_skew.q.out | 3 +-
.../clientpositive/spark/groupby5_noskew.q.out | 3 +-
.../results/clientpositive/spark/groupby6.q.out | 3 +-
.../clientpositive/spark/groupby6_map.q.out | 3 +-
.../spark/groupby6_map_skew.q.out | 3 +-
.../clientpositive/spark/groupby6_noskew.q.out | 3 +-
.../clientpositive/spark/groupby7_map.q.out | 6 +-
.../groupby7_map_multi_single_reducer.q.out | 6 +-
.../spark/groupby7_map_skew.q.out | 6 +-
.../clientpositive/spark/groupby7_noskew.q.out | 6 +-
.../groupby7_noskew_multi_single_reducer.q.out | 6 +-
.../results/clientpositive/spark/groupby8.q.out | 12 +-
.../clientpositive/spark/groupby8_map.q.out | 6 +-
.../spark/groupby8_map_skew.q.out | 6 +-
.../clientpositive/spark/groupby8_noskew.q.out | 6 +-
.../results/clientpositive/spark/groupby9.q.out | 30 +-
.../spark/groupby_complex_types.q.out | 9 +-
...pby_complex_types_multi_single_reducer.q.out | 6 +-
.../clientpositive/spark/groupby_cube1.q.out | 6 +-
.../clientpositive/spark/groupby_map_ppr.q.out | 3 +-
.../spark/groupby_map_ppr_multi_distinct.q.out | 3 +-
.../groupby_multi_insert_common_distinct.q.out | 6 +-
.../spark/groupby_multi_single_reducer.q.out | 24 +-
.../spark/groupby_multi_single_reducer2.q.out | 6 +-
.../spark/groupby_multi_single_reducer3.q.out | 24 +-
.../clientpositive/spark/groupby_position.q.out | 12 +-
.../clientpositive/spark/groupby_ppr.q.out | 3 +-
.../spark/groupby_ppr_multi_distinct.q.out | 6 +-
.../clientpositive/spark/groupby_rollup1.q.out | 6 +-
.../spark/groupby_sort_1_23.q.out | 60 +-
.../spark/groupby_sort_skew_1_23.q.out | 60 +-
.../spark/infer_bucket_sort_map_operators.q.out | 12 +-
.../spark/infer_bucket_sort_num_buckets.q.out | 3 +-
.../clientpositive/spark/innerjoin.q.out | 3 +-
.../results/clientpositive/spark/input12.q.out | 9 +-
.../results/clientpositive/spark/input13.q.out | 9 +-
.../results/clientpositive/spark/input14.q.out | 3 +-
.../results/clientpositive/spark/input17.q.out | 3 +-
.../results/clientpositive/spark/input18.q.out | 3 +-
.../clientpositive/spark/input1_limit.q.out | 6 +-
.../clientpositive/spark/input_part2.q.out | 6 +-
.../results/clientpositive/spark/insert1.q.out | 18 +-
.../clientpositive/spark/insert_into1.q.out | 15 +-
.../clientpositive/spark/insert_into2.q.out | 9 +-
.../clientpositive/spark/insert_into3.q.out | 12 +-
.../results/clientpositive/spark/join1.q.out | 3 +-
.../results/clientpositive/spark/join14.q.out | 3 +-
.../results/clientpositive/spark/join17.q.out | 3 +-
.../results/clientpositive/spark/join2.q.out | 3 +-
.../results/clientpositive/spark/join25.q.out | 3 +-
.../results/clientpositive/spark/join26.q.out | 3 +-
.../results/clientpositive/spark/join27.q.out | 3 +-
.../results/clientpositive/spark/join28.q.out | 3 +-
.../results/clientpositive/spark/join29.q.out | 3 +-
.../results/clientpositive/spark/join3.q.out | 3 +-
.../results/clientpositive/spark/join30.q.out | 3 +-
.../results/clientpositive/spark/join31.q.out | 3 +-
.../results/clientpositive/spark/join32.q.out | 3 +-
.../clientpositive/spark/join32_lessSize.q.out | 18 +-
.../results/clientpositive/spark/join33.q.out | 3 +-
.../results/clientpositive/spark/join34.q.out | 3 +-
.../results/clientpositive/spark/join35.q.out | 3 +-
.../results/clientpositive/spark/join36.q.out | 3 +-
.../results/clientpositive/spark/join37.q.out | 3 +-
.../results/clientpositive/spark/join39.q.out | 3 +-
.../results/clientpositive/spark/join4.q.out | 3 +-
.../results/clientpositive/spark/join5.q.out | 3 +-
.../results/clientpositive/spark/join6.q.out | 3 +-
.../results/clientpositive/spark/join7.q.out | 3 +-
.../results/clientpositive/spark/join8.q.out | 3 +-
.../results/clientpositive/spark/join9.q.out | 3 +-
.../clientpositive/spark/join_map_ppr.q.out | 6 +-
.../spark/list_bucket_dml_10.q.out | 3 +-
.../spark/list_bucket_dml_2.q.out | 3 +-
.../clientpositive/spark/load_dyn_part1.q.out | 6 +-
.../clientpositive/spark/load_dyn_part10.q.out | 3 +-
.../clientpositive/spark/load_dyn_part13.q.out | 3 +-
.../clientpositive/spark/load_dyn_part14.q.out | 3 +-
.../clientpositive/spark/load_dyn_part2.q.out | 3 +-
.../clientpositive/spark/load_dyn_part3.q.out | 3 +-
.../clientpositive/spark/load_dyn_part4.q.out | 3 +-
.../clientpositive/spark/load_dyn_part5.q.out | 3 +-
.../clientpositive/spark/load_dyn_part8.q.out | 6 +-
.../clientpositive/spark/load_dyn_part9.q.out | 3 +-
.../clientpositive/spark/mapreduce1.q.out | 3 +-
.../clientpositive/spark/mapreduce2.q.out | 3 +-
.../results/clientpositive/spark/merge1.q.out | 9 +-
.../results/clientpositive/spark/merge2.q.out | 9 +-
.../spark/metadata_only_queries.q.out | 14 +
.../metadata_only_queries_with_filters.q.out | 8 +
.../clientpositive/spark/multi_insert.q.out | 72 +-
.../clientpositive/spark/multi_insert_gby.q.out | 12 +-
.../spark/multi_insert_gby2.q.out | 6 +-
.../spark/multi_insert_gby3.q.out | 27 +-
.../spark/multi_insert_lateral_view.q.out | 42 +-
.../spark/multi_insert_mixed.q.out | 9 +-
...i_insert_move_tasks_share_dependencies.q.out | 96 +-
.../spark/multigroupby_singlemr.q.out | 33 +-
.../clientpositive/spark/orc_merge1.q.out | 9 +-
.../clientpositive/spark/orc_merge2.q.out | 3 +-
.../clientpositive/spark/orc_merge3.q.out | 3 +-
.../clientpositive/spark/orc_merge4.q.out | 3 +-
.../clientpositive/spark/orc_merge5.q.out | 9 +-
.../clientpositive/spark/orc_merge6.q.out | 9 +-
.../clientpositive/spark/orc_merge7.q.out | 9 +-
.../spark/orc_merge_diff_fs.q.out | 9 +-
.../spark/orc_merge_incompat1.q.out | 3 +-
.../spark/orc_merge_incompat2.q.out | 6 +-
.../results/clientpositive/spark/parallel.q.out | 6 +-
.../clientpositive/spark/parallel_join1.q.out | 3 +-
.../clientpositive/spark/parallel_orderby.q.out | 3 +-
.../test/results/clientpositive/spark/pcr.q.out | 12 +-
.../clientpositive/spark/ppd_multi_insert.q.out | 18 +-
.../test/results/clientpositive/spark/ptf.q.out | 6 +-
.../spark/reduce_deduplicate.q.out | 6 +-
.../results/clientpositive/spark/sample1.q.out | 3 +-
.../results/clientpositive/spark/sample2.q.out | 3 +-
.../results/clientpositive/spark/sample4.q.out | 3 +-
.../results/clientpositive/spark/sample5.q.out | 3 +-
.../results/clientpositive/spark/sample6.q.out | 3 +-
.../results/clientpositive/spark/sample7.q.out | 3 +-
.../results/clientpositive/spark/skewjoin.q.out | 3 +-
.../clientpositive/spark/skewjoin_noskew.q.out | 3 +-
.../clientpositive/spark/smb_mapjoin_1.q.out | 43 +
.../clientpositive/spark/smb_mapjoin_11.q.out | 3 +-
.../clientpositive/spark/smb_mapjoin_12.q.out | 6 +-
.../clientpositive/spark/smb_mapjoin_18.q.out | 9 +-
.../clientpositive/spark/smb_mapjoin_19.q.out | 3 +-
.../clientpositive/spark/smb_mapjoin_20.q.out | 9 +-
.../clientpositive/spark/smb_mapjoin_21.q.out | 18 +-
.../clientpositive/spark/smb_mapjoin_22.q.out | 6 +-
.../clientpositive/spark/smb_mapjoin_6.q.out | 12 +-
.../clientpositive/spark/smb_mapjoin_7.q.out | 3 +-
.../spark/spark_dynamic_partition_pruning.q.out | 3 +-
.../spark/spark_explainuser_1.q.out | 14 +-
.../spark_multi_insert_parallel_orderby.q.out | 18 +-
.../spark/spark_use_ts_stats_for_mapjoin.q.out | 3 +-
...k_vectorized_dynamic_partition_pruning.q.out | 3 +-
.../results/clientpositive/spark/stats0.q.out | 12 +-
.../results/clientpositive/spark/stats1.q.out | 3 +-
.../results/clientpositive/spark/stats10.q.out | 6 +-
.../results/clientpositive/spark/stats12.q.out | 5 +-
.../results/clientpositive/spark/stats13.q.out | 5 +-
.../results/clientpositive/spark/stats2.q.out | 3 +-
.../results/clientpositive/spark/stats3.q.out | 3 +-
.../results/clientpositive/spark/stats5.q.out | 3 +-
.../results/clientpositive/spark/stats7.q.out | 3 +-
.../results/clientpositive/spark/stats8.q.out | 15 +-
.../results/clientpositive/spark/stats9.q.out | 3 +-
.../clientpositive/spark/stats_noscan_1.q.out | 6 +-
.../clientpositive/spark/stats_only_null.q.out | 16 +
.../spark/stats_partscan_1_23.q.out | 188 +++
.../spark/subquery_multiinsert.q.out | 12 +-
.../clientpositive/spark/temp_table.q.out | 6 +-
.../results/clientpositive/spark/union10.q.out | 3 +-
.../results/clientpositive/spark/union12.q.out | 3 +-
.../results/clientpositive/spark/union17.q.out | 6 +-
.../results/clientpositive/spark/union18.q.out | 6 +-
.../results/clientpositive/spark/union19.q.out | 6 +-
.../results/clientpositive/spark/union22.q.out | 3 +-
.../results/clientpositive/spark/union25.q.out | 3 +-
.../results/clientpositive/spark/union28.q.out | 3 +-
.../results/clientpositive/spark/union29.q.out | 3 +-
.../results/clientpositive/spark/union30.q.out | 3 +-
.../results/clientpositive/spark/union31.q.out | 18 +-
.../results/clientpositive/spark/union33.q.out | 6 +-
.../results/clientpositive/spark/union4.q.out | 3 +-
.../results/clientpositive/spark/union6.q.out | 3 +-
.../spark/union_lateralview.q.out | 3 +-
.../clientpositive/spark/union_top_level.q.out | 9 +-
.../clientpositive/spark/vector_char_4.q.out | 3 +-
.../spark/vector_outer_join1.q.out | 2 +
.../spark/vector_outer_join2.q.out | 2 +
.../spark/vector_outer_join3.q.out | 2 +
.../spark/vector_outer_join4.q.out | 2 +
.../spark/vector_outer_join5.q.out | 8 +
.../clientpositive/spark/vector_varchar_4.q.out | 3 +-
.../clientpositive/spark/vectorized_ptf.q.out | 6 +-
.../special_character_in_tabnames_2.q.out | 2 +
ql/src/test/results/clientpositive/stats0.q.out | 12 +-
ql/src/test/results/clientpositive/stats1.q.out | 3 +-
.../test/results/clientpositive/stats10.q.out | 6 +-
.../test/results/clientpositive/stats12.q.out | 5 +-
.../test/results/clientpositive/stats13.q.out | 5 +-
ql/src/test/results/clientpositive/stats2.q.out | 3 +-
ql/src/test/results/clientpositive/stats3.q.out | 3 +-
ql/src/test/results/clientpositive/stats4.q.out | 6 +-
ql/src/test/results/clientpositive/stats5.q.out | 3 +-
ql/src/test/results/clientpositive/stats7.q.out | 3 +-
ql/src/test/results/clientpositive/stats8.q.out | 15 +-
ql/src/test/results/clientpositive/stats9.q.out | 3 +-
.../clientpositive/stats_empty_dyn_part.q.out | 3 +-
.../clientpositive/stats_invalidation.q.out | 2 +
.../clientpositive/stats_missing_warning.q.out | 6 +
.../results/clientpositive/stats_noscan_1.q.out | 6 +-
.../clientpositive/stats_only_null.q.out | 16 +
.../clientpositive/stats_partial_size.q.out | 2 +
.../clientpositive/stats_partscan_1_23.q.out | 191 +++
.../results/clientpositive/stats_ppr_all.q.out | 8 +
.../clientpositive/subquery_multiinsert.q.out | 12 +-
.../temp_table_display_colstats_tbllvl.q.out | 164 +-
.../tez/acid_vectorization_original_tez.q.out | 3 +-
.../clientpositive/tez/explainanalyze_1.q.out | 4 +-
.../clientpositive/tez/explainanalyze_2.q.out | 24 +-
.../clientpositive/tez/explainanalyze_3.q.out | 34 +-
.../clientpositive/tez/explainanalyze_5.q.out | 162 +-
.../clientpositive/tez/explainuser_3.q.out | 38 +-
.../results/clientpositive/tunable_ndv.q.out | 22 +
ql/src/test/results/clientpositive/udf1.q.out | 3 +-
ql/src/test/results/clientpositive/udf3.q.out | 3 +-
.../results/clientpositive/udf_10_trims.q.out | 3 +-
.../clientpositive/udf_character_length.q.out | 3 +-
.../results/clientpositive/udf_length.q.out | 3 +-
.../clientpositive/udf_octet_length.q.out | 3 +-
.../results/clientpositive/udf_reverse.q.out | 3 +-
.../test/results/clientpositive/union10.q.out | 3 +-
.../test/results/clientpositive/union12.q.out | 3 +-
.../test/results/clientpositive/union17.q.out | 6 +-
.../test/results/clientpositive/union18.q.out | 6 +-
.../test/results/clientpositive/union19.q.out | 6 +-
.../test/results/clientpositive/union22.q.out | 3 +-
.../test/results/clientpositive/union25.q.out | 3 +-
.../test/results/clientpositive/union28.q.out | 3 +-
.../test/results/clientpositive/union29.q.out | 3 +-
.../test/results/clientpositive/union30.q.out | 3 +-
.../test/results/clientpositive/union31.q.out | 18 +-
.../test/results/clientpositive/union33.q.out | 6 +-
ql/src/test/results/clientpositive/union4.q.out | 3 +-
ql/src/test/results/clientpositive/union6.q.out | 3 +-
.../clientpositive/union_lateralview.q.out | 3 +-
.../results/clientpositive/union_stats.q.out | 3 +-
.../clientpositive/updateAccessTime.q.out | 2 +
.../results/clientpositive/vector_bucket.q.out | 3 +-
.../results/clientpositive/vector_char_4.q.out | 3 +-
.../results/clientpositive/vector_const.q.out | 2 +-
.../clientpositive/vector_decimal_6.q.out | 3 +-
.../clientpositive/vector_groupby4.q.out | 3 +-
.../clientpositive/vector_groupby6.q.out | 3 +-
.../clientpositive/vector_if_expr_2.q.out | 10 +-
.../results/clientpositive/vector_like_2.q.out | 10 +-
.../clientpositive/vector_multi_insert.q.out | 9 +-
.../clientpositive/vector_outer_join1.q.out | 2 +
.../clientpositive/vector_outer_join2.q.out | 2 +
.../clientpositive/vector_outer_join3.q.out | 2 +
.../clientpositive/vector_outer_join4.q.out | 2 +
.../vector_outer_reference_windowed.q.out | 176 +-
.../vector_tablesample_rows.q.out | 6 +-
.../vector_udf_character_length.q.out | 7 +-
.../vector_udf_octet_length.q.out | 5 +-
.../clientpositive/vector_varchar_4.q.out | 3 +-
.../clientpositive/vector_varchar_simple.q.out | 3 +-
.../clientpositive/vector_windowing.q.out | 9 +-
.../vector_windowing_expressions.q.out | 6 +-
.../vector_windowing_streaming.q.out | 3 +-
.../clientpositive/vectorized_mapjoin2.q.out | 14 +-
.../hadoop/hive/common/StatsSetupConst.java | 12 +-
.../hadoop/hive/metastore/HiveAlterHandler.java | 18 +-
.../hadoop/hive/metastore/ObjectStore.java | 2 +-
.../TransactionalValidationListener.java | 5 +
.../merge/ColumnStatsMergerFactory.java | 2 +-
885 files changed, 13156 insertions(+), 4184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
----------------------------------------------------------------------
diff --git a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out b/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
index 59bca50..f14c3d6 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
@@ -119,7 +119,8 @@ STAGE PLANS:
name: default.src_x1
Stage: Stage-5
- Stats-Aggr Operator
+ Stats Work
+ Basic Stats Work:
Stage: Stage-6
Map Reduce
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java
index a73893f..5379754 100644
--- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java
+++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java
@@ -249,7 +249,8 @@ public final class Vertex implements Comparable<Vertex>{
// find the right op
Op choose = null;
for (Op op : this.outputOps) {
- if (op.outputVertexName.equals(callingVertex.name)) {
+ // op.outputVertexName may be null
+ if (callingVertex.name.equals(op.outputVertexName)) {
choose = op;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/contrib/src/test/results/clientpositive/serde_typedbytes.q.out
----------------------------------------------------------------------
diff --git a/contrib/src/test/results/clientpositive/serde_typedbytes.q.out b/contrib/src/test/results/clientpositive/serde_typedbytes.q.out
index 6876ca8..c844a70 100644
--- a/contrib/src/test/results/clientpositive/serde_typedbytes.q.out
+++ b/contrib/src/test/results/clientpositive/serde_typedbytes.q.out
@@ -93,7 +93,8 @@ STAGE PLANS:
name: default.dest1
Stage: Stage-2
- Stats-Aggr Operator
+ Stats Work
+ Basic Stats Work:
Stage: Stage-3
Map Reduce
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out
----------------------------------------------------------------------
diff --git a/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out b/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out
index 79cf8fe..c0228aa 100644
--- a/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out
+++ b/contrib/src/test/results/clientpositive/serde_typedbytes2.q.out
@@ -89,7 +89,8 @@ STAGE PLANS:
name: default.dest1
Stage: Stage-2
- Stats-Aggr Operator
+ Stats Work
+ Basic Stats Work:
Stage: Stage-3
Map Reduce
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out
----------------------------------------------------------------------
diff --git a/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out b/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out
index fec58ef..0b7541a 100644
--- a/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out
+++ b/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out
@@ -89,7 +89,8 @@ STAGE PLANS:
name: default.dest1
Stage: Stage-2
- Stats-Aggr Operator
+ Stats Work
+ Basic Stats Work:
Stage: Stage-3
Map Reduce
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out
----------------------------------------------------------------------
diff --git a/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out b/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out
index 1131478..981ff21 100644
--- a/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out
+++ b/contrib/src/test/results/clientpositive/serde_typedbytes4.q.out
@@ -89,7 +89,8 @@ STAGE PLANS:
name: default.dest1
Stage: Stage-2
- Stats-Aggr Operator
+ Stats Work
+ Basic Stats Work:
PREHOOK: query: FROM (
FROM src
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out
----------------------------------------------------------------------
diff --git a/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out b/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out
index 8d3b95e..5a7df3c 100644
--- a/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out
+++ b/contrib/src/test/results/clientpositive/serde_typedbytes5.q.out
@@ -93,7 +93,8 @@ STAGE PLANS:
name: default.dest1
Stage: Stage-2
- Stats-Aggr Operator
+ Stats Work
+ Basic Stats Work:
Stage: Stage-3
Map Reduce
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/data/scripts/q_test_init_src.sql
----------------------------------------------------------------------
diff --git a/data/scripts/q_test_init_src.sql b/data/scripts/q_test_init_src.sql
index 56b44e0..2a62d29 100644
--- a/data/scripts/q_test_init_src.sql
+++ b/data/scripts/q_test_init_src.sql
@@ -3,3 +3,8 @@ DROP TABLE IF EXISTS src PURGE;
CREATE TABLE src(key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE;
LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/kv1.txt" OVERWRITE INTO TABLE src;
+
+ANALYZE TABLE src COMPUTE STATISTICS;
+
+ANALYZE TABLE src COMPUTE STATISTICS FOR COLUMNS key,value;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out b/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
index e8927e9..efc4a11 100644
--- a/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
@@ -119,7 +119,8 @@ STAGE PLANS:
name: default.src_x1
Stage: Stage-5
- Stats-Aggr Operator
+ Stats Work
+ Basic Stats Work:
Stage: Stage-6
Map Reduce
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/itests/hive-blobstore/src/test/results/clientpositive/explain.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/explain.q.out b/itests/hive-blobstore/src/test/results/clientpositive/explain.q.out
index 09197f9..cae2a13 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/explain.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/explain.q.out
@@ -88,7 +88,8 @@ STAGE PLANS:
name: default.blobstore_table
Stage: Stage-2
- Stats-Aggr Operator
+ Stats Work
+ Basic Stats Work:
PREHOOK: query: SELECT * FROM blobstore_table
PREHOOK: type: QUERY
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
index e55b1c2..daf95c3 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
@@ -208,8 +208,9 @@ STAGE PLANS:
name: default.table1
Stage: Stage-2
- Stats-Aggr Operator
- Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+ Stats Work
+ Basic Stats Work:
+ Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
PREHOOK: query: DROP TABLE table1
PREHOOK: type: DROPTABLE
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
index f50f4af..5349210 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
@@ -168,8 +168,9 @@ STAGE PLANS:
name: default.table1
Stage: Stage-2
- Stats-Aggr Operator
- Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+ Stats Work
+ Basic Stats Work:
+ Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
Stage: Stage-3
Map Reduce
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
index 660cebb..42b9821 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
@@ -226,8 +226,9 @@ STAGE PLANS:
name: default.table1
Stage: Stage-2
- Stats-Aggr Operator
- Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+ Stats Work
+ Basic Stats Work:
+ Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
PREHOOK: query: DROP TABLE table1
PREHOOK: type: DROPTABLE
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
index ba0e83d..cae1a5b 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
@@ -176,8 +176,9 @@ STAGE PLANS:
name: default.table1
Stage: Stage-2
- Stats-Aggr Operator
- Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+ Stats Work
+ Basic Stats Work:
+ Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
Stage: Stage-3
Map Reduce
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out b/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
index 2ababb1..45e6d25 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
@@ -245,7 +245,8 @@ STAGE PLANS:
name: default.blobstore_table
Stage: Stage-3
- Stats-Aggr Operator
+ Stats Work
+ Basic Stats Work:
#### A masked pattern was here ####
PREHOOK: query: EXPLAIN EXTENDED FROM hdfs_table INSERT OVERWRITE TABLE blobstore_table SELECT hdfs_table.key GROUP BY hdfs_table.key ORDER BY hdfs_table.key
@@ -469,8 +470,9 @@ STAGE PLANS:
name: default.blobstore_table
Stage: Stage-3
- Stats-Aggr Operator
- Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
+ Stats Work
+ Basic Stats Work:
+ Stats Aggregation Key Prefix: ### BLOBSTORE_STAGING_PATH ###
PREHOOK: query: DROP TABLE hdfs_table
PREHOOK: type: DROPTABLE
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
index ad2baa2..e8ef4b9 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
@@ -44,6 +44,7 @@ public class TestMTQueries extends BaseTestQueries {
util.getConf().setBoolean("hive.exec.submit.local.task.via.child", true);
util.getConf().set("hive.stats.dbclass", "fs");
util.getConf().set("hive.mapred.mode", "nonstrict");
+ util.getConf().set("hive.stats.column.autogather", "false");
}
boolean success = QTestUtil.queryListRunnerMultiThreaded(qfiles, qts);
if (!success) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
index 2edf749..f5ed735 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java
@@ -203,6 +203,7 @@ public class TestJdbcWithMiniHS2 {
private static void startMiniHS2(HiveConf conf, boolean httpMode) throws Exception {
conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
conf.setBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false);
+ conf.setBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER, false);
MiniHS2.Builder builder = new MiniHS2.Builder().withConf(conf).cleanupLocalDirOnStartup(false);
if (httpMode) {
builder = builder.withHTTPTransport();
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 46abf8a..5f92321 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -116,6 +116,9 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
auto_sortmerge_join_7.q,\
auto_sortmerge_join_8.q,\
auto_sortmerge_join_9.q,\
+ autoColumnStats_1.q,\
+ autoColumnStats_10.q,\
+ autoColumnStats_2.q,\
bucket2.q,\
bucket3.q,\
bucket4.q,\
@@ -486,8 +489,6 @@ minillaplocal.query.files=\
auto_sortmerge_join_6.q,\
auto_sortmerge_join_8.q,\
auto_sortmerge_join_9.q,\
- autoColumnStats_1.q,\
- autoColumnStats_2.q,\
bucket4.q,\
bucket_groupby.q,\
bucket_many.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index f29a20d..ed58b41 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -33,6 +33,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -61,6 +62,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimaps;
+
import org.apache.commons.cli.OptionBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -6923,39 +6925,80 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (request.isSetNeedMerge() && request.isNeedMerge()) {
// one single call to get all column stats
ColumnStatistics csOld = getMS().getTableColumnStatistics(dbName, tableName, colNames);
- if (csOld != null && csOld.getStatsObjSize() != 0) {
+ Table t = getTable(dbName, tableName);
+ // we first use t.getParameters() to prune the stats
+ MetaStoreUtils.getMergableCols(firstColStats, t.getParameters());
+ // we merge those that can be merged
+ if (csOld != null && csOld.getStatsObjSize() != 0
+ && !firstColStats.getStatsObj().isEmpty()) {
MetaStoreUtils.mergeColStats(firstColStats, csOld);
}
+ if (!firstColStats.getStatsObj().isEmpty()) {
+ return update_table_column_statistics(firstColStats);
+ } else {
+ LOG.debug("All the column stats are not accurate to merge.");
+ return true;
+ }
+ } else {
+ // This is the overwrite case, we do not care about the accuracy.
+ return update_table_column_statistics(firstColStats);
}
- return update_table_column_statistics(firstColStats);
}
} else {
// partition level column stats merging
- List<String> partitionNames = new ArrayList<>();
+ List<Partition> partitions = new ArrayList<>();
+ // note that we may have two or more duplicate partition names.
+ // see autoColumnStats_2.q under TestMiniLlapLocalCliDriver
+ Map<String, ColumnStatistics> newStatsMap = new HashMap<>();
for (ColumnStatistics csNew : csNews) {
- partitionNames.add(csNew.getStatsDesc().getPartName());
+ String partName = csNew.getStatsDesc().getPartName();
+ if (newStatsMap.containsKey(partName)) {
+ MetaStoreUtils.mergeColStats(csNew, newStatsMap.get(partName));
+ }
+ newStatsMap.put(partName, csNew);
}
- Map<String, ColumnStatistics> map = new HashMap<>();
+
+ Map<String, ColumnStatistics> oldStatsMap = new HashMap<>();
+ Map<String, Partition> mapToPart = new HashMap<>();
if (request.isSetNeedMerge() && request.isNeedMerge()) {
// a single call to get all column stats for all partitions
+ List<String> partitionNames = new ArrayList<>();
+ partitionNames.addAll(newStatsMap.keySet());
List<ColumnStatistics> csOlds = getMS().getPartitionColumnStatistics(dbName, tableName,
partitionNames, colNames);
- if (csNews.size() != csOlds.size()) {
+ if (newStatsMap.values().size() != csOlds.size()) {
// some of the partitions miss stats.
LOG.debug("Some of the partitions miss stats.");
}
for (ColumnStatistics csOld : csOlds) {
- map.put(csOld.getStatsDesc().getPartName(), csOld);
+ oldStatsMap.put(csOld.getStatsDesc().getPartName(), csOld);
+ }
+ // another single call to get all the partition objects
+ partitions = getMS().getPartitionsByNames(dbName, tableName, partitionNames);
+ for (int index = 0; index < partitionNames.size(); index++) {
+ mapToPart.put(partitionNames.get(index), partitions.get(index));
}
}
Table t = getTable(dbName, tableName);
- for (int index = 0; index < csNews.size(); index++) {
- ColumnStatistics csNew = csNews.get(index);
- ColumnStatistics csOld = map.get(csNew.getStatsDesc().getPartName());
- if (csOld != null && csOld.getStatsObjSize() != 0) {
- MetaStoreUtils.mergeColStats(csNew, csOld);
+ for (Entry<String, ColumnStatistics> entry : newStatsMap.entrySet()) {
+ ColumnStatistics csNew = entry.getValue();
+ ColumnStatistics csOld = oldStatsMap.get(entry.getKey());
+ if (request.isSetNeedMerge() && request.isNeedMerge()) {
+ // we first use getParameters() to prune the stats
+ MetaStoreUtils.getMergableCols(csNew, mapToPart.get(entry.getKey()).getParameters());
+ // we merge those that can be merged
+ if (csOld != null && csOld.getStatsObjSize() != 0 && !csNew.getStatsObj().isEmpty()) {
+ MetaStoreUtils.mergeColStats(csNew, csOld);
+ }
+ if (!csNew.getStatsObj().isEmpty()) {
+ ret = ret && updatePartitonColStats(t, csNew);
+ } else {
+ LOG.debug("All the column stats " + csNew.getStatsDesc().getPartName()
+ + " are not accurate to merge.");
+ }
+ } else {
+ ret = ret && updatePartitonColStats(t, csNew);
}
- ret = ret && updatePartitonColStats(t, csNew);
}
}
return ret;
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index a491789..24590b9 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -38,11 +38,10 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Maps;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.slf4j.Logger;
@@ -1596,6 +1595,19 @@ public class MetaStoreUtils {
return new URLClassLoader(curPath.toArray(new URL[0]), loader);
}
+ protected static void getMergableCols(ColumnStatistics csNew, Map<String, String> parameters) {
+ List<ColumnStatisticsObj> list = new ArrayList<>();
+ for (int index = 0; index < csNew.getStatsObj().size(); index++) {
+ ColumnStatisticsObj statsObjNew = csNew.getStatsObj().get(index);
+ // canColumnStatsMerge guarantees that it is accurate before we do merge
+ if (StatsSetupConst.canColumnStatsMerge(parameters, statsObjNew.getColName())) {
+ list.add(statsObjNew);
+ }
+ // in all the other cases, we can not merge
+ }
+ csNew.setStatsObj(list);
+ }
+
// this function will merge csOld into csNew.
public static void mergeColStats(ColumnStatistics csNew, ColumnStatistics csOld)
throws InvalidObjectException {
@@ -1619,13 +1631,20 @@ public class MetaStoreUtils {
ColumnStatisticsObj statsObjNew = csNew.getStatsObj().get(index);
ColumnStatisticsObj statsObjOld = map.get(statsObjNew.getColName());
if (statsObjOld != null) {
+ // because we already confirm that the stats is accurate
+ // it is impossible that the column types have been changed while the
+ // column stats is still accurate.
+ assert (statsObjNew.getStatsData().getSetField() == statsObjOld.getStatsData()
+ .getSetField());
// If statsObjOld is found, we can merge.
ColumnStatsMerger merger = ColumnStatsMergerFactory.getColumnStatsMerger(statsObjNew,
statsObjOld);
merger.merge(statsObjNew, statsObjOld);
}
+ // If statsObjOld is not found, we just use statsObjNew as it is accurate.
list.add(statsObjNew);
}
+ // in all the other cases, we can not merge
csNew.setStatsObj(list);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index dc31bd5..9cd32d2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1000,6 +1000,7 @@
<configuration>
<downloadJavadocs>true</downloadJavadocs>
<downloadSources>true</downloadSources>
+ <buildOutputDirectory>target/eclipse/classes</buildOutputDirectory>
<workspaceActiveCodeStyleProfileName>Hive</workspaceActiveCodeStyleProfileName>
<workspaceCodeStylesURL>${basedir}/dev-support/eclipse-styles.xml</workspaceCodeStylesURL>
</configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/ec9cc0bc/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java b/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
index 583d3d3..1789cc3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverContext.java
@@ -18,11 +18,11 @@
package org.apache.hadoop.hive.ql;
+import org.apache.hadoop.hive.ql.exec.StatsTask;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.NodeUtils;
import org.apache.hadoop.hive.ql.exec.NodeUtils.Function;
import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.StatsTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskRunner;
import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
@@ -64,7 +64,7 @@ public class DriverContext {
private Context ctx;
private boolean shutdown;
- final Map<String, StatsTask> statsTasks = new HashMap<String, StatsTask>(1);
+ final Map<String, StatsTask> statsTasks = new HashMap<>(1);
public DriverContext() {
}
@@ -191,7 +191,9 @@ public class DriverContext {
NodeUtils.iterateTask(rootTasks, StatsTask.class, new Function<StatsTask>() {
@Override
public void apply(StatsTask statsTask) {
- statsTasks.put(statsTask.getWork().getAggKey(), statsTask);
+ if (statsTask.getWork().isAggregating()) {
+ statsTasks.put(statsTask.getWork().getAggKey(), statsTask);
+ }
}
});
}