You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dj...@apache.org on 2018/05/08 18:43:02 UTC
[57/58] [abbrv] hive git commit: HIVE-18910 : Migrate to Murmur hash
for shuffle and bucketing (Deepak Jaiswal, reviewed by Jasone Dere)
HIVE-18910 : Migrate to Murmur hash for shuffle and bucketing (Deepak Jaiswal, reviewed by Jasone Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/091fd962
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/091fd962
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/091fd962
Branch: refs/heads/storage-branch-2.6
Commit: 091fd962b8b1615545588028727b0e4c2b36e0b9
Parents: 4c0475f
Author: Deepak Jaiswal <dj...@apache.org>
Authored: Wed May 2 11:32:07 2018 -0700
Committer: Deepak Jaiswal <dj...@apache.org>
Committed: Tue May 8 11:37:23 2018 -0700
----------------------------------------------------------------------
.../results/positive/external_table_ppd.q.out | 1 +
.../positive/hbase_binary_storage_queries.q.out | 2 +
.../src/test/results/positive/hbase_ddl.q.out | 2 +
.../src/test/results/positive/hbasestats.q.out | 5 +
.../streaming/AbstractRecordWriter.java | 9 +-
.../hive/hcatalog/streaming/TestStreaming.java | 10 +-
.../mutate/worker/TestBucketIdResolverImpl.java | 2 +-
.../org/apache/hive/hcatalog/api/HCatTable.java | 4 +
.../hive/hcatalog/api/TestHCatClient.java | 2 +
.../insert_into_dynamic_partitions.q.out | 4 +
.../clientpositive/insert_into_table.q.out | 11 +
.../insert_overwrite_directory.q.out | 4 +
.../insert_overwrite_dynamic_partitions.q.out | 4 +
.../clientpositive/insert_overwrite_table.q.out | 11 +
.../write_final_output_blobstore.q.out | 8 +
.../hive/ql/txn/compactor/TestCompactor.java | 2 +-
.../test/resources/testconfiguration.properties | 6 +-
.../hadoop/hive/ql/exec/FileSinkOperator.java | 16 +-
.../hadoop/hive/ql/exec/FunctionRegistry.java | 1 +
.../apache/hadoop/hive/ql/exec/Operator.java | 9 +
.../hadoop/hive/ql/exec/OperatorFactory.java | 3 +
.../hadoop/hive/ql/exec/ReduceSinkOperator.java | 44 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 13 +
.../VectorKeySeriesSerializedImpl.java | 4 +-
.../VectorReduceSinkObjectHashOperator.java | 135 +-
.../hadoop/hive/ql/io/HiveFileFormatUtils.java | 11 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 8 +-
.../apache/hadoop/hive/ql/metadata/Table.java | 11 +
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 63 +-
.../optimizer/FixedBucketPruningOptimizer.java | 18 +-
.../ql/optimizer/PrunerOperatorFactory.java | 3 +-
.../optimizer/SortedDynPartitionOptimizer.java | 6 +-
...tedDynPartitionTimeGranularityOptimizer.java | 9 +-
.../annotation/OpTraitsRulesProcFactory.java | 37 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 16 +-
.../optimizer/spark/SparkMapJoinOptimizer.java | 3 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 33 +-
.../apache/hadoop/hive/ql/plan/OpTraits.java | 24 +-
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 2 +-
.../hadoop/hive/ql/plan/ReduceSinkDesc.java | 10 +-
.../apache/hadoop/hive/ql/plan/TableDesc.java | 5 +
.../hive/ql/plan/VectorReduceSinkDesc.java | 9 +
.../hive/ql/udf/generic/GenericUDFHash.java | 3 +-
.../ql/udf/generic/GenericUDFMurmurHash.java | 64 +
.../hadoop/hive/ql/TestTxnAddPartition.java | 11 +-
.../apache/hadoop/hive/ql/TestTxnCommands.java | 20 +-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 63 +-
.../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 14 +-
.../parse/authorization/TestPrivilegesV1.java | 2 +-
.../clientpositive/archive_excludeHadoop20.q | 4 +-
.../test/queries/clientpositive/bucket_many.q | 4 +-
.../clientpositive/bucket_map_join_tez1.q | 2 +-
.../clientpositive/bucket_num_reducers.q | 5 +-
.../clientpositive/bucket_num_reducers2.q | 6 +-
.../clientpositive/bucket_num_reducers_acid2.q | 3 -
.../clientpositive/insert_update_delete.q | 1 -
ql/src/test/queries/clientpositive/sample10.q | 4 +-
.../clientpositive/tez_dynpart_hashjoin_3.q | 4 +-
ql/src/test/queries/clientpositive/tez_smb_1.q | 2 +-
.../clientnegative/alter_file_format.q.out | 1 +
.../alter_view_as_select_with_partition.q.out | 1 +
.../clientnegative/unset_table_property.q.out | 1 +
.../clientnegative/unset_view_property.q.out | 1 +
.../results/clientpositive/acid_mapjoin.q.out | 20 +-
.../results/clientpositive/acid_nullscan.q.out | 2 +
.../clientpositive/acid_table_stats.q.out | 2 +
.../clientpositive/alterColumnStats.q.out | 3 +
.../clientpositive/alterColumnStatsPart.q.out | 1 +
.../clientpositive/alter_file_format.q.out | 7 +
...lter_numbuckets_partitioned_table2_h23.q.out | 9 +
...alter_numbuckets_partitioned_table_h23.q.out | 8 +
.../alter_partition_clusterby_sortby.q.out | 1 +
.../alter_partition_coltype.q.out | 4 +
.../clientpositive/alter_skewed_table.q.out | 6 +
.../alter_table_add_partition.q.out | 3 +
.../alter_table_column_stats.q.out | 22 +
.../clientpositive/alter_table_not_sorted.q.out | 2 +
.../clientpositive/alter_table_serde2.q.out | 2 +
.../clientpositive/alter_view_as_select.q.out | 3 +
.../clientpositive/alter_view_col_type.q.out | 2 +
.../analyze_table_null_partition.q.out | 4 +
.../clientpositive/analyze_tbl_date.q.out | 1 +
.../archive_excludeHadoop20.q.out | 33 +-
.../clientpositive/autoColumnStats_1.q.out | 2 +
.../clientpositive/autoColumnStats_10.q.out | 8 +
.../clientpositive/autoColumnStats_2.q.out | 1 +
.../clientpositive/autoColumnStats_3.q.out | 1 +
.../clientpositive/autoColumnStats_4.q.out | 3 +
.../clientpositive/autoColumnStats_5a.q.out | 21 +
.../clientpositive/autoColumnStats_8.q.out | 4 +
.../clientpositive/autoColumnStats_9.q.out | 1 +
.../auto_join_reordering_values.q.out | 8 +
.../avro_schema_evolution_native.q.out | 2 +
.../clientpositive/basicstat_partval.q.out | 1 +
.../beeline/escape_comments.q.out | 3 +
.../materialized_view_create_rewrite.q.out | 2 +
.../clientpositive/beeline/smb_mapjoin_1.q.out | 1 +
.../clientpositive/beeline/smb_mapjoin_11.q.out | 229 +-
.../clientpositive/beeline/smb_mapjoin_12.q.out | 10 +-
.../clientpositive/beeline/smb_mapjoin_13.q.out | 4 +
.../clientpositive/binary_output_format.q.out | 10 +
.../test/results/clientpositive/bucket1.q.out | 4 +
.../test/results/clientpositive/bucket2.q.out | 256 +-
.../test/results/clientpositive/bucket3.q.out | 256 +-
.../clientpositive/bucket_map_join_1.q.out | 2 +
.../clientpositive/bucket_map_join_2.q.out | 2 +
.../clientpositive/bucket_map_join_spark1.q.out | 10 +
.../clientpositive/bucket_map_join_spark2.q.out | 10 +
.../clientpositive/bucket_map_join_spark3.q.out | 10 +
.../clientpositive/bucket_map_join_spark4.q.out | 12 +
.../clientpositive/bucket_num_reducers.q.out | 17 -
.../clientpositive/bucket_num_reducers2.q.out | 13 -
.../bucket_num_reducers_acid2.q.out | 48 -
.../clientpositive/bucketcontext_1.q.out | 5 +
.../clientpositive/bucketcontext_2.q.out | 5 +
.../clientpositive/bucketcontext_3.q.out | 4 +
.../clientpositive/bucketcontext_4.q.out | 4 +
.../clientpositive/bucketcontext_5.q.out | 4 +
.../clientpositive/bucketcontext_6.q.out | 4 +
.../clientpositive/bucketcontext_7.q.out | 6 +
.../clientpositive/bucketcontext_8.q.out | 6 +
.../clientpositive/bucketmapjoin10.q.out | 4 +
.../clientpositive/bucketmapjoin11.q.out | 8 +
.../clientpositive/bucketmapjoin12.q.out | 4 +
.../clientpositive/bucketmapjoin13.q.out | 9 +
.../results/clientpositive/bucketmapjoin5.q.out | 20 +
.../results/clientpositive/bucketmapjoin8.q.out | 4 +
.../results/clientpositive/bucketmapjoin9.q.out | 4 +
.../clientpositive/bucketmapjoin_negative.q.out | 11 +
.../bucketmapjoin_negative2.q.out | 12 +
.../bucketmapjoin_negative3.q.out | 18 +
.../bucketsortoptimize_insert_1.q.out | 8 +-
.../bucketsortoptimize_insert_3.q.out | 4 +-
.../bucketsortoptimize_insert_4.q.out | 24 +-
.../bucketsortoptimize_insert_5.q.out | 44 +-
.../bucketsortoptimize_insert_8.q.out | 44 +-
.../clientpositive/cbo_rp_outer_join_ppr.q.out | 8 +
.../columnStatsUpdateForStatsOptimizer_2.q.out | 4 +
...names_with_leading_and_trailing_spaces.q.out | 1 +
.../column_pruner_multiple_children.q.out | 1 +
.../clientpositive/columnstats_infinity.q.out | 3 +
.../clientpositive/columnstats_partlvl.q.out | 2 +
.../clientpositive/columnstats_tbllvl.q.out | 4 +
.../test/results/clientpositive/combine3.q.out | 48 +-
.../test/results/clientpositive/comments.q.out | 2 +
.../constantPropagateForSubQuery.q.out | 4 +
.../create_alter_list_bucketing_table1.q.out | 6 +
.../results/clientpositive/create_like.q.out | 3 +
.../clientpositive/create_like_tbl_props.q.out | 1 +
.../clientpositive/create_like_view.q.out | 4 +
.../clientpositive/create_or_replace_view.q.out | 5 +
.../clientpositive/create_skewed_table1.q.out | 3 +
.../create_table_like_stats.q.out | 1 +
.../results/clientpositive/create_view.q.out | 38 +-
.../create_view_defaultformats.q.out | 2 +
.../create_view_partitioned.q.out | 3 +
.../clientpositive/create_view_translate.q.out | 2 +
.../create_with_constraints.q.out | 44 +
.../create_with_constraints2.q.out | 2 +
ql/src/test/results/clientpositive/ctas.q.out | 5 +
.../results/clientpositive/ctas_colname.q.out | 7 +
.../ctas_uses_database_location.q.out | 1 +
.../clientpositive/database_location.q.out | 2 +
.../results/clientpositive/decimal_serde.q.out | 2 +
.../clientpositive/default_file_format.q.out | 12 +
.../results/clientpositive/deleteAnalyze.q.out | 2 +
.../describe_comment_indent.q.out | 1 +
.../describe_comment_nonascii.q.out | 1 +
.../clientpositive/describe_syntax.q.out | 2 +
.../results/clientpositive/describe_table.q.out | 2 +
.../display_colstats_tbllvl.q.out | 2 +
.../druid/druidkafkamini_basic.q.out | 4 +
.../clientpositive/druid/druidmini_mv.q.out | 3 +
.../results/clientpositive/druid_basic1.q.out | 2 +
.../results/clientpositive/druid_basic2.q.out | 7 +
.../clientpositive/druid_intervals.q.out | 1 +
.../results/clientpositive/druid_topn.q.out | 1 +
.../dynamic_partition_skip_default.q.out | 4 +
.../dynpart_sort_opt_bucketing.q.out | 88 +-
.../encryption_join_unencrypted_tbl.q.out | 4 +
...on_join_with_different_encryption_keys.q.out | 4 +
.../clientpositive/escape_comments.q.out | 3 +
.../clientpositive/exim_hidden_files.q.out | 1 +
.../extrapolate_part_stats_date.q.out | 4 +
.../extrapolate_part_stats_full.q.out | 12 +
.../extrapolate_part_stats_partial.q.out | 38 +
.../results/clientpositive/filter_aggr.q.out | 2 +
.../clientpositive/filter_join_breaktask.q.out | 2 +
.../results/clientpositive/filter_union.q.out | 4 +
.../clientpositive/fouter_join_ppr.q.out | 16 +
.../clientpositive/groupby_map_ppr.q.out | 4 +
.../groupby_map_ppr_multi_distinct.q.out | 4 +
.../results/clientpositive/groupby_ppr.q.out | 4 +
.../groupby_ppr_multi_distinct.q.out | 8 +
.../clientpositive/groupby_sort_1_23.q.out | 132 +
.../results/clientpositive/groupby_sort_6.q.out | 7 +
.../clientpositive/groupby_sort_skew_1_23.q.out | 132 +
.../infer_bucket_sort_dyn_part.q.out | 2 +
.../infer_bucket_sort_map_operators.q.out | 2 +
.../infer_bucket_sort_num_buckets.q.out | 2 +
.../infer_bucket_sort_reducers_power_two.q.out | 6 +-
.../test/results/clientpositive/input23.q.out | 1 +
.../test/results/clientpositive/input30.q.out | 4 +-
.../test/results/clientpositive/input31.q.out | 2 +-
.../test/results/clientpositive/input39.q.out | 2 +-
.../test/results/clientpositive/input42.q.out | 6 +
.../results/clientpositive/input_part1.q.out | 9 +
.../results/clientpositive/input_part2.q.out | 12 +
.../results/clientpositive/input_part7.q.out | 2 +
.../results/clientpositive/input_part9.q.out | 2 +
ql/src/test/results/clientpositive/join17.q.out | 4 +
ql/src/test/results/clientpositive/join26.q.out | 7 +
ql/src/test/results/clientpositive/join32.q.out | 8 +
ql/src/test/results/clientpositive/join33.q.out | 8 +
ql/src/test/results/clientpositive/join34.q.out | 7 +
ql/src/test/results/clientpositive/join35.q.out | 9 +
ql/src/test/results/clientpositive/join9.q.out | 5 +
.../clientpositive/join_filters_overlap.q.out | 10 +
.../results/clientpositive/join_map_ppr.q.out | 18 +
.../clientpositive/list_bucket_dml_1.q.out | 5 +
.../clientpositive/list_bucket_dml_11.q.out | 5 +
.../clientpositive/list_bucket_dml_12.q.out | 6 +
.../clientpositive/list_bucket_dml_13.q.out | 5 +
.../clientpositive/list_bucket_dml_14.q.out | 7 +
.../clientpositive/list_bucket_dml_2.q.out | 5 +
.../clientpositive/list_bucket_dml_3.q.out | 5 +
.../clientpositive/list_bucket_dml_4.q.out | 13 +
.../clientpositive/list_bucket_dml_5.q.out | 6 +
.../clientpositive/list_bucket_dml_6.q.out | 14 +
.../clientpositive/list_bucket_dml_7.q.out | 14 +
.../clientpositive/list_bucket_dml_8.q.out | 6 +
.../clientpositive/list_bucket_dml_9.q.out | 13 +
.../list_bucket_query_multiskew_1.q.out | 4 +
.../list_bucket_query_multiskew_2.q.out | 3 +
.../list_bucket_query_multiskew_3.q.out | 3 +
.../list_bucket_query_oneskew_1.q.out | 3 +
.../list_bucket_query_oneskew_2.q.out | 4 +
.../list_bucket_query_oneskew_3.q.out | 1 +
.../llap/acid_bucket_pruning.q.out | 159 -
.../llap/acid_vectorization_original.q.out | 729 -
.../llap/alter_merge_stats_orc.q.out | 2 +
.../clientpositive/llap/autoColumnStats_1.q.out | 2 +
.../llap/autoColumnStats_10.q.out | 8 +
.../clientpositive/llap/autoColumnStats_2.q.out | 1 +
.../llap/auto_sortmerge_join_1.q.out | 9 +
.../llap/auto_sortmerge_join_11.q.out | 14 +
.../llap/auto_sortmerge_join_12.q.out | 5 +
.../llap/auto_sortmerge_join_2.q.out | 6 +
.../llap/auto_sortmerge_join_3.q.out | 9 +
.../llap/auto_sortmerge_join_4.q.out | 9 +
.../llap/auto_sortmerge_join_5.q.out | 12 +
.../llap/auto_sortmerge_join_7.q.out | 12 +
.../llap/auto_sortmerge_join_8.q.out | 12 +
.../results/clientpositive/llap/bucket2.q.out | 256 +-
.../results/clientpositive/llap/bucket3.q.out | 256 +-
.../results/clientpositive/llap/bucket4.q.out | 256 +-
.../results/clientpositive/llap/bucket5.q.out | 39 +-
.../results/clientpositive/llap/bucket6.q.out | 136 +-
.../clientpositive/llap/bucket_groupby.q.out | 470 +-
.../clientpositive/llap/bucket_many.q.out | 27 +-
.../llap/bucket_map_join_tez1.q.out | 7075 ----
.../llap/bucket_num_reducers.q.out | 202 +
.../llap/bucket_num_reducers2.q.out | 242 +
.../llap/bucket_num_reducers_acid2.q.out | 84 +
.../clientpositive/llap/bucketmapjoin1.q.out | 10 +
.../clientpositive/llap/bucketmapjoin2.q.out | 13 +
.../clientpositive/llap/bucketmapjoin3.q.out | 8 +
.../clientpositive/llap/bucketmapjoin4.q.out | 12 +
.../clientpositive/llap/bucketmapjoin7.q.out | 2 +
.../clientpositive/llap/bucketpruning1.q.out | 30 +-
.../llap/bucketsortoptimize_insert_2.q.out | 148 +-
.../llap/bucketsortoptimize_insert_6.q.out | 88 +-
.../llap/bucketsortoptimize_insert_7.q.out | 54 +-
.../clientpositive/llap/cbo_rp_views.q.out | 2 +-
.../results/clientpositive/llap/cbo_views.q.out | 2 +-
.../clientpositive/llap/check_constraint.q.out | 37 +-
.../results/clientpositive/llap/cluster.q.out | 754 +-
.../columnStatsUpdateForStatsOptimizer_1.q.out | 8 +
...names_with_leading_and_trailing_spaces.q.out | 1 +
.../llap/column_table_stats.q.out | 15 +
.../llap/column_table_stats_orc.q.out | 11 +
.../llap/correlationoptimizer1.q.out | 40 +-
.../test/results/clientpositive/llap/ctas.q.out | 5 +
.../results/clientpositive/llap/cte_1.q.out | 37328 ++++++++---------
.../llap/default_constraint.q.out | 2793 --
.../clientpositive/llap/deleteAnalyze.q.out | 2 +
.../llap/disable_merge_for_bucketing.q.out | 256 +-
.../llap/dynamic_partition_pruning.q.out | 6 +-
.../llap/dynamic_semijoin_reduction.q.out | 4 +
.../llap/dynamic_semijoin_user_level.q.out | 4 +
.../llap/dynpart_sort_optimization_acid.q.out | 1718 -
.../llap/enforce_constraint_notnull.q.out | 6010 ---
.../clientpositive/llap/except_distinct.q.out | 2 +-
.../clientpositive/llap/explainuser_4.q.out | 2 +-
.../extrapolate_part_stats_partial_ndv.q.out | 19 +
.../llap/filter_join_breaktask.q.out | 3 +
.../clientpositive/llap/filter_union.q.out | 4 +
.../llap/hybridgrace_hashjoin_2.q.out | 4 +-
.../llap/infer_bucket_sort_bucketed_table.q.out | 12 +-
.../llap/insert_into_default_keyword.q.out | 3078 --
.../insert_values_orig_table_use_metadata.q.out | 5 +
.../clientpositive/llap/intersect_all.q.out | 1528 +-
.../llap/intersect_distinct.q.out | 922 +-
.../clientpositive/llap/join32_lessSize.q.out | 31 +
.../clientpositive/llap/lateral_view.q.out | 14 +-
.../results/clientpositive/llap/lineage2.q.out | 60 +-
.../llap/list_bucket_dml_10.q.out | 4 +
.../clientpositive/llap/llap_nullscan.q.out | 2 +
.../results/clientpositive/llap/llap_smb.q.out | 44 +-
.../clientpositive/llap/mapjoin_mapjoin.q.out | 8 +
.../llap/materialized_view_create.q.out | 5 +-
.../llap/materialized_view_create_rewrite.q.out | 2 +
.../materialized_view_create_rewrite_2.q.out | 10 +-
.../materialized_view_create_rewrite_3.q.out | 6 +-
.../materialized_view_create_rewrite_4.q.out | 9 +-
.../materialized_view_create_rewrite_5.q.out | 1 +
...materialized_view_create_rewrite_dummy.q.out | 2 +
...erialized_view_create_rewrite_multi_db.q.out | 2 +
...ized_view_create_rewrite_rebuild_dummy.q.out | 6 +-
...alized_view_create_rewrite_time_window.q.out | 13 +-
.../llap/materialized_view_describe.q.out | 7 +-
.../clientpositive/llap/metadataonly1.q.out | 22 +-
.../results/clientpositive/llap/mm_all.q.out | 3 +
.../clientpositive/llap/multiMapJoin1.q.out | 4 +-
.../clientpositive/llap/multi_column_in.q.out | 4 +-
.../llap/multi_count_distinct_null.q.out | 28 +-
.../clientpositive/llap/optimize_nullscan.q.out | 28 +
.../clientpositive/llap/orc_analyze.q.out | 21 +-
.../clientpositive/llap/orc_create.q.out | 6 +
.../clientpositive/llap/orc_llap_counters.q.out | 1 +
.../llap/orc_llap_counters1.q.out | 1 +
.../clientpositive/llap/parquet_types.q.out | 4 +-
.../llap/parquet_types_vectorization.q.out | 4 +-
.../llap/partition_multilevels.q.out | 324 +-
.../clientpositive/llap/partition_pruning.q.out | 5 +
.../clientpositive/llap/ppd_union_view.q.out | 3 +
.../clientpositive/llap/quotedid_smb.q.out | 16 +-
.../llap/reduce_deduplicate.q.out | 6 +
.../llap/reduce_deduplicate_extended.q.out | 36 +-
.../clientpositive/llap/results_cache_1.q.out | 8 +-
.../llap/results_cache_capacity.q.out | 8 +-
.../llap/results_cache_with_masking.q.out | 8 +-
.../results/clientpositive/llap/sample1.q.out | 3 +
.../results/clientpositive/llap/sample10.q.out | 36 +-
.../clientpositive/llap/schema_evol_stats.q.out | 2 +
.../llap/selectDistinctStar.q.out | 2 +
.../clientpositive/llap/skiphf_aggr.q.out | 6 +-
.../clientpositive/llap/smb_mapjoin_15.q.out | 16 +
.../clientpositive/llap/smb_mapjoin_18.q.out | 8 +-
.../clientpositive/llap/smb_mapjoin_19.q.out | 8 +-
.../results/clientpositive/llap/stats11.q.out | 10 +
.../clientpositive/llap/subquery_notin.q.out | 112 +-
.../results/clientpositive/llap/sysdb.q.out | 190 +-
.../llap/tez_dynpart_hashjoin_3.q.out | 59 +-
.../llap/tez_join_result_complex.q.out | 8 +
.../clientpositive/llap/tez_smb_main.q.out | 4 +-
.../clientpositive/llap/tez_union2.q.out | 922 +-
.../llap/tez_vector_dynpart_hashjoin_1.q.out | 6 +-
.../llap/udaf_collect_set_2.q.out | 60 +-
.../clientpositive/llap/unionDistinct_1.q.out | 2365 +-
.../clientpositive/llap/unionDistinct_3.q.out | 1276 +-
.../clientpositive/llap/union_fast_stats.q.out | 12 +-
.../clientpositive/llap/union_stats.q.out | 7 +
.../clientpositive/llap/vector_coalesce_2.q.out | 8 +-
.../llap/vector_complex_all.q.out | 6 +-
.../llap/vector_grouping_sets.q.out | 8 +-
.../llap/vector_partitioned_date_time.q.out | 128 +-
.../llap/vector_ptf_part_simple.q.out | 788 +-
.../llap/vector_windowing_expressions.q.out | 64 +-
.../vector_windowing_multipartitioning.q.out | 42 +-
.../vector_windowing_range_multiorder.q.out | 19084 ++++-----
.../clientpositive/llap/vectorization_0.q.out | 10 +
.../llap/vectorized_distinct_gby.q.out | 2 +-
.../vectorized_dynamic_partition_pruning.q.out | 6 +-
.../vectorized_insert_into_bucketed_table.q.out | 6 +-
.../llap/vectorized_parquet.q.out | 190 +-
.../results/clientpositive/load_dyn_part8.q.out | 4 +
.../clientpositive/louter_join_ppr.q.out | 16 +
.../clientpositive/mapjoin_mapjoin.q.out | 8 +
.../test/results/clientpositive/masking_5.q.out | 5 -
ql/src/test/results/clientpositive/merge3.q.out | 7 +
ql/src/test/results/clientpositive/mm_all.q.out | 3 +
.../results/clientpositive/mm_buckets.q.out | 34 +-
.../results/clientpositive/mm_default.q.out | 4 +
.../multi_insert_partitioned.q.out | 5 +
.../clientpositive/named_column_join.q.out | 3 +
.../results/clientpositive/nonmr_fetch.q.out | 105 +-
.../results/clientpositive/nullformat.q.out | 1 +
.../results/clientpositive/nullformatCTAS.q.out | 1 +
.../offset_limit_global_optimizer.q.out | 32 +
.../results/clientpositive/outer_join_ppr.q.out | 8 +
.../outer_reference_windowed.q.out | 1 +
.../clientpositive/parallel_orderby.q.out | 6 +-
.../clientpositive/parquet_analyze.q.out | 2 +
.../parquet_array_null_element.q.out | 1 +
.../results/clientpositive/parquet_create.q.out | 1 +
.../parquet_mixed_partition_formats.q.out | 1 +
.../clientpositive/parquet_partitioned.q.out | 1 +
.../results/clientpositive/parquet_serde.q.out | 1 +
.../parquet_vectorization_0.q.out | 10 +
.../part_inherit_tbl_props_with_star.q.out | 1 +
ql/src/test/results/clientpositive/pcr.q.out | 61 +
ql/src/test/results/clientpositive/pcs.q.out | 19 +
.../results/clientpositive/pointlookup2.q.out | 28 +
.../results/clientpositive/pointlookup3.q.out | 18 +
.../results/clientpositive/pointlookup4.q.out | 4 +
.../clientpositive/ppd_join_filter.q.out | 16 +
ql/src/test/results/clientpositive/ppd_vc.q.out | 10 +
.../clientpositive/ppr_allchildsarenull.q.out | 6 +
.../test/results/clientpositive/push_or.q.out | 2 +
.../results/clientpositive/quotedid_stats.q.out | 2 +
.../clientpositive/rand_partitionpruner1.q.out | 2 +
.../clientpositive/rand_partitionpruner2.q.out | 10 +
.../clientpositive/rand_partitionpruner3.q.out | 2 +
.../clientpositive/rcfile_default_format.q.out | 8 +
.../results/clientpositive/regexp_extract.q.out | 4 +
.../rename_external_partition_location.q.out | 2 +
.../clientpositive/repl_2_exim_basic.q.out | 4 +
.../clientpositive/repl_3_exim_metadata.q.out | 2 +
.../clientpositive/router_join_ppr.q.out | 16 +
.../test/results/clientpositive/row__id.q.out | 162 -
.../test/results/clientpositive/sample1.q.out | 9 +
.../test/results/clientpositive/sample2.q.out | 261 +-
.../test/results/clientpositive/sample3.q.out | 317 +-
.../test/results/clientpositive/sample4.q.out | 261 +-
.../test/results/clientpositive/sample5.q.out | 327 +-
.../test/results/clientpositive/sample6.q.out | 1889 +-
.../test/results/clientpositive/sample7.q.out | 184 +-
.../test/results/clientpositive/sample8.q.out | 660 +-
.../test/results/clientpositive/sample9.q.out | 253 +-
.../sample_islocalmode_hook_use_metadata.q.out | 1 +
.../clientpositive/serde_user_properties.q.out | 6 +
.../show_create_table_alter.q.out | 5 +
.../show_create_table_db_table.q.out | 3 +
.../show_create_table_delimited.q.out | 1 +
.../show_create_table_partitioned.q.out | 1 +
.../show_create_table_serde.q.out | 4 +
.../show_create_table_temp_table.q.out | 2 +-
.../results/clientpositive/show_functions.q.out | 2 +
.../clientpositive/show_tblproperties.q.out | 5 +
.../results/clientpositive/smb_mapjoin_1.q.out | 1 +
.../results/clientpositive/smb_mapjoin_11.q.out | 229 +-
.../results/clientpositive/smb_mapjoin_12.q.out | 10 +-
.../results/clientpositive/smb_mapjoin_13.q.out | 4 +
.../results/clientpositive/smb_mapjoin_20.q.out | 1016 +-
.../results/clientpositive/smb_mapjoin_22.q.out | 8 +-
.../results/clientpositive/smb_mapjoin_46.q.out | 36 +-
.../results/clientpositive/smb_mapjoin_47.q.out | 46 +-
.../clientpositive/sort_merge_join_desc_5.q.out | 1 +
.../clientpositive/sort_merge_join_desc_6.q.out | 2 +
.../clientpositive/sort_merge_join_desc_7.q.out | 4 +
.../spark/alter_merge_stats_orc.q.out | 2 +
.../spark/auto_join_reordering_values.q.out | 10 +
.../spark/auto_sortmerge_join_1.q.out | 6 +
.../spark/auto_sortmerge_join_12.q.out | 5 +
.../spark/auto_sortmerge_join_2.q.out | 4 +
.../spark/auto_sortmerge_join_3.q.out | 3 +
.../spark/auto_sortmerge_join_4.q.out | 3 +
.../spark/auto_sortmerge_join_5.q.out | 6 +
.../spark/auto_sortmerge_join_7.q.out | 6 +
.../spark/auto_sortmerge_join_8.q.out | 6 +
.../results/clientpositive/spark/bucket2.q.out | 256 +-
.../results/clientpositive/spark/bucket3.q.out | 256 +-
.../results/clientpositive/spark/bucket4.q.out | 256 +-
.../clientpositive/spark/bucket4.q.out_spark | 256 +-
.../results/clientpositive/spark/bucket5.q.out | 35 +-
.../results/clientpositive/spark/bucket6.q.out | 136 +-
.../spark/bucket_map_join_1.q.out | 4 +
.../spark/bucket_map_join_2.q.out | 4 +
.../spark/bucket_map_join_spark1.q.out | 8 +
.../spark/bucket_map_join_spark2.q.out | 8 +
.../spark/bucket_map_join_spark3.q.out | 8 +
.../spark/bucket_map_join_spark4.q.out | 12 +
.../spark/bucket_map_join_tez1.q.out | 288 +-
.../clientpositive/spark/bucketmapjoin1.q.out | 10 +
.../clientpositive/spark/bucketmapjoin10.q.out | 4 +
.../clientpositive/spark/bucketmapjoin11.q.out | 8 +
.../clientpositive/spark/bucketmapjoin12.q.out | 4 +
.../clientpositive/spark/bucketmapjoin13.q.out | 9 +
.../clientpositive/spark/bucketmapjoin2.q.out | 13 +
.../clientpositive/spark/bucketmapjoin3.q.out | 8 +
.../clientpositive/spark/bucketmapjoin4.q.out | 12 +
.../clientpositive/spark/bucketmapjoin5.q.out | 12 +
.../clientpositive/spark/bucketmapjoin7.q.out | 2 +
.../spark/bucketmapjoin7.q.out_spark | 2 +
.../clientpositive/spark/bucketmapjoin8.q.out | 4 +
.../clientpositive/spark/bucketmapjoin9.q.out | 4 +
.../spark/bucketmapjoin_negative.q.out | 5 +
.../spark/bucketmapjoin_negative2.q.out | 6 +
.../spark/bucketmapjoin_negative3.q.out | 36 +
.../spark/bucketsortoptimize_insert_2.q.out | 148 +-
.../spark/bucketsortoptimize_insert_4.q.out | 24 +-
.../spark/bucketsortoptimize_insert_6.q.out | 88 +-
.../spark/bucketsortoptimize_insert_7.q.out | 54 +-
.../spark/bucketsortoptimize_insert_8.q.out | 44 +-
.../clientpositive/spark/cbo_semijoin.q.out | 4 +-
.../results/clientpositive/spark/ctas.q.out | 5 +
.../spark/disable_merge_for_bucketing.q.out | 256 +-
.../disable_merge_for_bucketing.q.out_spark | 256 +-
.../spark/filter_join_breaktask.q.out | 3 +
.../clientpositive/spark/groupby_map_ppr.q.out | 4 +
.../spark/groupby_map_ppr_multi_distinct.q.out | 4 +
.../clientpositive/spark/groupby_ppr.q.out | 4 +
.../spark/groupby_ppr_multi_distinct.q.out | 8 +
.../spark/groupby_sort_1_23.q.out | 76 +
.../spark/groupby_sort_skew_1_23.q.out | 76 +
.../infer_bucket_sort_bucketed_table.q.out | 11 +-
.../spark/infer_bucket_sort_num_buckets.q.out | 2 +-
.../clientpositive/spark/input_part2.q.out | 6 +
.../results/clientpositive/spark/join17.q.out | 6 +
.../results/clientpositive/spark/join26.q.out | 7 +
.../results/clientpositive/spark/join32.q.out | 7 +
.../clientpositive/spark/join32_lessSize.q.out | 31 +
.../results/clientpositive/spark/join33.q.out | 7 +
.../results/clientpositive/spark/join34.q.out | 8 +
.../results/clientpositive/spark/join35.q.out | 8 +
.../results/clientpositive/spark/join9.q.out | 5 +
.../spark/join_filters_overlap.q.out | 34 +
.../clientpositive/spark/join_map_ppr.q.out | 14 +
.../spark/list_bucket_dml_10.q.out | 4 +
.../spark/list_bucket_dml_2.q.out | 5 +
.../clientpositive/spark/load_dyn_part8.q.out | 4 +
.../clientpositive/spark/louter_join_ppr.q.out | 16 +
.../clientpositive/spark/mapjoin_mapjoin.q.out | 8 +
.../clientpositive/spark/mapreduce1.q.out | 448 +-
.../spark/optimize_nullscan.q.out | 28 +
.../clientpositive/spark/outer_join_ppr.q.out | 8 +
.../clientpositive/spark/parallel_orderby.q.out | 2 +
.../spark/parquet_vectorization_0.q.out | 10 +
.../test/results/clientpositive/spark/pcr.q.out | 50 +
.../clientpositive/spark/ppd_join_filter.q.out | 16 +
.../clientpositive/spark/quotedid_smb.q.out | 16 +-
.../spark/reduce_deduplicate.q.out | 6 +
.../clientpositive/spark/router_join_ppr.q.out | 16 +
.../results/clientpositive/spark/sample1.q.out | 3 +
.../results/clientpositive/spark/sample10.q.out | 40 +-
.../results/clientpositive/spark/sample2.q.out | 509 +-
.../results/clientpositive/spark/sample3.q.out | 317 +-
.../results/clientpositive/spark/sample4.q.out | 509 +-
.../results/clientpositive/spark/sample5.q.out | 321 +-
.../results/clientpositive/spark/sample6.q.out | 2589 +-
.../results/clientpositive/spark/sample7.q.out | 288 +-
.../results/clientpositive/spark/sample8.q.out | 661 +-
.../results/clientpositive/spark/sample9.q.out | 505 +-
.../clientpositive/spark/smb_mapjoin_1.q.out | 1 +
.../clientpositive/spark/smb_mapjoin_11.q.out | 229 +-
.../clientpositive/spark/smb_mapjoin_12.q.out | 10 +-
.../clientpositive/spark/smb_mapjoin_13.q.out | 6 +
.../clientpositive/spark/smb_mapjoin_15.q.out | 16 +
.../clientpositive/spark/smb_mapjoin_18.q.out | 8 +-
.../clientpositive/spark/smb_mapjoin_19.q.out | 8 +-
.../clientpositive/spark/smb_mapjoin_20.q.out | 1016 +-
.../clientpositive/spark/smb_mapjoin_22.q.out | 8 +-
.../spark_dynamic_partition_pruning_7.q.out | 329 -
.../spark/spark_union_merge.q.out | 8 +
.../results/clientpositive/spark/stats0.q.out | 8 +
.../results/clientpositive/spark/stats1.q.out | 2 +
.../results/clientpositive/spark/stats10.q.out | 253 +-
.../results/clientpositive/spark/stats16.q.out | 2 +
.../results/clientpositive/spark/stats3.q.out | 3 +
.../results/clientpositive/spark/stats5.q.out | 1 +
.../clientpositive/spark/stats_noscan_2.q.out | 2 +
.../results/clientpositive/spark/statsfs.q.out | 2 +
.../spark/subquery_nested_subquery.q.out | 24 +-
.../clientpositive/spark/subquery_notin.q.out | 1368 +-
.../clientpositive/spark/subquery_select.q.out | 370 +-
.../spark/subquery_shared_alias.q.out | 8 +-
.../clientpositive/spark/transform_ppr1.q.out | 4 +
.../clientpositive/spark/transform_ppr2.q.out | 2 +
.../spark/truncate_column_buckets.q.out | 8 +-
.../results/clientpositive/spark/union22.q.out | 6 +
.../results/clientpositive/spark/union24.q.out | 24 +
.../clientpositive/spark/union_ppr.q.out | 4 +
.../clientpositive/spark/union_remove_1.q.out | 1 +
.../clientpositive/spark/union_remove_10.q.out | 3 +-
.../clientpositive/spark/union_remove_11.q.out | 1 +
.../clientpositive/spark/union_remove_12.q.out | 1 +
.../clientpositive/spark/union_remove_13.q.out | 3 +-
.../clientpositive/spark/union_remove_14.q.out | 1 +
.../clientpositive/spark/union_remove_15.q.out | 3 +-
.../clientpositive/spark/union_remove_16.q.out | 3 +-
.../clientpositive/spark/union_remove_17.q.out | 1 +
.../clientpositive/spark/union_remove_18.q.out | 1 +
.../clientpositive/spark/union_remove_19.q.out | 1 +
.../clientpositive/spark/union_remove_2.q.out | 1 +
.../clientpositive/spark/union_remove_20.q.out | 1 +
.../clientpositive/spark/union_remove_21.q.out | 1 +
.../clientpositive/spark/union_remove_22.q.out | 1 +
.../clientpositive/spark/union_remove_23.q.out | 1 +
.../clientpositive/spark/union_remove_24.q.out | 1 +
.../clientpositive/spark/union_remove_3.q.out | 1 +
.../clientpositive/spark/union_remove_4.q.out | 1 +
.../clientpositive/spark/union_remove_5.q.out | 1 +
.../clientpositive/spark/union_remove_7.q.out | 3 +-
.../clientpositive/spark/union_remove_8.q.out | 3 +-
.../clientpositive/spark/union_remove_9.q.out | 3 +-
.../clientpositive/spark/vectorization_0.q.out | 10 +
ql/src/test/results/clientpositive/stats0.q.out | 14 +
ql/src/test/results/clientpositive/stats1.q.out | 2 +
.../test/results/clientpositive/stats10.q.out | 253 +-
.../test/results/clientpositive/stats16.q.out | 2 +
ql/src/test/results/clientpositive/stats3.q.out | 3 +
ql/src/test/results/clientpositive/stats5.q.out | 1 +
.../clientpositive/stats_empty_partition2.q.out | 1 +
.../clientpositive/stats_invalidation.q.out | 2 +
.../clientpositive/stats_list_bucket.q.out | 1 +
.../results/clientpositive/stats_noscan_2.q.out | 2 +
.../test/results/clientpositive/statsfs.q.out | 2 +
.../temp_table_display_colstats_tbllvl.q.out | 5 +
.../tez/acid_vectorization_original_tez.q.out | 758 -
.../clientpositive/tez/explainanalyze_4.q.out | 2 +-
.../clientpositive/tez/explainanalyze_5.q.out | 8 +-
.../results/clientpositive/transform_ppr1.q.out | 4 +
.../results/clientpositive/transform_ppr2.q.out | 2 +
.../clientpositive/truncate_column.q.out | 7 +
.../truncate_column_list_bucket.q.out | 2 +
.../results/clientpositive/udf_explode.q.out | 4 +
.../results/clientpositive/udtf_explode.q.out | 4 +
.../clientpositive/unicode_comments.q.out | 3 +
.../clientpositive/unicode_notation.q.out | 3 +
.../test/results/clientpositive/union22.q.out | 9 +
.../test/results/clientpositive/union24.q.out | 24 +
.../clientpositive/union_pos_alias.q.out | 1 +
.../test/results/clientpositive/union_ppr.q.out | 2 +
.../results/clientpositive/union_remove_1.q.out | 1 +
.../clientpositive/union_remove_10.q.out | 1 +
.../clientpositive/union_remove_11.q.out | 1 +
.../clientpositive/union_remove_12.q.out | 1 +
.../clientpositive/union_remove_13.q.out | 1 +
.../clientpositive/union_remove_14.q.out | 1 +
.../clientpositive/union_remove_15.q.out | 1 +
.../clientpositive/union_remove_16.q.out | 1 +
.../clientpositive/union_remove_17.q.out | 1 +
.../clientpositive/union_remove_18.q.out | 1 +
.../clientpositive/union_remove_19.q.out | 1 +
.../results/clientpositive/union_remove_2.q.out | 1 +
.../clientpositive/union_remove_20.q.out | 1 +
.../clientpositive/union_remove_21.q.out | 1 +
.../clientpositive/union_remove_22.q.out | 1 +
.../clientpositive/union_remove_23.q.out | 1 +
.../clientpositive/union_remove_24.q.out | 1 +
.../results/clientpositive/union_remove_3.q.out | 1 +
.../results/clientpositive/union_remove_4.q.out | 1 +
.../results/clientpositive/union_remove_5.q.out | 1 +
.../results/clientpositive/union_remove_7.q.out | 1 +
.../results/clientpositive/union_remove_8.q.out | 1 +
.../results/clientpositive/union_remove_9.q.out | 1 +
.../results/clientpositive/union_stats.q.out | 5 +
.../unset_table_view_property.q.out | 17 +
.../results/clientpositive/view_alias.q.out | 6 +
.../test/results/clientpositive/view_cbo.q.out | 10 +
.../objectinspector/ObjectInspectorUtils.java | 163 +-
.../TestObjectInspectorUtils.java | 4 +-
.../metastore/api/hive_metastoreConstants.java | 1 +
.../org/apache/hive/common/util/Murmur3.java | 40 +-
.../apache/hive/common/util/TestMurmur3.java | 2 +-
.../apache/hive/streaming/TestStreaming.java | 10 +-
657 files changed, 47500 insertions(+), 67463 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/hbase-handler/src/test/results/positive/external_table_ppd.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/external_table_ppd.q.out b/hbase-handler/src/test/results/positive/external_table_ppd.q.out
index cdc43ee..b59dfd7 100644
--- a/hbase-handler/src/test/results/positive/external_table_ppd.q.out
+++ b/hbase-handler/src/test/results/positive/external_table_ppd.q.out
@@ -56,6 +56,7 @@ Retention: 0
Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigint_col\":\"true\",\"boolean_col\":\"true\",\"double_col\":\"true\",\"float_col\":\"true\",\"int_col\":\"true\",\"key\":\"true\",\"smallint_col\":\"true\",\"tinyint_col\":\"true\"}}
+ bucketing_version 2
hbase.table.default.storage.type binary
hbase.table.name t_hive
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out b/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
index 153613e..172db75 100644
--- a/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_binary_storage_queries.q.out
@@ -56,6 +56,7 @@ Retention: 0
Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigint_col\":\"true\",\"boolean_col\":\"true\",\"double_col\":\"true\",\"float_col\":\"true\",\"int_col\":\"true\",\"key\":\"true\",\"smallint_col\":\"true\",\"tinyint_col\":\"true\"}}
+ bucketing_version 2
hbase.table.default.storage.type binary
hbase.table.name t_hive
numFiles 0
@@ -236,6 +237,7 @@ Table Type: EXTERNAL_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bigint_col\":\"true\",\"boolean_col\":\"true\",\"double_col\":\"true\",\"float_col\":\"true\",\"int_col\":\"true\",\"key\":\"true\",\"smallint_col\":\"true\",\"tinyint_col\":\"true\"}}
EXTERNAL TRUE
+ bucketing_version 2
hbase.table.name t_hive
numFiles 0
numRows 0
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/hbase-handler/src/test/results/positive/hbase_ddl.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_ddl.q.out b/hbase-handler/src/test/results/positive/hbase_ddl.q.out
index ef3f5f7..598582c 100644
--- a/hbase-handler/src/test/results/positive/hbase_ddl.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_ddl.q.out
@@ -116,6 +116,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
hbase.mapred.output.outputtable kkk
hbase.table.name hbase_table_0
#### A masked pattern was here ####
@@ -162,6 +163,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
hbase.table.name hbase_table_0
#### A masked pattern was here ####
numFiles 0
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/hbase-handler/src/test/results/positive/hbasestats.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbasestats.q.out b/hbase-handler/src/test/results/positive/hbasestats.q.out
index 5d000d2..8a7edc2 100644
--- a/hbase-handler/src/test/results/positive/hbasestats.q.out
+++ b/hbase-handler/src/test/results/positive/hbasestats.q.out
@@ -38,6 +38,7 @@ Retention: 0
Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"country\":\"true\",\"country_id\":\"true\",\"key\":\"true\",\"state\":\"true\"}}
+ bucketing_version 2
numFiles 0
numRows 0
rawDataSize 0
@@ -129,6 +130,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -193,6 +195,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
numFiles 0
numRows 0
@@ -249,6 +252,7 @@ Retention: 0
Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ bucketing_version 2
#### A masked pattern was here ####
numFiles 0
numRows 2
@@ -318,6 +322,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: MANAGED_TABLE
Table Parameters:
+ bucketing_version 2
#### A masked pattern was here ####
numFiles 0
numRows 2
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
index 924e233..2a68220 100644
--- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
+++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java
@@ -19,6 +19,7 @@
package org.apache.hive.hcatalog.streaming;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -27,6 +28,7 @@ import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -185,7 +187,12 @@ public abstract class AbstractRecordWriter implements RecordWriter {
}
ObjectInspector[] inspectors = getBucketObjectInspectors();
Object[] bucketFields = getBucketFields(row);
- return ObjectInspectorUtils.getBucketNumber(bucketFields, inspectors, totalBuckets);
+ int bucketingVersion = Utilities.getBucketingVersion(
+ tbl.getParameters().get(hive_metastoreConstants.TABLE_BUCKETING_VERSION));
+
+ return bucketingVersion == 2 ?
+ ObjectInspectorUtils.getBucketNumber(bucketFields, inspectors, totalBuckets) :
+ ObjectInspectorUtils.getBucketNumberOld(bucketFields, inspectors, totalBuckets);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index fe2b1c1..90dbdac 100644
--- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -1491,9 +1491,9 @@ public class TestStreaming {
// assert bucket listing is as expected
Assert.assertEquals("number of buckets does not match expectation", actual1.values().size(), 3);
- Assert.assertEquals("records in bucket does not match expectation", actual1.get(0).size(), 2);
+ Assert.assertTrue("bucket 0 shouldn't have been created", actual1.get(0) == null);
Assert.assertEquals("records in bucket does not match expectation", actual1.get(1).size(), 1);
- Assert.assertTrue("bucket 2 shouldn't have been created", actual1.get(2) == null);
+ Assert.assertEquals("records in bucket does not match expectation", actual1.get(2).size(), 2);
Assert.assertEquals("records in bucket does not match expectation", actual1.get(3).size(), 1);
}
private void runCmdOnDriver(String cmd) throws QueryFailedException {
@@ -1624,7 +1624,7 @@ public class TestStreaming {
} else if (file.contains("bucket_00001")) {
corruptDataFile(file, conf, -1);
} else if (file.contains("bucket_00002")) {
- Assert.assertFalse("bucket 2 shouldn't have been created", true);
+ corruptDataFile(file, conf, 100);
} else if (file.contains("bucket_00003")) {
corruptDataFile(file, conf, 100);
}
@@ -1654,9 +1654,9 @@ public class TestStreaming {
System.setErr(origErr);
errDump = new String(myErr.toByteArray());
- Assert.assertEquals(true, errDump.contains("bucket_00000 recovered successfully!"));
- Assert.assertEquals(true, errDump.contains("No readable footers found. Creating empty orc file."));
Assert.assertEquals(true, errDump.contains("bucket_00001 recovered successfully!"));
+ Assert.assertEquals(true, errDump.contains("No readable footers found. Creating empty orc file."));
+ Assert.assertEquals(true, errDump.contains("bucket_00002 recovered successfully!"));
Assert.assertEquals(true, errDump.contains("bucket_00003 recovered successfully!"));
Assert.assertEquals(false, errDump.contains("Exception"));
Assert.assertEquals(false, errDump.contains("is still open for writes."));
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestBucketIdResolverImpl.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestBucketIdResolverImpl.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestBucketIdResolverImpl.java
index 03c28a3..e890c52 100644
--- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestBucketIdResolverImpl.java
+++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestBucketIdResolverImpl.java
@@ -42,7 +42,7 @@ public class TestBucketIdResolverImpl {
public void testAttachBucketIdToRecord() {
MutableRecord record = new MutableRecord(1, "hello");
capturingBucketIdResolver.attachBucketIdToRecord(record);
- assertThat(record.rowId, is(new RecordIdentifier(-1L,
+ assertThat(record.rowId, is(new RecordIdentifier(-1L,
BucketCodec.V1.encode(new AcidOutputFormat.Options(null).bucket(1)),
-1L)));
assertThat(record.id, is(1));
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java
index 9963291..ed2aef4 100644
--- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java
+++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java
@@ -176,6 +176,10 @@ public class HCatTable {
newTable.putToParameters("comment", comment);
}
+ if (newTable.getParameters().get("bucketing_version") == null) {
+ newTable.putToParameters("bucketing_version", "2");
+ }
+
newTable.setSd(sd);
if (partCols != null) {
ArrayList<FieldSchema> hivePtnCols = new ArrayList<FieldSchema>();
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
index f9ee9d9..da08d2f 100644
--- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
+++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
@@ -297,6 +297,8 @@ public class TestHCatClient {
assertNotNull(inner);
assertNotNull(outer);
for ( Map.Entry<String,String> e : inner.entrySet()){
+ // If it is bucketing version, skip it
+ if (e.getKey().equals("bucketing_version")) continue;
assertTrue(outer.containsKey(e.getKey()));
assertEquals(outer.get(e.getKey()), e.getValue());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
index caa0029..a0ebed7 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
@@ -122,6 +122,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns
columns.comments
@@ -137,6 +138,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns
columns.comments
@@ -171,6 +173,7 @@ STAGE PLANS:
properties:
bucket_count 2
bucket_field_name id
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -203,6 +206,7 @@ STAGE PLANS:
properties:
bucket_count 2
bucket_field_name id
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
index ab8ad77..f6b6fee 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
@@ -87,6 +87,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -133,6 +134,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns
columns.comments
@@ -148,6 +150,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns
columns.comments
@@ -206,6 +209,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -250,6 +254,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -281,6 +286,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -303,6 +309,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -335,6 +342,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -369,6 +377,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -400,6 +409,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -422,6 +432,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out
index 2b28a66..61d8450 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out
@@ -113,6 +113,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true","key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id,key
columns.comments
@@ -135,6 +136,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true","key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id,key
columns.comments
@@ -364,6 +366,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true","key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id,key
columns.comments
@@ -386,6 +389,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true","key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id,key
columns.comments
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
index cdb67dd..6dba301 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
@@ -140,6 +140,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns
columns.comments
@@ -155,6 +156,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns
columns.comments
@@ -189,6 +191,7 @@ STAGE PLANS:
properties:
bucket_count 2
bucket_field_name id
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -221,6 +224,7 @@ STAGE PLANS:
properties:
bucket_count 2
bucket_field_name id
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
index 2c23a7e..fdfe44c 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
@@ -95,6 +95,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -141,6 +142,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns
columns.comments
@@ -156,6 +158,7 @@ STAGE PLANS:
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns
columns.comments
@@ -214,6 +217,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -258,6 +262,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -289,6 +294,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -311,6 +317,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -343,6 +350,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -377,6 +385,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -408,6 +417,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
@@ -430,6 +440,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"id":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns id
columns.comments
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
----------------------------------------------------------------------
diff --git a/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out b/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
index a1be085..1cd023b 100644
--- a/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
+++ b/itests/hive-blobstore/src/test/results/clientpositive/write_final_output_blobstore.q.out
@@ -70,6 +70,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key
columns.comments
@@ -91,6 +92,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key
columns.comments
@@ -194,6 +196,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key
columns.comments
@@ -261,6 +264,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key
columns.comments
@@ -335,6 +339,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key
columns.comments
@@ -356,6 +361,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key
columns.comments
@@ -459,6 +465,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key
columns.comments
@@ -526,6 +533,7 @@ STAGE PLANS:
properties:
COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}}
bucket_count -1
+ bucketing_version 2
column.name.delimiter ,
columns key
columns.comments
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index 4ebd096..8ee033d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -1107,7 +1107,7 @@ public class TestCompactor {
}
String name = stat[0].getPath().getName();
Assert.assertEquals(name, "base_0000004");
- checkExpectedTxnsPresent(stat[0].getPath(), null, columnNamesProperty, columnTypesProperty, 0, 1L, 4L, 2);
+ checkExpectedTxnsPresent(stat[0].getPath(), null, columnNamesProperty, columnTypesProperty, 1, 1L, 4L, 2);
} finally {
connection.close();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 1a34659..2ca7b5f 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -8,14 +8,11 @@ minimr.query.files=infer_bucket_sort_map_operators.q,\
infer_bucket_sort_num_buckets.q,\
root_dir_external_table.q,\
parallel_orderby.q,\
- bucket_num_reducers.q,\
udf_using.q,\
index_bitmap3.q,\
index_bitmap_auto.q,\
scriptfile1.q,\
- bucket_num_reducers2.q,\
bucket_num_reducers_acid.q,\
- bucket_num_reducers_acid2.q,\
scriptfile1_win.q
# These tests are disabled for minimr
@@ -464,6 +461,9 @@ minillaplocal.query.files=\
bucket_map_join_tez1.q,\
bucket_map_join_tez2.q,\
bucket_map_join_tez_empty.q,\
+ bucket_num_reducers.q,\
+ bucket_num_reducers2.q,\
+ bucket_num_reducers_acid2.q,\
bucketizedhiveinputformat.q,\
bucketmapjoin6.q,\
bucketmapjoin7.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index c084fa0..962fc5d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -66,10 +66,7 @@ import org.apache.hadoop.hive.ql.plan.SkewedColumnPositionPair;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.ql.stats.StatsCollectionContext;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
-import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.SerDeStats;
-import org.apache.hadoop.hive.serde2.Serializer;
+import org.apache.hadoop.hive.serde2.*;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
@@ -86,6 +83,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hive.common.util.HiveStringUtils;
+import org.apache.hive.common.util.Murmur3;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -99,6 +97,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.function.BiFunction;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_TEMPORARY_TABLE_STORAGE;
@@ -144,6 +143,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
private transient Path destTablePath;
private transient boolean isInsertOverwrite;
private transient String counterGroup;
+ private transient BiFunction<Object[], ObjectInspector[], Integer> hashFunc;
/**
* Counters.
*/
@@ -587,6 +587,11 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
logEveryNRows = HiveConf.getLongVar(hconf, HiveConf.ConfVars.HIVE_LOG_N_RECORDS);
statsMap.put(getCounterName(Counter.RECORDS_OUT), row_count);
+
+ // Setup hashcode
+ hashFunc = conf.getTableInfo().getBucketingVersion() == 2 ?
+ ObjectInspectorUtils::getBucketHashCode :
+ ObjectInspectorUtils::getBucketHashCodeOld;
} catch (HiveException e) {
throw e;
} catch (Exception e) {
@@ -1050,7 +1055,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
for(int i = 0; i < partitionEval.length; i++) {
bucketFieldValues[i] = partitionEval[i].evaluate(row);
}
- int keyHashCode = ObjectInspectorUtils.getBucketHashCode(bucketFieldValues, partitionObjectInspectors);
+ int keyHashCode = hashFunc.apply(bucketFieldValues, partitionObjectInspectors);
key.setHashCode(keyHashCode);
int bucketNum = prtner.getBucket(key, null, totalFiles);
return bucketMap.get(bucketNum);
@@ -1578,4 +1583,5 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
private boolean isNativeTable() {
return !conf.getTableInfo().isNonNative();
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index d59bf1f..bb91eea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -478,6 +478,7 @@ public final class FunctionRegistry {
system.registerGenericUDF("when", GenericUDFWhen.class);
system.registerGenericUDF("nullif", GenericUDFNullif.class);
system.registerGenericUDF("hash", GenericUDFHash.class);
+ system.registerGenericUDF("murmur_hash", GenericUDFMurmurHash.class);
system.registerGenericUDF("coalesce", GenericUDFCoalesce.class);
system.registerGenericUDF("index", GenericUDFIndex.class);
system.registerGenericUDF("in_file", GenericUDFInFile.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
index c28ef99..108bb57 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
@@ -90,6 +90,7 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C
private transient Configuration hconf;
protected final transient Collection<Future<?>> asyncInitOperations = new HashSet<>();
+ protected int bucketingVersion = -1;
// It can be optimized later so that an operator operator (init/close) is performed
// only after that operation has been performed on all the parents. This will require
// initializing the whole tree in all the mappers (which might be required for mappers
@@ -1600,4 +1601,12 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C
}
return true;
}
+
+ public void setBucketingVersion(int bucketingVersion) {
+ this.bucketingVersion = bucketingVersion;
+ }
+
+ public int getBucketingVersion() {
+ return bucketingVersion;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
index 21ca04d..7bb6590 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
@@ -265,6 +265,9 @@ public final class OperatorFactory {
Operator<T> ret = get(oplist0.getCompilationOpContext(), (Class<T>) conf.getClass());
ret.setConf(conf);
+ // Set the bucketing Version
+ ret.setBucketingVersion(oplist0.getBucketingVersion());
+
// Add the new operator as child of each of the passed in operators
List<Operator> children = oplist0.getChildOperators();
children.add(ret);
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
index d4363fd..caaf543 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
@@ -27,10 +27,12 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
+import java.util.function.BiFunction;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CompilationOpContext;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
@@ -39,20 +41,19 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
+import org.apache.hadoop.hive.serde2.ByteStream;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.Serializer;
+import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
-import org.apache.hadoop.io.BinaryComparable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.util.hash.MurmurHash;
+import org.apache.hive.common.util.Murmur3;
/**
* Reduce Sink Operator sends output to the reduce stage.
@@ -61,7 +62,6 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
implements Serializable, TopNHash.BinaryCollector {
private static final long serialVersionUID = 1L;
- private static final MurmurHash hash = (MurmurHash) MurmurHash.getInstance();
private transient ObjectInspector[] partitionObjectInspectors;
private transient ObjectInspector[] bucketObjectInspectors;
@@ -114,11 +114,13 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
protected transient List<List<Integer>> distinctColIndices;
protected transient Random random;
+ protected transient BiFunction<Object[], ObjectInspector[], Integer> hashFunc;
+
/**
* This two dimensional array holds key data and a corresponding Union object
* which contains the tag identifying the aggregate expression for distinct columns.
*
- * If there is no distict expression, cachedKeys is simply like this.
+ * If there is no distinct expression, cachedKeys is simply like this.
* cachedKeys[0] = [col0][col1]
*
* with two distict expression, union(tag:key) is attatched for each distinct expression
@@ -228,6 +230,14 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
useUniformHash = conf.getReducerTraits().contains(UNIFORM);
firstRow = true;
+ // acidOp flag has to be checked to use JAVA hash which works like
+ // identity function for integers, necessary to read RecordIdentifier
+ // incase of ACID updates/deletes.
+ boolean acidOp = conf.getWriteType() == AcidUtils.Operation.UPDATE ||
+ conf.getWriteType() == AcidUtils.Operation.DELETE;
+ hashFunc = bucketingVersion == 2 && !acidOp ?
+ ObjectInspectorUtils::getBucketHashCode :
+ ObjectInspectorUtils::getBucketHashCodeOld;
} catch (Exception e) {
String msg = "Error initializing ReduceSinkOperator: " + e.getMessage();
LOG.error(msg, e);
@@ -308,7 +318,7 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
}
// Determine distKeyLength (w/o distincts), and then add the first if present.
- populateCachedDistributionKeys(row, 0);
+ populateCachedDistributionKeys(row);
// replace bucketing columns with hashcode % numBuckets
int bucketNumber = -1;
@@ -335,7 +345,6 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
} else {
hashCode = computeHashCode(row, bucketNumber);
}
-
firstKey.setHashCode(hashCode);
/*
@@ -379,20 +388,22 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
}
}
- private int computeBucketNumber(Object row, int numBuckets) throws HiveException {
+ private int computeBucketNumber(Object row, int numBuckets)
+ throws HiveException, SerDeException {
Object[] bucketFieldValues = new Object[bucketEval.length];
for (int i = 0; i < bucketEval.length; i++) {
bucketFieldValues[i] = bucketEval[i].evaluate(row);
}
- return ObjectInspectorUtils.getBucketNumber(bucketFieldValues, bucketObjectInspectors, numBuckets);
+ return ObjectInspectorUtils.getBucketNumber(
+ hashFunc.apply(bucketFieldValues, bucketObjectInspectors), numBuckets);
}
- private void populateCachedDistributionKeys(Object row, int index) throws HiveException {
+ private void populateCachedDistributionKeys(Object row) throws HiveException {
for (int i = 0; i < numDistributionKeys; i++) {
- cachedKeys[index][i] = keyEval[i].evaluate(row);
+ cachedKeys[0][i] = keyEval[i].evaluate(row);
}
if (cachedKeys[0].length > numDistributionKeys) {
- cachedKeys[index][numDistributionKeys] = null;
+ cachedKeys[0][numDistributionKeys] = null;
}
}
@@ -414,7 +425,7 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
}
protected final int computeMurmurHash(HiveKey firstKey) {
- return hash.hash(firstKey.getBytes(), firstKey.getDistKeyLength(), 0);
+ return Murmur3.hash32(firstKey.getBytes(), firstKey.getDistKeyLength(), 0);
}
/**
@@ -439,7 +450,7 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
for(int i = 0; i < partitionEval.length; i++) {
bucketFieldValues[i] = partitionEval[i].evaluate(row);
}
- keyHashCode = ObjectInspectorUtils.getBucketHashCode(bucketFieldValues, partitionObjectInspectors);
+ keyHashCode = hashFunc.apply(bucketFieldValues, partitionObjectInspectors);
}
int hashCode = buckNum < 0 ? keyHashCode : keyHashCode * 31 + buckNum;
if (LOG.isTraceEnabled()) {
@@ -586,4 +597,5 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
public void setOutputCollector(OutputCollector _out) {
this.out = _out;
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/091fd962/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 2503543..b5a7853 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -4475,4 +4475,17 @@ public final class Utilities {
+ " on HDFS should be writable. Current permissions are: " + currentHDFSDirPermission);
}
}
+
+ // Get the bucketing version stored in the string format
+ public static int getBucketingVersion(final String versionStr) {
+ int bucketingVersion = 1;
+ if (versionStr != null) {
+ try {
+ bucketingVersion = Integer.parseInt(versionStr);
+ } catch (NumberFormatException e) {
+ // Do nothing
+ }
+ }
+ return bucketingVersion;
+ }
}