You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hive.apache.org by j....@gmail.com on 2018/10/12 23:42:01 UTC
Review Request 69011: HIVE-20713: Use percentage for join conversion
size thresholds
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/69011/
-----------------------------------------------------------
Review request for hive and Ashutosh Chauhan.
Bugs: HIVE-20713
https://issues.apache.org/jira/browse/HIVE-20713
Repository: hive-git
Description
-------
HIVE-20713: Use percentage for join conversion size thresholds
Diffs
-----
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java cc6239c3135714fb65aa1afc2882852460b68b37
data/conf/hive-site.xml 0daf9adc717bc1c4413d2e34691c26a3e2585c77
data/conf/llap/hive-site.xml 44ca6c9daf092a35f1c58c26dfa3575c303464ce
data/conf/tez/hive-site.xml 236adc7087b43f4e9ab95b2fa57436cf75c679aa
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java 40dd992455f2fa6bae85d9d02338bc820a370ebe
ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryInfo.java PRE-CREATION
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java a6b0dbc0dc956d81d027f08a55fbdf0ca452638f
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 54377428eafdb79e1bbdc8a182eafb46f8febd23
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 151d1b39eb17bba336bb02ab06f216361e4f6ff2
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java 5217208611a6ab63b218a585e820516221022ec6
ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java 7ff92edd91018f55dd90200f184869a74868dd46
ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java 5269eb6f6a77ac031d144a96a2687331e3444f4b
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinResolver.java a84534a9495b44ac319ad52a1f841428861d62e9
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java a52e5e60d97d24996c813b463caffb946ce5b61a
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java 0d1990a8cb3882c2c8b271c90e2675e88c79a7ea
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java 64f1e7b830f85e898252f964bdbfb73267224ef6
ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java ad6db21fe6041e18adc646ca20759ca3cde58713
ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java 22f3266c87f1d42c254893b424b68e757fb2953b
ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java fdc963506cabd96c125ec15b6a1eacb71fd33d42
ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java c7cd4ad3f6deba320b343cda071640cd1c78743e
ql/src/test/queries/clientpositive/acid_mapjoin.q e3d14cc97d1e193a6bbb2a0e25e0741a192e2638
ql/src/test/queries/clientpositive/autoColumnStats_1.q 9a62127dabee743b8880f0effdd5184fb24975d0
ql/src/test/queries/clientpositive/autoColumnStats_2.q 51f252b303fb579d315aabcabb75b4429c7b5d4d
ql/src/test/queries/clientpositive/autoColumnStats_3.q 7a419118902ffab767780057d8e83fbde3fbdba9
ql/src/test/queries/clientpositive/auto_join_reordering_values.q 39b2e1d9a78f77c50d60a749085b05ba9acd6b20
ql/src/test/queries/clientpositive/auto_join_stats.q 8b377bf4ca70ccaf43b625afc51b14399168888f
ql/src/test/queries/clientpositive/auto_join_without_localtask.q 15697f2f11ac4669c7bdf6e4144f02fd3cb465b2
ql/src/test/queries/clientpositive/auto_sortmerge_join_1.q 1fbe8f79cff97ef7baf8eb8afce36665b425b23d
ql/src/test/queries/clientpositive/auto_sortmerge_join_10.q 1b15a740120862a7d56e7b36e07a0e3c4d7c835d
ql/src/test/queries/clientpositive/auto_sortmerge_join_11.q 76e615fbde1d8d52eb2a8de5d08d69d26e304a99
ql/src/test/queries/clientpositive/auto_sortmerge_join_12.q d68efea393af588527a698dda2a578433195472e
ql/src/test/queries/clientpositive/auto_sortmerge_join_13.q bbd7afab07b62d1e6b63b408a5cce71db5ef55f1
ql/src/test/queries/clientpositive/auto_sortmerge_join_14.q 1e21c92e59f56688d05d3fbc21184a70c9779996
ql/src/test/queries/clientpositive/auto_sortmerge_join_15.q b3dd8e50d7cd4e3f95f8b595c10acbfb15732943
ql/src/test/queries/clientpositive/auto_sortmerge_join_16.q 0a72ddfc6b859912962b30719637645648d625ad
ql/src/test/queries/clientpositive/auto_sortmerge_join_2.q c6d5318f43daa419b8ad0fedcf6782ea3f333f5d
ql/src/test/queries/clientpositive/auto_sortmerge_join_3.q f41b0970c9ff98f5d72bfa2b3ad07b4b7581e6ab
ql/src/test/queries/clientpositive/auto_sortmerge_join_4.q e28e0ab4a09cf0214ce946ec79bb34063705e9ad
ql/src/test/queries/clientpositive/auto_sortmerge_join_5.q 9550e21382237f26f9b7187bc024053221fe365f
ql/src/test/queries/clientpositive/auto_sortmerge_join_6.q 0aeec0e50fa3dcff0e20f77cf9541d67c141d4db
ql/src/test/queries/clientpositive/auto_sortmerge_join_7.q 8c9dbacbbad8efe42986556a7d2f6cb62fd3b906
ql/src/test/queries/clientpositive/auto_sortmerge_join_8.q 51647e86b2bc1d680b59c30aeeaef6f949f21427
ql/src/test/queries/clientpositive/auto_sortmerge_join_9.q 5696682fd5c3f0ab214873bf2714ebab3ab0959c
ql/src/test/queries/clientpositive/bucket_map_join_tez1.q 049a4d9571cdca6231587f64a8d20261381fb38f
ql/src/test/queries/clientpositive/bucket_map_join_tez2.q ec32e9c820778a6d907e599f3b0bf21b485e5c4a
ql/src/test/queries/clientpositive/bucket_map_join_tez_empty.q 359560aab79b1dd6e59f084df65bab6ad7472827
ql/src/test/queries/clientpositive/bucketsortoptimize_insert_2.q 0b913caa8e0a190643bf3caa1de2667a37d58727
ql/src/test/queries/clientpositive/bucketsortoptimize_insert_6.q 2d4907c5cf1b820e77813f291090137512415cd2
ql/src/test/queries/clientpositive/bucketsortoptimize_insert_7.q bef48b2d0a49f3ee27281a7c6047c7c155baff98
ql/src/test/queries/clientpositive/cbo_rp_cross_product_check_2.q 00c19c74ad45fc13e0a2cf74af3f0fb33b73a1a3
ql/src/test/queries/clientpositive/convert_decimal64_to_decimal.q c76057cb0654a7e2f716dbd4bd36d8ebfed7339f
ql/src/test/queries/clientpositive/correlationoptimizer5.q d75a48f6c036a426d5788cd68aacd28f3b0f0b18
ql/src/test/queries/clientpositive/correlationoptimizer7.q 0d57d7e7cb9ff98c561309bbdcb851f7bac85b43
ql/src/test/queries/clientpositive/cross_join.q 041f0c38a67a0f178cf0854031dc4780187492f3
ql/src/test/queries/clientpositive/cross_product_check_1.q 23c663849165dcd0f790460215b83accefa6c711
ql/src/test/queries/clientpositive/cross_product_check_2.q 329e878f66ea8283b4088d8179378cd182ecebf5
ql/src/test/queries/clientpositive/dynamic_partition_pruning.q de339f892a4733c9b23a800f290796b08e605a2f
ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q f3d94d56d16baf2d6dce8342af2a258d78954903
ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q 32c28540ad60730233af60002fa0e8b7467dc406
ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_2.q 20b657e78f5870b21f6614339924ab9fd50efd26
ql/src/test/queries/clientpositive/dynamic_semijoin_user_level.q 5e3a1b84fbaafc0bb77be1ffa8111387cb9eb13c
ql/src/test/queries/clientpositive/explainanalyze_1.q 90438263138048ba556148610fcfe5a4242994a2
ql/src/test/queries/clientpositive/explainanalyze_2.q 7e48a8a2b81410f47bc3415af2fe3aa14ead16bd
ql/src/test/queries/clientpositive/explainanalyze_3.q 1f31218a8520b397376dad4cbf1bb54a7b3bd4d3
ql/src/test/queries/clientpositive/explainanalyze_4.q 6daba22caa8adf6de86fb312e5e15ca6d1fb30ec
ql/src/test/queries/clientpositive/explainuser_1.q 23bdb79196f57da244ab6e51098c333571cb0e34
ql/src/test/queries/clientpositive/explainuser_2.q bc795cfd3156375c47f296fb89ab06ac35001e40
ql/src/test/queries/clientpositive/explainuser_3.q 0c110aea1acb9b4d1d73eada6e028d869f32b50b
ql/src/test/queries/clientpositive/explainuser_4.q 2125d09d31aa6a85b4459c3d37f5d1b972d4d2b1
ql/src/test/queries/clientpositive/fullouter_mapjoin_1_optimized.q 32b2e0495f3a9500d1cc8055e394917f630f0cc9
ql/src/test/queries/clientpositive/hybridgrace_hashjoin_1.q 30cf8c4bb9807d22fbca5bd28152294f827cdc49
ql/src/test/queries/clientpositive/hybridgrace_hashjoin_2.q d989ca7dc883fa071cf5772f358c68bff78f659f
ql/src/test/queries/clientpositive/join28.q 868ce9debd657e08253a77a8c6dd9a4d07da642b
ql/src/test/queries/clientpositive/join29.q 20aadbbc26fcbaff369062a203e8c3b16d398fec
ql/src/test/queries/clientpositive/join31.q 208340713c97f52324a78efad48808f6fa5c0ec6
ql/src/test/queries/clientpositive/join32.q c55e730ac11663fb180a9d84f3522009e19402c4
ql/src/test/queries/clientpositive/join32_lessSize.q 5c7be2691a612950054e5e6e6696605d20819c17
ql/src/test/queries/clientpositive/join33.q 6ddf0eb6b1d0092368091a077cf55ed2bf0ecf03
ql/src/test/queries/clientpositive/join34.q e0234c61d556128b42ca4d9980fa86573e2b4ce2
ql/src/test/queries/clientpositive/join35.q 29b6b000c67f2f028e79852aea2bb8134d4ec163
ql/src/test/queries/clientpositive/join41.q 33dfcb087e1f47f614bf33b914b98c6706ec6e1d
ql/src/test/queries/clientpositive/join_grp_diff_keys.q fb110b4ae4829395dd0c4ce7e448d5e91aa84e3c
ql/src/test/queries/clientpositive/join_star.q 70cc8f7c07ee1f5a776ad42f98a07140f8880455
ql/src/test/queries/clientpositive/llap_smb.q edec3010d11b5de0db31e8256e9b0b25ca73b6e8
ql/src/test/queries/clientpositive/llap_smb_ptf.q 037b97dc5154d359c0249deb6cbf80670d3baa6a
ql/src/test/queries/clientpositive/llap_vector_nohybridgrace.q d637d364300bf3030fa0503a3a976242d7c71c8b
ql/src/test/queries/clientpositive/load_data_using_job.q 970a7521f54edc070e5bbeabd47dc166b11f801f
ql/src/test/queries/clientpositive/mapjoin_decimal.q c84267a049fe28dc3fbfe0aa1b8608df4a96421d
ql/src/test/queries/clientpositive/mapjoin_hint.q 7189f070e1839aab37f1f462dc203e4c9e836382
ql/src/test/queries/clientpositive/mapjoin_mapjoin.q de0523805603df21799bf7feb6fca52aab572630
ql/src/test/queries/clientpositive/mapjoin_subquery.q 79fe197330b760aa7c23e6ea451207fd66c2afdc
ql/src/test/queries/clientpositive/mapjoin_subquery2.q 95051d4cf8198a43d1117532d03759102799b560
ql/src/test/queries/clientpositive/mrr.q 0f2dd8474158a621349982c31cfda98afc2cfd67
ql/src/test/queries/clientpositive/multiMapJoin1.q a65ed72c27ea1815916eed916116b35c38f0c5d5
ql/src/test/queries/clientpositive/multiMapJoin2.q 166cb092714956805de0afaddc15fc5d35191c8f
ql/src/test/queries/clientpositive/murmur_hash_migration.q 2b8da9f68368961ca7dfe74c6481be126a0e9170
ql/src/test/queries/clientpositive/quotedid_smb.q acc7626407131118c446149c247821a604d33bdc
ql/src/test/queries/clientpositive/reduce_deduplicate_extended2.q f88fd50250ac25a83b9b28ddfcdf0f1f8dfb1c34
ql/src/test/queries/clientpositive/reopt_semijoin.q 75db410e629300a6b7042159a022ac1a4eed02ae
ql/src/test/queries/clientpositive/runtime_skewjoin_mapjoin_spark.q ac93eedd246ac34dfd8562a6fed5890ae542282c
ql/src/test/queries/clientpositive/runtime_stats_merge.q e69410159aaa54c34ae2f0e192a60a63df3fabc7
ql/src/test/queries/clientpositive/smb_cache.q d87ba6ce8af186fdcf01eae3f1bbf4d24adc8089
ql/src/test/queries/clientpositive/smb_mapjoin_25.q 56aa331eade9d2ae0d0998736d99adb05779b04c
ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q 0b6ecc4e9d638c8842bdf5ede4abe939bbde5ef0
ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_2.q 532a50a74671b4a5a36cbc73c0c769cce5207405
ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_3.q 4a245f9277263fc9dff59171e6324eece3ec432d
ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_5.q d0814199cc00b8756a22031bfa7461f13b25c139
ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_mapjoin_only.q ba44aefbc4ae6c48d37439e5c666cce12969a851
ql/src/test/queries/clientpositive/spark_explainuser_1.q 27183292ada69584fe2d55ada339d776ada0b0ae
ql/src/test/queries/clientpositive/spark_use_ts_stats_for_mapjoin.q 5ee09fc6845c7ae2bdecf19fac6810314737405b
ql/src/test/queries/clientpositive/spark_vectorized_dynamic_partition_pruning.q 22bb5021293f221b80ed622fc7fb14ff758744be
ql/src/test/queries/clientpositive/subq_where_serialization.q de68c8aa29d322501bffd139abd34ff4085b5463
ql/src/test/queries/clientpositive/tez-tag.q b50eed929367cdfddfeca51298b97fadad49d86d
ql/src/test/queries/clientpositive/tez_bmj_schema_evolution.q de89c2735303105e7a4facc0cba3c2e09879e2d0
ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_1.q 47c00381d67504511a7600a3ead44dcbee62b806
ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_2.q 80765935acbf4176853343b484756e5e81e36325
ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_3.q 10e982e1f6c6375cc361dae416a1c46fb4480329
ql/src/test/queries/clientpositive/tez_fixed_bucket_pruning.q cbc39977da13d83ae2d660237db676e8e141c78e
ql/src/test/queries/clientpositive/tez_join_hash.q c5818958a2cdab0a0ba564f788bbf1c813fb57f2
ql/src/test/queries/clientpositive/tez_join_result_complex.q a77c44dc024401c1c08800ae3a5745f61d5f8d4d
ql/src/test/queries/clientpositive/tez_smb_1.q e121d52fd08b55562ecce0f7c372fba5babdd441
ql/src/test/queries/clientpositive/tez_smb_empty.q ffe30cc8f2c7b85a83f5dbe88f8f6fb624482477
ql/src/test/queries/clientpositive/tez_smb_main.q 3a75d614e6be469269da57dbaf5bf283ce7481d1
ql/src/test/queries/clientpositive/tez_vector_dynpart_hashjoin_1.q ecb2249f96f0c329cf610ff4e202e304f3835ffd
ql/src/test/queries/clientpositive/tez_vector_dynpart_hashjoin_2.q 04d68293cc002e6c1aba72287c7de9d64eda665a
ql/src/test/queries/clientpositive/union22.q 5d41351941c7a34275959e0fc3d621fb709ffae5
ql/src/test/queries/clientpositive/unionDistinct_1.q 75c66b02788a200e5a97e501eb58cb8132495c7f
ql/src/test/queries/clientpositive/vector_binary_join_groupby.q f7a14e7abbd60579ca1aecfba10dcaf5b7d26086
ql/src/test/queries/clientpositive/vector_char_mapjoin1.q f5c05a4a9c01495b672990e60a28137946c57ad0
ql/src/test/queries/clientpositive/vector_decimal_mapjoin.q ef769fbd8fcf5c13f7c0e4d45d8146d8e02adf72
ql/src/test/queries/clientpositive/vector_full_outer_join.q cc774887a0fb545a2c76adf90042c5037e7f011a
ql/src/test/queries/clientpositive/vector_fullouter_mapjoin_1_fast.q 1685f35c2a2109041db0d3bf247f8d463598911e
ql/src/test/queries/clientpositive/vector_fullouter_mapjoin_1_optimized.q 8b5926691630d5701c0a154278f1db138f097c1d
ql/src/test/queries/clientpositive/vector_fullouter_mapjoin_1_optimized_passthru.q 869668edb81607800b461bf7f32b6ab1ff9cca54
ql/src/test/queries/clientpositive/vector_groupby_mapjoin.q b1d3f40ee73a3412bd27d0fb6fe2932d51bcc150
ql/src/test/queries/clientpositive/vector_include_no_sel.q e68db1fcdee225b32b1b34c0ff178c7d27d36d99
ql/src/test/queries/clientpositive/vector_join30.q 74c44336432aefac9012bf52bba16eb3595d7956
ql/src/test/queries/clientpositive/vector_join_filters.q b9f3740b5c6be941273bc70379c3632a37fad59b
ql/src/test/queries/clientpositive/vector_join_nulls.q f87dc44e1c3261022b3faa9e73232886bb57c007
ql/src/test/queries/clientpositive/vector_left_outer_join2.q 84f656b99e14e7f08a0cff321d3082cdfc74c0c1
ql/src/test/queries/clientpositive/vector_leftsemi_mapjoin.q f9b4222a239f5d1354b8facbfdeb8f145e1eb000
ql/src/test/queries/clientpositive/vector_llap_text_1.q f5c805eea82930fc12890c2497738a85a3c6c25c
ql/src/test/queries/clientpositive/vector_outer_join5.q 19aefdcb56252ee2f8ce23f32e0c58b585cd5a83
ql/src/test/queries/clientpositive/vector_reduce_groupby_duplicate_cols.q c82c960ce6201842988060ed10243694d2b65030
ql/src/test/queries/clientpositive/vector_varchar_mapjoin1.q 285d2ac64f3f6383e8bc71b21dbe160ec04ae75d
ql/src/test/queries/clientpositive/vectorized_dynamic_partition_pruning.q 03e3d7b3cebb1a9cf200e6439b0b63d437f73486
ql/src/test/queries/clientpositive/vectorized_mapjoin.q 0b4e65df59a9d4ca2418153c6535e13ad7d1b5a4
ql/src/test/queries/clientpositive/vectorized_mapjoin2.q d259547ac9af721def23e19271c8940663e64d21
ql/src/test/queries/clientpositive/vectorized_mapjoin3.q c216499cdbca6cc6918431e7f34509d37b133cce
ql/src/test/queries/clientpositive/vectorized_multi_output_select.q 75a85c7e748bca66f55d5e8913ec14f432c6fffb
ql/src/test/queries/clientpositive/vectorized_nested_mapjoin.q a70ba025c3705606d3c0994e0fca0b46aed37eb7
Diff: https://reviews.apache.org/r/69011/diff/1/
Testing
-------
Thanks,
Prasanth_J
Re: Review Request 69011: HIVE-20713: Use percentage for join
conversion size thresholds
Posted by Ashutosh Chauhan <ha...@apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/69011/#review212674
-----------------------------------------------------------
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
Lines 2017 (patched)
<https://reviews.apache.org/r/69011/#comment298514>
Default value set by Ambari is 0.24. Shall we use 0.25 here instead of 0.3.
https://github.com/hortonworks/hdp_ambari_definitions/blob/AMBARI-2.7-maint/src/main/resources/stacks/HDP/3.0/services/YARN/service_advisor.py#L1416
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
Lines 2022 (patched)
<https://reviews.apache.org/r/69011/#comment298515>
Currently factor = large table / executor memory. I think reducer count should also part of the equation since its memory / reducer which matters here.
i.e. large_table / (executor_memory * reducer_count) In perfect world where all our estimates meets reality this should be 1. That is shuffle data is perfectly divided for each executor. This is also more intuitive to think about for end user. If they want to be aggressive set this to more than 1, and to be conservative < 1. We can pick default of 1, which is also more intutitive than current 10. Although, this assumes reducer count info is available.
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
Lines 2038 (patched)
<https://reviews.apache.org/r/69011/#comment298516>
Shall we just delete these configs. Leaving them in code doesnt serve any purpose other than to confuse code reader :)
ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryInfo.java
Lines 60-62 (patched)
<https://reviews.apache.org/r/69011/#comment298517>
Didnt follow this logic. If tezcontainersize is < 0 then we will pick default_map_memory and if that is < 0 then we again pick that. So, you will still end up with -ve value.
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
Line 103 (original), 104 (patched)
<https://reviews.apache.org/r/69011/#comment298518>
LOG.debug
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
Lines 370 (patched)
<https://reviews.apache.org/r/69011/#comment298519>
LOG.debug
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
Line 1577 (original), 1580 (patched)
<https://reviews.apache.org/r/69011/#comment298520>
LOG.debug
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java
Lines 435 (patched)
<https://reviews.apache.org/r/69011/#comment298521>
LOG.debug
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java
Lines 79 (patched)
<https://reviews.apache.org/r/69011/#comment298522>
LOG.debug
ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
Line 194 (original), 195 (patched)
<https://reviews.apache.org/r/69011/#comment298523>
LOG.debug
ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
Line 405 (original), 406 (patched)
<https://reviews.apache.org/r/69011/#comment298524>
LOG.debug
ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
Lines 600 (patched)
<https://reviews.apache.org/r/69011/#comment298525>
LOG.debug
- Ashutosh Chauhan
On Oct. 12, 2018, 11:41 p.m., Prasanth_J wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/69011/
> -----------------------------------------------------------
>
> (Updated Oct. 12, 2018, 11:41 p.m.)
>
>
> Review request for hive and Ashutosh Chauhan.
>
>
> Bugs: HIVE-20713
> https://issues.apache.org/jira/browse/HIVE-20713
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-20713: Use percentage for join conversion size thresholds
>
>
> Diffs
> -----
>
> common/src/java/org/apache/hadoop/hive/conf/HiveConf.java cc6239c3135714fb65aa1afc2882852460b68b37
> data/conf/hive-site.xml 0daf9adc717bc1c4413d2e34691c26a3e2585c77
> data/conf/llap/hive-site.xml 44ca6c9daf092a35f1c58c26dfa3575c303464ce
> data/conf/tez/hive-site.xml 236adc7087b43f4e9ab95b2fa57436cf75c679aa
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java 40dd992455f2fa6bae85d9d02338bc820a370ebe
> ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryInfo.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java a6b0dbc0dc956d81d027f08a55fbdf0ca452638f
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 54377428eafdb79e1bbdc8a182eafb46f8febd23
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 151d1b39eb17bba336bb02ab06f216361e4f6ff2
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java 5217208611a6ab63b218a585e820516221022ec6
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java 7ff92edd91018f55dd90200f184869a74868dd46
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java 5269eb6f6a77ac031d144a96a2687331e3444f4b
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinResolver.java a84534a9495b44ac319ad52a1f841428861d62e9
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java a52e5e60d97d24996c813b463caffb946ce5b61a
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapClusterStateForCompile.java 0d1990a8cb3882c2c8b271c90e2675e88c79a7ea
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MemoryDecider.java 64f1e7b830f85e898252f964bdbfb73267224ef6
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java ad6db21fe6041e18adc646ca20759ca3cde58713
> ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java 22f3266c87f1d42c254893b424b68e757fb2953b
> ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java fdc963506cabd96c125ec15b6a1eacb71fd33d42
> ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java c7cd4ad3f6deba320b343cda071640cd1c78743e
> ql/src/test/queries/clientpositive/acid_mapjoin.q e3d14cc97d1e193a6bbb2a0e25e0741a192e2638
> ql/src/test/queries/clientpositive/autoColumnStats_1.q 9a62127dabee743b8880f0effdd5184fb24975d0
> ql/src/test/queries/clientpositive/autoColumnStats_2.q 51f252b303fb579d315aabcabb75b4429c7b5d4d
> ql/src/test/queries/clientpositive/autoColumnStats_3.q 7a419118902ffab767780057d8e83fbde3fbdba9
> ql/src/test/queries/clientpositive/auto_join_reordering_values.q 39b2e1d9a78f77c50d60a749085b05ba9acd6b20
> ql/src/test/queries/clientpositive/auto_join_stats.q 8b377bf4ca70ccaf43b625afc51b14399168888f
> ql/src/test/queries/clientpositive/auto_join_without_localtask.q 15697f2f11ac4669c7bdf6e4144f02fd3cb465b2
> ql/src/test/queries/clientpositive/auto_sortmerge_join_1.q 1fbe8f79cff97ef7baf8eb8afce36665b425b23d
> ql/src/test/queries/clientpositive/auto_sortmerge_join_10.q 1b15a740120862a7d56e7b36e07a0e3c4d7c835d
> ql/src/test/queries/clientpositive/auto_sortmerge_join_11.q 76e615fbde1d8d52eb2a8de5d08d69d26e304a99
> ql/src/test/queries/clientpositive/auto_sortmerge_join_12.q d68efea393af588527a698dda2a578433195472e
> ql/src/test/queries/clientpositive/auto_sortmerge_join_13.q bbd7afab07b62d1e6b63b408a5cce71db5ef55f1
> ql/src/test/queries/clientpositive/auto_sortmerge_join_14.q 1e21c92e59f56688d05d3fbc21184a70c9779996
> ql/src/test/queries/clientpositive/auto_sortmerge_join_15.q b3dd8e50d7cd4e3f95f8b595c10acbfb15732943
> ql/src/test/queries/clientpositive/auto_sortmerge_join_16.q 0a72ddfc6b859912962b30719637645648d625ad
> ql/src/test/queries/clientpositive/auto_sortmerge_join_2.q c6d5318f43daa419b8ad0fedcf6782ea3f333f5d
> ql/src/test/queries/clientpositive/auto_sortmerge_join_3.q f41b0970c9ff98f5d72bfa2b3ad07b4b7581e6ab
> ql/src/test/queries/clientpositive/auto_sortmerge_join_4.q e28e0ab4a09cf0214ce946ec79bb34063705e9ad
> ql/src/test/queries/clientpositive/auto_sortmerge_join_5.q 9550e21382237f26f9b7187bc024053221fe365f
> ql/src/test/queries/clientpositive/auto_sortmerge_join_6.q 0aeec0e50fa3dcff0e20f77cf9541d67c141d4db
> ql/src/test/queries/clientpositive/auto_sortmerge_join_7.q 8c9dbacbbad8efe42986556a7d2f6cb62fd3b906
> ql/src/test/queries/clientpositive/auto_sortmerge_join_8.q 51647e86b2bc1d680b59c30aeeaef6f949f21427
> ql/src/test/queries/clientpositive/auto_sortmerge_join_9.q 5696682fd5c3f0ab214873bf2714ebab3ab0959c
> ql/src/test/queries/clientpositive/bucket_map_join_tez1.q 049a4d9571cdca6231587f64a8d20261381fb38f
> ql/src/test/queries/clientpositive/bucket_map_join_tez2.q ec32e9c820778a6d907e599f3b0bf21b485e5c4a
> ql/src/test/queries/clientpositive/bucket_map_join_tez_empty.q 359560aab79b1dd6e59f084df65bab6ad7472827
> ql/src/test/queries/clientpositive/bucketsortoptimize_insert_2.q 0b913caa8e0a190643bf3caa1de2667a37d58727
> ql/src/test/queries/clientpositive/bucketsortoptimize_insert_6.q 2d4907c5cf1b820e77813f291090137512415cd2
> ql/src/test/queries/clientpositive/bucketsortoptimize_insert_7.q bef48b2d0a49f3ee27281a7c6047c7c155baff98
> ql/src/test/queries/clientpositive/cbo_rp_cross_product_check_2.q 00c19c74ad45fc13e0a2cf74af3f0fb33b73a1a3
> ql/src/test/queries/clientpositive/convert_decimal64_to_decimal.q c76057cb0654a7e2f716dbd4bd36d8ebfed7339f
> ql/src/test/queries/clientpositive/correlationoptimizer5.q d75a48f6c036a426d5788cd68aacd28f3b0f0b18
> ql/src/test/queries/clientpositive/correlationoptimizer7.q 0d57d7e7cb9ff98c561309bbdcb851f7bac85b43
> ql/src/test/queries/clientpositive/cross_join.q 041f0c38a67a0f178cf0854031dc4780187492f3
> ql/src/test/queries/clientpositive/cross_product_check_1.q 23c663849165dcd0f790460215b83accefa6c711
> ql/src/test/queries/clientpositive/cross_product_check_2.q 329e878f66ea8283b4088d8179378cd182ecebf5
> ql/src/test/queries/clientpositive/dynamic_partition_pruning.q de339f892a4733c9b23a800f290796b08e605a2f
> ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q f3d94d56d16baf2d6dce8342af2a258d78954903
> ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q 32c28540ad60730233af60002fa0e8b7467dc406
> ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_2.q 20b657e78f5870b21f6614339924ab9fd50efd26
> ql/src/test/queries/clientpositive/dynamic_semijoin_user_level.q 5e3a1b84fbaafc0bb77be1ffa8111387cb9eb13c
> ql/src/test/queries/clientpositive/explainanalyze_1.q 90438263138048ba556148610fcfe5a4242994a2
> ql/src/test/queries/clientpositive/explainanalyze_2.q 7e48a8a2b81410f47bc3415af2fe3aa14ead16bd
> ql/src/test/queries/clientpositive/explainanalyze_3.q 1f31218a8520b397376dad4cbf1bb54a7b3bd4d3
> ql/src/test/queries/clientpositive/explainanalyze_4.q 6daba22caa8adf6de86fb312e5e15ca6d1fb30ec
> ql/src/test/queries/clientpositive/explainuser_1.q 23bdb79196f57da244ab6e51098c333571cb0e34
> ql/src/test/queries/clientpositive/explainuser_2.q bc795cfd3156375c47f296fb89ab06ac35001e40
> ql/src/test/queries/clientpositive/explainuser_3.q 0c110aea1acb9b4d1d73eada6e028d869f32b50b
> ql/src/test/queries/clientpositive/explainuser_4.q 2125d09d31aa6a85b4459c3d37f5d1b972d4d2b1
> ql/src/test/queries/clientpositive/fullouter_mapjoin_1_optimized.q 32b2e0495f3a9500d1cc8055e394917f630f0cc9
> ql/src/test/queries/clientpositive/hybridgrace_hashjoin_1.q 30cf8c4bb9807d22fbca5bd28152294f827cdc49
> ql/src/test/queries/clientpositive/hybridgrace_hashjoin_2.q d989ca7dc883fa071cf5772f358c68bff78f659f
> ql/src/test/queries/clientpositive/join28.q 868ce9debd657e08253a77a8c6dd9a4d07da642b
> ql/src/test/queries/clientpositive/join29.q 20aadbbc26fcbaff369062a203e8c3b16d398fec
> ql/src/test/queries/clientpositive/join31.q 208340713c97f52324a78efad48808f6fa5c0ec6
> ql/src/test/queries/clientpositive/join32.q c55e730ac11663fb180a9d84f3522009e19402c4
> ql/src/test/queries/clientpositive/join32_lessSize.q 5c7be2691a612950054e5e6e6696605d20819c17
> ql/src/test/queries/clientpositive/join33.q 6ddf0eb6b1d0092368091a077cf55ed2bf0ecf03
> ql/src/test/queries/clientpositive/join34.q e0234c61d556128b42ca4d9980fa86573e2b4ce2
> ql/src/test/queries/clientpositive/join35.q 29b6b000c67f2f028e79852aea2bb8134d4ec163
> ql/src/test/queries/clientpositive/join41.q 33dfcb087e1f47f614bf33b914b98c6706ec6e1d
> ql/src/test/queries/clientpositive/join_grp_diff_keys.q fb110b4ae4829395dd0c4ce7e448d5e91aa84e3c
> ql/src/test/queries/clientpositive/join_star.q 70cc8f7c07ee1f5a776ad42f98a07140f8880455
> ql/src/test/queries/clientpositive/llap_smb.q edec3010d11b5de0db31e8256e9b0b25ca73b6e8
> ql/src/test/queries/clientpositive/llap_smb_ptf.q 037b97dc5154d359c0249deb6cbf80670d3baa6a
> ql/src/test/queries/clientpositive/llap_vector_nohybridgrace.q d637d364300bf3030fa0503a3a976242d7c71c8b
> ql/src/test/queries/clientpositive/load_data_using_job.q 970a7521f54edc070e5bbeabd47dc166b11f801f
> ql/src/test/queries/clientpositive/mapjoin_decimal.q c84267a049fe28dc3fbfe0aa1b8608df4a96421d
> ql/src/test/queries/clientpositive/mapjoin_hint.q 7189f070e1839aab37f1f462dc203e4c9e836382
> ql/src/test/queries/clientpositive/mapjoin_mapjoin.q de0523805603df21799bf7feb6fca52aab572630
> ql/src/test/queries/clientpositive/mapjoin_subquery.q 79fe197330b760aa7c23e6ea451207fd66c2afdc
> ql/src/test/queries/clientpositive/mapjoin_subquery2.q 95051d4cf8198a43d1117532d03759102799b560
> ql/src/test/queries/clientpositive/mrr.q 0f2dd8474158a621349982c31cfda98afc2cfd67
> ql/src/test/queries/clientpositive/multiMapJoin1.q a65ed72c27ea1815916eed916116b35c38f0c5d5
> ql/src/test/queries/clientpositive/multiMapJoin2.q 166cb092714956805de0afaddc15fc5d35191c8f
> ql/src/test/queries/clientpositive/murmur_hash_migration.q 2b8da9f68368961ca7dfe74c6481be126a0e9170
> ql/src/test/queries/clientpositive/quotedid_smb.q acc7626407131118c446149c247821a604d33bdc
> ql/src/test/queries/clientpositive/reduce_deduplicate_extended2.q f88fd50250ac25a83b9b28ddfcdf0f1f8dfb1c34
> ql/src/test/queries/clientpositive/reopt_semijoin.q 75db410e629300a6b7042159a022ac1a4eed02ae
> ql/src/test/queries/clientpositive/runtime_skewjoin_mapjoin_spark.q ac93eedd246ac34dfd8562a6fed5890ae542282c
> ql/src/test/queries/clientpositive/runtime_stats_merge.q e69410159aaa54c34ae2f0e192a60a63df3fabc7
> ql/src/test/queries/clientpositive/smb_cache.q d87ba6ce8af186fdcf01eae3f1bbf4d24adc8089
> ql/src/test/queries/clientpositive/smb_mapjoin_25.q 56aa331eade9d2ae0d0998736d99adb05779b04c
> ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning.q 0b6ecc4e9d638c8842bdf5ede4abe939bbde5ef0
> ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_2.q 532a50a74671b4a5a36cbc73c0c769cce5207405
> ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_3.q 4a245f9277263fc9dff59171e6324eece3ec432d
> ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_5.q d0814199cc00b8756a22031bfa7461f13b25c139
> ql/src/test/queries/clientpositive/spark_dynamic_partition_pruning_mapjoin_only.q ba44aefbc4ae6c48d37439e5c666cce12969a851
> ql/src/test/queries/clientpositive/spark_explainuser_1.q 27183292ada69584fe2d55ada339d776ada0b0ae
> ql/src/test/queries/clientpositive/spark_use_ts_stats_for_mapjoin.q 5ee09fc6845c7ae2bdecf19fac6810314737405b
> ql/src/test/queries/clientpositive/spark_vectorized_dynamic_partition_pruning.q 22bb5021293f221b80ed622fc7fb14ff758744be
> ql/src/test/queries/clientpositive/subq_where_serialization.q de68c8aa29d322501bffd139abd34ff4085b5463
> ql/src/test/queries/clientpositive/tez-tag.q b50eed929367cdfddfeca51298b97fadad49d86d
> ql/src/test/queries/clientpositive/tez_bmj_schema_evolution.q de89c2735303105e7a4facc0cba3c2e09879e2d0
> ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_1.q 47c00381d67504511a7600a3ead44dcbee62b806
> ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_2.q 80765935acbf4176853343b484756e5e81e36325
> ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_3.q 10e982e1f6c6375cc361dae416a1c46fb4480329
> ql/src/test/queries/clientpositive/tez_fixed_bucket_pruning.q cbc39977da13d83ae2d660237db676e8e141c78e
> ql/src/test/queries/clientpositive/tez_join_hash.q c5818958a2cdab0a0ba564f788bbf1c813fb57f2
> ql/src/test/queries/clientpositive/tez_join_result_complex.q a77c44dc024401c1c08800ae3a5745f61d5f8d4d
> ql/src/test/queries/clientpositive/tez_smb_1.q e121d52fd08b55562ecce0f7c372fba5babdd441
> ql/src/test/queries/clientpositive/tez_smb_empty.q ffe30cc8f2c7b85a83f5dbe88f8f6fb624482477
> ql/src/test/queries/clientpositive/tez_smb_main.q 3a75d614e6be469269da57dbaf5bf283ce7481d1
> ql/src/test/queries/clientpositive/tez_vector_dynpart_hashjoin_1.q ecb2249f96f0c329cf610ff4e202e304f3835ffd
> ql/src/test/queries/clientpositive/tez_vector_dynpart_hashjoin_2.q 04d68293cc002e6c1aba72287c7de9d64eda665a
> ql/src/test/queries/clientpositive/union22.q 5d41351941c7a34275959e0fc3d621fb709ffae5
> ql/src/test/queries/clientpositive/unionDistinct_1.q 75c66b02788a200e5a97e501eb58cb8132495c7f
> ql/src/test/queries/clientpositive/vector_binary_join_groupby.q f7a14e7abbd60579ca1aecfba10dcaf5b7d26086
> ql/src/test/queries/clientpositive/vector_char_mapjoin1.q f5c05a4a9c01495b672990e60a28137946c57ad0
> ql/src/test/queries/clientpositive/vector_decimal_mapjoin.q ef769fbd8fcf5c13f7c0e4d45d8146d8e02adf72
> ql/src/test/queries/clientpositive/vector_full_outer_join.q cc774887a0fb545a2c76adf90042c5037e7f011a
> ql/src/test/queries/clientpositive/vector_fullouter_mapjoin_1_fast.q 1685f35c2a2109041db0d3bf247f8d463598911e
> ql/src/test/queries/clientpositive/vector_fullouter_mapjoin_1_optimized.q 8b5926691630d5701c0a154278f1db138f097c1d
> ql/src/test/queries/clientpositive/vector_fullouter_mapjoin_1_optimized_passthru.q 869668edb81607800b461bf7f32b6ab1ff9cca54
> ql/src/test/queries/clientpositive/vector_groupby_mapjoin.q b1d3f40ee73a3412bd27d0fb6fe2932d51bcc150
> ql/src/test/queries/clientpositive/vector_include_no_sel.q e68db1fcdee225b32b1b34c0ff178c7d27d36d99
> ql/src/test/queries/clientpositive/vector_join30.q 74c44336432aefac9012bf52bba16eb3595d7956
> ql/src/test/queries/clientpositive/vector_join_filters.q b9f3740b5c6be941273bc70379c3632a37fad59b
> ql/src/test/queries/clientpositive/vector_join_nulls.q f87dc44e1c3261022b3faa9e73232886bb57c007
> ql/src/test/queries/clientpositive/vector_left_outer_join2.q 84f656b99e14e7f08a0cff321d3082cdfc74c0c1
> ql/src/test/queries/clientpositive/vector_leftsemi_mapjoin.q f9b4222a239f5d1354b8facbfdeb8f145e1eb000
> ql/src/test/queries/clientpositive/vector_llap_text_1.q f5c805eea82930fc12890c2497738a85a3c6c25c
> ql/src/test/queries/clientpositive/vector_outer_join5.q 19aefdcb56252ee2f8ce23f32e0c58b585cd5a83
> ql/src/test/queries/clientpositive/vector_reduce_groupby_duplicate_cols.q c82c960ce6201842988060ed10243694d2b65030
> ql/src/test/queries/clientpositive/vector_varchar_mapjoin1.q 285d2ac64f3f6383e8bc71b21dbe160ec04ae75d
> ql/src/test/queries/clientpositive/vectorized_dynamic_partition_pruning.q 03e3d7b3cebb1a9cf200e6439b0b63d437f73486
> ql/src/test/queries/clientpositive/vectorized_mapjoin.q 0b4e65df59a9d4ca2418153c6535e13ad7d1b5a4
> ql/src/test/queries/clientpositive/vectorized_mapjoin2.q d259547ac9af721def23e19271c8940663e64d21
> ql/src/test/queries/clientpositive/vectorized_mapjoin3.q c216499cdbca6cc6918431e7f34509d37b133cce
> ql/src/test/queries/clientpositive/vectorized_multi_output_select.q 75a85c7e748bca66f55d5e8913ec14f432c6fffb
> ql/src/test/queries/clientpositive/vectorized_nested_mapjoin.q a70ba025c3705606d3c0994e0fca0b46aed37eb7
>
>
> Diff: https://reviews.apache.org/r/69011/diff/1/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Prasanth_J
>
>