You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hive.apache.org by j....@gmail.com on 2017/04/27 02:12:52 UTC
Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if hash
table memory exceeds threshold
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/
-----------------------------------------------------------
Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
Bugs: HIVE-16546
https://issues.apache.org/jira/browse/HIVE-16546
Repository: hive-git
Description
-------
HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
Diffs
-----
common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824c21f2fbf98177cb12a18019416f36a3f9
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd1a556a25f9cc68dc3ba00e315bf16cd53
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6b4f34bc05ba854c6df1bf117bcc23bc25
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8e7494e85ee057bd7886b89da028c6cc20
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f541e4113fbc8e4dd69338115f81126ce8c
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef65c801ee3e9f4b9e11565a40e0db9cc4e
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90125343669cb7429740e3722901b0f78db
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242dafb002721624b7c882967f7b5347cccef97
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961ebaf90f386cb736a101e251ea7f0e162c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c54af867d5a18af913d25409c02c1b01fb
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e9779e6ae030855105cdd7f76dc5ba8665c94
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bcc1df9020a19d483925a4c68e925540369
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1580bb762b757a73c1121935bf28d9d8ef
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693c557382bc55bbf611a3b05a51603126fd
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9aed81c7fec5396a38104f78902d33562f
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee1cb2d9b4ba5e480720ca787b19c751193
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb118edda493362f79b690ecc30540b07d86a
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c78d046227e10fc6ebd237fd6c17386a30
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788b61a355e9971b27687da06b9438e6ddf2
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb4365e296cf83ee29d51df381ba8831b15
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 532891079542ee5c8c1fe3c6b4923a525bd8a730
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034ff6bef9003abf81ff36240baa34b9a718b
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4b65f22c35c459e6dba94ad0d83912caea
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729bcdb47aad47f372f6d384046929d5858f
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1b3ff6a099fed9b8d419811edd5edc4860
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b340d5d254ec703dfb5f31d3b662876e4ea3
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c29ba27450072c8eeeb00416fd0efee107
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7716e7200a4a09cf228108d4f2d78f8b37
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861cb118e092f4635488dea3435129e74fb8
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9ca5464958f1c7f351000b6ec17df8fab39
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87a05b6d00a370a8ca225778300141bb025
ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e76908243c70ce804f222594a296a1f4afd
ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b58da12645eae246a058582c38f1c40b07
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d1370350d1aea258b821f94db3db9103f6
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca887c0a0de33d53ee866554b1f758f231b
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bdf7e0d8dc95f5e065cf7dd6a5e03d41602
ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb28d48032a4792084d0ab2488de48105dc
ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c23659906698249d9b4f8feddf3b595711
serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9fc84321940ca79d1534c2b997c89f64d4f
Diff: https://reviews.apache.org/r/58777/diff/1/
Testing
-------
Thanks,
Prasanth_J
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by Sergey Shelukhin <se...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/#review173228
-----------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
Lines 148 (patched)
<https://reviews.apache.org/r/58777/#comment246275>
what is this based on?
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
Lines 228 (patched)
<https://reviews.apache.org/r/58777/#comment246277>
hmm.. given that both are multiplied by inflation factor, this basically compares the size of the table with nocondtasksize, multiplication is not necessary.
Wouldn't it always fail on any kind of a compressed table?
- Sergey Shelukhin
On April 27, 2017, 8:43 a.m., Prasanth_J wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58777/
> -----------------------------------------------------------
>
> (Updated April 27, 2017, 8:43 a.m.)
>
>
> Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
>
>
> Bugs: HIVE-16546
> https://issues.apache.org/jira/browse/HIVE-16546
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
>
>
> Diffs
> -----
>
> common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
> common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java 72dcdd3
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
> ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bd
> ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
> ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
> serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
>
>
> Diff: https://reviews.apache.org/r/58777/diff/2/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Prasanth_J
>
>
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by j....@gmail.com.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/#review173243
-----------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java
Lines 148 (patched)
<https://reviews.apache.org/r/58777/#comment246289>
Key and Value for the non-optimized hash table loader is Object[] which will hold serialized binary objects or deserialized object corresponding to column values. It is very intrusive to add memory estimation for all types, OIs, writable etc. so the assumption here is that each entry in the hash table is of size 1KB. In most cases, we use optimized hash table which is pretty much flat and can provide better in-memory estimates. Best way to find deep object size is to iterate all declared fields and used instrumentation object size to find the actual size but it needs a separate agent combined with reflection :)
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
Lines 228 (patched)
<https://reviews.apache.org/r/58777/#comment246291>
Good catch. My bad. Both should not be multiplied by inflation factor. Only no conditional task size has to be multiplied by inflation factor.
Regd. compressed tables. It actually depends. ORC for example, even if the table is compressed the raw data size returned by ORC reader represents uncompressed data size. Metastore stores file size (compressed) and raw data size. Statistics annotation will use raw data size when available else hive.stats.deserialization.factor can be set to account for inflation.
- Prasanth_J
On April 27, 2017, 8:43 a.m., Prasanth_J wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58777/
> -----------------------------------------------------------
>
> (Updated April 27, 2017, 8:43 a.m.)
>
>
> Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
>
>
> Bugs: HIVE-16546
> https://issues.apache.org/jira/browse/HIVE-16546
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
>
>
> Diffs
> -----
>
> common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
> common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java 72dcdd3
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
> ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bd
> ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
> ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
> serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
>
>
> Diff: https://reviews.apache.org/r/58777/diff/2/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Prasanth_J
>
>
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by j....@gmail.com.
> On April 30, 2017, 8:12 a.m., Lefty Leverenz wrote:
> > common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
> > Lines 3155-3158 (patched)
> > <https://reviews.apache.org/r/58777/diff/4/?file=1702345#file1702345line3155>
> >
> > "after every these many rows" -> "after every interval of this many rows"
> >
> > "exceed" -> "exceeds"
> >
> > Also, please add another \n to the description after "when running" (line 3157 in diff revision 4).
Thanks Lefty! Fixed in the new patch.
- Prasanth_J
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/#review173439
-----------------------------------------------------------
On May 2, 2017, 12:22 a.m., Prasanth_J wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58777/
> -----------------------------------------------------------
>
> (Updated May 2, 2017, 12:22 a.m.)
>
>
> Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
>
>
> Bugs: HIVE-16546
> https://issues.apache.org/jira/browse/HIVE-16546
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
>
>
> Diffs
> -----
>
> common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
> common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionError.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionException.java dbe00b6
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java 7fc3226
> ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java 595d1bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java 486d43a
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
> ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bd
> ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
> ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
> ql/src/test/org/apache/hadoop/hive/ql/exec/mapjoin/TestMapJoinMemoryExhaustionHandler.java 16b5b17
> serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
>
>
> Diff: https://reviews.apache.org/r/58777/diff/5/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Prasanth_J
>
>
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by Lefty Leverenz <le...@gmail.com>.
> On April 30, 2017, 8:12 a.m., Lefty Leverenz wrote:
> > common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
> > Lines 3155-3158 (patched)
> > <https://reviews.apache.org/r/58777/diff/4/?file=1702345#file1702345line3155>
> >
> > "after every these many rows" -> "after every interval of this many rows"
> >
> > "exceed" -> "exceeds"
> >
> > Also, please add another \n to the description after "when running" (line 3157 in diff revision 4).
>
> Prasanth_J wrote:
> Thanks Lefty! Fixed in the new patch.
+1 Thanks for the fixes, Prasanth.
- Lefty
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/#review173439
-----------------------------------------------------------
On May 2, 2017, 8:22 a.m., Prasanth_J wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58777/
> -----------------------------------------------------------
>
> (Updated May 2, 2017, 8:22 a.m.)
>
>
> Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
>
>
> Bugs: HIVE-16546
> https://issues.apache.org/jira/browse/HIVE-16546
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
>
>
> Diffs
> -----
>
> common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
> common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 3400560
> llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java ff6e7ce
> ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt 4393c3b
> ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt 7468c2f
> ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt 57b7ea5
> ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt 749e97e
> ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt 9dfc147
> ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt 32ecb34
> ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt bd0f14d
> ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt dc9d4b1
> ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt 01062a9
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionError.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionException.java dbe00b6
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java 7fc3226
> ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java 595d1bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java 486d43a
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAggregationBufferBatch.java 630046d
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java 5b4c7c3
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java 0866f63
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java 74e25ae
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java 483d9dc
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilter.java 2139eae
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilterMerge.java d2446d5
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java 494febc
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java dec88cb
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java 337ba0a
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java 8cd3506
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java 61d6977
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java b10f66f
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java 2709b07
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java 03dce1e
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
> ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 654f3b1
> ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
> ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
> ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java ec80641
> ql/src/test/org/apache/hadoop/hive/ql/exec/mapjoin/TestMapJoinMemoryExhaustionHandler.java 16b5b17
> serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
> storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java 4a745e4
>
>
> Diff: https://reviews.apache.org/r/58777/diff/6/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Prasanth_J
>
>
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by Lefty Leverenz <le...@gmail.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/#review173439
-----------------------------------------------------------
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
Lines 3155-3158 (patched)
<https://reviews.apache.org/r/58777/#comment246428>
"after every these many rows" -> "after every interval of this many rows"
"exceed" -> "exceeds"
Also, please add another \n to the description after "when running" (line 3157 in diff revision 4).
- Lefty Leverenz
On April 28, 2017, 4:53 a.m., Prasanth_J wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58777/
> -----------------------------------------------------------
>
> (Updated April 28, 2017, 4:53 a.m.)
>
>
> Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
>
>
> Bugs: HIVE-16546
> https://issues.apache.org/jira/browse/HIVE-16546
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
>
>
> Diffs
> -----
>
> common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
> common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionError.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionException.java dbe00b6
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java 7fc3226
> ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java 595d1bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java 24d3526
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java 486d43a
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
> ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bd
> ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
> ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
> ql/src/test/org/apache/hadoop/hive/ql/exec/mapjoin/TestMapJoinMemoryExhaustionHandler.java 16b5b17
> serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
>
>
> Diff: https://reviews.apache.org/r/58777/diff/4/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Prasanth_J
>
>
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by j....@gmail.com.
> On May 1, 2017, 9:13 p.m., Sergey Shelukhin wrote:
> > ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
> > Lines 225 (patched)
> > <https://reviews.apache.org/r/58777/diff/4/?file=1702355#file1702355line225>
> >
> > is it possible to handle 0/negative values differently, to have a complete off switch in case something goes wrong?
Setting value of 0 will not completely turn off the check.
- Prasanth_J
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/#review173510
-----------------------------------------------------------
On May 2, 2017, 12:22 a.m., Prasanth_J wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58777/
> -----------------------------------------------------------
>
> (Updated May 2, 2017, 12:22 a.m.)
>
>
> Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
>
>
> Bugs: HIVE-16546
> https://issues.apache.org/jira/browse/HIVE-16546
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
>
>
> Diffs
> -----
>
> common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
> common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionError.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionException.java dbe00b6
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java 7fc3226
> ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java 595d1bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java 486d43a
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
> ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bd
> ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
> ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
> ql/src/test/org/apache/hadoop/hive/ql/exec/mapjoin/TestMapJoinMemoryExhaustionHandler.java 16b5b17
> serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
>
>
> Diff: https://reviews.apache.org/r/58777/diff/5/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Prasanth_J
>
>
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by Sergey Shelukhin <se...@hortonworks.com>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/#review173510
-----------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
Lines 225 (patched)
<https://reviews.apache.org/r/58777/#comment246502>
is it possible to handle 0/negative values differently, to have a complete off switch in case something goes wrong?
- Sergey Shelukhin
On April 28, 2017, 4:53 a.m., Prasanth_J wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/58777/
> -----------------------------------------------------------
>
> (Updated April 28, 2017, 4:53 a.m.)
>
>
> Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
>
>
> Bugs: HIVE-16546
> https://issues.apache.org/jira/browse/HIVE-16546
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
>
>
> Diffs
> -----
>
> common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
> common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionError.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionException.java dbe00b6
> ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java 7fc3226
> ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java 595d1bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
> ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java 24d3526
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java 486d43a
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
> ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
> ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
> ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bd
> ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
> ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
> ql/src/test/org/apache/hadoop/hive/ql/exec/mapjoin/TestMapJoinMemoryExhaustionHandler.java 16b5b17
> serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
>
>
> Diff: https://reviews.apache.org/r/58777/diff/4/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Prasanth_J
>
>
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by j....@gmail.com.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/
-----------------------------------------------------------
(Updated May 2, 2017, 8:22 a.m.)
Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
Changes
-------
Fix for integer overflows.
Bugs: HIVE-16546
https://issues.apache.org/jira/browse/HIVE-16546
Repository: hive-git
Description
-------
HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
Diffs (updated)
-----
common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 3400560
llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java ff6e7ce
ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt 4393c3b
ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt 7468c2f
ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt 57b7ea5
ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxIntervalDayTime.txt 749e97e
ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt 9dfc147
ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxTimestamp.txt 32ecb34
ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt bd0f14d
ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt dc9d4b1
ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVarDecimal.txt 01062a9
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionError.java PRE-CREATION
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionException.java dbe00b6
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java 7fc3226
ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java 595d1bd
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java 486d43a
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAggregationBufferBatch.java 630046d
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java 5b4c7c3
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorAggregateExpression.java 0866f63
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java 74e25ae
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgTimestamp.java 483d9dc
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilter.java 2139eae
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFBloomFilterMerge.java d2446d5
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java 494febc
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java dec88cb
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountStar.java 337ba0a
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdPopTimestamp.java 8cd3506
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFStdSampTimestamp.java 61d6977
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFSumDecimal.java b10f66f
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarPopTimestamp.java 2709b07
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFVarSampTimestamp.java 03dce1e
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 654f3b1
ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java ec80641
ql/src/test/org/apache/hadoop/hive/ql/exec/mapjoin/TestMapJoinMemoryExhaustionHandler.java 16b5b17
serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java 4a745e4
Diff: https://reviews.apache.org/r/58777/diff/6/
Changes: https://reviews.apache.org/r/58777/diff/5-6/
Testing
-------
Thanks,
Prasanth_J
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by j....@gmail.com.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/
-----------------------------------------------------------
(Updated May 2, 2017, 12:22 a.m.)
Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
Changes
-------
Addressed Lefty's and Sergey's review comments.
Bugs: HIVE-16546
https://issues.apache.org/jira/browse/HIVE-16546
Repository: hive-git
Description
-------
HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
Diffs (updated)
-----
common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionError.java PRE-CREATION
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionException.java dbe00b6
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java 7fc3226
ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java 595d1bd
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java 486d43a
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bd
ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
ql/src/test/org/apache/hadoop/hive/ql/exec/mapjoin/TestMapJoinMemoryExhaustionHandler.java 16b5b17
serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
Diff: https://reviews.apache.org/r/58777/diff/5/
Changes: https://reviews.apache.org/r/58777/diff/4-5/
Testing
-------
Thanks,
Prasanth_J
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by j....@gmail.com.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/
-----------------------------------------------------------
(Updated April 28, 2017, 4:53 a.m.)
Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
Changes
-------
Minor update to fix test failures.
Bugs: HIVE-16546
https://issues.apache.org/jira/browse/HIVE-16546
Repository: hive-git
Description
-------
HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
Diffs (updated)
-----
common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionError.java PRE-CREATION
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionException.java dbe00b6
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java 7fc3226
ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java 595d1bd
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java 24d3526
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java 486d43a
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bd
ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
ql/src/test/org/apache/hadoop/hive/ql/exec/mapjoin/TestMapJoinMemoryExhaustionHandler.java 16b5b17
serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
Diff: https://reviews.apache.org/r/58777/diff/4/
Changes: https://reviews.apache.org/r/58777/diff/3-4/
Testing
-------
Thanks,
Prasanth_J
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by j....@gmail.com.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/
-----------------------------------------------------------
(Updated April 27, 2017, 10:40 p.m.)
Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
Changes
-------
Addressed Sergey's review comments. Also removed some unnecessary changes to ObjectCache. Added reporting fatal error so that task attempts are not retried on map join memory exhaustion.
Bugs: HIVE-16546
https://issues.apache.org/jira/browse/HIVE-16546
Repository: hive-git
Description
-------
HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
Diffs (updated)
-----
common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionError.java PRE-CREATION
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionException.java dbe00b6
ql/src/java/org/apache/hadoop/hive/ql/exec/mapjoin/MapJoinMemoryExhaustionHandler.java 7fc3226
ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java 595d1bd
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java 24d3526
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java 486d43a
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bd
ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
ql/src/test/org/apache/hadoop/hive/ql/exec/mapjoin/TestMapJoinMemoryExhaustionHandler.java 16b5b17
serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
Diff: https://reviews.apache.org/r/58777/diff/3/
Changes: https://reviews.apache.org/r/58777/diff/2-3/
Testing
-------
Thanks,
Prasanth_J
Re: Review Request 58777: HIVE-16546: LLAP: Fail map join tasks if
hash table memory exceeds threshold
Posted by j....@gmail.com.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/58777/
-----------------------------------------------------------
(Updated April 27, 2017, 8:43 a.m.)
Review request for hive, Gunther Hagleitner, Sergey Shelukhin, and Siddharth Seth.
Bugs: HIVE-16546
https://issues.apache.org/jira/browse/HIVE-16546
Repository: hive-git
Description
-------
HIVE-16546: LLAP: Fail map join tasks if hash table memory exceeds threshold
Diffs (updated)
-----
common/src/java/org/apache/hadoop/hive/common/MemoryEstimate.java PRE-CREATION
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java d3ea824
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java 04e24bd
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java a3bccc6
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java 04e89e8
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java c86e5f5
ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainer.java 6d71fef
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java 7b13e90
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ObjectCache.java 72dcdd3
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java 6242daf
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java 1a41961
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashSet.java 331867c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b93e977
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTable.java b6db3bc
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java 49ecdd1
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java be51693
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java 6fe98f9
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMultiSet.java 9140aee
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashSet.java d3efb11
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java 8bfa07c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMap.java add4788
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashMultiSet.java faefdbb
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastMultiKeyHashSet.java 5328910
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMap.java f13034f
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashMultiSet.java 53ad7b4
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastStringHashSet.java 723c729
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java 05f1cf1
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java f9c5b34
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/hashtable/VectorMapJoinHashTable.java c7e585c
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashSet.java 93a89d7
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedHashTable.java 5fe7861
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedStringHashSet.java f921b9c
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java ad77e87
ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b2893e7
ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java d375d1b
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java 93b8a5d
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java 405c3ca
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java d39b8bd
ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java 032c7bb
ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java 940630c
serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java a4ecd9f
Diff: https://reviews.apache.org/r/58777/diff/2/
Changes: https://reviews.apache.org/r/58777/diff/1-2/
Testing
-------
Thanks,
Prasanth_J