You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/02/23 19:30:54 UTC
hive git commit: HIVE-16012 : BytesBytes hash table - better capacity
exhaustion handling (Sergey Shelukhin, reviewed by Wei Zheng)
Repository: hive
Updated Branches:
refs/heads/master e941e63c7 -> 6ca79e3aa
HIVE-16012 : BytesBytes hash table - better capacity exhaustion handling (Sergey Shelukhin, reviewed by Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6ca79e3a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6ca79e3a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6ca79e3a
Branch: refs/heads/master
Commit: 6ca79e3aa2b5a8812e5c4aaee80c4115e2b9def8
Parents: e941e63
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Feb 23 11:25:18 2017 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Feb 23 11:25:18 2017 -0800
----------------------------------------------------------------------
.../hive/ql/exec/persistence/BytesBytesMultiHashMap.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6ca79e3a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
index 6b89e98..04e24bd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/BytesBytesMultiHashMap.java
@@ -26,6 +26,7 @@ import java.util.TreeMap;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.debug.Utils;
import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput;
import org.apache.hadoop.hive.serde2.SerDeException;
@@ -553,6 +554,12 @@ public final class BytesBytesMultiHashMap {
if (capacity <= 0) {
throw new AssertionError("Invalid capacity " + capacity);
}
+ if (capacity > Integer.MAX_VALUE) {
+ throw new RuntimeException("Attempting to expand the hash table to " + capacity
+ + " that overflows maximum array size. For this query, you may want to disable "
+ + ConfVars.HIVEDYNAMICPARTITIONHASHJOIN.varname + " or reduce "
+ + ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD.varname);
+ }
}
/**
@@ -715,8 +722,7 @@ public final class BytesBytesMultiHashMap {
}
private void expandAndRehash() {
- long capacity = refs.length << 1;
- expandAndRehashImpl(capacity);
+ expandAndRehashImpl(((long)refs.length) << 1);
}
private void expandAndRehashImpl(long capacity) {