You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/08/18 00:00:58 UTC
[28/50] [abbrv] hive git commit: HIVE-11467 : WriteBuffers rounding
wbSize to next power of 2 may cause OOM (Wei Zheng,
reviewed by Sergey Shelukhin)
HIVE-11467 : WriteBuffers rounding wbSize to next power of 2 may cause OOM (Wei Zheng, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/433ea9cd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/433ea9cd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/433ea9cd
Branch: refs/heads/hbase-metastore
Commit: 433ea9cda59a986689cf99953826512e4ff07e4d
Parents: 1a75644
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Aug 13 14:46:07 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Aug 13 14:46:07 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/persistence/HybridHashTableContainer.java | 4 ++++
serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/433ea9cd/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
index ad1246d..ff64f52 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
@@ -290,6 +290,10 @@ public class HybridHashTableContainer
}
}
+ // Round to power of 2 here, as is required by WriteBuffers
+ writeBufferSize = Integer.bitCount(writeBufferSize) == 1 ?
+ writeBufferSize : Integer.highestOneBit(writeBufferSize);
+
// Cap WriteBufferSize to avoid large preallocations
writeBufferSize = writeBufferSize < minWbSize ? minWbSize : Math.min(maxWbSize, writeBufferSize);
LOG.info("Write buffer size: " + writeBufferSize);
http://git-wip-us.apache.org/repos/asf/hive/blob/433ea9cd/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java b/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
index acb51f9..05d9359 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
@@ -56,7 +56,7 @@ public final class WriteBuffers implements RandomAccessOutput {
public WriteBuffers(int wbSize, long maxSize) {
- this.wbSize = Integer.bitCount(wbSize) == 1 ? wbSize : (Integer.highestOneBit(wbSize) << 1);
+ this.wbSize = Integer.bitCount(wbSize) == 1 ? wbSize : Integer.highestOneBit(wbSize);
this.wbSizeLog2 = 31 - Integer.numberOfLeadingZeros(this.wbSize);
this.offsetMask = this.wbSize - 1;
this.maxSize = maxSize;