You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/05/05 20:10:55 UTC
[16/20] hive git commit: HIVE-13653 : improve config error messages
for LLAP cache size/etc (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
HIVE-13653 : improve config error messages for LLAP cache size/etc (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f41d693b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f41d693b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f41d693b
Branch: refs/heads/llap
Commit: f41d693b5b984ea55b01394af0dbb6c7121db90a
Parents: 96f2dc7
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu May 5 10:41:47 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu May 5 10:41:47 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/llap/cache/BuddyAllocator.java | 43 +++++++++++++++-----
1 file changed, 32 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f41d693b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
index d78c1e0..1d5a7db 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
@@ -44,6 +44,8 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
// We don't know the acceptable size for Java array, so we'll use 1Gb boundary.
// That is guaranteed to fit any maximum allocation.
private static final int MAX_ARENA_SIZE = 1024*1024*1024;
+ // Don't try to operate with less than MIN_SIZE allocator space, it will just give you grief.
+ private static final int MIN_TOTAL_MEMORY_SIZE = 64*1024*1024;
public BuddyAllocator(Configuration conf, MemoryManager mm, LlapDaemonCacheMetrics metrics) {
@@ -51,8 +53,19 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
(int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC),
(int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MAX_ALLOC),
HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_ARENA_COUNT),
- HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE),
- mm, metrics);
+ getMaxTotalMemorySize(conf), mm, metrics);
+ }
+
+ private static long getMaxTotalMemorySize(Configuration conf) {
+ long maxSize = HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
+ if (maxSize > MIN_TOTAL_MEMORY_SIZE || HiveConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST)) {
+ return maxSize;
+ }
+ throw new RuntimeException("Allocator space is too small for reasonable operation; "
+ + ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname + "=" + maxSize + ", but at least "
+ + MIN_TOTAL_MEMORY_SIZE + " is required. If you cannot spare any memory, you can "
+ + "disable LLAP IO entirely via " + ConfVars.LLAP_IO_ENABLED.varname + "; or set "
+ + ConfVars.LLAP_IO_MEMORY_MODE.varname + " to 'none'");
}
@VisibleForTesting
@@ -69,16 +82,19 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
+ ", arena size " + arenaSizeVal + ". total size " + maxSizeVal);
}
+ String minName = ConfVars.LLAP_ALLOCATOR_MIN_ALLOC.varname,
+ maxName = ConfVars.LLAP_ALLOCATOR_MAX_ALLOC.varname;
if (minAllocation < 8) {
- throw new AssertionError("Min allocation must be at least 8 bytes: " + minAllocation);
+ throw new RuntimeException(minName + " must be at least 8 bytes: " + minAllocation);
}
- if (maxSizeVal < arenaSizeVal || maxAllocation < minAllocation) {
- throw new AssertionError("Inconsistent sizes of cache, arena and allocations: "
- + minAllocation + ", " + maxAllocation + ", " + arenaSizeVal + ", " + maxSizeVal);
+ if (maxSizeVal < maxAllocation || maxAllocation < minAllocation) {
+ throw new RuntimeException("Inconsistent sizes; expecting " + minName + " <= " + maxName
+ + " <= " + ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname + "; configured with min="
+ + minAllocation + ", max=" + maxAllocation + " and total=" + maxSizeVal);
}
if ((Integer.bitCount(minAllocation) != 1) || (Integer.bitCount(maxAllocation) != 1)) {
- throw new AssertionError("Allocation sizes must be powers of two: "
- + minAllocation + ", " + maxAllocation);
+ throw new RuntimeException("Allocation sizes must be powers of two; configured with "
+ + minName + "=" + minAllocation + ", " + maxName + "=" + maxAllocation);
}
if ((arenaSizeVal % maxAllocation) > 0) {
long oldArenaSize = arenaSizeVal;
@@ -94,8 +110,8 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
+ " to be divisible by arena size " + arenaSize);
}
if ((maxSizeVal / arenaSize) > Integer.MAX_VALUE) {
- throw new AssertionError(
- "Too many arenas needed to allocate the cache: " + arenaSize + "," + maxSizeVal);
+ throw new RuntimeException(
+ "Too many arenas needed to allocate the cache: " + arenaSize + ", " + maxSizeVal);
}
maxSize = maxSizeVal;
memoryManager.updateMaxSize(maxSize);
@@ -280,7 +296,12 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
private FreeList[] freeLists;
void init() {
- data = isDirect ? ByteBuffer.allocateDirect(arenaSize) : ByteBuffer.allocate(arenaSize);
+ try {
+ data = isDirect ? ByteBuffer.allocateDirect(arenaSize) : ByteBuffer.allocate(arenaSize);
+ } catch (OutOfMemoryError oom) {
+ throw new OutOfMemoryError("Cannot allocate " + arenaSize + " bytes: " + oom.getMessage()
+ + "; make sure your xmx and process size are set correctly.");
+ }
int maxMinAllocs = 1 << (arenaSizeLog2 - minAllocLog2);
headers = new byte[maxMinAllocs];
int allocLog2Diff = maxAllocLog2 - minAllocLog2, freeListCount = allocLog2Diff + 1;