You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/07/02 01:52:19 UTC
hive git commit: HIVE-11156 : LLAP: Cache arena size rounding (Sergey
Shelukhin)
Repository: hive
Updated Branches:
refs/heads/llap 5dc48d596 -> 4339fd86c
HIVE-11156 : LLAP: Cache arena size rounding (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4339fd86
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4339fd86
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4339fd86
Branch: refs/heads/llap
Commit: 4339fd86cc243be0c4cc5580a496785c9b0753af
Parents: 5dc48d5
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Jul 1 16:52:10 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Jul 1 16:52:10 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/llap/cache/BuddyAllocator.java | 20 ++++++++++++++------
.../llap/cache/LowLevelCacheMemoryManager.java | 8 ++++++--
.../hadoop/hive/llap/cache/MemoryManager.java | 1 +
.../hive/llap/cache/TestBuddyAllocator.java | 4 ++++
.../hive/llap/cache/TestOrcMetadataCache.java | 4 ++++
5 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4339fd86/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
index 9f1472f..3631418 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
@@ -47,19 +47,19 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
minAllocation = HiveConf.getIntVar(conf, ConfVars.LLAP_ORC_CACHE_MIN_ALLOC);
maxAllocation = HiveConf.getIntVar(conf, ConfVars.LLAP_ORC_CACHE_MAX_ALLOC);
arenaSize = HiveConf.getIntVar(conf, ConfVars.LLAP_ORC_CACHE_ARENA_SIZE);
- maxSize = HiveConf.getLongVar(conf, ConfVars.LLAP_ORC_CACHE_MAX_SIZE);
+ long maxSizeVal = HiveConf.getLongVar(conf, ConfVars.LLAP_ORC_CACHE_MAX_SIZE);
if (LlapIoImpl.LOGL.isInfoEnabled()) {
LlapIoImpl.LOG.info("Buddy allocator with " + (isDirect ? "direct" : "byte")
+ " buffers; allocation sizes " + minAllocation + " - " + maxAllocation
- + ", arena size " + arenaSize + ". total size " + maxSize);
+ + ", arena size " + arenaSize + ". total size " + maxSizeVal);
}
if (minAllocation < 8) {
throw new AssertionError("Min allocation must be at least 8: " + minAllocation);
}
- if (maxSize < arenaSize || arenaSize < maxAllocation || maxAllocation < minAllocation) {
+ if (maxSizeVal < arenaSize || arenaSize < maxAllocation || maxAllocation < minAllocation) {
throw new AssertionError("Inconsistent sizes of cache, arena and allocations: "
- + minAllocation + ", " + maxAllocation + ", " + arenaSize + ", " + maxSize);
+ + minAllocation + ", " + maxAllocation + ", " + arenaSize + ", " + maxSizeVal);
}
if ((Integer.bitCount(minAllocation) != 1) || (Integer.bitCount(maxAllocation) != 1)
|| (Long.bitCount(arenaSize) != 1)) {
@@ -67,10 +67,18 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
throw new AssertionError("Allocation and arena sizes must be powers of two: "
+ minAllocation + ", " + maxAllocation + ", " + arenaSize);
}
- if ((maxSize % arenaSize) > 0 || (maxSize / arenaSize) > Integer.MAX_VALUE) {
+ if ((maxSizeVal % arenaSize) > 0) {
+ long oldMaxSize = maxSizeVal;
+ maxSizeVal = (maxSizeVal / arenaSize) * arenaSize;
+ LlapIoImpl.LOG.warn("Rounding cache size to " + maxSizeVal + " from " + oldMaxSize
+ + " to be divisible by arena size " + arenaSize);
+ }
+ if ((maxSizeVal / arenaSize) > Integer.MAX_VALUE) {
throw new AssertionError(
- "Cache size not consistent with arena size: " + arenaSize + "," + maxSize);
+ "Too many arenas needed to allocate the cache: " + arenaSize + "," + maxSizeVal);
}
+ maxSize = maxSizeVal;
+ memoryManager.updateMaxSize(maxSize);
minAllocLog2 = 31 - Integer.numberOfLeadingZeros(minAllocation);
maxAllocLog2 = 31 - Integer.numberOfLeadingZeros(maxAllocation);
arenaSizeLog2 = 63 - Long.numberOfLeadingZeros(arenaSize);
http://git-wip-us.apache.org/repos/asf/hive/blob/4339fd86/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
index 85f66f8..4a256ee 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
@@ -33,9 +33,9 @@ import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
*/
public class LowLevelCacheMemoryManager implements MemoryManager {
private final AtomicLong usedMemory;
- protected final long maxSize;
private final LowLevelCachePolicy evictor;
- private LlapDaemonCacheMetrics metrics;
+ private final LlapDaemonCacheMetrics metrics;
+ private long maxSize;
public LowLevelCacheMemoryManager(Configuration conf, LowLevelCachePolicy evictor,
LlapDaemonCacheMetrics metrics) {
@@ -104,4 +104,8 @@ public class LowLevelCacheMemoryManager implements MemoryManager {
return "cache state\n" + evictor.debugDumpForOom();
}
+ @Override
+ public void updateMaxSize(long maxSize) {
+ this.maxSize = maxSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4339fd86/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
index 8e167ec..e1b0cb4 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
@@ -21,4 +21,5 @@ package org.apache.hadoop.hive.llap.cache;
public interface MemoryManager extends LlapOomDebugDump {
boolean reserveMemory(long memoryToReserve, boolean waitForEviction);
void releaseMemory(long memUsage);
+ void updateMaxSize(long maxSize);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4339fd86/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
index d35edb7..7d265c1 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
@@ -54,6 +54,10 @@ public class TestBuddyAllocator {
public String debugDumpForOom() {
return "";
}
+
+ @Override
+ public void updateMaxSize(long maxSize) {
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/hive/blob/4339fd86/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
index 513aedf..3549fd9 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
@@ -74,6 +74,10 @@ public class TestOrcMetadataCache {
public String debugDumpForOom() {
return "";
}
+
+ @Override
+ public void updateMaxSize(long maxSize) {
+ }
}
@Test