You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/03/29 19:44:10 UTC
[3/3] hive git commit: HIVE-16318 : LLAP cache: address some issues
in 2.2/2.3 (Sergey Shelukhin,
reviewed by Siddharth Seth and Prasanth Jayachandran)
HIVE-16318 : LLAP cache: address some issues in 2.2/2.3 (Sergey Shelukhin, reviewed by Siddharth Seth and Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ae3fef4d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ae3fef4d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ae3fef4d
Branch: refs/heads/branch-2.2
Commit: ae3fef4d0e1237eecd5aa71d771e290ba30811c1
Parents: f31f749
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Mar 29 12:09:48 2017 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Mar 29 12:44:00 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 8 ++++-
.../llap/cache/LowLevelCacheMemoryManager.java | 7 ----
.../llap/cache/LowLevelFifoCachePolicy.java | 2 +-
.../llap/cache/LowLevelLrfuCachePolicy.java | 6 ----
.../hive/llap/io/api/impl/LlapIoImpl.java | 37 ++++++++++++++++----
5 files changed, 39 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ae3fef4d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index de2e9d6..ce5ec43 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2868,13 +2868,19 @@ public class HiveConf extends Configuration {
"LLAP IO memory usage; 'cache' (the default) uses data and metadata cache with a\n" +
"custom off-heap allocator, 'none' doesn't use either (this mode may result in\n" +
"significant performance degradation)"),
- LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", "16Kb", new SizeValidator(),
+ LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", "256Kb", new SizeValidator(),
"Minimum allocation possible from LLAP buddy allocator. Allocations below that are\n" +
"padded to minimum allocation. For ORC, should generally be the same as the expected\n" +
"compression buffer size, or next lowest power of 2. Must be a power of 2."),
LLAP_ALLOCATOR_MAX_ALLOC("hive.llap.io.allocator.alloc.max", "16Mb", new SizeValidator(),
"Maximum allocation possible from LLAP buddy allocator. For ORC, should be as large as\n" +
"the largest expected ORC compression buffer size. Must be a power of 2."),
+ @Deprecated
+ LLAP_IO_METADATA_FRACTION("hive.llap.io.metadata.fraction", 0.1f,
+ "Temporary setting for on-heap metadata cache fraction of xmx, set to avoid potential\n" +
+ "heap problems on very large datasets when on-heap metadata cache takes over\n" +
+ "everything. -1 managed metadata and data together (which is more flexible). This\n" +
+ "setting will be removed (in effect become -1) once ORC metadata cache is moved off-heap."),
LLAP_ALLOCATOR_ARENA_COUNT("hive.llap.io.allocator.arena.count", 8,
"Arena count for LLAP low-level cache; cache will be allocated in the steps of\n" +
"(size/arena_count) bytes. This size must be <= 1Gb and >= max allocation; if it is\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/ae3fef4d/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
index 88bfa8b..91545ed 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
@@ -38,18 +38,11 @@ public class LowLevelCacheMemoryManager implements MemoryManager {
private long maxSize;
public LowLevelCacheMemoryManager(
- Configuration conf, LowLevelCachePolicy evictor, LlapDaemonCacheMetrics metrics) {
- this(HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE), evictor, metrics);
- }
-
- @VisibleForTesting
- public LowLevelCacheMemoryManager(
long maxSize, LowLevelCachePolicy evictor, LlapDaemonCacheMetrics metrics) {
this.maxSize = maxSize;
this.evictor = evictor;
this.usedMemory = new AtomicLong(0);
this.metrics = metrics;
- metrics.setCacheCapacityTotal(maxSize);
if (LlapIoImpl.LOG.isInfoEnabled()) {
LlapIoImpl.LOG.info("Memory manager initialized with max size {} and" +
" {} ability to evict blocks", maxSize, ((evictor == null) ? "no " : ""));
http://git-wip-us.apache.org/repos/asf/hive/blob/ae3fef4d/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
index 430a5f8..8006502 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
@@ -34,7 +34,7 @@ public class LowLevelFifoCachePolicy implements LowLevelCachePolicy {
private EvictionListener evictionListener;
private LlapOomDebugDump parentDebugDump;
- public LowLevelFifoCachePolicy(Configuration conf) {
+ public LowLevelFifoCachePolicy() {
LlapIoImpl.LOG.info("FIFO cache policy");
buffers = new LinkedList<LlapCacheableBuffer>();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ae3fef4d/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
index 4cd2c18..1bf5593 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
@@ -69,12 +69,6 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
private EvictionListener evictionListener;
private LlapOomDebugDump parentDebugDump;
- public LowLevelLrfuCachePolicy(Configuration conf) {
- this((int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC),
- HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE), conf);
- }
-
- @VisibleForTesting
public LowLevelLrfuCachePolicy(int minBufferSize, long maxSize, Configuration conf) {
lambda = HiveConf.getFloatVar(conf, HiveConf.ConfVars.LLAP_LRFU_LAMBDA);
int maxBuffers = (int)Math.ceil((maxSize * 1.0) / minBufferSize);
http://git-wip-us.apache.org/repos/asf/hive/blob/ae3fef4d/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
index 7c309a4..5a86114 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
@@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit;
import javax.management.ObjectName;
+import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
import org.apache.hadoop.hive.llap.daemon.impl.StatsRecordingThreadPool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,11 +122,32 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
if (useLowLevelCache) {
// Memory manager uses cache policy to trigger evictions, so create the policy first.
boolean useLrfu = HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_USE_LRFU);
- LowLevelCachePolicy cachePolicy =
- useLrfu ? new LowLevelLrfuCachePolicy(conf) : new LowLevelFifoCachePolicy(conf);
+ long totalMemorySize = HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
+ int minAllocSize = (int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC);
+ float metadataFraction = HiveConf.getFloatVar(conf, ConfVars.LLAP_IO_METADATA_FRACTION);
+ long metaMem = 0;
+ // TODO: this split a workaround until HIVE-15665.
+ // Technically we don't have to do it for on-heap data cache but we'd do for testing.
+ boolean isSplitCache = metadataFraction > 0f;
+ if (isSplitCache) {
+ metaMem = (long)(LlapDaemon.getTotalHeapSize() * metadataFraction);
+ }
+ LowLevelCachePolicy cachePolicy = useLrfu ? new LowLevelLrfuCachePolicy(
+ minAllocSize, totalMemorySize, conf) : new LowLevelFifoCachePolicy();
// Allocator uses memory manager to request memory, so create the manager next.
LowLevelCacheMemoryManager memManager = new LowLevelCacheMemoryManager(
- conf, cachePolicy, cacheMetrics);
+ totalMemorySize, cachePolicy, cacheMetrics);
+ LowLevelCachePolicy metaCachePolicy = null;
+ LowLevelCacheMemoryManager metaMemManager = null;
+ if (isSplitCache) {
+ metaCachePolicy = useLrfu ? new LowLevelLrfuCachePolicy(
+ minAllocSize, metaMem, conf) : new LowLevelFifoCachePolicy();
+ metaMemManager = new LowLevelCacheMemoryManager(metaMem, metaCachePolicy, cacheMetrics);
+ } else {
+ metaCachePolicy = cachePolicy;
+ metaMemManager = memManager;
+ }
+ cacheMetrics.setCacheCapacityTotal(totalMemorySize + metaMem);
// Cache uses allocator to allocate and deallocate, create allocator and then caches.
EvictionAwareAllocator allocator = new BuddyAllocator(conf, memManager, cacheMetrics);
this.allocator = allocator;
@@ -138,10 +160,13 @@ public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
serdeCache = serdeCacheImpl;
}
boolean useGapCache = HiveConf.getBoolVar(conf, ConfVars.LLAP_CACHE_ENABLE_ORC_GAP_CACHE);
- metadataCache = new OrcMetadataCache(memManager, cachePolicy, useGapCache);
+ metadataCache = new OrcMetadataCache(metaMemManager, metaCachePolicy, useGapCache);
// And finally cache policy uses cache to notify it of eviction. The cycle is complete!
- cachePolicy.setEvictionListener(new EvictionDispatcher(
- cache, serdeCache, metadataCache, allocator));
+ EvictionDispatcher e = new EvictionDispatcher(cache, serdeCache, metadataCache, allocator);
+ if (isSplitCache) {
+ metaCachePolicy.setEvictionListener(e);
+ }
+ cachePolicy.setEvictionListener(e);
cachePolicy.setParentDebugDumper(cacheImpl);
cacheImpl.startThreads(); // Start the cache threads.
bufferManager = cacheImpl; // Cache also serves as buffer manager.