You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/10/03 19:57:49 UTC
hive git commit: HIVE-20657 : pre-allocate LLAP cache at init time
(Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master a06a3703b -> d9d431cfe
HIVE-20657 : pre-allocate LLAP cache at init time (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d9d431cf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d9d431cf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d9d431cf
Branch: refs/heads/master
Commit: d9d431cfe96cee3829cf8fbee45c9760f2367a7f
Parents: a06a370
Author: sergey <se...@apache.org>
Authored: Wed Oct 3 12:57:51 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Wed Oct 3 12:57:51 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 ++
.../hadoop/hive/llap/cache/BuddyAllocator.java | 19 ++++++++++++-------
.../hive/llap/cache/TestBuddyAllocator.java | 12 ++++++------
.../llap/cache/TestBuddyAllocatorForceEvict.java | 4 ++--
.../hive/llap/cache/TestOrcMetadataCache.java | 4 ++--
5 files changed, 24 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d9d431cf/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index d1e6631..a49f8af 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3897,6 +3897,8 @@ public class HiveConf extends Configuration {
"Maximum size for IO allocator or ORC low-level cache.", "hive.llap.io.cache.orc.size"),
LLAP_ALLOCATOR_DIRECT("hive.llap.io.allocator.direct", true,
"Whether ORC low-level cache should use direct allocation."),
+ LLAP_ALLOCATOR_PREALLOCATE("hive.llap.io.allocator.preallocate", true,
+ "Whether to preallocate the entire IO memory at init time."),
LLAP_ALLOCATOR_MAPPED("hive.llap.io.allocator.mmap", false,
"Whether ORC low-level cache should use memory mapped allocation (direct I/O). \n" +
"This is recommended to be used along-side NVDIMM (DAX) or NVMe flash storage."),
http://git-wip-us.apache.org/repos/asf/hive/blob/d9d431cf/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
index 5dee2f9..a5d20ee 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
@@ -101,7 +101,9 @@ public final class BuddyAllocator
getMaxTotalMemorySize(conf),
HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_DEFRAG_HEADROOM),
HiveConf.getVar(conf, ConfVars.LLAP_ALLOCATOR_MAPPED_PATH),
- mm, metrics, HiveConf.getVar(conf, ConfVars.LLAP_ALLOCATOR_DISCARD_METHOD));
+ mm, metrics, HiveConf.getVar(conf, ConfVars.LLAP_ALLOCATOR_DISCARD_METHOD),
+ HiveConf.getBoolVar(conf, ConfVars.LLAP_ALLOCATOR_PREALLOCATE)
+ );
}
private static boolean areAssertsEnabled() {
@@ -122,9 +124,10 @@ public final class BuddyAllocator
}
@VisibleForTesting
- public BuddyAllocator(boolean isDirectVal, boolean isMappedVal, int minAllocVal,
- int maxAllocVal, int arenaCount, long maxSizeVal, long defragHeadroom, String mapPath,
- MemoryManager memoryManager, LlapDaemonCacheMetrics metrics, String discardMethod) {
+ public BuddyAllocator(boolean isDirectVal, boolean isMappedVal, int minAllocVal, int maxAllocVal,
+ int arenaCount, long maxSizeVal, long defragHeadroom, String mapPath,
+ MemoryManager memoryManager, LlapDaemonCacheMetrics metrics, String discardMethod,
+ boolean doPreallocate) {
isDirect = isDirectVal;
isMapped = isMappedVal;
minAllocation = minAllocVal;
@@ -156,9 +159,11 @@ public final class BuddyAllocator
for (int i = 0; i < maxArenas; ++i) {
arenas[i] = new Arena();
}
- Arena firstArena = arenas[0];
- firstArena.init(0);
- allocatedArenas.set(1);
+ int initCount = doPreallocate && !isMapped ? maxArenas : 1;
+ for (int i = 0; i < initCount; ++i) {
+ arenas[i].init(i);
+ }
+ allocatedArenas.set(initCount);
this.memoryManager = memoryManager;
defragCounters = new AtomicLong[maxAllocLog2 - minAllocLog2 + 1];
for (int i = 0; i < defragCounters.length; ++i) {
http://git-wip-us.apache.org/repos/asf/hive/blob/d9d431cf/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
index 51577bd..bdaa12a 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
@@ -87,7 +87,7 @@ public class TestBuddyAllocator {
int min = 3, max = 8, maxAlloc = 1 << max;
BuddyAllocator a = new BuddyAllocator(isDirect, isMapped, 1 << min, maxAlloc, maxAlloc,
maxAlloc, 0, tmpDir, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"), null);
+ LlapDaemonCacheMetrics.create("test", "1"), null, true);
for (int i = max; i >= min; --i) {
allocSameSize(a, 1 << (max - i), i);
}
@@ -98,7 +98,7 @@ public class TestBuddyAllocator {
int max = 8, maxAlloc = 1 << max, allocLog2 = max - 1, arenaCount = 5;
BuddyAllocator a = new BuddyAllocator(isDirect, isMapped, 1 << 3, maxAlloc, maxAlloc,
maxAlloc * arenaCount, 0, tmpDir, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"), null);
+ LlapDaemonCacheMetrics.create("test", "1"), null, true);
allocSameSize(a, arenaCount * 2, allocLog2);
}
@@ -107,7 +107,7 @@ public class TestBuddyAllocator {
final int min = 3, max = 8, maxAlloc = 1 << max, allocsPerSize = 3;
final BuddyAllocator a = new BuddyAllocator(isDirect, isMapped, 1 << min, maxAlloc,
maxAlloc * 8, maxAlloc * 24, 0, tmpDir, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"), null);
+ LlapDaemonCacheMetrics.create("test", "1"), null, true);
ExecutorService executor = Executors.newFixedThreadPool(3);
final CountDownLatch cdlIn = new CountDownLatch(3), cdlOut = new CountDownLatch(1);
FutureTask<Void> upTask = new FutureTask<Void>(new Callable<Void>() {
@@ -152,7 +152,7 @@ public class TestBuddyAllocator {
final int min = 3, max = 4, maxAlloc = 1 << max, minAllocCount = 2048, threadCount = 4;
final BuddyAllocator a = new BuddyAllocator(isDirect, isMapped, 1 << min, maxAlloc, maxAlloc,
(1 << min) * minAllocCount, 0, tmpDir, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"), null);
+ LlapDaemonCacheMetrics.create("test", "1"), null, true);
ExecutorService executor = Executors.newFixedThreadPool(threadCount);
final CountDownLatch cdlIn = new CountDownLatch(threadCount), cdlOut = new CountDownLatch(1);
Callable<Void> testCallable = new Callable<Void>() {
@@ -183,7 +183,7 @@ public class TestBuddyAllocator {
public void testCachedirCreated() throws Exception {
int min = 3, max = 8, maxAlloc = 1 << max;
new BuddyAllocator(isDirect, isMapped, 1 << min, maxAlloc, maxAlloc, maxAlloc, 0, tmpDir + "/testifcreated",
- new DummyMemoryManager(), LlapDaemonCacheMetrics.create("test", "1"), null);
+ new DummyMemoryManager(), LlapDaemonCacheMetrics.create("test", "1"), null, false);
}
static void syncThreadStart(final CountDownLatch cdlIn, final CountDownLatch cdlOut) {
@@ -200,7 +200,7 @@ public class TestBuddyAllocator {
int min = 3, max = 8, maxAlloc = 1 << max, arenaSize = maxAlloc * arenaSizeMult;
BuddyAllocator a = new BuddyAllocator(isDirect, isMapped, 1 << min, maxAlloc, arenaSize,
arenaSize * arenaCount, 0, tmpDir, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"), null);
+ LlapDaemonCacheMetrics.create("test", "1"), null, true);
allocateUp(a, min, max, allocCount, true);
allocateDown(a, min, max, allocCount, true);
allocateDown(a, min, max, allocCount, false);
http://git-wip-us.apache.org/repos/asf/hive/blob/d9d431cf/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocatorForceEvict.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocatorForceEvict.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocatorForceEvict.java
index b3617a6..c9e41e6 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocatorForceEvict.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocatorForceEvict.java
@@ -228,7 +228,7 @@ public class TestBuddyAllocatorForceEvict {
cdlIn.await(); // Wait for all threads to be ready.
} catch (InterruptedException e) {
throw new RuntimeException(e);
- }
+ }
cdlOut.countDown(); // Release them at the same time.
for (int i = 0; i < allocTasks.length; ++i) {
try {
@@ -417,7 +417,7 @@ public class TestBuddyAllocatorForceEvict {
public static BuddyAllocator create(int max, int arenas, int total, boolean isShortcut,
boolean isBruteForceOnly) {
BuddyAllocator result = new BuddyAllocator(false, false, 8, max, arenas, total, 0,
- null, MM, METRICS, isBruteForceOnly ? "brute" : null);
+ null, MM, METRICS, isBruteForceOnly ? "brute" : null, true);
if (!isShortcut) {
result.disableDefragShortcutForTest();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d9d431cf/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
index aa9d6ed..ad6b90e 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
@@ -102,7 +102,7 @@ public class TestOrcMetadataCache {
final int MAX_ALLOC = 64;
LlapDaemonCacheMetrics metrics = LlapDaemonCacheMetrics.create("", "");
BuddyAllocator alloc = new BuddyAllocator(
- false, false, 8, MAX_ALLOC, 1, 4096, 0, null, mm, metrics, null);
+ false, false, 8, MAX_ALLOC, 1, 4096, 0, null, mm, metrics, null, true);
MetadataCache cache = new MetadataCache(alloc, mm, cp, true, metrics);
Object fileKey1 = new Object();
Random rdm = new Random();
@@ -163,7 +163,7 @@ public class TestOrcMetadataCache {
final int MAX_ALLOC = 64;
LlapDaemonCacheMetrics metrics = LlapDaemonCacheMetrics.create("", "");
BuddyAllocator alloc = new BuddyAllocator(
- false, false, 8, MAX_ALLOC, 1, 4096, 0, null, mm, metrics, null);
+ false, false, 8, MAX_ALLOC, 1, 4096, 0, null, mm, metrics, null, true);
MetadataCache cache = new MetadataCache(alloc, mm, cp, true, metrics);
DataCache.BooleanRef gotAllData = new DataCache.BooleanRef();
Object fileKey1 = new Object();