You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/01/21 03:15:42 UTC
[1/2] hive git commit: HIVE-12220 : LLAP: Usability issues with
hive.llap.io.cache.orc.size (Sergey Shelukhin,
reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/branch-2.0 bdb5c9acd -> 797b4f267
refs/heads/master a9f370160 -> ceb57733f
HIVE-12220 : LLAP: Usability issues with hive.llap.io.cache.orc.size (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ceb57733
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ceb57733
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ceb57733
Branch: refs/heads/master
Commit: ceb57733f5ce444eddde884b4087e5226c80419f
Parents: a9f3701
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Jan 20 18:09:43 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Jan 20 18:09:43 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 45 ++++++++++--
.../org/apache/hadoop/hive/conf/Validator.java | 65 +++++++++++++++++
.../hadoop/hive/llap/cache/BuddyAllocator.java | 28 +++++---
.../llap/cache/LowLevelCacheMemoryManager.java | 14 ++--
.../llap/cache/LowLevelLrfuCachePolicy.java | 10 ++-
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 3 +-
.../hive/llap/daemon/impl/LlapDaemon.java | 2 +-
.../llap/io/encoded/OrcEncodedDataReader.java | 2 +-
.../hive/llap/cache/TestBuddyAllocator.java | 32 +++------
.../llap/cache/TestLowLevelLrfuCachePolicy.java | 74 +++++++++-----------
10 files changed, 186 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ceb57733/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 32049eb..cf3280f 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -36,15 +36,14 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-
import javax.security.auth.login.LoginException;
-
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.hive.conf.Validator.PatternSet;
import org.apache.hadoop.hive.conf.Validator.RangeValidator;
import org.apache.hadoop.hive.conf.Validator.RatioValidator;
+import org.apache.hadoop.hive.conf.Validator.SizeValidator;
import org.apache.hadoop.hive.conf.Validator.StringSet;
import org.apache.hadoop.hive.conf.Validator.TimeValidator;
import org.apache.hadoop.hive.shims.Utils;
@@ -2346,18 +2345,18 @@ public class HiveConf extends Configuration {
"LLAP IO memory usage; 'cache' (the default) uses data and metadata cache with a\n" +
"custom off-heap allocator, 'allocator' uses the custom allocator without the caches,\n" +
"'none' doesn't use either (this mode may result in significant performance degradation)"),
- LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", 128 * 1024,
+ LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", "128Kb", new SizeValidator(),
"Minimum allocation possible from LLAP buddy allocator. Allocations below that are\n" +
"padded to minimum allocation. For ORC, should generally be the same as the expected\n" +
"compression buffer size, or next lowest power of 2. Must be a power of 2."),
- LLAP_ALLOCATOR_MAX_ALLOC("hive.llap.io.allocator.alloc.max", 16 * 1024 * 1024,
+ LLAP_ALLOCATOR_MAX_ALLOC("hive.llap.io.allocator.alloc.max", "16Mb", new SizeValidator(),
"Maximum allocation possible from LLAP buddy allocator. For ORC, should be as large as\n" +
"the largest expected ORC compression buffer size. Must be a power of 2."),
LLAP_ALLOCATOR_ARENA_COUNT("hive.llap.io.allocator.arena.count", 8,
"Arena count for LLAP low-level cache; cache will be allocated in the steps of\n" +
"(size/arena_count) bytes. This size must be <= 1Gb and >= max allocation; if it is\n" +
"not the case, an adjusted size will be used. Using powers of 2 is recommended."),
- LLAP_IO_MEMORY_MAX_SIZE("hive.llap.io.memory.size", 1024L * 1024 * 1024,
+ LLAP_IO_MEMORY_MAX_SIZE("hive.llap.io.memory.size", "1Gb", new SizeValidator(),
"Maximum size for IO allocator or ORC low-level cache.", "hive.llap.io.cache.orc.size"),
LLAP_ALLOCATOR_DIRECT("hive.llap.io.allocator.direct", true,
"Whether ORC low-level cache should use direct allocation."),
@@ -2937,6 +2936,14 @@ public class HiveConf extends Configuration {
setTimeVar(this, var, time, outUnit);
}
+ public static long getSizeVar(Configuration conf, ConfVars var) {
+ return toSizeBytes(getVar(conf, var));
+ }
+
+ public long getSizeVar(ConfVars var) {
+ return getSizeVar(this, var);
+ }
+
private static TimeUnit getDefaultTimeUnit(ConfVars var) {
TimeUnit inputUnit = null;
if (var.validator instanceof TimeValidator) {
@@ -2946,11 +2953,16 @@ public class HiveConf extends Configuration {
}
public static long toTime(String value, TimeUnit inputUnit, TimeUnit outUnit) {
- String[] parsed = parseTime(value.trim());
+ String[] parsed = parseNumberFollowedByUnit(value.trim());
return outUnit.convert(Long.valueOf(parsed[0].trim().trim()), unitFor(parsed[1].trim(), inputUnit));
}
- private static String[] parseTime(String value) {
+ public static long toSizeBytes(String value) {
+ String[] parsed = parseNumberFollowedByUnit(value.trim());
+ return Long.valueOf(parsed[0].trim()) * multiplierFor(parsed[1].trim());
+ }
+
+ private static String[] parseNumberFollowedByUnit(String value) {
char[] chars = value.toCharArray();
int i = 0;
for (; i < chars.length && (chars[i] == '-' || Character.isDigit(chars[i])); i++) {
@@ -2983,6 +2995,25 @@ public class HiveConf extends Configuration {
throw new IllegalArgumentException("Invalid time unit " + unit);
}
+
+ public static long multiplierFor(String unit) {
+ unit = unit.trim().toLowerCase();
+ if (unit.isEmpty() || unit.equals("b") || unit.equals("bytes")) {
+ return 1;
+ } else if (unit.equals("kb")) {
+ return 1024;
+ } else if (unit.equals("mb")) {
+ return 1024*1024;
+ } else if (unit.equals("gb")) {
+ return 1024*1024*1024;
+ } else if (unit.equals("tb")) {
+ return 1024*1024*1024*1024;
+ } else if (unit.equals("pb")) {
+ return 1024*1024*1024*1024*1024;
+ }
+ throw new IllegalArgumentException("Invalid size unit " + unit);
+ }
+
public static String stringFor(TimeUnit timeunit) {
switch (timeunit) {
case DAYS: return "day";
http://git-wip-us.apache.org/repos/asf/hive/blob/ceb57733/common/src/java/org/apache/hadoop/hive/conf/Validator.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/Validator.java b/common/src/java/org/apache/hadoop/hive/conf/Validator.java
index 04a305d..3fb09b9 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/Validator.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/Validator.java
@@ -281,4 +281,69 @@ public interface Validator {
return time + " " + HiveConf.stringFor(timeUnit);
}
}
+
+
+ class SizeValidator implements Validator {
+
+ private final Long min;
+ private final boolean minInclusive;
+
+ private final Long max;
+ private final boolean maxInclusive;
+
+ public SizeValidator() {
+ this(null, false, null, false);
+ }
+
+ public SizeValidator(Long min, boolean minInclusive, Long max, boolean maxInclusive) {
+ this.min = min;
+ this.minInclusive = minInclusive;
+ this.max = max;
+ this.maxInclusive = maxInclusive;
+ }
+
+ @Override
+ public String validate(String value) {
+ try {
+ long size = HiveConf.toSizeBytes(value);
+ if (min != null && (minInclusive ? size < min : size <= min)) {
+ return value + " is smaller than " + sizeString(min);
+ }
+ if (max != null && (maxInclusive ? size > max : size >= max)) {
+ return value + " is bigger than " + sizeString(max);
+ }
+ } catch (Exception e) {
+ return e.toString();
+ }
+ return null;
+ }
+
+ public String toDescription() {
+ String description =
+ "Expects a byte size value with unit (blank for bytes, kb, mb, gb, tb, pb)";
+ if (min != null && max != null) {
+ description += ".\nThe size should be in between " +
+ sizeString(min) + (minInclusive ? " (inclusive)" : " (exclusive)") + " and " +
+ sizeString(max) + (maxInclusive ? " (inclusive)" : " (exclusive)");
+ } else if (min != null) {
+ description += ".\nThe time should be bigger than " +
+ (minInclusive ? "or equal to " : "") + sizeString(min);
+ } else if (max != null) {
+ description += ".\nThe size should be smaller than " +
+ (maxInclusive ? "or equal to " : "") + sizeString(max);
+ }
+ return description;
+ }
+
+ private String sizeString(long size) {
+ final String[] units = { " bytes", "Kb", "Mb", "Gb", "Tb" };
+ long current = 1;
+ for (int i = 0; i < units.length && current > 0; ++i) {
+ long next = current << 10;
+ if ((size & (next - 1)) != 0) return (long)(size / current) + units[i];
+ current = next;
+ }
+ return current > 0 ? ((long)(size / current) + "Pb") : (size + units[0]);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ceb57733/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
index ab4df5d..824ff33 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
@@ -17,10 +17,10 @@
*/
package org.apache.hadoop.hive.llap.cache;
+import com.google.common.annotations.VisibleForTesting;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -44,13 +44,23 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
// We don't know the acceptable size for Java array, so we'll use 1Gb boundary.
// That is guaranteed to fit any maximum allocation.
private static final int MAX_ARENA_SIZE = 1024*1024*1024;
- public BuddyAllocator(Configuration conf, MemoryManager memoryManager,
- LlapDaemonCacheMetrics metrics) {
- isDirect = HiveConf.getBoolVar(conf, ConfVars.LLAP_ALLOCATOR_DIRECT);
- minAllocation = HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC);
- maxAllocation = HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_MAX_ALLOC);
- int arenaCount = HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_ARENA_COUNT);
- long maxSizeVal = HiveConf.getLongVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
+
+
+ public BuddyAllocator(Configuration conf, MemoryManager mm, LlapDaemonCacheMetrics metrics) {
+ this(HiveConf.getBoolVar(conf, ConfVars.LLAP_ALLOCATOR_DIRECT),
+ (int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC),
+ (int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MAX_ALLOC),
+ HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_ARENA_COUNT),
+ HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE),
+ mm, metrics);
+ }
+
+ @VisibleForTesting
+ public BuddyAllocator(boolean isDirectVal, int minAllocVal, int maxAllocVal, int arenaCount,
+ long maxSizeVal, MemoryManager memoryManager, LlapDaemonCacheMetrics metrics) {
+ isDirect = isDirectVal;
+ minAllocation = minAllocVal;
+ maxAllocation = maxAllocVal;
int arenaSizeVal = (arenaCount == 0) ? MAX_ARENA_SIZE : (int)(maxSizeVal / arenaCount);
arenaSizeVal = Math.max(maxAllocation, Math.min(arenaSizeVal, MAX_ARENA_SIZE));
if (LlapIoImpl.LOG.isInfoEnabled()) {
@@ -60,7 +70,7 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
}
if (minAllocation < 8) {
- throw new AssertionError("Min allocation must be at least 8: " + minAllocation);
+ throw new AssertionError("Min allocation must be at least 8 bytes: " + minAllocation);
}
if (maxSizeVal < arenaSizeVal || maxAllocation < minAllocation) {
throw new AssertionError("Inconsistent sizes of cache, arena and allocations: "
http://git-wip-us.apache.org/repos/asf/hive/blob/ceb57733/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
index 992da8e..1cfe2bc 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.hive.llap.cache;
+import com.google.common.annotations.VisibleForTesting;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -37,9 +37,15 @@ public class LowLevelCacheMemoryManager implements MemoryManager {
private final LlapDaemonCacheMetrics metrics;
private long maxSize;
- public LowLevelCacheMemoryManager(Configuration conf, LowLevelCachePolicy evictor,
- LlapDaemonCacheMetrics metrics) {
- this.maxSize = HiveConf.getLongVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
+ public LowLevelCacheMemoryManager(
+ Configuration conf, LowLevelCachePolicy evictor, LlapDaemonCacheMetrics metrics) {
+ this(HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE), evictor, metrics);
+ }
+
+ @VisibleForTesting
+ public LowLevelCacheMemoryManager(
+ long maxSize, LowLevelCachePolicy evictor, LlapDaemonCacheMetrics metrics) {
+ this.maxSize = maxSize;
this.evictor = evictor;
this.usedMemory = new AtomicLong(0);
this.metrics = metrics;
http://git-wip-us.apache.org/repos/asf/hive/blob/ceb57733/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
index 84910d7..6f52b86 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.hive.llap.cache;
+import com.google.common.annotations.VisibleForTesting;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
-
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -67,8 +67,12 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
private LlapOomDebugDump parentDebugDump;
public LowLevelLrfuCachePolicy(Configuration conf) {
- long maxSize = HiveConf.getLongVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
- int minBufferSize = HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC);
+ this((int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC),
+ HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE), conf);
+ }
+
+ @VisibleForTesting
+ public LowLevelLrfuCachePolicy(int minBufferSize, long maxSize, Configuration conf) {
lambda = HiveConf.getFloatVar(conf, HiveConf.ConfVars.LLAP_LRFU_LAMBDA);
int maxBuffers = (int)Math.ceil((maxSize * 1.0) / minBufferSize);
int maxHeapSize = -1;
http://git-wip-us.apache.org/repos/asf/hive/blob/ceb57733/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index d6e1a6e..358fde9 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -172,7 +172,8 @@ public class LlapServiceDriver {
}
if (options.getCache() != -1) {
- conf.setLong(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname, options.getCache());
+ conf.set(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname,
+ Long.toString(options.getCache()));
}
if (options.getXmx() != -1) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ceb57733/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index c5759d6..110fb5a 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -311,7 +311,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
long executorMemoryBytes = HiveConf.getIntVar(
daemonConf, ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB) * 1024l * 1024l;
- long ioMemoryBytes = HiveConf.getLongVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
+ long ioMemoryBytes = HiveConf.getSizeVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
boolean isDirectCache = HiveConf.getBoolVar(daemonConf, ConfVars.LLAP_ALLOCATOR_DIRECT);
boolean llapIoEnabled = HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.LLAP_IO_ENABLED);
llapDaemon = new LlapDaemon(daemonConf, numExecutors, executorMemoryBytes, llapIoEnabled,
http://git-wip-us.apache.org/repos/asf/hive/blob/ceb57733/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 3ddfc29..1f1aac7 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -513,7 +513,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
private void validateFileMetadata() throws IOException {
if (fileMetadata.getCompressionKind() == CompressionKind.NONE) return;
int bufferSize = fileMetadata.getCompressionBufferSize();
- int minAllocSize = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_ALLOCATOR_MIN_ALLOC);
+ long minAllocSize = HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC);
if (bufferSize < minAllocSize) {
LOG.warn("ORC compression buffer size (" + bufferSize + ") is smaller than LLAP low-level "
+ "cache minimum allocation size (" + minAllocSize + "). Decrease the value for "
http://git-wip-us.apache.org/repos/asf/hive/blob/ceb57733/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
index fc014a7..7b04103 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
@@ -78,9 +78,8 @@ public class TestBuddyAllocator {
@Test
public void testSameSizes() throws Exception {
int min = 3, max = 8, maxAlloc = 1 << max;
- Configuration conf = createConf(1 << min, maxAlloc, maxAlloc, maxAlloc);
- BuddyAllocator a = new BuddyAllocator(conf, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"));
+ BuddyAllocator a = new BuddyAllocator(false, 1 << min, maxAlloc, maxAlloc, maxAlloc,
+ new DummyMemoryManager(), LlapDaemonCacheMetrics.create("test", "1"));
for (int i = max; i >= min; --i) {
allocSameSize(a, 1 << (max - i), i);
}
@@ -89,18 +88,16 @@ public class TestBuddyAllocator {
@Test
public void testMultipleArenas() throws Exception {
int max = 8, maxAlloc = 1 << max, allocLog2 = max - 1, arenaCount = 5;
- Configuration conf = createConf(1 << 3, maxAlloc, maxAlloc, maxAlloc * arenaCount);
- BuddyAllocator a = new BuddyAllocator(conf, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"));
+ BuddyAllocator a = new BuddyAllocator(false, 1 << 3, maxAlloc, maxAlloc, maxAlloc * arenaCount,
+ new DummyMemoryManager(), LlapDaemonCacheMetrics.create("test", "1"));
allocSameSize(a, arenaCount * 2, allocLog2);
}
@Test
public void testMTT() {
final int min = 3, max = 8, maxAlloc = 1 << max, allocsPerSize = 3;
- Configuration conf = createConf(1 << min, maxAlloc, maxAlloc * 8, maxAlloc * 24);
- final BuddyAllocator a = new BuddyAllocator(conf, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"));
+ final BuddyAllocator a = new BuddyAllocator(false, 1 << min, maxAlloc, maxAlloc * 8,
+ maxAlloc * 24, new DummyMemoryManager(), LlapDaemonCacheMetrics.create("test", "1"));
ExecutorService executor = Executors.newFixedThreadPool(3);
final CountDownLatch cdlIn = new CountDownLatch(3), cdlOut = new CountDownLatch(1);
FutureTask<Void> upTask = new FutureTask<Void>(new Callable<Void>() {
@@ -143,8 +140,8 @@ public class TestBuddyAllocator {
@Test
public void testMTTArenas() {
final int min = 3, max = 4, maxAlloc = 1 << max, minAllocCount = 2048, threadCount = 4;
- Configuration conf = createConf(1 << min, maxAlloc, maxAlloc, (1 << min) * minAllocCount);
- final BuddyAllocator a = new BuddyAllocator(conf, new DummyMemoryManager(),
+ final BuddyAllocator a = new BuddyAllocator(false, 1 << min, maxAlloc, maxAlloc,
+ (1 << min) * minAllocCount, new DummyMemoryManager(),
LlapDaemonCacheMetrics.create("test", "1"));
ExecutorService executor = Executors.newFixedThreadPool(threadCount);
final CountDownLatch cdlIn = new CountDownLatch(threadCount), cdlOut = new CountDownLatch(1);
@@ -183,8 +180,8 @@ public class TestBuddyAllocator {
private void testVariableSizeInternal(
int allocCount, int arenaSizeMult, int arenaCount) throws Exception {
int min = 3, max = 8, maxAlloc = 1 << max, arenaSize = maxAlloc * arenaSizeMult;
- Configuration conf = createConf(1 << min, maxAlloc, arenaSize, arenaSize * arenaCount);
- BuddyAllocator a = new BuddyAllocator(conf, new DummyMemoryManager(),
+ BuddyAllocator a = new BuddyAllocator(false, 1 << min, maxAlloc, arenaSize,
+ arenaSize * arenaCount, new DummyMemoryManager(),
LlapDaemonCacheMetrics.create("test", "1"));
allocateUp(a, min, max, allocCount, true);
allocateDown(a, min, max, allocCount, true);
@@ -279,13 +276,4 @@ public class TestBuddyAllocator {
a.deallocate(mem);
}
}
-
- private Configuration createConf(int min, int max, int arena, int total) {
- Configuration conf = new Configuration();
- conf.setInt(ConfVars.LLAP_ALLOCATOR_MIN_ALLOC.varname, min);
- conf.setInt(ConfVars.LLAP_ALLOCATOR_MAX_ALLOC.varname, max);
- conf.setInt(ConfVars.LLAP_ALLOCATOR_ARENA_COUNT.varname, total/arena);
- conf.setLong(ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname, total);
- return conf;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ceb57733/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
index 46e9547..616c040 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
@@ -17,30 +17,34 @@
*/
package org.apache.hadoop.hive.llap.cache;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
-
-import org.mockito.stubbing.Answer;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.mockito.invocation.InvocationOnMock;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
import org.junit.Assume;
import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestLowLevelLrfuCachePolicy {
private static final Logger LOG = LoggerFactory.getLogger(TestLowLevelLrfuCachePolicy.class);
@@ -49,13 +53,15 @@ public class TestLowLevelLrfuCachePolicy {
public void testRegression_HIVE_12178() throws Exception {
LOG.info("Testing wrong list status after eviction");
EvictionTracker et = new EvictionTracker();
- int memSize = 2, lambda = 1; // Set lambda to 1 so the heap size becomes 1 (LRU).
- Configuration conf = createConf(1, memSize, (double)lambda);
- final LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(conf);
+ int memSize = 2;
+ Configuration conf = new Configuration();
+ // Set lambda to 1 so the heap size becomes 1 (LRU).
+ conf.setDouble(HiveConf.ConfVars.LLAP_LRFU_LAMBDA.varname, 1.0f);
+ final LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(1, memSize, conf);
Field f = LowLevelLrfuCachePolicy.class.getDeclaredField("listLock");
f.setAccessible(true);
ReentrantLock listLock = (ReentrantLock)f.get(lrfu);
- LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(conf, lrfu,
+ LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(memSize, lrfu,
LlapDaemonCacheMetrics.create("test", "1"));
lrfu.setEvictionListener(et);
final LlapDataBuffer buffer1 = LowLevelCacheImpl.allocateFake();
@@ -118,12 +124,12 @@ public class TestLowLevelLrfuCachePolicy {
int heapSize = 4;
LOG.info("Testing lambda 0 (LFU)");
Random rdm = new Random(1234);
- Configuration conf = createConf(1, heapSize);
+ Configuration conf = new Configuration();
ArrayList<LlapDataBuffer> inserted = new ArrayList<LlapDataBuffer>(heapSize);
conf.setFloat(HiveConf.ConfVars.LLAP_LRFU_LAMBDA.varname, 0.0f);
EvictionTracker et = new EvictionTracker();
- LowLevelLrfuCachePolicy lfu = new LowLevelLrfuCachePolicy(conf);
- LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(conf, lfu,
+ LowLevelLrfuCachePolicy lfu = new LowLevelLrfuCachePolicy(1, heapSize, conf);
+ LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(heapSize, lfu,
LlapDaemonCacheMetrics.create("test", "1"));
lfu.setEvictionListener(et);
for (int i = 0; i < heapSize; ++i) {
@@ -143,31 +149,17 @@ public class TestLowLevelLrfuCachePolicy {
verifyOrder(mm, lfu, et, inserted, null);
}
- private Configuration createConf(int min, int heapSize, Double lambda) {
- Configuration conf = new Configuration();
- conf.setInt(HiveConf.ConfVars.LLAP_ALLOCATOR_MIN_ALLOC.varname, min);
- conf.setInt(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname, heapSize);
- if (lambda != null) {
- conf.setDouble(HiveConf.ConfVars.LLAP_LRFU_LAMBDA.varname, lambda.doubleValue());
- }
- return conf;
- }
-
- private Configuration createConf(int min, int heapSize) {
- return createConf(min, heapSize, null);
- }
-
@Test
public void testLruExtreme() {
int heapSize = 4;
LOG.info("Testing lambda 1 (LRU)");
Random rdm = new Random(1234);
- Configuration conf = createConf(1, heapSize);
+ Configuration conf = new Configuration();
ArrayList<LlapDataBuffer> inserted = new ArrayList<LlapDataBuffer>(heapSize);
conf.setFloat(HiveConf.ConfVars.LLAP_LRFU_LAMBDA.varname, 1.0f);
EvictionTracker et = new EvictionTracker();
- LowLevelLrfuCachePolicy lru = new LowLevelLrfuCachePolicy(conf);
- LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(conf, lru,
+ LowLevelLrfuCachePolicy lru = new LowLevelLrfuCachePolicy(1, heapSize, conf);
+ LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(heapSize, lru,
LlapDaemonCacheMetrics.create("test", "1"));
lru.setEvictionListener(et);
for (int i = 0; i < heapSize; ++i) {
@@ -192,9 +184,9 @@ public class TestLowLevelLrfuCachePolicy {
LOG.info("Testing deadlock resolution");
ArrayList<LlapDataBuffer> inserted = new ArrayList<LlapDataBuffer>(heapSize);
EvictionTracker et = new EvictionTracker();
- Configuration conf = createConf(1, heapSize);
- LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(conf);
- LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(conf, lrfu,
+ Configuration conf = new Configuration();
+ LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(1, heapSize, conf);
+ LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(heapSize, lrfu,
LlapDaemonCacheMetrics.create("test", "1"));
lrfu.setEvictionListener(et);
for (int i = 0; i < heapSize; ++i) {
@@ -267,12 +259,12 @@ public class TestLowLevelLrfuCachePolicy {
private void testHeapSize(int heapSize) {
LOG.info("Testing heap size " + heapSize);
Random rdm = new Random(1234);
- Configuration conf = createConf(1, heapSize);
+ Configuration conf = new Configuration();
conf.setFloat(HiveConf.ConfVars.LLAP_LRFU_LAMBDA.varname, 0.2f); // very small heap, 14 elements
EvictionTracker et = new EvictionTracker();
- LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(conf);
+ LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(1, heapSize, conf);
MetricsMock m = createMetricsMock();
- LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(conf, lrfu, m.metricsMock);
+ LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(heapSize, lrfu, m.metricsMock);
lrfu.setEvictionListener(et);
// Insert the number of elements plus 2, to trigger 2 evictions.
int toEvict = 2;
[2/2] hive git commit: HIVE-12220 : LLAP: Usability issues with
hive.llap.io.cache.orc.size (Sergey Shelukhin,
reviewed by Prasanth Jayachandran)
Posted by se...@apache.org.
HIVE-12220 : LLAP: Usability issues with hive.llap.io.cache.orc.size (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/797b4f26
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/797b4f26
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/797b4f26
Branch: refs/heads/branch-2.0
Commit: 797b4f2672e9ab4d65a04ed446ce11158f1fb7a4
Parents: bdb5c9a
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Jan 20 18:09:43 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Jan 20 18:10:00 2016 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 45 ++++++++++--
.../org/apache/hadoop/hive/conf/Validator.java | 65 +++++++++++++++++
.../hadoop/hive/llap/cache/BuddyAllocator.java | 28 +++++---
.../llap/cache/LowLevelCacheMemoryManager.java | 14 ++--
.../llap/cache/LowLevelLrfuCachePolicy.java | 10 ++-
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 3 +-
.../hive/llap/daemon/impl/LlapDaemon.java | 2 +-
.../llap/io/encoded/OrcEncodedDataReader.java | 2 +-
.../hive/llap/cache/TestBuddyAllocator.java | 32 +++------
.../llap/cache/TestLowLevelLrfuCachePolicy.java | 74 +++++++++-----------
10 files changed, 186 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/797b4f26/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 38da64a..33b3f3f 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -36,15 +36,14 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-
import javax.security.auth.login.LoginException;
-
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.hive.conf.Validator.PatternSet;
import org.apache.hadoop.hive.conf.Validator.RangeValidator;
import org.apache.hadoop.hive.conf.Validator.RatioValidator;
+import org.apache.hadoop.hive.conf.Validator.SizeValidator;
import org.apache.hadoop.hive.conf.Validator.StringSet;
import org.apache.hadoop.hive.conf.Validator.TimeValidator;
import org.apache.hadoop.hive.shims.Utils;
@@ -2335,18 +2334,18 @@ public class HiveConf extends Configuration {
"LLAP IO memory usage; 'cache' (the default) uses data and metadata cache with a\n" +
"custom off-heap allocator, 'allocator' uses the custom allocator without the caches,\n" +
"'none' doesn't use either (this mode may result in significant performance degradation)"),
- LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", 128 * 1024,
+ LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", "128Kb", new SizeValidator(),
"Minimum allocation possible from LLAP buddy allocator. Allocations below that are\n" +
"padded to minimum allocation. For ORC, should generally be the same as the expected\n" +
"compression buffer size, or next lowest power of 2. Must be a power of 2."),
- LLAP_ALLOCATOR_MAX_ALLOC("hive.llap.io.allocator.alloc.max", 16 * 1024 * 1024,
+ LLAP_ALLOCATOR_MAX_ALLOC("hive.llap.io.allocator.alloc.max", "16Mb", new SizeValidator(),
"Maximum allocation possible from LLAP buddy allocator. For ORC, should be as large as\n" +
"the largest expected ORC compression buffer size. Must be a power of 2."),
LLAP_ALLOCATOR_ARENA_COUNT("hive.llap.io.allocator.arena.count", 8,
"Arena count for LLAP low-level cache; cache will be allocated in the steps of\n" +
"(size/arena_count) bytes. This size must be <= 1Gb and >= max allocation; if it is\n" +
"not the case, an adjusted size will be used. Using powers of 2 is recommended."),
- LLAP_IO_MEMORY_MAX_SIZE("hive.llap.io.memory.size", 1024L * 1024 * 1024,
+ LLAP_IO_MEMORY_MAX_SIZE("hive.llap.io.memory.size", "1Gb", new SizeValidator(),
"Maximum size for IO allocator or ORC low-level cache.", "hive.llap.io.cache.orc.size"),
LLAP_ALLOCATOR_DIRECT("hive.llap.io.allocator.direct", true,
"Whether ORC low-level cache should use direct allocation."),
@@ -2922,6 +2921,14 @@ public class HiveConf extends Configuration {
setTimeVar(this, var, time, outUnit);
}
+ public static long getSizeVar(Configuration conf, ConfVars var) {
+ return toSizeBytes(getVar(conf, var));
+ }
+
+ public long getSizeVar(ConfVars var) {
+ return getSizeVar(this, var);
+ }
+
private static TimeUnit getDefaultTimeUnit(ConfVars var) {
TimeUnit inputUnit = null;
if (var.validator instanceof TimeValidator) {
@@ -2931,11 +2938,16 @@ public class HiveConf extends Configuration {
}
public static long toTime(String value, TimeUnit inputUnit, TimeUnit outUnit) {
- String[] parsed = parseTime(value.trim());
+ String[] parsed = parseNumberFollowedByUnit(value.trim());
return outUnit.convert(Long.valueOf(parsed[0].trim().trim()), unitFor(parsed[1].trim(), inputUnit));
}
- private static String[] parseTime(String value) {
+ public static long toSizeBytes(String value) {
+ String[] parsed = parseNumberFollowedByUnit(value.trim());
+ return Long.valueOf(parsed[0].trim()) * multiplierFor(parsed[1].trim());
+ }
+
+ private static String[] parseNumberFollowedByUnit(String value) {
char[] chars = value.toCharArray();
int i = 0;
for (; i < chars.length && (chars[i] == '-' || Character.isDigit(chars[i])); i++) {
@@ -2968,6 +2980,25 @@ public class HiveConf extends Configuration {
throw new IllegalArgumentException("Invalid time unit " + unit);
}
+
+ public static long multiplierFor(String unit) {
+ unit = unit.trim().toLowerCase();
+ if (unit.isEmpty() || unit.equals("b") || unit.equals("bytes")) {
+ return 1;
+ } else if (unit.equals("kb")) {
+ return 1024;
+ } else if (unit.equals("mb")) {
+ return 1024*1024;
+ } else if (unit.equals("gb")) {
+ return 1024*1024*1024;
+ } else if (unit.equals("tb")) {
+ return 1024*1024*1024*1024;
+ } else if (unit.equals("pb")) {
+ return 1024*1024*1024*1024*1024;
+ }
+ throw new IllegalArgumentException("Invalid size unit " + unit);
+ }
+
public static String stringFor(TimeUnit timeunit) {
switch (timeunit) {
case DAYS: return "day";
http://git-wip-us.apache.org/repos/asf/hive/blob/797b4f26/common/src/java/org/apache/hadoop/hive/conf/Validator.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/Validator.java b/common/src/java/org/apache/hadoop/hive/conf/Validator.java
index 04a305d..3fb09b9 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/Validator.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/Validator.java
@@ -281,4 +281,69 @@ public interface Validator {
return time + " " + HiveConf.stringFor(timeUnit);
}
}
+
+
+ class SizeValidator implements Validator {
+
+ private final Long min;
+ private final boolean minInclusive;
+
+ private final Long max;
+ private final boolean maxInclusive;
+
+ public SizeValidator() {
+ this(null, false, null, false);
+ }
+
+ public SizeValidator(Long min, boolean minInclusive, Long max, boolean maxInclusive) {
+ this.min = min;
+ this.minInclusive = minInclusive;
+ this.max = max;
+ this.maxInclusive = maxInclusive;
+ }
+
+ @Override
+ public String validate(String value) {
+ try {
+ long size = HiveConf.toSizeBytes(value);
+ if (min != null && (minInclusive ? size < min : size <= min)) {
+ return value + " is smaller than " + sizeString(min);
+ }
+ if (max != null && (maxInclusive ? size > max : size >= max)) {
+ return value + " is bigger than " + sizeString(max);
+ }
+ } catch (Exception e) {
+ return e.toString();
+ }
+ return null;
+ }
+
+ public String toDescription() {
+ String description =
+ "Expects a byte size value with unit (blank for bytes, kb, mb, gb, tb, pb)";
+ if (min != null && max != null) {
+ description += ".\nThe size should be in between " +
+ sizeString(min) + (minInclusive ? " (inclusive)" : " (exclusive)") + " and " +
+ sizeString(max) + (maxInclusive ? " (inclusive)" : " (exclusive)");
+ } else if (min != null) {
+ description += ".\nThe time should be bigger than " +
+ (minInclusive ? "or equal to " : "") + sizeString(min);
+ } else if (max != null) {
+ description += ".\nThe size should be smaller than " +
+ (maxInclusive ? "or equal to " : "") + sizeString(max);
+ }
+ return description;
+ }
+
+ private String sizeString(long size) {
+ final String[] units = { " bytes", "Kb", "Mb", "Gb", "Tb" };
+ long current = 1;
+ for (int i = 0; i < units.length && current > 0; ++i) {
+ long next = current << 10;
+ if ((size & (next - 1)) != 0) return (long)(size / current) + units[i];
+ current = next;
+ }
+ return current > 0 ? ((long)(size / current) + "Pb") : (size + units[0]);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/797b4f26/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
index ab4df5d..824ff33 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
@@ -17,10 +17,10 @@
*/
package org.apache.hadoop.hive.llap.cache;
+import com.google.common.annotations.VisibleForTesting;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -44,13 +44,23 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
// We don't know the acceptable size for Java array, so we'll use 1Gb boundary.
// That is guaranteed to fit any maximum allocation.
private static final int MAX_ARENA_SIZE = 1024*1024*1024;
- public BuddyAllocator(Configuration conf, MemoryManager memoryManager,
- LlapDaemonCacheMetrics metrics) {
- isDirect = HiveConf.getBoolVar(conf, ConfVars.LLAP_ALLOCATOR_DIRECT);
- minAllocation = HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC);
- maxAllocation = HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_MAX_ALLOC);
- int arenaCount = HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_ARENA_COUNT);
- long maxSizeVal = HiveConf.getLongVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
+
+
+ public BuddyAllocator(Configuration conf, MemoryManager mm, LlapDaemonCacheMetrics metrics) {
+ this(HiveConf.getBoolVar(conf, ConfVars.LLAP_ALLOCATOR_DIRECT),
+ (int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC),
+ (int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MAX_ALLOC),
+ HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_ARENA_COUNT),
+ HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE),
+ mm, metrics);
+ }
+
+ @VisibleForTesting
+ public BuddyAllocator(boolean isDirectVal, int minAllocVal, int maxAllocVal, int arenaCount,
+ long maxSizeVal, MemoryManager memoryManager, LlapDaemonCacheMetrics metrics) {
+ isDirect = isDirectVal;
+ minAllocation = minAllocVal;
+ maxAllocation = maxAllocVal;
int arenaSizeVal = (arenaCount == 0) ? MAX_ARENA_SIZE : (int)(maxSizeVal / arenaCount);
arenaSizeVal = Math.max(maxAllocation, Math.min(arenaSizeVal, MAX_ARENA_SIZE));
if (LlapIoImpl.LOG.isInfoEnabled()) {
@@ -60,7 +70,7 @@ public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAlloca
}
if (minAllocation < 8) {
- throw new AssertionError("Min allocation must be at least 8: " + minAllocation);
+ throw new AssertionError("Min allocation must be at least 8 bytes: " + minAllocation);
}
if (maxSizeVal < arenaSizeVal || maxAllocation < minAllocation) {
throw new AssertionError("Inconsistent sizes of cache, arena and allocations: "
http://git-wip-us.apache.org/repos/asf/hive/blob/797b4f26/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
index 992da8e..1cfe2bc 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.hive.llap.cache;
+import com.google.common.annotations.VisibleForTesting;
import java.util.concurrent.atomic.AtomicLong;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -37,9 +37,15 @@ public class LowLevelCacheMemoryManager implements MemoryManager {
private final LlapDaemonCacheMetrics metrics;
private long maxSize;
- public LowLevelCacheMemoryManager(Configuration conf, LowLevelCachePolicy evictor,
- LlapDaemonCacheMetrics metrics) {
- this.maxSize = HiveConf.getLongVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
+ public LowLevelCacheMemoryManager(
+ Configuration conf, LowLevelCachePolicy evictor, LlapDaemonCacheMetrics metrics) {
+ this(HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE), evictor, metrics);
+ }
+
+ @VisibleForTesting
+ public LowLevelCacheMemoryManager(
+ long maxSize, LowLevelCachePolicy evictor, LlapDaemonCacheMetrics metrics) {
+ this.maxSize = maxSize;
this.evictor = evictor;
this.usedMemory = new AtomicLong(0);
this.metrics = metrics;
http://git-wip-us.apache.org/repos/asf/hive/blob/797b4f26/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
index 84910d7..6f52b86 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.hive.llap.cache;
+import com.google.common.annotations.VisibleForTesting;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
-
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -67,8 +67,12 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
private LlapOomDebugDump parentDebugDump;
public LowLevelLrfuCachePolicy(Configuration conf) {
- long maxSize = HiveConf.getLongVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
- int minBufferSize = HiveConf.getIntVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC);
+ this((int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC),
+ HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE), conf);
+ }
+
+ @VisibleForTesting
+ public LowLevelLrfuCachePolicy(int minBufferSize, long maxSize, Configuration conf) {
lambda = HiveConf.getFloatVar(conf, HiveConf.ConfVars.LLAP_LRFU_LAMBDA);
int maxBuffers = (int)Math.ceil((maxSize * 1.0) / minBufferSize);
int maxHeapSize = -1;
http://git-wip-us.apache.org/repos/asf/hive/blob/797b4f26/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index d6e1a6e..358fde9 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -172,7 +172,8 @@ public class LlapServiceDriver {
}
if (options.getCache() != -1) {
- conf.setLong(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname, options.getCache());
+ conf.set(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname,
+ Long.toString(options.getCache()));
}
if (options.getXmx() != -1) {
http://git-wip-us.apache.org/repos/asf/hive/blob/797b4f26/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index ddedfbf..4f1299d 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -298,7 +298,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
long executorMemoryBytes = HiveConf.getIntVar(
daemonConf, ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB) * 1024l * 1024l;
- long ioMemoryBytes = HiveConf.getLongVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
+ long ioMemoryBytes = HiveConf.getSizeVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
boolean isDirectCache = HiveConf.getBoolVar(daemonConf, ConfVars.LLAP_ALLOCATOR_DIRECT);
boolean llapIoEnabled = HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.LLAP_IO_ENABLED);
llapDaemon = new LlapDaemon(daemonConf, numExecutors, executorMemoryBytes, llapIoEnabled,
http://git-wip-us.apache.org/repos/asf/hive/blob/797b4f26/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 3ddfc29..1f1aac7 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -513,7 +513,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
private void validateFileMetadata() throws IOException {
if (fileMetadata.getCompressionKind() == CompressionKind.NONE) return;
int bufferSize = fileMetadata.getCompressionBufferSize();
- int minAllocSize = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_ALLOCATOR_MIN_ALLOC);
+ long minAllocSize = HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC);
if (bufferSize < minAllocSize) {
LOG.warn("ORC compression buffer size (" + bufferSize + ") is smaller than LLAP low-level "
+ "cache minimum allocation size (" + minAllocSize + "). Decrease the value for "
http://git-wip-us.apache.org/repos/asf/hive/blob/797b4f26/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
index fc014a7..7b04103 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestBuddyAllocator.java
@@ -78,9 +78,8 @@ public class TestBuddyAllocator {
@Test
public void testSameSizes() throws Exception {
int min = 3, max = 8, maxAlloc = 1 << max;
- Configuration conf = createConf(1 << min, maxAlloc, maxAlloc, maxAlloc);
- BuddyAllocator a = new BuddyAllocator(conf, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"));
+ BuddyAllocator a = new BuddyAllocator(false, 1 << min, maxAlloc, maxAlloc, maxAlloc,
+ new DummyMemoryManager(), LlapDaemonCacheMetrics.create("test", "1"));
for (int i = max; i >= min; --i) {
allocSameSize(a, 1 << (max - i), i);
}
@@ -89,18 +88,16 @@ public class TestBuddyAllocator {
@Test
public void testMultipleArenas() throws Exception {
int max = 8, maxAlloc = 1 << max, allocLog2 = max - 1, arenaCount = 5;
- Configuration conf = createConf(1 << 3, maxAlloc, maxAlloc, maxAlloc * arenaCount);
- BuddyAllocator a = new BuddyAllocator(conf, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"));
+ BuddyAllocator a = new BuddyAllocator(false, 1 << 3, maxAlloc, maxAlloc, maxAlloc * arenaCount,
+ new DummyMemoryManager(), LlapDaemonCacheMetrics.create("test", "1"));
allocSameSize(a, arenaCount * 2, allocLog2);
}
@Test
public void testMTT() {
final int min = 3, max = 8, maxAlloc = 1 << max, allocsPerSize = 3;
- Configuration conf = createConf(1 << min, maxAlloc, maxAlloc * 8, maxAlloc * 24);
- final BuddyAllocator a = new BuddyAllocator(conf, new DummyMemoryManager(),
- LlapDaemonCacheMetrics.create("test", "1"));
+ final BuddyAllocator a = new BuddyAllocator(false, 1 << min, maxAlloc, maxAlloc * 8,
+ maxAlloc * 24, new DummyMemoryManager(), LlapDaemonCacheMetrics.create("test", "1"));
ExecutorService executor = Executors.newFixedThreadPool(3);
final CountDownLatch cdlIn = new CountDownLatch(3), cdlOut = new CountDownLatch(1);
FutureTask<Void> upTask = new FutureTask<Void>(new Callable<Void>() {
@@ -143,8 +140,8 @@ public class TestBuddyAllocator {
@Test
public void testMTTArenas() {
final int min = 3, max = 4, maxAlloc = 1 << max, minAllocCount = 2048, threadCount = 4;
- Configuration conf = createConf(1 << min, maxAlloc, maxAlloc, (1 << min) * minAllocCount);
- final BuddyAllocator a = new BuddyAllocator(conf, new DummyMemoryManager(),
+ final BuddyAllocator a = new BuddyAllocator(false, 1 << min, maxAlloc, maxAlloc,
+ (1 << min) * minAllocCount, new DummyMemoryManager(),
LlapDaemonCacheMetrics.create("test", "1"));
ExecutorService executor = Executors.newFixedThreadPool(threadCount);
final CountDownLatch cdlIn = new CountDownLatch(threadCount), cdlOut = new CountDownLatch(1);
@@ -183,8 +180,8 @@ public class TestBuddyAllocator {
private void testVariableSizeInternal(
int allocCount, int arenaSizeMult, int arenaCount) throws Exception {
int min = 3, max = 8, maxAlloc = 1 << max, arenaSize = maxAlloc * arenaSizeMult;
- Configuration conf = createConf(1 << min, maxAlloc, arenaSize, arenaSize * arenaCount);
- BuddyAllocator a = new BuddyAllocator(conf, new DummyMemoryManager(),
+ BuddyAllocator a = new BuddyAllocator(false, 1 << min, maxAlloc, arenaSize,
+ arenaSize * arenaCount, new DummyMemoryManager(),
LlapDaemonCacheMetrics.create("test", "1"));
allocateUp(a, min, max, allocCount, true);
allocateDown(a, min, max, allocCount, true);
@@ -279,13 +276,4 @@ public class TestBuddyAllocator {
a.deallocate(mem);
}
}
-
- private Configuration createConf(int min, int max, int arena, int total) {
- Configuration conf = new Configuration();
- conf.setInt(ConfVars.LLAP_ALLOCATOR_MIN_ALLOC.varname, min);
- conf.setInt(ConfVars.LLAP_ALLOCATOR_MAX_ALLOC.varname, max);
- conf.setInt(ConfVars.LLAP_ALLOCATOR_ARENA_COUNT.varname, total/arena);
- conf.setLong(ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname, total);
- return conf;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/797b4f26/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
index 46e9547..616c040 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelLrfuCachePolicy.java
@@ -17,30 +17,34 @@
*/
package org.apache.hadoop.hive.llap.cache;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
-
-import org.mockito.stubbing.Answer;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.mockito.invocation.InvocationOnMock;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
import org.junit.Assume;
import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestLowLevelLrfuCachePolicy {
private static final Logger LOG = LoggerFactory.getLogger(TestLowLevelLrfuCachePolicy.class);
@@ -49,13 +53,15 @@ public class TestLowLevelLrfuCachePolicy {
public void testRegression_HIVE_12178() throws Exception {
LOG.info("Testing wrong list status after eviction");
EvictionTracker et = new EvictionTracker();
- int memSize = 2, lambda = 1; // Set lambda to 1 so the heap size becomes 1 (LRU).
- Configuration conf = createConf(1, memSize, (double)lambda);
- final LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(conf);
+ int memSize = 2;
+ Configuration conf = new Configuration();
+ // Set lambda to 1 so the heap size becomes 1 (LRU).
+ conf.setDouble(HiveConf.ConfVars.LLAP_LRFU_LAMBDA.varname, 1.0f);
+ final LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(1, memSize, conf);
Field f = LowLevelLrfuCachePolicy.class.getDeclaredField("listLock");
f.setAccessible(true);
ReentrantLock listLock = (ReentrantLock)f.get(lrfu);
- LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(conf, lrfu,
+ LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(memSize, lrfu,
LlapDaemonCacheMetrics.create("test", "1"));
lrfu.setEvictionListener(et);
final LlapDataBuffer buffer1 = LowLevelCacheImpl.allocateFake();
@@ -118,12 +124,12 @@ public class TestLowLevelLrfuCachePolicy {
int heapSize = 4;
LOG.info("Testing lambda 0 (LFU)");
Random rdm = new Random(1234);
- Configuration conf = createConf(1, heapSize);
+ Configuration conf = new Configuration();
ArrayList<LlapDataBuffer> inserted = new ArrayList<LlapDataBuffer>(heapSize);
conf.setFloat(HiveConf.ConfVars.LLAP_LRFU_LAMBDA.varname, 0.0f);
EvictionTracker et = new EvictionTracker();
- LowLevelLrfuCachePolicy lfu = new LowLevelLrfuCachePolicy(conf);
- LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(conf, lfu,
+ LowLevelLrfuCachePolicy lfu = new LowLevelLrfuCachePolicy(1, heapSize, conf);
+ LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(heapSize, lfu,
LlapDaemonCacheMetrics.create("test", "1"));
lfu.setEvictionListener(et);
for (int i = 0; i < heapSize; ++i) {
@@ -143,31 +149,17 @@ public class TestLowLevelLrfuCachePolicy {
verifyOrder(mm, lfu, et, inserted, null);
}
- private Configuration createConf(int min, int heapSize, Double lambda) {
- Configuration conf = new Configuration();
- conf.setInt(HiveConf.ConfVars.LLAP_ALLOCATOR_MIN_ALLOC.varname, min);
- conf.setInt(HiveConf.ConfVars.LLAP_IO_MEMORY_MAX_SIZE.varname, heapSize);
- if (lambda != null) {
- conf.setDouble(HiveConf.ConfVars.LLAP_LRFU_LAMBDA.varname, lambda.doubleValue());
- }
- return conf;
- }
-
- private Configuration createConf(int min, int heapSize) {
- return createConf(min, heapSize, null);
- }
-
@Test
public void testLruExtreme() {
int heapSize = 4;
LOG.info("Testing lambda 1 (LRU)");
Random rdm = new Random(1234);
- Configuration conf = createConf(1, heapSize);
+ Configuration conf = new Configuration();
ArrayList<LlapDataBuffer> inserted = new ArrayList<LlapDataBuffer>(heapSize);
conf.setFloat(HiveConf.ConfVars.LLAP_LRFU_LAMBDA.varname, 1.0f);
EvictionTracker et = new EvictionTracker();
- LowLevelLrfuCachePolicy lru = new LowLevelLrfuCachePolicy(conf);
- LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(conf, lru,
+ LowLevelLrfuCachePolicy lru = new LowLevelLrfuCachePolicy(1, heapSize, conf);
+ LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(heapSize, lru,
LlapDaemonCacheMetrics.create("test", "1"));
lru.setEvictionListener(et);
for (int i = 0; i < heapSize; ++i) {
@@ -192,9 +184,9 @@ public class TestLowLevelLrfuCachePolicy {
LOG.info("Testing deadlock resolution");
ArrayList<LlapDataBuffer> inserted = new ArrayList<LlapDataBuffer>(heapSize);
EvictionTracker et = new EvictionTracker();
- Configuration conf = createConf(1, heapSize);
- LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(conf);
- LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(conf, lrfu,
+ Configuration conf = new Configuration();
+ LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(1, heapSize, conf);
+ LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(heapSize, lrfu,
LlapDaemonCacheMetrics.create("test", "1"));
lrfu.setEvictionListener(et);
for (int i = 0; i < heapSize; ++i) {
@@ -267,12 +259,12 @@ public class TestLowLevelLrfuCachePolicy {
private void testHeapSize(int heapSize) {
LOG.info("Testing heap size " + heapSize);
Random rdm = new Random(1234);
- Configuration conf = createConf(1, heapSize);
+ Configuration conf = new Configuration();
conf.setFloat(HiveConf.ConfVars.LLAP_LRFU_LAMBDA.varname, 0.2f); // very small heap, 14 elements
EvictionTracker et = new EvictionTracker();
- LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(conf);
+ LowLevelLrfuCachePolicy lrfu = new LowLevelLrfuCachePolicy(1, heapSize, conf);
MetricsMock m = createMetricsMock();
- LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(conf, lrfu, m.metricsMock);
+ LowLevelCacheMemoryManager mm = new LowLevelCacheMemoryManager(heapSize, lrfu, m.metricsMock);
lrfu.setEvictionListener(et);
// Insert the number of elements plus 2, to trigger 2 evictions.
int toEvict = 2;