You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/04/28 02:40:42 UTC
hive git commit: HIVE-13346 : LLAP doesn't update metadata priority
when reusing from cache;
some tweaks in LRFU policy (Sergey Shelukhin, reviewed by Prasanth
Jayachandran)
Repository: hive
Updated Branches:
refs/heads/branch-2.0 2f3134a12 -> 2d335ccb8
HIVE-13346 : LLAP doesn't update metadata priority when reusing from cache; some tweaks in LRFU policy (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Conflicts:
llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2d335ccb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2d335ccb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2d335ccb
Branch: refs/heads/branch-2.0
Commit: 2d335ccb8f1ab572fe6a22d070cba49227596066
Parents: 2f3134a
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Apr 27 16:54:05 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Apr 27 17:24:53 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 4 +-
data/conf/hive-site.xml | 2 +-
.../llap/cache/LowLevelLrfuCachePolicy.java | 9 +++--
.../hive/llap/io/metadata/OrcMetadataCache.java | 41 ++++++++++++--------
.../hive/llap/cache/TestOrcMetadataCache.java | 17 +++++++-
5 files changed, 48 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2d335ccb/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index ed3f539..af69b01 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2371,7 +2371,7 @@ public class HiveConf extends Configuration {
"LLAP IO memory usage; 'cache' (the default) uses data and metadata cache with a\n" +
"custom off-heap allocator, 'allocator' uses the custom allocator without the caches,\n" +
"'none' doesn't use either (this mode may result in significant performance degradation)"),
- LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", "128Kb", new SizeValidator(),
+ LLAP_ALLOCATOR_MIN_ALLOC("hive.llap.io.allocator.alloc.min", "16Kb", new SizeValidator(),
"Minimum allocation possible from LLAP buddy allocator. Allocations below that are\n" +
"padded to minimum allocation. For ORC, should generally be the same as the expected\n" +
"compression buffer size, or next lowest power of 2. Must be a power of 2."),
@@ -2386,7 +2386,7 @@ public class HiveConf extends Configuration {
"Maximum size for IO allocator or ORC low-level cache.", "hive.llap.io.cache.orc.size"),
LLAP_ALLOCATOR_DIRECT("hive.llap.io.allocator.direct", true,
"Whether ORC low-level cache should use direct allocation."),
- LLAP_USE_LRFU("hive.llap.io.use.lrfu", false,
+ LLAP_USE_LRFU("hive.llap.io.use.lrfu", true,
"Whether ORC low-level cache should use LRFU cache policy instead of default (FIFO)."),
LLAP_LRFU_LAMBDA("hive.llap.io.lrfu.lambda", 0.01f,
"Lambda for ORC low-level cache LRFU cache policy. Must be in [0, 1]. 0 makes LRFU\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/2d335ccb/data/conf/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml
index cbb5546..041b67d 100644
--- a/data/conf/hive-site.xml
+++ b/data/conf/hive-site.xml
@@ -288,7 +288,7 @@
<property>
<name>hive.llap.io.use.lrfu</name>
- <value>false</value>
+ <value>true</value>
</property>
http://git-wip-us.apache.org/repos/asf/hive/blob/2d335ccb/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
index 6f52b86..a6c704e 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
@@ -102,7 +102,11 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
buffer.priority = F0;
buffer.lastUpdate = time;
if (priority == Priority.HIGH) {
- buffer.priority *= 8; // this is arbitrary
+ // This is arbitrary. Note that metadata may come from a big scan and nuke all the data
+ // from some small frequently accessed tables, because it gets such a large priority boost
+ // to start with. Think of the multiplier as the number of accesses after which the data
+ // becomes more important than some random read-once metadata, in a pure-LFU scheme.
+ buffer.priority *= 3;
} else {
assert priority == Priority.NORMAL;
}
@@ -258,7 +262,7 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
heap[ix] = buffer;
}
- // Note: almost never called (unless buffers are very large or we evict a lot).
+ // Note: almost never called (unless buffers are very large or we evict a lot, or LFU).
private LlapCacheableBuffer evictFromHeapUnderLock(long time) {
while (true) {
if (heapSize == 0) return null;
@@ -289,7 +293,6 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
// down; therefore, we can update priorities of other blocks as we go for part of the heap -
// we correct any discrepancy w/the parent after expiring priority, and any block we expire
// the priority for already has lower priority than that of its children.
- // TODO: avoid expiring priorities if times are close? might be needlessly expensive.
int ix = buffer.indexInHeap;
double priority = buffer.priority;
while (true) {
http://git-wip-us.apache.org/repos/asf/hive/blob/2d335ccb/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
index 43c8fb3..72e563a 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.llap.io.metadata;
+import org.apache.hadoop.hive.llap.cache.LlapCacheableBuffer;
+
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
@@ -44,15 +46,7 @@ public class OrcMetadataCache {
memoryManager.reserveMemory(memUsage, false);
OrcFileMetadata val = metadata.putIfAbsent(metaData.getFileId(), metaData);
// See OrcFileMetadata; it is always unlocked, so we just "touch" it here to simulate use.
- if (val == null) {
- val = metaData;
- policy.cache(val, Priority.HIGH);
- } else {
- memoryManager.releaseMemory(memUsage);
- policy.notifyLock(val);
- }
- policy.notifyUnlock(val);
- return val;
+ return touchOnPut(metaData, val, memUsage);
}
public OrcStripeMetadata putStripeMetadata(OrcStripeMetadata metaData) {
@@ -60,23 +54,36 @@ public class OrcMetadataCache {
memoryManager.reserveMemory(memUsage, false);
OrcStripeMetadata val = stripeMetadata.putIfAbsent(metaData.getKey(), metaData);
// See OrcStripeMetadata; it is always unlocked, so we just "touch" it here to simulate use.
- if (val == null) {
- val = metaData;
- policy.cache(val, Priority.HIGH);
+ return touchOnPut(metaData, val, memUsage);
+ }
+
+ private <T extends LlapCacheableBuffer> T touchOnPut(T newVal, T oldVal, long memUsage) {
+ if (oldVal == null) {
+ oldVal = newVal;
+ policy.cache(oldVal, Priority.HIGH);
} else {
memoryManager.releaseMemory(memUsage);
- policy.notifyLock(val);
+ policy.notifyLock(oldVal);
}
- policy.notifyUnlock(val);
- return val;
+ policy.notifyUnlock(oldVal);
+ return oldVal;
}
public OrcStripeMetadata getStripeMetadata(OrcBatchKey stripeKey) throws IOException {
- return stripeMetadata.get(stripeKey);
+ return touchOnGet(stripeMetadata.get(stripeKey));
}
public OrcFileMetadata getFileMetadata(long fileId) throws IOException {
- return metadata.get(fileId);
+ return touchOnGet(metadata.get(fileId));
+ }
+
+
+ private <T extends LlapCacheableBuffer> T touchOnGet(T result) {
+ if (result != null) {
+ policy.notifyLock(result);
+ policy.notifyUnlock(result); // Never locked for eviction; Java object.
+ }
+ return result;
}
public void notifyEvicted(OrcFileMetadata buffer) {
http://git-wip-us.apache.org/repos/asf/hive/blob/2d335ccb/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
index 901e58a..3c854bb 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java
@@ -29,16 +29,18 @@ import org.junit.Test;
public class TestOrcMetadataCache {
private static class DummyCachePolicy implements LowLevelCachePolicy {
- public DummyCachePolicy() {
- }
+ int lockCount = 0, unlockCount = 0;
public void cache(LlapCacheableBuffer buffer, Priority pri) {
+ ++lockCount;
}
public void notifyLock(LlapCacheableBuffer buffer) {
+ ++lockCount;
}
public void notifyUnlock(LlapCacheableBuffer buffer) {
+ ++unlockCount;
}
public long evictSomeBlocks(long memoryToReserve) {
@@ -54,6 +56,11 @@ public class TestOrcMetadataCache {
public void setParentDebugDumper(LlapOomDebugDump dumper) {
}
+
+ public void verifyEquals(int i) {
+ assertEquals(i, lockCount);
+ assertEquals(i, unlockCount);
+ }
}
private static class DummyMemoryManager implements MemoryManager {
@@ -92,14 +99,19 @@ public class TestOrcMetadataCache {
OrcFileMetadata ofm1 = OrcFileMetadata.createDummy(1), ofm2 = OrcFileMetadata.createDummy(2);
assertSame(ofm1, cache.putFileMetadata(ofm1));
assertEquals(1, mm.allocs);
+ cp.verifyEquals(1);
assertSame(ofm2, cache.putFileMetadata(ofm2));
assertEquals(2, mm.allocs);
+ cp.verifyEquals(2);
assertSame(ofm1, cache.getFileMetadata(1));
assertSame(ofm2, cache.getFileMetadata(2));
+ cp.verifyEquals(4);
OrcFileMetadata ofm3 = OrcFileMetadata.createDummy(1);
assertSame(ofm1, cache.putFileMetadata(ofm3));
assertEquals(2, mm.allocs);
+ cp.verifyEquals(5);
assertSame(ofm1, cache.getFileMetadata(1));
+ cp.verifyEquals(6);
OrcStripeMetadata osm1 = OrcStripeMetadata.createDummy(1), osm2 = OrcStripeMetadata.createDummy(2);
assertSame(osm1, cache.putStripeMetadata(osm1));
@@ -112,5 +124,6 @@ public class TestOrcMetadataCache {
assertSame(osm1, cache.putStripeMetadata(osm3));
assertEquals(4, mm.allocs);
assertSame(osm1, cache.getStripeMetadata(osm3.getKey()));
+ cp.verifyEquals(12);
}
}