You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by bs...@apache.org on 2019/05/31 16:21:26 UTC
[hive] 01/02: Revert "Merge pull request #554 from
b-slim/HIVE-21332"
This is an automated email from the ASF dual-hosted git repository.
bslim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit cc647fb33e949def12cf5e1c529d7c1186c4520e
Author: Slim Bouguerra <bs...@apache.org>
AuthorDate: Fri May 31 09:14:04 2019 -0700
Revert "Merge pull request #554 from b-slim/HIVE-21332"
This reverts commit 38c20ba5f772e53d8c07fda40845d88facca3bbb, reversing
changes made to a94c0306f48215d96be3b842525c4341772d26f9.
---
.../hadoop/hive/llap/cache/LowLevelCacheImpl.java | 33 ++-------------
.../hive/llap/cache/LowLevelCachePolicy.java | 48 ----------------------
.../hive/llap/cache/LowLevelLrfuCachePolicy.java | 2 +-
.../hadoop/hive/llap/cache/MemoryManager.java | 33 ---------------
.../hive/llap/io/metadata/MetadataCache.java | 19 ++-------
5 files changed, 8 insertions(+), 127 deletions(-)
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
index 62d7e55..e012d7d 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
@@ -453,13 +453,11 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
public void debugDumpShort(StringBuilder sb) {
sb.append("\nORC cache state ");
int allLocked = 0, allUnlocked = 0, allEvicted = 0, allMoving = 0;
- long totalUsedSpace = 0;
for (Map.Entry<Object, FileCache<ConcurrentSkipListMap<Long, LlapDataBuffer>>> e :
cache.entrySet()) {
if (!e.getValue().incRef()) continue;
try {
int fileLocked = 0, fileUnlocked = 0, fileEvicted = 0, fileMoving = 0;
- long fileMemoryUsage = 0;
if (e.getValue().getCache().isEmpty()) continue;
List<LlapDataBuffer> lockedBufs = null;
if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
@@ -485,7 +483,6 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
++fileUnlocked;
}
} finally {
- fileMemoryUsage += e2.getValue().allocSize;
e2.getValue().decRef();
}
}
@@ -493,21 +490,8 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
allUnlocked += fileUnlocked;
allEvicted += fileEvicted;
allMoving += fileMoving;
- totalUsedSpace += fileMemoryUsage;
-
- sb.append("\n file "
- + e.getKey()
- + ": "
- + fileLocked
- + " locked, "
- + fileUnlocked
- + " unlocked, "
- + fileEvicted
- + " evicted, "
- + fileMoving
- + " being moved,"
- + fileMemoryUsage
- + " total used byte");
+ sb.append("\n file " + e.getKey() + ": " + fileLocked + " locked, " + fileUnlocked
+ + " unlocked, " + fileEvicted + " evicted, " + fileMoving + " being moved");
if (fileLocked > 0 && LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) {
LlapIoImpl.LOCKING_LOGGER.trace("locked-buffers: {}", lockedBufs);
}
@@ -515,16 +499,7 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
e.getValue().decRef();
}
}
- sb.append("\nORC cache summary: "
- + allLocked
- + " locked, "
- + allUnlocked
- + " unlocked, "
- + allEvicted
- + " evicted, "
- + allMoving
- + " being moved,"
- + totalUsedSpace
- + "total used space");
+ sb.append("\nORC cache summary: " + allLocked + " locked, " + allUnlocked + " unlocked, "
+ + allEvicted + " evicted, " + allMoving + " being moved");
}
}
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java
index aa5ad66..45829dd 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java
@@ -20,59 +20,11 @@ package org.apache.hadoop.hive.llap.cache;
import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority;
-/**
- * Actor managing the eviction requests.
- * Cache policy relies notifications from the actual {@link LowLevelCache} to keep track of buffer access.
- */
public interface LowLevelCachePolicy extends LlapIoDebugDump {
-
- /**
- * Signals to the policy the addition of a new page to the cache directory.
- *
- * @param buffer buffer to be cached
- * @param priority the priority of cached element
- */
void cache(LlapCacheableBuffer buffer, Priority priority);
-
- /**
- * Notifies the policy that this buffer is locked, thus take it out of the free list.
- * Note that this notification is a hint and can not be the source of truth about what can be evicted
- * currently the source of truth is the counter of reference to the buffer see {@link LlapCacheableBuffer#isLocked()}.
- *
- * @param buffer buffer to be locked.
- */
void notifyLock(LlapCacheableBuffer buffer);
-
- /**
- * Notifies the policy that a buffer is unlocked after been used. This notification signals to the policy that an
- * access to this page occurred thus can be used to track what page got a read request
- *
- * @param buffer buffer that just got unlocked
- */
void notifyUnlock(LlapCacheableBuffer buffer);
-
- /**
- * Signals to the policy that it has to evict some pages to make room incoming buffers.
- * Policy has to at least evict the amount requested.
- * Policy does not now about the shape of evicted buffers and only can reason about total size.
- * Not that is method will block until at least {@code memoryToReserve} bytes are evicted.
- *
- * @param memoryToReserve amount of bytes to be evicted
- * @return actual amount of evicted bytes.
- */
long evictSomeBlocks(long memoryToReserve);
-
- /**
- * Sets the eviction listener dispatcher.
- *
- * @param listener eviction listener actor
- */
void setEvictionListener(EvictionListener listener);
-
- /**
- * Signals to the policy to evict all the unlocked used buffers.
- *
- * @return amount (bytes) of memory evicted.
- */
long purge();
}
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
index 759819d..e6f293b 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
@@ -206,7 +206,7 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
LlapCacheableBuffer current = listTail;
oldTail = listTail;
while (current != null) {
- boolean canEvict = LlapCacheableBuffer.INVALIDATE_OK == current.invalidate();
+ boolean canEvict = LlapCacheableBuffer.INVALIDATE_OK != current.invalidate();
current.indexInHeap = LlapCacheableBuffer.NOT_IN_CACHE;
if (canEvict) {
current = current.prev;
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
index 18da2c4..2190139 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
@@ -20,42 +20,9 @@ package org.apache.hadoop.hive.llap.cache;
import java.util.concurrent.atomic.AtomicBoolean;
-/**
- * Memory Manager is an accountant over a fixed size of memory.
- * It does is the following.
- * 1 - tracks the amount of memory (bytes) reserved out of a given maximum size to be shared between IO Threads.
- * 2 - when a reservation can not be fulfilled form the current free space it has to notify Evictor to free up some
- * space.
- * <p>
- * Note that it does not know about the actual shape, content or owners of memory, all it cares about is bytes usage.
- */
public interface MemoryManager {
-
- /**
- * Signals to the Memory manager the release of some memory bytes that are free to be used.
- *
- * @param memUsage amount of memory bytes that are released to be added to the ledger as free.
- */
void releaseMemory(long memUsage);
-
- /**
- * Sets the amount of bytes that the memory manager is managing.
- *
- * @param maxSize total amount of available bytes to be allocated.
- */
void updateMaxSize(long maxSize);
-
- /**
- * Reserves some amount of bytes within the managed pool of memory.
- * <p>
- * Callers expect that the memory manager will always fulfill the request by notifying the Evictor about how much
- * need to be evicted to accommodate the reserve request.
- * Note that this method will block until reservation is fulfilled.
- *
- * @param memoryToReserve Amount of bytes to reserve.
- * @param isStopped Caller state to indicate if it is still running while the memory manager is trying to
- * allocate the space.
- */
void reserveMemory(long memoryToReserve, AtomicBoolean isStopped);
/**
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java
index 8400fe9..2b3bca6 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java
@@ -127,22 +127,9 @@ public class MetadataCache implements LlapIoDebugDump, FileMetadataCache {
@Override
public void debugDumpShort(StringBuilder sb) {
- sb.append("\nMetadata cache state: ")
- .append(metadata.size())
- .append(" files and stripes, ")
- .append(metadata.values().parallelStream().mapToLong(value -> {
- if (value.getSingleLlapBuffer() != null) {
- return value.getSingleLlapBuffer().allocSize;
- }
- long sum = 0;
- for (LlapAllocatorBuffer llapMetadataBuffer : value.getMultipleLlapBuffers()) {
- sum += llapMetadataBuffer.allocSize;
- }
- return sum;
- }).sum())
- .append(" total used bytes, ")
- .append(estimateErrors.size())
- .append(" files w/ORC estimate");
+ // TODO: perhaps add counters for separate things and multiple buffer cases.
+ sb.append("\nMetadata cache state: ").append(metadata.size()).append(
+ " files and stripes, ").append(estimateErrors.size()).append(" files w/ORC estimate");
}
@Override