You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/04/27 20:07:55 UTC
[7/8] hive git commit: HIVE-16547 : LLAP: may not unlock buffers in
some cases (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
HIVE-16547 : LLAP: may not unlock buffers in some cases (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c84468e3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c84468e3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c84468e3
Branch: refs/heads/branch-2.2
Commit: c84468e3567261181df8fbbf86ac39fd2242ea71
Parents: 9bedccc
Author: sergey <se...@apache.org>
Authored: Thu Apr 27 11:52:07 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Apr 27 12:44:20 2017 -0700
----------------------------------------------------------------------
.../ql/io/orc/encoded/EncodedReaderImpl.java | 24 ++++++++++++++++----
1 file changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c84468e3/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index 9b31988..3e26d9a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -374,8 +374,8 @@ class EncodedReaderImpl implements EncodedReader {
if (sctx.stripeLevelStream == null) {
sctx.stripeLevelStream = POOLS.csdPool.take();
// We will be using this for each RG while also sending RGs to processing.
- // To avoid buffers being unlocked, run refcount one ahead; we will not increase
- // it when building the last RG, so each RG processing will decref once, and the
+ // To avoid buffers being unlocked, run refcount one ahead; so each RG
+ // processing will decref once, and the
// last one will unlock the buffers.
sctx.stripeLevelStream.incRef();
// For stripe-level streams we don't need the extra refcount on the block.
@@ -388,9 +388,7 @@ class EncodedReaderImpl implements EncodedReader {
iter = lastCached;
}
}
- if (!isLastRg) {
- sctx.stripeLevelStream.incRef();
- }
+ sctx.stripeLevelStream.incRef();
cb = sctx.stripeLevelStream;
} else {
// This stream can be separated by RG using index. Let's do that.
@@ -437,6 +435,22 @@ class EncodedReaderImpl implements EncodedReader {
}
// Release the unreleased buffers. See class comment about refcounts.
+ for (int colIx = 0; colIx < colCtxs.length; ++colIx) {
+ ColumnReadContext ctx = colCtxs[colIx];
+ if (ctx == null) continue; // This column is not included.
+ for (int streamIx = 0; streamIx < ctx.streamCount; ++streamIx) {
+ StreamContext sctx = ctx.streams[streamIx];
+ if (sctx == null || sctx.stripeLevelStream == null) continue;
+ if (0 != sctx.stripeLevelStream.decRef()) continue;
+ for (MemoryBuffer buf : sctx.stripeLevelStream.getCacheBuffers()) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Unlocking {} at the end of processing", buf);
+ }
+ cacheWrapper.releaseBuffer(buf);
+ }
+ }
+ }
+
releaseInitialRefcounts(toRead.next);
releaseCacheChunksIntoObjectPool(toRead.next);
}