You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/04/27 20:07:53 UTC
[5/8] hive git commit: HIVE-16547 : LLAP: may not unlock buffers in
some cases (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
HIVE-16547 : LLAP: may not unlock buffers in some cases (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f71d446d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f71d446d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f71d446d
Branch: refs/heads/branch-2.3
Commit: f71d446d4d3bcf77e3249e2a68cff7366c5a1184
Parents: 346f708
Author: sergey <se...@apache.org>
Authored: Thu Apr 27 11:52:07 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Thu Apr 27 12:44:10 2017 -0700
----------------------------------------------------------------------
.../ql/io/orc/encoded/EncodedReaderImpl.java | 24 ++++++++++++++++----
1 file changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f71d446d/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
index 326b4b6..5b2e9b5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java
@@ -416,8 +416,8 @@ class EncodedReaderImpl implements EncodedReader {
if (sctx.stripeLevelStream == null) {
sctx.stripeLevelStream = POOLS.csdPool.take();
// We will be using this for each RG while also sending RGs to processing.
- // To avoid buffers being unlocked, run refcount one ahead; we will not increase
- // it when building the last RG, so each RG processing will decref once, and the
+ // To avoid buffers being unlocked, run refcount one ahead; so each RG
+ // processing will decref once, and the
// last one will unlock the buffers.
sctx.stripeLevelStream.incRef();
// For stripe-level streams we don't need the extra refcount on the block.
@@ -430,9 +430,7 @@ class EncodedReaderImpl implements EncodedReader {
iter = lastCached;
}
}
- if (!isLastRg) {
- sctx.stripeLevelStream.incRef();
- }
+ sctx.stripeLevelStream.incRef();
cb = sctx.stripeLevelStream;
} else {
// This stream can be separated by RG using index. Let's do that.
@@ -479,6 +477,22 @@ class EncodedReaderImpl implements EncodedReader {
}
// Release the unreleased buffers. See class comment about refcounts.
+ for (int colIx = 0; colIx < colCtxs.length; ++colIx) {
+ ColumnReadContext ctx = colCtxs[colIx];
+ if (ctx == null) continue; // This column is not included.
+ for (int streamIx = 0; streamIx < ctx.streamCount; ++streamIx) {
+ StreamContext sctx = ctx.streams[streamIx];
+ if (sctx == null || sctx.stripeLevelStream == null) continue;
+ if (0 != sctx.stripeLevelStream.decRef()) continue;
+ for (MemoryBuffer buf : sctx.stripeLevelStream.getCacheBuffers()) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Unlocking {} at the end of processing", buf);
+ }
+ cacheWrapper.releaseBuffer(buf);
+ }
+ }
+ }
+
releaseInitialRefcounts(toRead.next);
// Release buffers as we are done with all the streams... also see toRelease comment.
if (toRelease != null) {