You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/06/05 21:42:09 UTC
[2/2] hive git commit: HIVE-10946 : LLAP: recent optimization
introduced wrong assert to elevator causing test to faill (Sergey Shelukhin)
HIVE-10946 : LLAP: recent optimization introduced wrong assert to elevator causing test to faill (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8256e59a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8256e59a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8256e59a
Branch: refs/heads/llap
Commit: 8256e59a92424c58a0dfd75ed99bf9c1930f3332
Parents: da88a51
Author: Sergey Shelukhin <se...@apache.org>
Authored: Fri Jun 5 12:41:36 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Fri Jun 5 12:41:36 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java | 5 +++--
ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java | 3 +++
.../org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java | 3 +++
3 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8256e59a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 03a72a2..d6022b5 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -267,10 +267,11 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
+ stripe.getOffset() + ", " + stripe.getLength());
}
colRgs = readState[stripeIxMod];
- assert colRgs.length > 0;
// We assume that NO_RGS value is only set from SARG filter and for all columns;
// intermediate changes for individual columns will unset values in the array.
- if (colRgs[0] == SargApplier.READ_NO_RGS) continue;
+ // Skip this case for 0-column read. We could probably special-case it just like we do
+ // in EncodedReaderImpl, but for now it's not that important.
+ if (colRgs.length > 0 && colRgs[0] == SargApplier.READ_NO_RGS) continue;
// 6.1. Determine the columns to read (usually the same as requested).
if (cache == null || cols == null || cols.size() == colRgs.length) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8256e59a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
index 63eed9d..861f536 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.CancellationException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -406,6 +407,8 @@ public abstract class Operator<T extends OperatorDesc> implements Serializable,C
} else {
try {
os[i++] = f.get();
+ } catch (CancellationException ex) {
+ asyncEx = new InterruptedException("Future was canceled");
} catch (Throwable t) {
f.cancel(true);
asyncEx = t;
http://git-wip-us.apache.org/repos/asf/hive/blob/8256e59a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java
index 9c4d29e..e66955d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/EncodedReaderImpl.java
@@ -285,6 +285,8 @@ public class EncodedReaderImpl implements EncodedReader {
int colIx = stream.getColumn();
OrcProto.Stream.Kind streamKind = stream.getKind();
if (!included[colIx] || StreamName.getArea(streamKind) != StreamName.Area.DATA) {
+ // We have a stream for included column, but in future it might have no data streams.
+ // It's more like "has at least one column included that has an index stream".
hasIndexOnlyCols = hasIndexOnlyCols | included[colIx];
if (DebugUtils.isTraceOrcEnabled()) {
LOG.info("Skipping stream: " + streamKind + " at " + offset + ", " + length);
@@ -329,6 +331,7 @@ public class EncodedReaderImpl implements EncodedReader {
if (listToRead.get() == null) {
// No data to read for this stripe. Check if we have some included index-only columns.
+ // TODO: there may be a bug here. Could there be partial RG filtering on index-only column?
if (hasIndexOnlyCols && (includedRgs == null)) {
OrcEncodedColumnBatch ecb = ECB_POOL.take();
ecb.init(fileId, stripeIx, OrcEncodedColumnBatch.ALL_RGS, colRgs.length);