You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/06/18 10:08:21 UTC
hive git commit: HIVE-11033: BloomFilter index is not honored by ORC
reader (Prasanth Jayachandran reviewed by Gopal V)
Repository: hive
Updated Branches:
refs/heads/branch-1.2 749bbfcf6 -> 1537196a9
HIVE-11033: BloomFilter index is not honored by ORC reader (Prasanth Jayachandran reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1537196a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1537196a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1537196a
Branch: refs/heads/branch-1.2
Commit: 1537196a9394f61b5101a9455696e97b04448ec4
Parents: 749bbfc
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Thu Jun 18 01:08:05 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Thu Jun 18 01:08:05 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java | 4 ++++
.../org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java | 9 ++++-----
2 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1537196a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
index ba59b35..56df608 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
@@ -57,6 +57,10 @@ public class OrcUtils {
ObjectInspector inspector) {
int numFlattenedCols = getFlattenedColumnsCount(inspector);
boolean[] results = new boolean[numFlattenedCols];
+ if ("*".equals(selectedColumns)) {
+ Arrays.fill(results, true);
+ return results;
+ }
if (selectedColumns != null && !selectedColumns.isEmpty()) {
includeColumnsImpl(results, selectedColumns, allColumns, inspector);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1537196a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
index 58e19cb..beaf231 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
@@ -666,7 +666,6 @@ class RecordReaderImpl implements RecordReader {
private final List<PredicateLeaf> sargLeaves;
private final int[] filterColumns;
private final long rowIndexStride;
- private final OrcProto.BloomFilterIndex[] bloomFilterIndices;
// same as the above array, but indices are set to true
private final boolean[] sargColumns;
public SargApplier(SearchArgument sarg, String[] columnNames, long rowIndexStride,
@@ -674,7 +673,6 @@ class RecordReaderImpl implements RecordReader {
this.sarg = sarg;
sargLeaves = sarg.getLeaves();
filterColumns = mapSargColumns(sargLeaves, columnNames, 0);
- bloomFilterIndices = new OrcProto.BloomFilterIndex[types.size()];
this.rowIndexStride = rowIndexStride;
// included will not be null, row options will fill the array with trues if null
sargColumns = new boolean[includedCount];
@@ -694,7 +692,8 @@ class RecordReaderImpl implements RecordReader {
* @throws IOException
*/
public boolean[] pickRowGroups(
- StripeInformation stripe, OrcProto.RowIndex[] indexes) throws IOException {
+ StripeInformation stripe, OrcProto.RowIndex[] indexes,
+ OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException {
long rowsInStripe = stripe.getNumberOfRows();
int groupsInStripe = (int) ((rowsInStripe + rowIndexStride - 1) / rowIndexStride);
boolean[] result = new boolean[groupsInStripe]; // TODO: avoid alloc?
@@ -705,7 +704,7 @@ class RecordReaderImpl implements RecordReader {
OrcProto.ColumnStatistics stats =
indexes[filterColumns[pred]].getEntry(rowGroup).getStatistics();
OrcProto.BloomFilter bf = null;
- if (bloomFilterIndices[filterColumns[pred]] != null) {
+ if (bloomFilterIndices != null && bloomFilterIndices[filterColumns[pred]] != null) {
bf = bloomFilterIndices[filterColumns[pred]].getBloomFilter(rowGroup);
}
leafValues[pred] = evaluatePredicateProto(stats, sargLeaves.get(pred), bf);
@@ -749,7 +748,7 @@ class RecordReaderImpl implements RecordReader {
return null;
}
readRowIndex(currentStripe, included, sargApp.sargColumns);
- return sargApp.pickRowGroups(stripes.get(currentStripe), indexes);
+ return sargApp.pickRowGroups(stripes.get(currentStripe), indexes, bloomFilterIndices);
}
private void clearStreams() throws IOException {