You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/02/06 01:48:51 UTC
svn commit: r1565026 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
Author: gunther
Date: Thu Feb 6 00:48:50 2014
New Revision: 1565026
URL: http://svn.apache.org/r1565026
Log:
HIVE-6320: Row-based ORC reader with PPD turned on dies on BufferUnderFlowException (Patch by Prasanth J, reviewed by Owen O'Malley)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1565026&r1=1565025&r2=1565026&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Thu Feb 6 00:48:50 2014
@@ -2529,15 +2529,22 @@ class RecordReaderImpl implements Record
types.get(column).getKind(), stream.getKind(), isCompressed,
hasNull[column]);
long start = indexes[column].getEntry(group).getPositions(posn);
+ final long nextGroupOffset;
+ if (group < includedRowGroups.length - 1) {
+ nextGroupOffset = indexes[column].getEntry(group + 1).getPositions(posn);
+ } else {
+ nextGroupOffset = length;
+ }
+
// figure out the worst case last location
- long end = (group == includedRowGroups.length - 1) ?
- length : Math.min(length,
- indexes[column].getEntry(group + 1)
- .getPositions(posn)
- + (isCompressed ?
- (OutStream.HEADER_SIZE
- + compressionSize) :
- WORST_UNCOMPRESSED_SLOP));
+
+ // if adjacent groups have the same compressed block offset then stretch the slop
+ // by factor of 2 to safely accommodate the next compression block.
+ // One for the current compression block and another for the next compression block.
+ final long slop = isCompressed ? 2 * (OutStream.HEADER_SIZE + compressionSize)
+ : WORST_UNCOMPRESSED_SLOP;
+ long end = (group == includedRowGroups.length - 1) ? length : Math.min(length,
+ nextGroupOffset + slop);
result.add(new DiskRange(offset + start, offset + end));
}
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java?rev=1565026&r1=1565025&r2=1565026&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestRecordReaderImpl.java Thu Feb 6 00:48:50 2014
@@ -539,8 +539,8 @@ public class TestRecordReaderImpl {
result = RecordReaderImpl.planReadPartialDataStreams(streams, indexes,
columns, rowGroups, true, encodings, types, 32768);
assertThat(result, is(diskRanges(0, 1000, 100, 1000,
- 400, 1000, 1000, 11000+32771,
- 11000, 21000+32771, 41000, 51000+32771)));
+ 400, 1000, 1000, 11000+(2*32771),
+ 11000, 21000+(2*32771), 41000, 100000)));
rowGroups = new boolean[]{false, false, false, false, false, true};
result = RecordReaderImpl.planReadPartialDataStreams(streams, indexes,