You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2013/08/08 23:55:25 UTC
svn commit: r1512062 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
Author: omalley
Date: Thu Aug 8 21:55:25 2013
New Revision: 1512062
URL: http://svn.apache.org/r1512062
Log:
HIVE-4990. ORC seeks fail with non-zero offset or column project (omalley)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1512062&r1=1512061&r2=1512062&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Thu Aug 8 21:55:25 2013
@@ -839,7 +839,9 @@ class RecordReaderImpl implements Record
void seek(PositionProvider[] index) throws IOException {
super.seek(index);
for(TreeReader kid: fields) {
- kid.seek(index);
+ if (kid != null) {
+ kid.seek(index);
+ }
}
}
@@ -885,7 +887,9 @@ class RecordReaderImpl implements Record
void skipRows(long items) throws IOException {
items = countNonNulls(items);
for(TreeReader field: fields) {
- field.skipRows(items);
+ if (field != null) {
+ field.skipRows(items);
+ }
}
}
}
@@ -1357,7 +1361,7 @@ class RecordReaderImpl implements Record
readStripe();
}
readRowIndex();
- rowInStripe = rowNumber - rowBaseInStripe;
+ rowInStripe = rowNumber - rowBaseInStripe - firstRow;
if (rowIndexStride != 0) {
long entry = rowInStripe / rowIndexStride;
seekToRowEntry((int) entry);
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java?rev=1512062&r1=1512061&r2=1512062&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java Thu Aug 8 21:55:25 2013
@@ -58,6 +58,7 @@ import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
@@ -905,6 +906,40 @@ public class TestOrcFile {
compareList(expectedList, actualList);
compareList(expected.list, (List) row.getFieldValue(10));
}
+ rows.close();
+ Iterator<StripeInformation> stripeIterator =
+ reader.getStripes().iterator();
+ long offsetOfStripe2 = 0;
+ long offsetOfStripe4 = 0;
+ long lastRowOfStripe2 = 0;
+ for(int i = 0; i < 5; ++i) {
+ StripeInformation stripe = stripeIterator.next();
+ if (i < 2) {
+ lastRowOfStripe2 += stripe.getNumberOfRows();
+ } else if (i == 2) {
+ offsetOfStripe2 = stripe.getOffset();
+ lastRowOfStripe2 += stripe.getNumberOfRows() - 1;
+ } else if (i == 4) {
+ offsetOfStripe4 = stripe.getOffset();
+ }
+ }
+ boolean[] columns = new boolean[reader.getStatistics().length];
+ columns[5] = true; // long colulmn
+ columns[9] = true; // text column
+ rows = reader.rows(offsetOfStripe2, offsetOfStripe4 - offsetOfStripe2,
+ columns);
+ rows.seekToRow(lastRowOfStripe2);
+ for(int i = 0; i < 2; ++i) {
+ row = (OrcStruct) rows.next(row);
+ BigRow expected = createRandomRow(intValues, doubleValues,
+ stringValues, byteValues, words,
+ (int) (lastRowOfStripe2 + i));
+
+ assertEquals(expected.long1.longValue(),
+ ((LongWritable) row.getFieldValue(4)).get());
+ assertEquals(expected.string1, row.getFieldValue(8));
+ }
+ rows.close();
}
private void compareInner(InnerStruct expect,