You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/02/18 03:41:02 UTC
svn commit: r1660559 - in /hive/branches/llap:
llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/
llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/
ql/src/java/org/apache/hadoop/hive/ql/io/orc/ ql/src/te...
Author: prasanthj
Date: Wed Feb 18 02:41:02 2015
New Revision: 1660559
URL: http://svn.apache.org/r1660559
Log:
HIVE-9419p5: LLAP: ORC decoding of row-groups (Partial patch5)
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/StreamUtils.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/BinaryStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/BooleanStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/ByteStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/CharacterStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DateStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DecimalStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DoubleStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/FloatStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/IntStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/LongStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/ShortStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/StringStreamReader.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/TimestampStreamReader.java
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java
hive/branches/llap/ql/src/test/queries/clientpositive/orc_llap.q
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/StreamUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/StreamUtils.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/StreamUtils.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/StreamUtils.java Wed Feb 18 02:41:02 2015
@@ -26,6 +26,8 @@ import org.apache.hadoop.hive.llap.io.ap
import org.apache.hadoop.hive.llap.io.api.cache.LlapMemoryBuffer;
import org.apache.hadoop.hive.ql.io.orc.CompressionCodec;
import org.apache.hadoop.hive.ql.io.orc.InStream;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto;
+import org.apache.hadoop.hive.ql.io.orc.PositionProvider;
import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
/**
@@ -45,4 +47,10 @@ public class StreamUtils {
}
return InStream.create(fileName, streamName, input, totalLength, codec, bufferSize, null);
}
+
+
+ public static PositionProvider getPositionProvider(OrcProto.RowIndexEntry rowIndex) {
+ PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
+ return positionProvider;
+ }
}
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/BinaryStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/BinaryStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/BinaryStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/BinaryStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,22 +32,35 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class BinaryStreamReader extends RecordReaderImpl.BinaryTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private BinaryStreamReader(int columnId, InStream present,
InStream data, InStream length, boolean isFileCompressed,
OrcProto.ColumnEncoding encoding, OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data, length, encoding);
this.isFileCompressed = isFileCompressed;
- this.rowIndex = rowIndex;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ stream.seek(index);
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ lengths.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/BooleanStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/BooleanStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/BooleanStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/BooleanStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,22 +32,30 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class BooleanStreamReader extends RecordReaderImpl.BooleanTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private BooleanStreamReader(int columnId, InStream present,
InStream data, boolean isFileCompressed,
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data);
this.isFileCompressed = isFileCompressed;
- this.rowIndex = rowIndex;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ reader.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/ByteStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/ByteStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/ByteStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/ByteStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,22 +32,30 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class ByteStreamReader extends RecordReaderImpl.ByteTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private ByteStreamReader(int columnId, InStream present,
InStream data, boolean isFileCompressed,
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data);
this.isFileCompressed = isFileCompressed;
- this.rowIndex = rowIndex;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ reader.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/CharacterStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/CharacterStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/CharacterStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/CharacterStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class CharacterStreamReader extends RecordReaderImpl.StringTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
+ private boolean isDictionaryEncoding;
private CharacterStreamReader(int columnId, int maxLength, OrcProto.Type charType,
InStream present,
@@ -41,7 +41,7 @@ public class CharacterStreamReader exten
OrcProto.ColumnEncoding encoding,
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId);
-
+ this.isDictionaryEncoding = dictionary != null;
if (charType.getKind() == OrcProto.Type.Kind.CHAR) {
reader = new RecordReaderImpl.CharTreeReader(columnId, maxLength, present, data, length,
dictionary, encoding);
@@ -55,12 +55,34 @@ public class CharacterStreamReader exten
this.isFileCompressed = isFileCompressed;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- reader.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isDictionaryEncoding) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ ((RecordReaderImpl.StringDictionaryTreeReader)reader).reader.seek(index);
+ } else {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ ((RecordReaderImpl.StringDirectTreeReader)reader).stream.seek(index);
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ ((RecordReaderImpl.StringDirectTreeReader)reader).lengths.seek(index);
+ }
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DateStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DateStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DateStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DateStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,7 +32,6 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class DateStreamReader extends RecordReaderImpl.DateTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private DateStreamReader(int columnId, InStream present,
InStream data, boolean isFileCompressed,
@@ -40,15 +39,24 @@ public class DateStreamReader extends Re
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data, encoding);
this.isFileCompressed = isFileCompressed;
- this.rowIndex = rowIndex;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ reader.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DecimalStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DecimalStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DecimalStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DecimalStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,22 +32,35 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class DecimalStreamReader extends RecordReaderImpl.DecimalTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private DecimalStreamReader(int columnId, int precision, int scale, InStream presentStream,
InStream valueStream, InStream scaleStream, boolean isFileCompressed,
OrcProto.RowIndexEntry rowIndex, OrcProto.ColumnEncoding encoding) throws IOException {
super(columnId, precision, scale, presentStream, valueStream, scaleStream, encoding);
this.isFileCompressed = isFileCompressed;
- this.rowIndex = rowIndex;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ valueStream.seek(index);
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ scaleReader.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DoubleStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DoubleStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DoubleStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/DoubleStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,21 +32,30 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class DoubleStreamReader extends RecordReaderImpl.DoubleTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private DoubleStreamReader(int columnId, InStream present,
InStream data, boolean isFileCompressed,
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data);
- this.rowIndex = rowIndex;
+ this.isFileCompressed = isFileCompressed;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ stream.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/FloatStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/FloatStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/FloatStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/FloatStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,22 +32,30 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class FloatStreamReader extends RecordReaderImpl.FloatTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private FloatStreamReader(int columnId, InStream present,
InStream data, boolean isFileCompressed,
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data);
this.isFileCompressed = isFileCompressed;
- this.rowIndex = rowIndex;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ stream.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/IntStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/IntStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/IntStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/IntStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,7 +32,6 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class IntStreamReader extends RecordReaderImpl.IntTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private IntStreamReader(int columnId, InStream present,
InStream data, boolean isFileCompressed,
@@ -40,15 +39,24 @@ public class IntStreamReader extends Rec
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data, encoding);
this.isFileCompressed = isFileCompressed;
- this.rowIndex = rowIndex;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ reader.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/LongStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/LongStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/LongStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/LongStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,7 +32,6 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class LongStreamReader extends RecordReaderImpl.LongTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private LongStreamReader(int columnId, InStream present,
InStream data, boolean isFileCompressed,
@@ -40,15 +39,24 @@ public class LongStreamReader extends Re
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data, encoding, skipCorrupt);
this.isFileCompressed = isFileCompressed;
- this.rowIndex = rowIndex;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ reader.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/ShortStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/ShortStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/ShortStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/ShortStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,7 +32,6 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class ShortStreamReader extends RecordReaderImpl.ShortTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private ShortStreamReader(int columnId, InStream present,
InStream data, boolean isFileCompressed,
@@ -40,15 +39,24 @@ public class ShortStreamReader extends R
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data, encoding);
this.isFileCompressed = isFileCompressed;
- this.rowIndex = rowIndex;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ reader.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/StringStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/StringStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/StringStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/StringStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class StringStreamReader extends RecordReaderImpl.StringTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
+ private boolean isDictionaryEncoding;
private StringStreamReader(int columnId, InStream present,
InStream data, InStream length, InStream dictionary,
@@ -40,15 +40,38 @@ public class StringStreamReader extends
OrcProto.ColumnEncoding encoding,
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data, length, dictionary, encoding);
+ this.isDictionaryEncoding = dictionary != null;
this.isFileCompressed = isFileCompressed;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isDictionaryEncoding) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ ((RecordReaderImpl.StringDictionaryTreeReader)reader).reader.seek(index);
+ } else {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ ((RecordReaderImpl.StringDirectTreeReader)reader).stream.seek(index);
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ ((RecordReaderImpl.StringDirectTreeReader)reader).lengths.seek(index);
+ }
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/TimestampStreamReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/TimestampStreamReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/TimestampStreamReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/orc/stream/readers/TimestampStreamReader.java Wed Feb 18 02:41:02 2015
@@ -32,7 +32,6 @@ import org.apache.hadoop.hive.ql.io.orc.
*/
public class TimestampStreamReader extends RecordReaderImpl.TimestampTreeReader {
private boolean isFileCompressed;
- private OrcProto.RowIndexEntry rowIndex;
private TimestampStreamReader(int columnId, InStream present,
InStream data, InStream nanos, boolean isFileCompressed,
@@ -40,15 +39,29 @@ public class TimestampStreamReader exten
OrcProto.RowIndexEntry rowIndex) throws IOException {
super(columnId, present, data, nanos, encoding, skipCorrupt);
this.isFileCompressed = isFileCompressed;
- this.rowIndex = rowIndex;
// position the readers based on the specified row index
- PositionProvider positionProvider = new RecordReaderImpl.PositionProviderImpl(rowIndex);
- seek(positionProvider);
+ seek(StreamUtils.getPositionProvider(rowIndex));
}
- public void seek(PositionProvider positionProvider) throws IOException {
- super.seek(positionProvider);
+ @Override
+ public void seek(PositionProvider index) throws IOException {
+ if (present != null) {
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ present.seek(index);
+ }
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ data.seek(index);
+
+ if (isFileCompressed) {
+ index.getNext();
+ }
+ nanos.seek(index);
}
public static class StreamReaderBuilder {
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Wed Feb 18 02:41:02 2015
@@ -223,7 +223,7 @@ public class RecordReaderImpl implements
public abstract static class TreeReader {
protected final int columnId;
- private BitFieldReader present = null;
+ protected BitFieldReader present = null;
protected boolean valuePresent = false;
public TreeReader(int columnId) throws IOException {
@@ -348,7 +348,7 @@ public class RecordReaderImpl implements
}
public static class BooleanTreeReader extends TreeReader {
- private BitFieldReader reader = null;
+ protected BitFieldReader reader = null;
public BooleanTreeReader(int columnId) throws IOException {
this(columnId, null, null);
@@ -420,7 +420,7 @@ public class RecordReaderImpl implements
}
public static class ByteTreeReader extends TreeReader{
- private RunLengthByteReader reader = null;
+ protected RunLengthByteReader reader = null;
ByteTreeReader(int columnId) throws IOException {
this(columnId, null, null);
@@ -490,7 +490,7 @@ public class RecordReaderImpl implements
}
public static class ShortTreeReader extends TreeReader{
- private IntegerReader reader = null;
+ protected IntegerReader reader = null;
public ShortTreeReader(int columnId) throws IOException {
this(columnId, null, null, null);
@@ -576,7 +576,7 @@ public class RecordReaderImpl implements
}
public static class IntTreeReader extends TreeReader{
- private IntegerReader reader = null;
+ protected IntegerReader reader = null;
public IntTreeReader(int columnId) throws IOException {
this(columnId, null, null, null);
@@ -662,7 +662,7 @@ public class RecordReaderImpl implements
}
public static class LongTreeReader extends TreeReader{
- private IntegerReader reader = null;
+ protected IntegerReader reader = null;
LongTreeReader(int columnId, boolean skipCorrupt) throws IOException {
this(columnId, null, null, null, skipCorrupt);
@@ -749,7 +749,7 @@ public class RecordReaderImpl implements
}
public static class FloatTreeReader extends TreeReader{
- private InStream stream;
+ protected InStream stream;
private final SerializationUtils utils;
public FloatTreeReader(int columnId) throws IOException {
@@ -841,7 +841,7 @@ public class RecordReaderImpl implements
}
public static class DoubleTreeReader extends TreeReader{
- private InStream stream;
+ protected InStream stream;
private final SerializationUtils utils;
public DoubleTreeReader(int columnId) throws IOException {
@@ -1037,8 +1037,8 @@ public class RecordReaderImpl implements
}
public static class TimestampTreeReader extends TreeReader {
- private IntegerReader data = null;
- private IntegerReader nanos = null;
+ protected IntegerReader data = null;
+ protected IntegerReader nanos = null;
private final boolean skipCorrupt;
TimestampTreeReader(int columnId, boolean skipCorrupt) throws IOException {
@@ -1170,7 +1170,7 @@ public class RecordReaderImpl implements
}
public static class DateTreeReader extends TreeReader{
- private IntegerReader reader = null;
+ protected IntegerReader reader = null;
DateTreeReader(int columnId) throws IOException {
this(columnId, null, null, null);
@@ -1254,8 +1254,8 @@ public class RecordReaderImpl implements
}
public static class DecimalTreeReader extends TreeReader{
- private InStream valueStream;
- private IntegerReader scaleReader = null;
+ protected InStream valueStream;
+ protected IntegerReader scaleReader = null;
private LongColumnVector scratchScaleVector;
private final int precision;
@@ -1543,8 +1543,8 @@ public class RecordReaderImpl implements
* stripe.
*/
public static class StringDirectTreeReader extends TreeReader {
- private InStream stream;
- private IntegerReader lengths;
+ public InStream stream;
+ public IntegerReader lengths;
private final LongColumnVector scratchlcv;
StringDirectTreeReader(int columnId) throws IOException {
@@ -1655,7 +1655,7 @@ public class RecordReaderImpl implements
public static class StringDictionaryTreeReader extends TreeReader {
private DynamicByteArray dictionaryBuffer;
private int[] dictionaryOffsets;
- private IntegerReader reader;
+ public IntegerReader reader;
private byte[] dictionaryBufferInBytesCache = null;
private final LongColumnVector scratchlcv;
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java Wed Feb 18 02:41:02 2015
@@ -21,15 +21,13 @@ import java.io.EOFException;
import java.io.IOException;
import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
-import org.apache.hadoop.hive.ql.io.orc.LlapUtils.PresentStreamReadResult;
-import org.apache.hadoop.hive.ql.io.orc.RunLengthIntegerWriterV2.EncodingType;
/**
* A reader that reads a sequence of bytes. A control byte is read before
* each run with positive values 0 to 127 meaning 3 to 130 repetitions. If the
* byte is -1 to -128, 1 to 128 literal byte values follow.
*/
-class RunLengthByteReader {
+public class RunLengthByteReader {
private final InStream input;
private final byte[] literals =
new byte[RunLengthByteWriter.MAX_LITERAL_SIZE];
@@ -113,7 +111,7 @@ class RunLengthByteReader {
}
}
- void seek(PositionProvider index) throws IOException {
+ public void seek(PositionProvider index) throws IOException {
input.seek(index);
int consumed = (int) index.getNext();
if (consumed != 0) {
Modified: hive/branches/llap/ql/src/test/queries/clientpositive/orc_llap.q
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/queries/clientpositive/orc_llap.q?rev=1660559&r1=1660558&r2=1660559&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/queries/clientpositive/orc_llap.q (original)
+++ hive/branches/llap/ql/src/test/queries/clientpositive/orc_llap.q Wed Feb 18 02:41:02 2015
@@ -38,4 +38,4 @@ select count(*) from orc_llap;
SET hive.llap.io.enabled=true;
-select cint, csmallint, cbigint from orc_llap where cint > 10 and cbigint is not null order by cbigint limit 1000;
\ No newline at end of file
+select cint, csmallint, cbigint from orc_llap where cint > 10 and cbigint is not null order by cbigint limit 1000;