You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/08/16 03:22:02 UTC
svn commit: r1514554 [6/18] - in /hive/branches/vectorization: ./
beeline/src/java/org/apache/hive/beeline/
cli/src/java/org/apache/hadoop/hive/cli/
cli/src/test/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/conf/ conf/ contrib/src...
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Fri Aug 16 01:21:54 2013
@@ -134,6 +134,21 @@ class RecordReaderImpl implements Record
}
}
+ IntegerReader createIntegerReader(OrcProto.ColumnEncoding.Kind kind,
+ InStream in,
+ boolean signed) throws IOException {
+ switch (kind) {
+ case DIRECT_V2:
+ case DICTIONARY_V2:
+ return new RunLengthIntegerReaderV2(in, signed);
+ case DIRECT:
+ case DICTIONARY:
+ return new RunLengthIntegerReader(in, signed);
+ default:
+ throw new IllegalArgumentException("Unknown encoding " + kind);
+ }
+ }
+
void startStripe(Map<StreamName, InStream> streams,
List<OrcProto.ColumnEncoding> encoding
) throws IOException {
@@ -336,20 +351,29 @@ class RecordReaderImpl implements Record
}
private static class ShortTreeReader extends TreeReader{
- private RunLengthIntegerReader reader = null;
+ private IntegerReader reader = null;
ShortTreeReader(Path path, int columnId) {
super(path, columnId);
}
@Override
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if ((encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) &&
+ (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT_V2)) {
+ throw new IOException("Unknown encoding " + encoding + " in column " +
+ columnId + " of " + path);
+ }
+ }
+
+ @Override
void startStripe(Map<StreamName, InStream> streams,
List<OrcProto.ColumnEncoding> encodings
) throws IOException {
super.startStripe(streams, encodings);
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DATA);
- reader = new RunLengthIntegerReader(streams.get(name), true);
+ reader = createIntegerReader(encodings.get(columnId).getKind(), streams.get(name), true);
}
@Override
@@ -397,20 +421,29 @@ class RecordReaderImpl implements Record
}
private static class IntTreeReader extends TreeReader{
- private RunLengthIntegerReader reader = null;
+ private IntegerReader reader = null;
IntTreeReader(Path path, int columnId) {
super(path, columnId);
}
@Override
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if ((encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) &&
+ (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT_V2)) {
+ throw new IOException("Unknown encoding " + encoding + " in column " +
+ columnId + " of " + path);
+ }
+ }
+
+ @Override
void startStripe(Map<StreamName, InStream> streams,
List<OrcProto.ColumnEncoding> encodings
) throws IOException {
super.startStripe(streams, encodings);
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DATA);
- reader = new RunLengthIntegerReader(streams.get(name), true);
+ reader = createIntegerReader(encodings.get(columnId).getKind(), streams.get(name), true);
}
@Override
@@ -458,20 +491,29 @@ class RecordReaderImpl implements Record
}
private static class LongTreeReader extends TreeReader{
- private RunLengthIntegerReader reader = null;
+ private IntegerReader reader = null;
LongTreeReader(Path path, int columnId) {
super(path, columnId);
}
@Override
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if ((encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) &&
+ (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT_V2)) {
+ throw new IOException("Unknown encoding " + encoding + " in column " +
+ columnId + " of " + path);
+ }
+ }
+
+ @Override
void startStripe(Map<StreamName, InStream> streams,
List<OrcProto.ColumnEncoding> encodings
) throws IOException {
super.startStripe(streams, encodings);
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DATA);
- reader = new RunLengthIntegerReader(streams.get(name), true);
+ reader = createIntegerReader(encodings.get(columnId).getKind(), streams.get(name), true);
}
@Override
@@ -678,13 +720,22 @@ class RecordReaderImpl implements Record
private static class BinaryTreeReader extends TreeReader{
private InStream stream;
- private RunLengthIntegerReader lengths;
+ private IntegerReader lengths = null;
BinaryTreeReader(Path path, int columnId) {
super(path, columnId);
}
@Override
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if ((encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) &&
+ (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT_V2)) {
+ throw new IOException("Unknown encoding " + encoding + " in column " +
+ columnId + " of " + path);
+ }
+ }
+
+ @Override
void startStripe(Map<StreamName, InStream> streams,
List<OrcProto.ColumnEncoding> encodings
) throws IOException {
@@ -692,9 +743,8 @@ class RecordReaderImpl implements Record
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DATA);
stream = streams.get(name);
- lengths = new RunLengthIntegerReader(streams.get(new
- StreamName(columnId, OrcProto.Stream.Kind.LENGTH)),
- false);
+ lengths = createIntegerReader(encodings.get(columnId).getKind(), streams.get(new
+ StreamName(columnId, OrcProto.Stream.Kind.LENGTH)), false);
}
@Override
@@ -747,8 +797,8 @@ class RecordReaderImpl implements Record
}
private static class TimestampTreeReader extends TreeReader{
- private RunLengthIntegerReader data;
- private RunLengthIntegerReader nanos;
+ private IntegerReader data = null;
+ private IntegerReader nanos = null;
private final LongColumnVector nanoVector = new LongColumnVector();
TimestampTreeReader(Path path, int columnId) {
@@ -756,14 +806,25 @@ class RecordReaderImpl implements Record
}
@Override
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if ((encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) &&
+ (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT_V2)) {
+ throw new IOException("Unknown encoding " + encoding + " in column " +
+ columnId + " of " + path);
+ }
+ }
+
+ @Override
void startStripe(Map<StreamName, InStream> streams,
List<OrcProto.ColumnEncoding> encodings
) throws IOException {
super.startStripe(streams, encodings);
- data = new RunLengthIntegerReader(streams.get(new StreamName(columnId,
- OrcProto.Stream.Kind.DATA)), true);
- nanos = new RunLengthIntegerReader(streams.get(new StreamName(columnId,
- OrcProto.Stream.Kind.SECONDARY)), false);
+ data = createIntegerReader(encodings.get(columnId).getKind(),
+ streams.get(new StreamName(columnId,
+ OrcProto.Stream.Kind.DATA)), true);
+ nanos = createIntegerReader(encodings.get(columnId).getKind(),
+ streams.get(new StreamName(columnId,
+ OrcProto.Stream.Kind.SECONDARY)), false);
}
@Override
@@ -865,20 +926,29 @@ class RecordReaderImpl implements Record
}
private static class DateTreeReader extends TreeReader{
- private RunLengthIntegerReader reader = null;
+ private IntegerReader reader = null;
DateTreeReader(Path path, int columnId) {
super(path, columnId);
}
@Override
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if ((encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) &&
+ (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT_V2)) {
+ throw new IOException("Unknown encoding " + encoding + " in column " +
+ columnId + " of " + path);
+ }
+ }
+
+ @Override
void startStripe(Map<StreamName, InStream> streams,
List<OrcProto.ColumnEncoding> encodings
) throws IOException {
super.startStripe(streams, encodings);
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DATA);
- reader = new RunLengthIntegerReader(streams.get(name), true);
+ reader = createIntegerReader(encodings.get(columnId).getKind(), streams.get(name), true);
}
@Override
@@ -910,20 +980,29 @@ class RecordReaderImpl implements Record
private static class DecimalTreeReader extends TreeReader{
private InStream valueStream;
- private RunLengthIntegerReader scaleStream;
+ private IntegerReader scaleStream = null;
DecimalTreeReader(Path path, int columnId) {
super(path, columnId);
}
@Override
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if ((encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) &&
+ (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT_V2)) {
+ throw new IOException("Unknown encoding " + encoding + " in column " +
+ columnId + " of " + path);
+ }
+ }
+
+ @Override
void startStripe(Map<StreamName, InStream> streams,
List<OrcProto.ColumnEncoding> encodings
) throws IOException {
super.startStripe(streams, encodings);
valueStream = streams.get(new StreamName(columnId,
OrcProto.Stream.Kind.DATA));
- scaleStream = new RunLengthIntegerReader(streams.get(
+ scaleStream = createIntegerReader(encodings.get(columnId).getKind(), streams.get(
new StreamName(columnId, OrcProto.Stream.Kind.SECONDARY)), true);
}
@@ -960,22 +1039,234 @@ class RecordReaderImpl implements Record
}
}
+ /**
+ * A tree reader that will read string columns. At the start of the
+ * stripe, it creates an internal reader based on whether a direct or
+ * dictionary encoding was used.
+ */
private static class StringTreeReader extends TreeReader {
- private DynamicByteArray dictionaryBuffer = null;
- private int dictionarySize;
+ private TreeReader reader;
+
+ StringTreeReader(Path path, int columnId) {
+ super(path, columnId);
+ }
+
+ @Override
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ reader.checkEncoding(encoding);
+ }
+
+ @Override
+ void startStripe(Map<StreamName, InStream> streams,
+ List<OrcProto.ColumnEncoding> encodings
+ ) throws IOException {
+ // For each stripe, checks the encoding and initializes the appropriate
+ // reader
+ switch (encodings.get(columnId).getKind()) {
+ case DIRECT:
+ case DIRECT_V2:
+ reader = new StringDirectTreeReader(path, columnId);
+ break;
+ case DICTIONARY:
+ case DICTIONARY_V2:
+ reader = new StringDictionaryTreeReader(path, columnId);
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported encoding " +
+ encodings.get(columnId).getKind());
+ }
+ reader.startStripe(streams, encodings);
+ }
+
+ @Override
+ void seek(PositionProvider[] index) throws IOException {
+ reader.seek(index);
+ }
+
+ @Override
+ Object next(Object previous) throws IOException {
+ return reader.next(previous);
+ }
+
+ @Override
+ Object nextVector(Object previousVector, long batchSize) throws IOException {
+ return reader.nextVector(previousVector, batchSize);
+ }
+
+ @Override
+ void skipRows(long items) throws IOException {
+ reader.skipRows(items);
+ }
+ }
+
+ /**
+ * A reader for string columns that are direct encoded in the current
+ * stripe.
+ */
+ private static class StringDirectTreeReader extends TreeReader {
+ private InStream stream;
+ private IntegerReader lengths;
+
+ private final LongColumnVector scratchlcv;
+
+ StringDirectTreeReader(Path path, int columnId) {
+ super(path, columnId);
+ scratchlcv = new LongColumnVector();
+ }
+
+ @Override
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT &&
+ encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT_V2) {
+ throw new IOException("Unknown encoding " + encoding + " in column " +
+ columnId + " of " + path);
+ }
+ }
+
+ @Override
+ void startStripe(Map<StreamName, InStream> streams,
+ List<OrcProto.ColumnEncoding> encodings
+ ) throws IOException {
+ super.startStripe(streams, encodings);
+ StreamName name = new StreamName(columnId,
+ OrcProto.Stream.Kind.DATA);
+ stream = streams.get(name);
+ lengths = createIntegerReader(encodings.get(columnId).getKind(),
+ streams.get(new StreamName(columnId, OrcProto.Stream.Kind.LENGTH)),
+ false);
+ }
+
+ @Override
+ void seek(PositionProvider[] index) throws IOException {
+ super.seek(index);
+ stream.seek(index[columnId]);
+ lengths.seek(index[columnId]);
+ }
+
+ @Override
+ Object next(Object previous) throws IOException {
+ super.next(previous);
+ Text result = null;
+ if (valuePresent) {
+ if (previous == null) {
+ result = new Text();
+ } else {
+ result = (Text) previous;
+ }
+ int len = (int) lengths.next();
+ int offset = 0;
+ byte[] bytes = new byte[len];
+ while (len > 0) {
+ int written = stream.read(bytes, offset, len);
+ if (written < 0) {
+ throw new EOFException("Can't finish byte read from " + stream);
+ }
+ len -= written;
+ offset += written;
+ }
+ result.set(bytes);
+ }
+ return result;
+ }
+
+ @Override
+ Object nextVector(Object previousVector, long batchSize) throws IOException {
+ BytesColumnVector result = null;
+ if (previousVector == null) {
+ result = new BytesColumnVector();
+ } else {
+ result = (BytesColumnVector) previousVector;
+ }
+
+ // Read present/isNull stream
+ super.nextVector(result, batchSize);
+
+ // Read lengths
+ scratchlcv.isNull = result.isNull;
+ lengths.nextVector(scratchlcv, batchSize);
+ int totalLength = 0;
+ if (!scratchlcv.isRepeating) {
+ for (int i = 0; i < batchSize; i++) {
+ if (!scratchlcv.isNull[i]) {
+ totalLength += (int) scratchlcv.vector[i];
+ }
+ }
+ } else {
+ if (!scratchlcv.isNull[0]) {
+ totalLength = (int) (batchSize * scratchlcv.vector[0]);
+ }
+ }
+
+ //Read all the strings for this batch
+ byte[] allBytes = new byte[totalLength];
+ int offset = 0;
+ int len = totalLength;
+ while (len > 0) {
+ int bytesRead = stream.read(allBytes, offset, len);
+ if (bytesRead < 0) {
+ throw new EOFException("Can't finish byte read from " + stream);
+ }
+ len -= bytesRead;
+ offset += bytesRead;
+ }
+
+ // Too expensive to figure out 'repeating' by comparisons.
+ result.isRepeating = false;
+ offset = 0;
+ if (!scratchlcv.isRepeating) {
+ for (int i = 0; i < batchSize; i++) {
+ if (!scratchlcv.isNull[i]) {
+ result.setRef(i, allBytes, offset, (int) scratchlcv.vector[i]);
+ offset += scratchlcv.vector[i];
+ } else {
+ result.setRef(i, allBytes, 0, 0);
+ }
+ }
+ } else {
+ for (int i = 0; i < batchSize; i++) {
+ if (!scratchlcv.isNull[i]) {
+ result.setRef(i, allBytes, offset, (int) scratchlcv.vector[0]);
+ offset += scratchlcv.vector[0];
+ } else {
+ result.setRef(i, allBytes, 0, 0);
+ }
+ }
+ }
+ return result;
+ }
+
+ @Override
+ void skipRows(long items) throws IOException {
+ items = countNonNulls(items);
+ long lengthToSkip = 0;
+ for(int i=0; i < items; ++i) {
+ lengthToSkip += lengths.next();
+ }
+ stream.skip(lengthToSkip);
+ }
+ }
+
+ /**
+ * A reader for string columns that are dictionary encoded in the current
+ * stripe.
+ */
+ private static class StringDictionaryTreeReader extends TreeReader {
+ private DynamicByteArray dictionaryBuffer;
private int[] dictionaryOffsets;
- private RunLengthIntegerReader reader;
+ private IntegerReader reader;
private byte[] dictionaryBufferInBytesCache = null;
private final LongColumnVector scratchlcv;
- StringTreeReader(Path path, int columnId) {
+ StringDictionaryTreeReader(Path path, int columnId) {
super(path, columnId);
scratchlcv = new LongColumnVector();
}
+ @Override
void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
- if (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DICTIONARY) {
+ if (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DICTIONARY &&
+ encoding.getKind() != OrcProto.ColumnEncoding.Kind.DICTIONARY_V2) {
throw new IOException("Unknown encoding " + encoding + " in column " +
columnId + " of " + path);
}
@@ -988,7 +1279,7 @@ class RecordReaderImpl implements Record
super.startStripe(streams, encodings);
// read the dictionary blob
- dictionarySize = encodings.get(columnId).getDictionarySize();
+ int dictionarySize = encodings.get(columnId).getDictionarySize();
StreamName name = new StreamName(columnId,
OrcProto.Stream.Kind.DICTIONARY_DATA);
InStream in = streams.get(name);
@@ -1005,7 +1296,8 @@ class RecordReaderImpl implements Record
// read the lengths
name = new StreamName(columnId, OrcProto.Stream.Kind.LENGTH);
in = streams.get(name);
- RunLengthIntegerReader lenReader = new RunLengthIntegerReader(in, false);
+ IntegerReader lenReader = createIntegerReader(encodings.get(columnId)
+ .getKind(), in, false);
int offset = 0;
if (dictionaryOffsets == null ||
dictionaryOffsets.length < dictionarySize + 1) {
@@ -1020,7 +1312,8 @@ class RecordReaderImpl implements Record
// set up the row reader
name = new StreamName(columnId, OrcProto.Stream.Kind.DATA);
- reader = new RunLengthIntegerReader(streams.get(name), false);
+ reader = createIntegerReader(encodings.get(columnId).getKind(),
+ streams.get(name), false);
}
@Override
@@ -1153,7 +1446,9 @@ class RecordReaderImpl implements Record
void seek(PositionProvider[] index) throws IOException {
super.seek(index);
for(TreeReader kid: fields) {
- kid.seek(index);
+ if (kid != null) {
+ kid.seek(index);
+ }
}
}
@@ -1221,7 +1516,9 @@ class RecordReaderImpl implements Record
void skipRows(long items) throws IOException {
items = countNonNulls(items);
for(TreeReader field: fields) {
- field.skipRows(items);
+ if (field != null) {
+ field.skipRows(items);
+ }
}
}
}
@@ -1307,7 +1604,7 @@ class RecordReaderImpl implements Record
private static class ListTreeReader extends TreeReader {
private final TreeReader elementReader;
- private RunLengthIntegerReader lengths;
+ private IntegerReader lengths = null;
ListTreeReader(Path path, int columnId,
List<OrcProto.Type> types,
@@ -1361,13 +1658,22 @@ class RecordReaderImpl implements Record
"NextVector is not supported operation for List type");
}
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if ((encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) &&
+ (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT_V2)) {
+ throw new IOException("Unknown encoding " + encoding + " in column " +
+ columnId + " of " + path);
+ }
+ }
+
@Override
void startStripe(Map<StreamName, InStream> streams,
List<OrcProto.ColumnEncoding> encodings
) throws IOException {
super.startStripe(streams, encodings);
- lengths = new RunLengthIntegerReader(streams.get(new StreamName(columnId,
- OrcProto.Stream.Kind.LENGTH)), false);
+ lengths = createIntegerReader(encodings.get(columnId).getKind(),
+ streams.get(new StreamName(columnId,
+ OrcProto.Stream.Kind.LENGTH)), false);
if (elementReader != null) {
elementReader.startStripe(streams, encodings);
}
@@ -1387,7 +1693,7 @@ class RecordReaderImpl implements Record
private static class MapTreeReader extends TreeReader {
private final TreeReader keyReader;
private final TreeReader valueReader;
- private RunLengthIntegerReader lengths;
+ private IntegerReader lengths = null;
MapTreeReader(Path path,
int columnId,
@@ -1445,13 +1751,22 @@ class RecordReaderImpl implements Record
"NextVector is not supported operation for Map type");
}
+ void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {
+ if ((encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) &&
+ (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT_V2)) {
+ throw new IOException("Unknown encoding " + encoding + " in column " +
+ columnId + " of " + path);
+ }
+ }
+
@Override
void startStripe(Map<StreamName, InStream> streams,
List<OrcProto.ColumnEncoding> encodings
) throws IOException {
super.startStripe(streams, encodings);
- lengths = new RunLengthIntegerReader(streams.get(new StreamName(columnId,
- OrcProto.Stream.Kind.LENGTH)), false);
+ lengths = createIntegerReader(encodings.get(columnId).getKind(),
+ streams.get(new StreamName(columnId,
+ OrcProto.Stream.Kind.LENGTH)), false);
if (keyReader != null) {
keyReader.startStripe(streams, encodings);
}
@@ -1735,7 +2050,7 @@ class RecordReaderImpl implements Record
readStripe();
}
readRowIndex();
- rowInStripe = rowNumber - rowBaseInStripe;
+ rowInStripe = rowNumber - rowBaseInStripe - firstRow;
if (rowIndexStride != 0) {
long entry = rowInStripe / rowIndexStride;
seekToRowEntry((int) entry);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReader.java Fri Aug 16 01:21:54 2013
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.exec.ve
/**
* A reader that reads a sequence of integers.
* */
-class RunLengthIntegerReader {
+class RunLengthIntegerReader implements IntegerReader {
private final InStream input;
private final boolean signed;
private final long[] literals =
@@ -73,11 +73,13 @@ class RunLengthIntegerReader {
}
}
- boolean hasNext() throws IOException {
+ @Override
+ public boolean hasNext() throws IOException {
return used != numLiterals || input.available() > 0;
}
- long next() throws IOException {
+ @Override
+ public long next() throws IOException {
long result;
if (used == numLiterals) {
readValues();
@@ -90,7 +92,8 @@ class RunLengthIntegerReader {
return result;
}
- void nextVector(LongColumnVector previous, long previousLen)
+ @Override
+ public void nextVector(LongColumnVector previous, long previousLen)
throws IOException {
previous.isRepeating = true;
for (int i = 0; i < previousLen; i++) {
@@ -113,8 +116,8 @@ class RunLengthIntegerReader {
}
}
-
- void seek(PositionProvider index) throws IOException {
+ @Override
+ public void seek(PositionProvider index) throws IOException {
input.seek(index);
int consumed = (int) index.getNext();
if (consumed != 0) {
@@ -130,7 +133,8 @@ class RunLengthIntegerReader {
}
}
- void skip(long numValues) throws IOException {
+ @Override
+ public void skip(long numValues) throws IOException {
while (numValues > 0) {
if (used == numLiterals) {
readValues();
Copied: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java (from r1513659, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java)
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java?p2=hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java&p1=hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java&r1=1513659&r2=1514554&rev=1514554&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerReaderV2.java Fri Aug 16 01:21:54 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.io.orc
import java.io.EOFException;
import java.io.IOException;
+import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
import org.apache.hadoop.hive.ql.io.orc.RunLengthIntegerWriterV2.EncodingType;
/**
@@ -322,4 +323,28 @@ class RunLengthIntegerReaderV2 implement
numValues -= consume;
}
}
+
+ @Override
+ public void nextVector(LongColumnVector previous, long previousLen) throws IOException {
+ previous.isRepeating = true;
+ for (int i = 0; i < previousLen; i++) {
+ if (!previous.isNull[i]) {
+ previous.vector[i] = next();
+ } else {
+ // The default value of null for int type in vectorized
+ // processing is 1, so set that if the value is null
+ previous.vector[i] = 1;
+ }
+
+ // The default value for nulls in Vectorization for int types is 1
+ // and given that non null value can also be 1, we need to check for isNull also
+ // when determining the isRepeating flag.
+ if (previous.isRepeating
+ && i > 0
+ && (previous.vector[i - 1] != previous.vector[i] ||
+ previous.isNull[i - 1] != previous.isNull[i])) {
+ previous.isRepeating = false;
+ }
+ }
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriter.java Fri Aug 16 01:21:54 2013
@@ -25,7 +25,7 @@ import java.io.IOException;
* repetition is offset by a delta. If the control byte is -1 to -128, 1 to 128
* literal vint values follow.
*/
-class RunLengthIntegerWriter {
+class RunLengthIntegerWriter implements IntegerWriter {
static final int MIN_REPEAT_SIZE = 3;
static final int MAX_DELTA = 127;
static final int MIN_DELTA = -128;
@@ -71,12 +71,14 @@ class RunLengthIntegerWriter {
}
}
- void flush() throws IOException {
+ @Override
+ public void flush() throws IOException {
writeValues();
output.flush();
}
- void write(long value) throws IOException {
+ @Override
+ public void write(long value) throws IOException {
if (numLiterals == 0) {
literals[numLiterals++] = value;
tailRunLength = 1;
@@ -130,8 +132,10 @@ class RunLengthIntegerWriter {
}
}
- void getPosition(PositionRecorder recorder) throws IOException {
+ @Override
+ public void getPosition(PositionRecorder recorder) throws IOException {
output.getPosition(recorder);
recorder.addPosition(numLiterals);
}
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SerializationUtils.java Fri Aug 16 01:21:54 2013
@@ -185,4 +185,279 @@ final class SerializationUtils {
result = result.shiftRight(1);
return result;
}
+
+ enum FixedBitSizes {
+ ONE, TWO, THREE, FOUR, FIVE, SIX, SEVEN, EIGHT, NINE, TEN, ELEVEN, TWELVE,
+ THIRTEEN, FOURTEEN, FIFTEEN, SIXTEEN, SEVENTEEN, EIGHTEEN, NINETEEN,
+ TWENTY, TWENTYONE, TWENTYTWO, TWENTYTHREE, TWENTYFOUR, TWENTYSIX,
+ TWENTYEIGHT, THIRTY, THIRTYTWO, FORTY, FORTYEIGHT, FIFTYSIX, SIXTYFOUR;
+ }
+
+ /**
+ * Count the number of bits required to encode the given value
+ * @param value
+ * @return bits required to store value
+ */
+ static int findClosestNumBits(long value) {
+ int count = 0;
+ while (value > 0) {
+ count++;
+ value = value >>> 1;
+ }
+ return getClosestFixedBits(count);
+ }
+
+ /**
+ * zigzag encode the given value
+ * @param val
+ * @return zigzag encoded value
+ */
+ static long zigzagEncode(long val) {
+ return (val << 1) ^ (val >> 63);
+ }
+
+ /**
+ * zigzag decode the given value
+ * @param val
+ * @return zizag decoded value
+ */
+ static long zigzagDecode(long val) {
+ return (val >>> 1) ^ -(val & 1);
+ }
+
+ /**
+ * Compute the bits required to represent pth percentile value
+ * @param data - array
+ * @param p - percentile value (>=0.0 to <=1.0)
+ * @return pth percentile bits
+ */
+ static int percentileBits(long[] data, double p) {
+ if ((p > 1.0) || (p <= 0.0)) {
+ return -1;
+ }
+
+ // histogram that store the encoded bit requirement for each values.
+ // maximum number of bits that can encoded is 32 (refer FixedBitSizes)
+ int[] hist = new int[32];
+
+ // compute the histogram
+ for(long l : data) {
+ int idx = encodeBitWidth(findClosestNumBits(l));
+ hist[idx] += 1;
+ }
+
+ int len = data.length;
+ int perLen = (int) (len * (1.0 - p));
+
+ // return the bits required by pth percentile length
+ for(int i = hist.length - 1; i >= 0; i--) {
+ perLen -= hist[i];
+ if (perLen < 0) {
+ return decodeBitWidth(i);
+ }
+ }
+
+ return 0;
+ }
+
+ /**
+ * Read n bytes in big endian order and convert to long
+ * @param b - byte array
+ * @return long value
+ */
+ static long bytesToLongBE(InStream input, int n) throws IOException {
+ long out = 0;
+ long val = 0;
+ while (n > 0) {
+ n--;
+ // store it in a long and then shift else integer overflow will occur
+ val = input.read();
+ out |= (val << (n * 8));
+ }
+ return out;
+ }
+
+ /**
+ * Calculate the number of bytes required
+ * @param n - number of values
+ * @param numBits - bit width
+ * @return number of bytes required
+ */
+ static int getTotalBytesRequired(int n, int numBits) {
+ return (n * numBits + 7) / 8;
+ }
+
+ /**
+ * For a given fixed bit this function will return the closest available fixed
+ * bit
+ * @param n
+ * @return closest valid fixed bit
+ */
+ static int getClosestFixedBits(int n) {
+ if (n == 0) {
+ return 1;
+ }
+
+ if (n >= 1 && n <= 24) {
+ return n;
+ } else if (n > 24 && n <= 26) {
+ return 26;
+ } else if (n > 26 && n <= 28) {
+ return 28;
+ } else if (n > 28 && n <= 30) {
+ return 30;
+ } else if (n > 30 && n <= 32) {
+ return 32;
+ } else if (n > 32 && n <= 40) {
+ return 40;
+ } else if (n > 40 && n <= 48) {
+ return 48;
+ } else if (n > 48 && n <= 56) {
+ return 56;
+ } else {
+ return 64;
+ }
+ }
+
+ /**
+ * Finds the closest available fixed bit width match and returns its encoded
+ * value (ordinal)
+ * @param n - fixed bit width to encode
+ * @return encoded fixed bit width
+ */
+ static int encodeBitWidth(int n) {
+ n = getClosestFixedBits(n);
+
+ if (n >= 1 && n <= 24) {
+ return n - 1;
+ } else if (n > 24 && n <= 26) {
+ return FixedBitSizes.TWENTYSIX.ordinal();
+ } else if (n > 26 && n <= 28) {
+ return FixedBitSizes.TWENTYEIGHT.ordinal();
+ } else if (n > 28 && n <= 30) {
+ return FixedBitSizes.THIRTY.ordinal();
+ } else if (n > 30 && n <= 32) {
+ return FixedBitSizes.THIRTYTWO.ordinal();
+ } else if (n > 32 && n <= 40) {
+ return FixedBitSizes.FORTY.ordinal();
+ } else if (n > 40 && n <= 48) {
+ return FixedBitSizes.FORTYEIGHT.ordinal();
+ } else if (n > 48 && n <= 56) {
+ return FixedBitSizes.FIFTYSIX.ordinal();
+ } else {
+ return FixedBitSizes.SIXTYFOUR.ordinal();
+ }
+ }
+
+ /**
+ * Decodes the ordinal fixed bit value to actual fixed bit width value
+ * @param n - encoded fixed bit width
+ * @return decoded fixed bit width
+ */
+ static int decodeBitWidth(int n) {
+ if (n >= FixedBitSizes.ONE.ordinal()
+ && n <= FixedBitSizes.TWENTYFOUR.ordinal()) {
+ return n + 1;
+ } else if (n == FixedBitSizes.TWENTYSIX.ordinal()) {
+ return 26;
+ } else if (n == FixedBitSizes.TWENTYEIGHT.ordinal()) {
+ return 28;
+ } else if (n == FixedBitSizes.THIRTY.ordinal()) {
+ return 30;
+ } else if (n == FixedBitSizes.THIRTYTWO.ordinal()) {
+ return 32;
+ } else if (n == FixedBitSizes.FORTY.ordinal()) {
+ return 40;
+ } else if (n == FixedBitSizes.FORTYEIGHT.ordinal()) {
+ return 48;
+ } else if (n == FixedBitSizes.FIFTYSIX.ordinal()) {
+ return 56;
+ } else {
+ return 64;
+ }
+ }
+
+ /**
+ * Bitpack and write the input values to underlying output stream
+ * @param input - values to write
+ * @param offset - offset
+ * @param len - length
+ * @param bitSize - bit width
+ * @param output - output stream
+ * @throws IOException
+ */
+ static void writeInts(long[] input, int offset, int len, int bitSize,
+ OutputStream output) throws IOException {
+ if (input == null || input.length < 1 || offset < 0 || len < 1
+ || bitSize < 1) {
+ return;
+ }
+
+ int bitsLeft = 8;
+ byte current = 0;
+ for(int i = offset; i < (offset + len); i++) {
+ long value = input[i];
+ int bitsToWrite = bitSize;
+ while (bitsToWrite > bitsLeft) {
+ // add the bits to the bottom of the current word
+ current |= value >>> (bitsToWrite - bitsLeft);
+ // subtract out the bits we just added
+ bitsToWrite -= bitsLeft;
+ // zero out the bits above bitsToWrite
+ value &= (1L << bitsToWrite) - 1;
+ output.write(current);
+ current = 0;
+ bitsLeft = 8;
+ }
+ bitsLeft -= bitsToWrite;
+ current |= value << bitsLeft;
+ if (bitsLeft == 0) {
+ output.write(current);
+ current = 0;
+ bitsLeft = 8;
+ }
+ }
+
+ // flush
+ if (bitsLeft != 8) {
+ output.write(current);
+ current = 0;
+ bitsLeft = 8;
+ }
+ }
+
+ /**
+ * Read bitpacked integers from input stream
+ * @param buffer - input buffer
+ * @param offset - offset
+ * @param len - length
+ * @param bitSize - bit width
+ * @param input - input stream
+ * @throws IOException
+ */
+ static void readInts(long[] buffer, int offset, int len, int bitSize,
+ InStream input) throws IOException {
+ int bitsLeft = 0;
+ int current = 0;
+
+ for(int i = offset; i < (offset + len); i++) {
+ long result = 0;
+ int bitsLeftToRead = bitSize;
+ while (bitsLeftToRead > bitsLeft) {
+ result <<= bitsLeft;
+ result |= current & ((1 << bitsLeft) - 1);
+ bitsLeftToRead -= bitsLeft;
+ current = input.read();
+ bitsLeft = 8;
+ }
+
+ // handle the left over bits
+ if (bitsLeftToRead > 0) {
+ result <<= bitsLeftToRead;
+ bitsLeft -= bitsLeftToRead;
+ result |= (current >> bitsLeft) & ((1 << bitsLeftToRead) - 1);
+ }
+ buffer[i] = result;
+ }
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java Fri Aug 16 01:21:54 2013
@@ -17,11 +17,11 @@
*/
package org.apache.hadoop.hive.ql.io.orc;
-import org.apache.hadoop.io.Text;
-
import java.io.IOException;
import java.io.OutputStream;
+import org.apache.hadoop.io.Text;
+
/**
* A red-black tree that stores strings. The strings are stored as UTF-8 bytes
* and an offset for each entry.
@@ -147,7 +147,7 @@ class StringRedBlackTree extends RedBlac
/**
* Visit all of the nodes in the tree in sorted order.
- * @param visitor the action to be applied to each ndoe
+ * @param visitor the action to be applied to each node
* @throws IOException
*/
public void visit(Visitor visitor) throws IOException {
@@ -163,6 +163,17 @@ class StringRedBlackTree extends RedBlac
keyOffsets.clear();
}
+ public void getText(Text result, int originalPosition) {
+ int offset = keyOffsets.get(originalPosition);
+ int length;
+ if (originalPosition + 1 == keyOffsets.size()) {
+ length = byteArray.size() - offset;
+ } else {
+ length = keyOffsets.get(originalPosition + 1) - offset;
+ }
+ byteArray.setText(result, offset, length);
+ }
+
/**
* Get the size of the character data in the table.
* @return the bytes used by the table
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Fri Aug 16 01:21:54 2013
@@ -27,11 +27,16 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
import org.apache.hadoop.hive.serde2.io.DateWritable;
@@ -55,9 +60,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.io.BytesWritable;
-
-import com.google.protobuf.ByteString;
-import com.google.protobuf.CodedOutputStream;
+import org.apache.hadoop.io.Text;
/**
* An ORC file writer. The file is divided into stripes, which is the natural
@@ -111,8 +114,11 @@ class WriterImpl implements Writer, Memo
private final boolean buildIndex;
private final MemoryManager memoryManager;
+ private final Configuration conf;
+
WriterImpl(FileSystem fs,
Path path,
+ Configuration conf,
ObjectInspector inspector,
long stripeSize,
CompressionKind compress,
@@ -121,6 +127,7 @@ class WriterImpl implements Writer, Memo
MemoryManager memoryManager) throws IOException {
this.fs = fs;
this.path = path;
+ this.conf = conf;
this.stripeSize = stripeSize;
this.compress = compress;
this.bufferSize = bufferSize;
@@ -344,6 +351,14 @@ class WriterImpl implements Writer, Memo
public boolean isCompressed() {
return codec != null;
}
+
+ /**
+ * Get the writer's configuration.
+ * @return configuration
+ */
+ public Configuration getConfiguration() {
+ return conf;
+ }
}
/**
@@ -366,6 +381,7 @@ class WriterImpl implements Writer, Memo
private final PositionedOutputStream rowIndexStream;
private boolean foundNulls;
private OutStream isPresentOutStream;
+ protected final boolean useDirectV2Encoding;
/**
* Create a tree writer.
@@ -381,6 +397,7 @@ class WriterImpl implements Writer, Memo
this.isCompressed = streamFactory.isCompressed();
this.id = columnId;
this.inspector = inspector;
+ this.useDirectV2Encoding = true;
if (nullable) {
isPresentOutStream = streamFactory.createStream(id,
OrcProto.Stream.Kind.PRESENT);
@@ -415,6 +432,32 @@ class WriterImpl implements Writer, Memo
return rowIndexEntry;
}
+ IntegerWriter createIntegerWriter(PositionedOutputStream output,
+ boolean signed, boolean isDirectV2) {
+ if (isDirectV2) {
+ return new RunLengthIntegerWriterV2(output, signed);
+ } else {
+ return new RunLengthIntegerWriter(output, signed);
+ }
+ }
+
+ boolean isNewWriteFormat(StreamFactory writer) {
+ String writeFormat = writer.getConfiguration().get(
+ HiveConf.ConfVars.HIVE_ORC_WRITE_FORMAT.varname);
+ if (writeFormat == null) {
+ LOG.warn("ORC write format not defined. Using 0.12 ORC write format.");
+ return true;
+ }
+ if (writeFormat
+ .equals(HiveConf.ConfVars.HIVE_ORC_WRITE_FORMAT.defaultVal)) {
+ LOG.info("Using 0.11 ORC write format.");
+ return false;
+ }
+
+ LOG.info("Using 0.12 ORC write format.");
+ return true;
+ }
+
/**
* Add a new value to the column.
* @param obj
@@ -620,10 +663,11 @@ class WriterImpl implements Writer, Memo
}
private static class IntegerTreeWriter extends TreeWriter {
- private final RunLengthIntegerWriter writer;
+ private final IntegerWriter writer;
private final ShortObjectInspector shortInspector;
private final IntObjectInspector intInspector;
private final LongObjectInspector longInspector;
+ private boolean isDirectV2 = true;
IntegerTreeWriter(int columnId,
ObjectInspector inspector,
@@ -632,7 +676,8 @@ class WriterImpl implements Writer, Memo
super(columnId, inspector, writer, nullable);
PositionedOutputStream out = writer.createStream(id,
OrcProto.Stream.Kind.DATA);
- this.writer = new RunLengthIntegerWriter(out, true);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ this.writer = createIntegerWriter(out, true, isDirectV2);
if (inspector instanceof IntObjectInspector) {
intInspector = (IntObjectInspector) inspector;
shortInspector = null;
@@ -651,6 +696,16 @@ class WriterImpl implements Writer, Memo
}
@Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
void write(Object obj) throws IOException {
super.write(obj);
if (obj != null) {
@@ -760,31 +815,46 @@ class WriterImpl implements Writer, Memo
private static class StringTreeWriter extends TreeWriter {
private static final int INITIAL_DICTIONARY_SIZE = 4096;
- private final PositionedOutputStream stringOutput;
- private final RunLengthIntegerWriter lengthOutput;
- private final RunLengthIntegerWriter rowOutput;
+ private final OutStream stringOutput;
+ private final IntegerWriter lengthOutput;
+ private final IntegerWriter rowOutput;
private final StringRedBlackTree dictionary =
new StringRedBlackTree(INITIAL_DICTIONARY_SIZE);
private final DynamicIntArray rows = new DynamicIntArray();
+ private final PositionedOutputStream directStreamOutput;
+ private final IntegerWriter directLengthOutput;
private final List<OrcProto.RowIndexEntry> savedRowIndex =
new ArrayList<OrcProto.RowIndexEntry>();
private final boolean buildIndex;
private final List<Long> rowIndexValueCount = new ArrayList<Long>();
+ // If the number of keys in a dictionary is greater than this fraction of
+ //the total number of non-null rows, turn off dictionary encoding
+ private final float dictionaryKeySizeThreshold;
+ private boolean useDictionaryEncoding = true;
+ private boolean isDirectV2 = true;
StringTreeWriter(int columnId,
ObjectInspector inspector,
StreamFactory writer,
boolean nullable) throws IOException {
super(columnId, inspector, writer, nullable);
+ this.isDirectV2 = isNewWriteFormat(writer);
stringOutput = writer.createStream(id,
OrcProto.Stream.Kind.DICTIONARY_DATA);
- lengthOutput = new RunLengthIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.LENGTH), false);
- rowOutput = new RunLengthIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.DATA), false);
+ lengthOutput = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.LENGTH), false, isDirectV2);
+ rowOutput = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.DATA), false, isDirectV2);
recordPosition(rowIndexPosition);
rowIndexValueCount.add(0L);
buildIndex = writer.buildIndex();
+ directStreamOutput = writer.createStream(id, OrcProto.Stream.Kind.DATA);
+ directLengthOutput = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.LENGTH), false, isDirectV2);
+ dictionaryKeySizeThreshold = writer.getConfiguration().getFloat(
+ HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.varname,
+ HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.
+ defaultFloatVal);
}
@Override
@@ -801,22 +871,36 @@ class WriterImpl implements Writer, Memo
@Override
void writeStripe(OrcProto.StripeFooter.Builder builder,
int requiredIndexEntries) throws IOException {
- // Traverse the red-black tree writing out the bytes and lengths; and
- // creating the map from the original order to the final sorted order.
+ // Set the flag indicating whether or not to use dictionary encoding
+ // based on whether or not the fraction of distinct keys over number of
+ // non-null rows is less than the configured threshold
+ useDictionaryEncoding = rows.size() > 0 &&
+ (float)(dictionary.size()) / rows.size() <=
+ dictionaryKeySizeThreshold;
final int[] dumpOrder = new int[dictionary.size()];
- dictionary.visit(new StringRedBlackTree.Visitor() {
- private int currentId = 0;
- @Override
- public void visit(StringRedBlackTree.VisitorContext context
- ) throws IOException {
- context.writeBytes(stringOutput);
- lengthOutput.write(context.getLength());
- dumpOrder[context.getOriginalPosition()] = currentId++;
- }
- });
+
+ if (useDictionaryEncoding) {
+ // Write the dictionary by traversing the red-black tree writing out
+ // the bytes and lengths; and creating the map from the original order
+ // to the final sorted order.
+ dictionary.visit(new StringRedBlackTree.Visitor() {
+ private int currentId = 0;
+ @Override
+ public void visit(StringRedBlackTree.VisitorContext context
+ ) throws IOException {
+ context.writeBytes(stringOutput);
+ lengthOutput.write(context.getLength());
+ dumpOrder[context.getOriginalPosition()] = currentId++;
+ }
+ });
+ } else {
+ // for direct encoding, we don't want the dictionary data stream
+ stringOutput.suppress();
+ }
int length = rows.size();
int rowIndexEntry = 0;
OrcProto.RowIndex.Builder rowIndex = getRowIndex();
+ Text text = new Text();
// write the values translated into the dump order.
for(int i = 0; i <= length; ++i) {
// now that we are writing out the row values, we can finalize the
@@ -826,12 +910,24 @@ class WriterImpl implements Writer, Memo
rowIndexEntry < savedRowIndex.size()) {
OrcProto.RowIndexEntry.Builder base =
savedRowIndex.get(rowIndexEntry++).toBuilder();
- rowOutput.getPosition(new RowIndexPositionRecorder(base));
+ if (useDictionaryEncoding) {
+ rowOutput.getPosition(new RowIndexPositionRecorder(base));
+ } else {
+ PositionRecorder posn = new RowIndexPositionRecorder(base);
+ directStreamOutput.getPosition(posn);
+ directLengthOutput.getPosition(posn);
+ }
rowIndex.addEntry(base.build());
}
}
if (i != length) {
- rowOutput.write(dumpOrder[rows.get(i)]);
+ if (useDictionaryEncoding) {
+ rowOutput.write(dumpOrder[rows.get(i)]);
+ } else {
+ dictionary.getText(text, rows.get(i));
+ directStreamOutput.write(text.getBytes(), 0, text.getLength());
+ directLengthOutput.write(text.getLength());
+ }
}
}
// we need to build the rowindex before calling super, since it
@@ -840,6 +936,8 @@ class WriterImpl implements Writer, Memo
stringOutput.flush();
lengthOutput.flush();
rowOutput.flush();
+ directStreamOutput.flush();
+ directLengthOutput.flush();
// reset all of the fields to be ready for the next stripe.
dictionary.clear();
rows.clear();
@@ -851,9 +949,24 @@ class WriterImpl implements Writer, Memo
@Override
OrcProto.ColumnEncoding getEncoding() {
- return OrcProto.ColumnEncoding.newBuilder().setKind(
- OrcProto.ColumnEncoding.Kind.DICTIONARY).
- setDictionarySize(dictionary.size()).build();
+ // Returns the encoding used for the last call to writeStripe
+ if (useDictionaryEncoding) {
+ if(isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder().setKind(
+ OrcProto.ColumnEncoding.Kind.DICTIONARY_V2).
+ setDictionarySize(dictionary.size()).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder().setKind(
+ OrcProto.ColumnEncoding.Kind.DICTIONARY).
+ setDictionarySize(dictionary.size()).build();
+ } else {
+ if(isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder().setKind(
+ OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder().setKind(
+ OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
}
/**
@@ -883,7 +996,8 @@ class WriterImpl implements Writer, Memo
private static class BinaryTreeWriter extends TreeWriter {
private final PositionedOutputStream stream;
- private final RunLengthIntegerWriter length;
+ private final IntegerWriter length;
+ private boolean isDirectV2 = true;
BinaryTreeWriter(int columnId,
ObjectInspector inspector,
@@ -892,12 +1006,23 @@ class WriterImpl implements Writer, Memo
super(columnId, inspector, writer, nullable);
this.stream = writer.createStream(id,
OrcProto.Stream.Kind.DATA);
- this.length = new RunLengthIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.LENGTH), false);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ this.length = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.LENGTH), false, isDirectV2);
recordPosition(rowIndexPosition);
}
@Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
void write(Object obj) throws IOException {
super.write(obj);
if (obj != null) {
@@ -930,22 +1055,34 @@ class WriterImpl implements Writer, Memo
Timestamp.valueOf("2015-01-01 00:00:00").getTime() / MILLIS_PER_SECOND;
private static class TimestampTreeWriter extends TreeWriter {
- private final RunLengthIntegerWriter seconds;
- private final RunLengthIntegerWriter nanos;
+ private final IntegerWriter seconds;
+ private final IntegerWriter nanos;
+ private final boolean isDirectV2;
TimestampTreeWriter(int columnId,
ObjectInspector inspector,
StreamFactory writer,
boolean nullable) throws IOException {
super(columnId, inspector, writer, nullable);
- this.seconds = new RunLengthIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.DATA), true);
- this.nanos = new RunLengthIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.SECONDARY), false);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ this.seconds = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.DATA), true, isDirectV2);
+ this.nanos = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.SECONDARY), false, isDirectV2);
recordPosition(rowIndexPosition);
}
@Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
void write(Object obj) throws IOException {
super.write(obj);
if (obj != null) {
@@ -991,7 +1128,8 @@ class WriterImpl implements Writer, Memo
}
private static class DateTreeWriter extends TreeWriter {
- private final RunLengthIntegerWriter writer;
+ private final IntegerWriter writer;
+ private final boolean isDirectV2;
DateTreeWriter(int columnId,
ObjectInspector inspector,
@@ -1000,7 +1138,8 @@ class WriterImpl implements Writer, Memo
super(columnId, inspector, writer, nullable);
PositionedOutputStream out = writer.createStream(id,
OrcProto.Stream.Kind.DATA);
- this.writer = new RunLengthIntegerWriter(out, true);
+ this.isDirectV2 = isNewWriteFormat(writer);
+ this.writer = createIntegerWriter(out, true, isDirectV2);
recordPosition(rowIndexPosition);
}
@@ -1028,24 +1167,46 @@ class WriterImpl implements Writer, Memo
super.recordPosition(recorder);
writer.getPosition(recorder);
}
+
+ @Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
}
private static class DecimalTreeWriter extends TreeWriter {
private final PositionedOutputStream valueStream;
- private final RunLengthIntegerWriter scaleStream;
+ private final IntegerWriter scaleStream;
+ private final boolean isDirectV2;
DecimalTreeWriter(int columnId,
ObjectInspector inspector,
StreamFactory writer,
boolean nullable) throws IOException {
super(columnId, inspector, writer, nullable);
+ this.isDirectV2 = isNewWriteFormat(writer);
valueStream = writer.createStream(id, OrcProto.Stream.Kind.DATA);
- scaleStream = new RunLengthIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.SECONDARY), true);
+ this.scaleStream = createIntegerWriter(writer.createStream(id,
+ OrcProto.Stream.Kind.SECONDARY), true, isDirectV2);
recordPosition(rowIndexPosition);
}
@Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
void write(Object obj) throws IOException {
super.write(obj);
if (obj != null) {
@@ -1118,25 +1279,36 @@ class WriterImpl implements Writer, Memo
}
private static class ListTreeWriter extends TreeWriter {
- private final RunLengthIntegerWriter lengths;
+ private final IntegerWriter lengths;
+ private final boolean isDirectV2;
ListTreeWriter(int columnId,
ObjectInspector inspector,
StreamFactory writer,
boolean nullable) throws IOException {
super(columnId, inspector, writer, nullable);
+ this.isDirectV2 = isNewWriteFormat(writer);
ListObjectInspector listObjectInspector = (ListObjectInspector) inspector;
childrenWriters = new TreeWriter[1];
childrenWriters[0] =
createTreeWriter(listObjectInspector.getListElementObjectInspector(),
writer, true);
- lengths =
- new RunLengthIntegerWriter(writer.createStream(columnId,
- OrcProto.Stream.Kind.LENGTH), false);
+ lengths = createIntegerWriter(writer.createStream(columnId,
+ OrcProto.Stream.Kind.LENGTH), false, isDirectV2);
recordPosition(rowIndexPosition);
}
@Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
void write(Object obj) throws IOException {
super.write(obj);
if (obj != null) {
@@ -1168,26 +1340,37 @@ class WriterImpl implements Writer, Memo
}
private static class MapTreeWriter extends TreeWriter {
- private final RunLengthIntegerWriter lengths;
+ private final IntegerWriter lengths;
+ private final boolean isDirectV2;
MapTreeWriter(int columnId,
ObjectInspector inspector,
StreamFactory writer,
boolean nullable) throws IOException {
super(columnId, inspector, writer, nullable);
+ this.isDirectV2 = isNewWriteFormat(writer);
MapObjectInspector insp = (MapObjectInspector) inspector;
childrenWriters = new TreeWriter[2];
childrenWriters[0] =
createTreeWriter(insp.getMapKeyObjectInspector(), writer, true);
childrenWriters[1] =
createTreeWriter(insp.getMapValueObjectInspector(), writer, true);
- lengths =
- new RunLengthIntegerWriter(writer.createStream(columnId,
- OrcProto.Stream.Kind.LENGTH), false);
+ lengths = createIntegerWriter(writer.createStream(columnId,
+ OrcProto.Stream.Kind.LENGTH), false, isDirectV2);
recordPosition(rowIndexPosition);
}
@Override
+ OrcProto.ColumnEncoding getEncoding() {
+ if (isDirectV2) {
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
+ }
+ return OrcProto.ColumnEncoding.newBuilder()
+ .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
+ }
+
+ @Override
void write(Object obj) throws IOException {
super.write(obj);
if (obj != null) {
@@ -1273,8 +1456,7 @@ class WriterImpl implements Writer, Memo
private static TreeWriter createTreeWriter(ObjectInspector inspector,
StreamFactory streamFactory,
- boolean nullable
- ) throws IOException {
+ boolean nullable) throws IOException {
switch (inspector.getCategory()) {
case PRIMITIVE:
switch (((PrimitiveObjectInspector) inspector).getPrimitiveCategory()) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java Fri Aug 16 01:21:54 2013
@@ -192,7 +192,7 @@ public class BlockMergeTask extends Task
try {
addInputPaths(job, work);
- Utilities.setMapWork(job, work, ctx.getMRTmpFileURI());
+ Utilities.setMapWork(job, work, ctx.getMRTmpFileURI(), true);
// remove the pwd from conf file so that job tracker doesn't show this
// logs
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java Fri Aug 16 01:21:54 2013
@@ -23,11 +23,8 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.util.ReflectionUtils;
/**
* PerfLogger.
@@ -55,7 +52,7 @@ public class PerfLogger {
static final private Log LOG = LogFactory.getLog(PerfLogger.class.getName());
- protected PerfLogger() {
+ public PerfLogger() {
// Use getPerfLogger to get an instance of PerfLogger
}
@@ -68,26 +65,17 @@ public class PerfLogger {
*
* Use resetPerfLogger to require a new instance. Useful at the beginning of execution.
*
- * @return Tries to return an instance of the class whose name is configured in
- * hive.exec.perf.logger, but if it can't it just returns an instance of
- * the base PerfLogger class
+ * @return Session perflogger if there's a sessionstate, otherwise return the thread local instance
*/
public static PerfLogger getPerfLogger(boolean resetPerfLogger) {
- if (perfLogger.get() == null || resetPerfLogger) {
- if (SessionState.get() == null) {
+ if (SessionState.get() == null) {
+ if (perfLogger.get() == null || resetPerfLogger) {
perfLogger.set(new PerfLogger());
- } else {
- HiveConf conf = SessionState.get().getConf();
- try {
- perfLogger.set((PerfLogger) ReflectionUtils.newInstance(conf.getClassByName(
- conf.getVar(ConfVars.HIVE_PERF_LOGGER)), conf));
- } catch (ClassNotFoundException e) {
- LOG.error("Performance Logger Class not found:" + e.getMessage());
- perfLogger.set(new PerfLogger());
- }
}
+ return perfLogger.get();
+ } else {
+ return SessionState.get().getPerfLogger(resetPerfLogger);
}
- return perfLogger.get();
}
/**
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri Aug 16 01:21:54 2013
@@ -1814,6 +1814,7 @@ private void constructOneLBLocationMap(F
List<Partition> partitions = new ArrayList<Partition>(partNames.size());
int batchSize = HiveConf.getIntVar(conf, HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX);
+ // TODO: might want to increase the default batch size. 1024 is viable; MS gets OOM if too high.
int nParts = partNames.size();
int nBatches = nParts / batchSize;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/metadata/VirtualColumn.java Fri Aug 16 01:21:54 2013
@@ -24,6 +24,10 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -47,6 +51,8 @@ public class VirtualColumn implements Se
public static VirtualColumn GROUPINGID =
new VirtualColumn("GROUPING__ID", (PrimitiveTypeInfo) TypeInfoFactory.intTypeInfo);
+ public static VirtualColumn[] VIRTUAL_COLUMNS =
+ new VirtualColumn[] {FILENAME, BLOCKOFFSET, ROWOFFSET, RAWDATASIZE, GROUPINGID};
private String name;
private PrimitiveTypeInfo typeInfo;
@@ -125,4 +131,15 @@ public class VirtualColumn implements Se
&& this.typeInfo.getTypeName().equals(c.getTypeInfo().getTypeName());
}
+
+ public static StructObjectInspector getVCSObjectInspector(List<VirtualColumn> vcs) {
+ List<String> names = new ArrayList<String>(vcs.size());
+ List<ObjectInspector> inspectors = new ArrayList<ObjectInspector>(vcs.size());
+ for (VirtualColumn vc : vcs) {
+ names.add(vc.getName());
+ inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ vc.getTypeInfo().getPrimitiveCategory()));
+ }
+ return ObjectInspectorFactory.getStandardStructObjectInspector(names, inspectors);
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java Fri Aug 16 01:21:54 2013
@@ -45,7 +45,6 @@ import org.apache.hadoop.hive.ql.lib.Nod
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.QB;
@@ -270,14 +269,7 @@ abstract public class AbstractBucketJoin
if (tbl.isPartitioned()) {
PrunedPartitionList prunedParts;
try {
- prunedParts = pGraphContext.getOpToPartList().get(tso);
- if (prunedParts == null) {
- prunedParts =
- PartitionPruner.prune(tbl, pGraphContext.getOpToPartPruner().get(tso),
- pGraphContext.getConf(), alias,
- pGraphContext.getPrunedPartitions());
- pGraphContext.getOpToPartList().put(tso, prunedParts);
- }
+ prunedParts = pGraphContext.getPrunedPartitions(alias, tso);
} catch (HiveException e) {
// Has to use full name to make sure it does not conflict with
// org.apache.commons.lang.StringUtils
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java Fri Aug 16 01:21:54 2013
@@ -42,7 +42,6 @@ import org.apache.hadoop.hive.ql.lib.Nod
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.QB;
@@ -310,15 +309,9 @@ abstract public class AbstractSMBJoinPro
Table tbl = topToTable.get(tso);
if (tbl.isPartitioned()) {
- PrunedPartitionList prunedParts = null;
+ PrunedPartitionList prunedParts;
try {
- prunedParts = pGraphContext.getOpToPartList().get(tso);
- if (prunedParts == null) {
- prunedParts = PartitionPruner.prune(tbl, pGraphContext
- .getOpToPartPruner().get(tso), pGraphContext.getConf(), alias,
- pGraphContext.getPrunedPartitions());
- pGraphContext.getOpToPartList().put(tso, prunedParts);
- }
+ prunedParts = pGraphContext.getPrunedPartitions(alias, tso);
} catch (HiveException e) {
LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
throw new SemanticException(e.getMessage(), e);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java Fri Aug 16 01:21:54 2013
@@ -70,10 +70,7 @@ public class AvgPartitionSizeBasedBigTab
}
else {
// For partitioned tables, get the size of all the partitions
- PrunedPartitionList partsList =
- PartitionPruner.prune(parseCtx.getTopToTable().get(topOp),
- parseCtx.getOpToPartPruner().get(topOp), parseCtx.getConf(),
- null, parseCtx.getPrunedPartitions());
+ PrunedPartitionList partsList = PartitionPruner.prune(topOp, parseCtx, null);
numPartitions = partsList.getNotDeniedPartns().size();
long totalSize = 0;
for (Partition part : partsList.getNotDeniedPartns()) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java Fri Aug 16 01:21:54 2013
@@ -457,7 +457,8 @@ public class BucketingSortingReduceSinkO
// nothing to be done for filters - the output schema does not change.
if (op instanceof TableScanOperator) {
assert !useBucketSortPositions;
- Table srcTable = pGraphContext.getTopToTable().get(op);
+ TableScanOperator ts = (TableScanOperator) op;
+ Table srcTable = pGraphContext.getTopToTable().get(ts);
// Find the positions of the bucketed columns in the table corresponding
// to the select list.
@@ -496,7 +497,7 @@ public class BucketingSortingReduceSinkO
}
if (srcTable.isPartitioned()) {
- PrunedPartitionList prunedParts = pGraphContext.getOpToPartList().get(op);
+ PrunedPartitionList prunedParts = pGraphContext.getOpToPartList().get(ts);
List<Partition> partitions = prunedParts.getNotDeniedPartns();
// Support for dynamic partitions can be added later
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java Fri Aug 16 01:21:54 2013
@@ -140,8 +140,7 @@ public class GenMRTableScan1 implements
}
if (confirmedPartns.size() > 0) {
Table source = parseCtx.getQB().getMetaData().getTableForAlias(alias);
- PrunedPartitionList partList = new PrunedPartitionList(source, confirmedPartns,
- new HashSet<Partition>(), null);
+ PrunedPartitionList partList = new PrunedPartitionList(source, confirmedPartns, false);
GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currTask, false, ctx, partList);
} else { // non-partitioned table
GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currTask, false, ctx);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Fri Aug 16 01:21:54 2013
@@ -25,6 +25,7 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
@@ -62,7 +63,6 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc;
-import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
import org.apache.hadoop.hive.ql.plan.MapredWork;
@@ -455,13 +455,8 @@ public final class GenMapRedUtils {
if (partsList == null) {
try {
- partsList = parseCtx.getOpToPartList().get((TableScanOperator) topOp);
- if (partsList == null) {
- partsList = PartitionPruner.prune(parseCtx.getTopToTable().get(topOp),
- parseCtx.getOpToPartPruner().get(topOp), opProcCtx.getConf(),
- alias_id, parseCtx.getPrunedPartitions());
- parseCtx.getOpToPartList().put((TableScanOperator) topOp, partsList);
- }
+ TableScanOperator tsOp = (TableScanOperator) topOp;
+ partsList = PartitionPruner.prune(tsOp, parseCtx, alias_id);
} catch (SemanticException e) {
throw e;
} catch (HiveException e) {
@@ -471,12 +466,9 @@ public final class GenMapRedUtils {
}
// Generate the map work for this alias_id
- Set<Partition> parts = null;
// pass both confirmed and unknown partitions through the map-reduce
// framework
-
- parts = partsList.getConfirmedPartns();
- parts.addAll(partsList.getUnknownPartns());
+ Set<Partition> parts = partsList.getPartitions();
PartitionDesc aliasPartnDesc = null;
try {
if (!parts.isEmpty()) {
@@ -987,6 +979,77 @@ public final class GenMapRedUtils {
return true;
}
+
+
+ /**
+ * Replace the Map-side operator tree associated with targetAlias in
+ * target with the Map-side operator tree associated with sourceAlias in source.
+ * @param sourceAlias
+ * @param targetAlias
+ * @param source
+ * @param target
+ */
+ public static void replaceMapWork(String sourceAlias, String targetAlias,
+ MapWork source, MapWork target) {
+ Map<String, ArrayList<String>> sourcePathToAliases = source.getPathToAliases();
+ Map<String, PartitionDesc> sourcePathToPartitionInfo = source.getPathToPartitionInfo();
+ Map<String, Operator<? extends OperatorDesc>> sourceAliasToWork = source.getAliasToWork();
+ Map<String, PartitionDesc> sourceAliasToPartnInfo = source.getAliasToPartnInfo();
+
+ Map<String, ArrayList<String>> targetPathToAliases = target.getPathToAliases();
+ Map<String, PartitionDesc> targetPathToPartitionInfo = target.getPathToPartitionInfo();
+ Map<String, Operator<? extends OperatorDesc>> targetAliasToWork = target.getAliasToWork();
+ Map<String, PartitionDesc> targetAliasToPartnInfo = target.getAliasToPartnInfo();
+
+ if (!sourceAliasToWork.containsKey(sourceAlias) ||
+ !targetAliasToWork.containsKey(targetAlias)) {
+ // Nothing to do if there is no operator tree associated with
+ // sourceAlias in source or there is not operator tree associated
+ // with targetAlias in target.
+ return;
+ }
+
+ if (sourceAliasToWork.size() > 1) {
+ // If there are multiple aliases in source, we do not know
+ // how to merge.
+ return;
+ }
+
+ // Remove unnecessary information from target
+ targetAliasToWork.remove(targetAlias);
+ targetAliasToPartnInfo.remove(targetAlias);
+ List<String> pathsToRemove = new ArrayList<String>();
+ for (Entry<String, ArrayList<String>> entry: targetPathToAliases.entrySet()) {
+ ArrayList<String> aliases = entry.getValue();
+ aliases.remove(targetAlias);
+ if (aliases.isEmpty()) {
+ pathsToRemove.add(entry.getKey());
+ }
+ }
+ for (String pathToRemove: pathsToRemove) {
+ targetPathToAliases.remove(pathToRemove);
+ targetPathToPartitionInfo.remove(pathToRemove);
+ }
+
+ // Add new information from source to target
+ targetAliasToWork.put(sourceAlias, sourceAliasToWork.get(sourceAlias));
+ targetAliasToPartnInfo.putAll(sourceAliasToPartnInfo);
+ targetPathToPartitionInfo.putAll(sourcePathToPartitionInfo);
+ List<String> pathsToAdd = new ArrayList<String>();
+ for (Entry<String, ArrayList<String>> entry: sourcePathToAliases.entrySet()) {
+ ArrayList<String> aliases = entry.getValue();
+ if (aliases.contains(sourceAlias)) {
+ pathsToAdd.add(entry.getKey());
+ }
+ }
+ for (String pathToAdd: pathsToAdd) {
+ if (!targetPathToAliases.containsKey(pathToAdd)) {
+ targetPathToAliases.put(pathToAdd, new ArrayList<String>());
+ }
+ targetPathToAliases.get(pathToAdd).add(sourceAlias);
+ }
+ }
+
private GenMapRedUtils() {
// prevent instantiation
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java Fri Aug 16 01:21:54 2013
@@ -61,8 +61,6 @@ public class GlobalLimitOptimizer implem
Map<String, Operator<? extends OperatorDesc>> topOps = pctx.getTopOps();
GlobalLimitCtx globalLimitCtx = pctx.getGlobalLimitCtx();
Map<TableScanOperator, ExprNodeDesc> opToPartPruner = pctx.getOpToPartPruner();
- Map<TableScanOperator, PrunedPartitionList> opToPartList = pctx.getOpToPartList();
- Map<String, PrunedPartitionList> prunedPartitions = pctx.getPrunedPartitions();
Map<String, SplitSample> nameToSplitSample = pctx.getNameToSplitSample();
Map<TableScanOperator, Table> topToTable = pctx.getTopToTable();
@@ -106,15 +104,10 @@ public class GlobalLimitOptimizer implem
if (PartitionPruner.onlyContainsPartnCols(tab,
opToPartPruner.get(ts))) {
- PrunedPartitionList partsList = null;
+ PrunedPartitionList partsList;
try {
- partsList = opToPartList.get(ts);
- if (partsList == null) {
- partsList = PartitionPruner.prune(tab,
- opToPartPruner.get(ts), conf, (String) topOps.keySet()
- .toArray()[0], prunedPartitions);
- opToPartList.put(ts, partsList);
- }
+ String alias = (String) topOps.keySet().toArray()[0];
+ partsList = PartitionPruner.prune(ts, pctx, alias);
} catch (HiveException e) {
// Has to use full name to make sure it does not conflict with
// org.apache.commons.lang.StringUtils
@@ -124,7 +117,7 @@ public class GlobalLimitOptimizer implem
// If there is any unknown partition, create a map-reduce job for
// the filter to prune correctly
- if ((partsList.getUnknownPartns().size() == 0)) {
+ if (!partsList.hasUnknownPartitions()) {
globalLimitCtx.enableOpt(tempGlobalLimit);
}
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java Fri Aug 16 01:21:54 2013
@@ -49,7 +49,6 @@ import org.apache.hadoop.hive.ql.lib.Rul
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -389,17 +388,9 @@ public class GroupByOptimizer implements
List<String> bucketCols = table.getBucketCols();
return matchBucketSortCols(groupByCols, bucketCols, sortCols);
} else {
- PrunedPartitionList partsList = null;
+ PrunedPartitionList partsList;
try {
- partsList = pGraphContext.getOpToPartList().get(tableScanOp);
- if (partsList == null) {
- partsList = PartitionPruner.prune(table,
- pGraphContext.getOpToPartPruner().get(tableScanOp),
- pGraphContext.getConf(),
- table.getTableName(),
- pGraphContext.getPrunedPartitions());
- pGraphContext.getOpToPartList().put(tableScanOp, partsList);
- }
+ partsList = pGraphContext.getPrunedPartitions(table.getTableName(), tableScanOp);
} catch (HiveException e) {
LOG.error(StringUtils.stringifyException(e));
throw new SemanticException(e.getMessage(), e);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java?rev=1514554&r1=1514553&r2=1514554&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/IndexUtils.java Fri Aug 16 01:21:54 2013
@@ -74,18 +74,10 @@ public final class IndexUtils {
Map<Table, List<Index>> indexes)
throws HiveException {
Hive hive = Hive.get(pctx.getConf());
- Set<Partition> queryPartitions = null;
// make sure each partition exists on the index table
PrunedPartitionList queryPartitionList = pctx.getOpToPartList().get(tableScan);
- if(queryPartitionList.getConfirmedPartns() != null
- && !queryPartitionList.getConfirmedPartns().isEmpty()){
- queryPartitions = queryPartitionList.getConfirmedPartns();
- }else if(queryPartitionList.getUnknownPartns() != null
- && !queryPartitionList.getUnknownPartns().isEmpty()){
- queryPartitions = queryPartitionList.getUnknownPartns();
- }
-
- if(queryPartitions == null) {
+ Set<Partition> queryPartitions = queryPartitionList.getPartitions();
+ if (queryPartitions == null || queryPartitions.isEmpty()) {
return null;
}