You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2015/07/06 23:52:51 UTC

[22/23] orc git commit: ORC-23. Simplify directory structure.

http://git-wip-us.apache.org/repos/asf/orc/blob/7f55b453/c++/src/ColumnReader.cc
----------------------------------------------------------------------
diff --git a/c++/src/ColumnReader.cc b/c++/src/ColumnReader.cc
new file mode 100644
index 0000000..78fb15f
--- /dev/null
+++ b/c++/src/ColumnReader.cc
@@ -0,0 +1,1558 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "orc/Int128.hh"
+
+#include "Adaptor.hh"
+#include "ByteRLE.hh"
+#include "ColumnReader.hh"
+#include "Exceptions.hh"
+#include "RLE.hh"
+
+#include <math.h>
+#include <iostream>
+
+namespace orc {
+
+  StripeStreams::~StripeStreams() {
+    // PASS
+  }
+
+  inline RleVersion convertRleVersion(proto::ColumnEncoding_Kind kind) {
+    switch (static_cast<int64_t>(kind)) {
+    case proto::ColumnEncoding_Kind_DIRECT:
+    case proto::ColumnEncoding_Kind_DICTIONARY:
+      return RleVersion_1;
+    case proto::ColumnEncoding_Kind_DIRECT_V2:
+    case proto::ColumnEncoding_Kind_DICTIONARY_V2:
+      return RleVersion_2;
+    default:
+      throw ParseError("Unknown encoding in convertRleVersion");
+    }
+  }
+
+  ColumnReader::ColumnReader(const Type& type,
+                             StripeStreams& stripe
+                             ): columnId(type.getColumnId()),
+                                memoryPool(stripe.getMemoryPool()) {
+    std::unique_ptr<SeekableInputStream> stream =
+      stripe.getStream(columnId, proto::Stream_Kind_PRESENT, true);
+    if (stream.get()) {
+      notNullDecoder = createBooleanRleDecoder(std::move(stream));
+    }
+  }
+
+  ColumnReader::~ColumnReader() {
+    // PASS
+  }
+
+  uint64_t ColumnReader::skip(uint64_t numValues) {
+    ByteRleDecoder* decoder = notNullDecoder.get();
+    if (decoder) {
+      // page through the values that we want to skip
+      // and count how many are non-null
+      const size_t MAX_BUFFER_SIZE = 32768;
+      size_t bufferSize = std::min(MAX_BUFFER_SIZE,
+                                   static_cast<size_t>(numValues));
+      char buffer[MAX_BUFFER_SIZE];
+      uint64_t remaining = numValues;
+      while (remaining > 0) {
+        uint64_t chunkSize =
+          std::min(remaining,
+                   static_cast<uint64_t>(bufferSize));
+        decoder->next(buffer, chunkSize, 0);
+        remaining -= chunkSize;
+        for(uint64_t i=0; i < chunkSize; ++i) {
+          if (!buffer[i]) {
+            numValues -= 1;
+          }
+        }
+      }
+    }
+    return numValues;
+  }
+
+  void ColumnReader::next(ColumnVectorBatch& rowBatch,
+                          uint64_t numValues,
+                          char* incomingMask) {
+    if (numValues > rowBatch.capacity) {
+      rowBatch.resize(numValues);
+    }
+    rowBatch.numElements = numValues;
+    ByteRleDecoder* decoder = notNullDecoder.get();
+    if (decoder) {
+      char* notNullArray = rowBatch.notNull.data();
+      decoder->next(notNullArray, numValues, incomingMask);
+      // check to see if there are nulls in this batch
+      for(uint64_t i=0; i < numValues; ++i) {
+        if (!notNullArray[i]) {
+          rowBatch.hasNulls = true;
+          return;
+        }
+      }
+    } else if (incomingMask) {
+      // If we don't have a notNull stream, copy the incomingMask
+      rowBatch.hasNulls = true;
+      memcpy(rowBatch.notNull.data(), incomingMask, numValues);
+      return;
+    }
+    rowBatch.hasNulls = false;
+  }
+
+  /**
+   * Expand an array of bytes in place to the corresponding array of longs.
+   * Has to work backwards so that they data isn't clobbered during the
+   * expansion.
+   * @param buffer the array of chars and array of longs that need to be
+   *        expanded
+   * @param numValues the number of bytes to convert to longs
+   */
+  void expandBytesToLongs(int64_t* buffer, uint64_t numValues) {
+    for(size_t i=numValues - 1; i < numValues; --i) {
+      buffer[i] = reinterpret_cast<char *>(buffer)[i];
+    }
+  }
+
+  class BooleanColumnReader: public ColumnReader {
+  private:
+    std::unique_ptr<orc::ByteRleDecoder> rle;
+
+  public:
+    BooleanColumnReader(const Type& type, StripeStreams& stipe);
+    ~BooleanColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char* notNull) override;
+  };
+
+  BooleanColumnReader::BooleanColumnReader(const Type& type,
+                                           StripeStreams& stripe
+                                           ): ColumnReader(type, stripe){
+    rle = createBooleanRleDecoder(stripe.getStream(columnId,
+                                                   proto::Stream_Kind_DATA,
+                                                   true));
+  }
+
+  BooleanColumnReader::~BooleanColumnReader() {
+    // PASS
+  }
+
+  uint64_t BooleanColumnReader::skip(uint64_t numValues) {
+    numValues = ColumnReader::skip(numValues);
+    rle->skip(numValues);
+    return numValues;
+  }
+
+  void BooleanColumnReader::next(ColumnVectorBatch& rowBatch,
+                                 uint64_t numValues,
+                                 char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    // Since the byte rle places the output in a char* instead of long*,
+    // we cheat here and use the long* and then expand it in a second pass.
+    int64_t *ptr = dynamic_cast<LongVectorBatch&>(rowBatch).data.data();
+    rle->next(reinterpret_cast<char*>(ptr),
+              numValues, rowBatch.hasNulls ? rowBatch.notNull.data() : 0);
+    expandBytesToLongs(ptr, numValues);
+  }
+
+  class ByteColumnReader: public ColumnReader {
+  private:
+    std::unique_ptr<orc::ByteRleDecoder> rle;
+
+  public:
+    ByteColumnReader(const Type& type, StripeStreams& stipe);
+    ~ByteColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char* notNull) override;
+  };
+
+  ByteColumnReader::ByteColumnReader(const Type& type,
+                                           StripeStreams& stripe
+                                           ): ColumnReader(type, stripe){
+    rle = createByteRleDecoder(stripe.getStream(columnId,
+                                                proto::Stream_Kind_DATA,
+                                                true));
+  }
+
+  ByteColumnReader::~ByteColumnReader() {
+    // PASS
+  }
+
+  uint64_t ByteColumnReader::skip(uint64_t numValues) {
+    numValues = ColumnReader::skip(numValues);
+    rle->skip(numValues);
+    return numValues;
+  }
+
+  void ByteColumnReader::next(ColumnVectorBatch& rowBatch,
+                              uint64_t numValues,
+                              char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    // Since the byte rle places the output in a char* instead of long*,
+    // we cheat here and use the long* and then expand it in a second pass.
+    int64_t *ptr = dynamic_cast<LongVectorBatch&>(rowBatch).data.data();
+    rle->next(reinterpret_cast<char*>(ptr),
+              numValues, rowBatch.hasNulls ? rowBatch.notNull.data() : 0);
+    expandBytesToLongs(ptr, numValues);
+  }
+
+  class IntegerColumnReader: public ColumnReader {
+  protected:
+    std::unique_ptr<orc::RleDecoder> rle;
+
+  public:
+    IntegerColumnReader(const Type& type, StripeStreams& stripe);
+    ~IntegerColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char* notNull) override;
+  };
+
+  IntegerColumnReader::IntegerColumnReader(const Type& type,
+                                           StripeStreams& stripe
+                                           ): ColumnReader(type, stripe) {
+    RleVersion vers = convertRleVersion(stripe.getEncoding(columnId).kind());
+    rle = createRleDecoder(stripe.getStream(columnId,
+                                            proto::Stream_Kind_DATA,
+                                            true),
+                           true, vers, memoryPool);
+  }
+
+  IntegerColumnReader::~IntegerColumnReader() {
+    // PASS
+  }
+
+  uint64_t IntegerColumnReader::skip(uint64_t numValues) {
+    numValues = ColumnReader::skip(numValues);
+    rle->skip(numValues);
+    return numValues;
+  }
+
+  void IntegerColumnReader::next(ColumnVectorBatch& rowBatch,
+                                 uint64_t numValues,
+                                 char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    rle->next(dynamic_cast<LongVectorBatch&>(rowBatch).data.data(),
+              numValues, rowBatch.hasNulls ? rowBatch.notNull.data() : 0);
+  }
+
+  class TimestampColumnReader: public IntegerColumnReader {
+  private:
+    std::unique_ptr<orc::RleDecoder> nanoRle;
+    DataBuffer<int64_t> nanoBuffer;
+
+  public:
+    TimestampColumnReader(const Type& type, StripeStreams& stripe);
+    ~TimestampColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char* notNull) override;
+  };
+
+
+  TimestampColumnReader::TimestampColumnReader(const Type& type,
+                                               StripeStreams& stripe
+                                               ): IntegerColumnReader(type,
+                                                                      stripe),
+                                                  nanoBuffer(memoryPool, 1024){
+    RleVersion vers = convertRleVersion(stripe.getEncoding(columnId).kind());
+    nanoRle = createRleDecoder(stripe.getStream(columnId,
+                                                proto::Stream_Kind_SECONDARY,
+                                                true),
+                               false, vers, memoryPool);
+  }
+
+  TimestampColumnReader::~TimestampColumnReader() {
+    // PASS
+  }
+
+  uint64_t TimestampColumnReader::skip(uint64_t numValues) {
+    numValues = IntegerColumnReader::skip(numValues);
+    nanoRle->skip(numValues);
+    return numValues;
+  }
+
+  void TimestampColumnReader::next(ColumnVectorBatch& rowBatch,
+                                 uint64_t numValues,
+                                 char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    notNull = rowBatch.hasNulls ? rowBatch.notNull.data() : nullptr;
+    int64_t* pStamp = dynamic_cast<LongVectorBatch&>(rowBatch).data.data();
+
+    // make sure that nanoBuffer is large enough
+    if (numValues > nanoBuffer.size()) {
+      nanoBuffer.resize(numValues);
+    }
+
+    rle->next(pStamp, numValues, notNull);
+    nanoRle->next(nanoBuffer.data(), numValues, notNull);
+
+    // Construct the values
+    for(uint64_t i=0; i < numValues; i++) {
+      if (notNull == nullptr || notNull[i]) {
+        int64_t nanosec =  nanoBuffer[i] >> 3;
+        uint64_t zeros = nanoBuffer[i] & 0x7;
+        if (zeros != 0) {
+          for(uint64_t j = 0; j <= zeros; ++j) {
+            nanosec *= 10;
+          }
+        }
+        pStamp[i] =  pStamp[i] * 1000000000 + 1420070400000000000;
+        if (pStamp[i] >= 0) {
+          pStamp[i] += nanosec;
+        } else {
+          pStamp[i] -= nanosec;
+        }
+      }
+    }
+  }
+
+  class DoubleColumnReader: public ColumnReader {
+  public:
+    DoubleColumnReader(const Type& type, StripeStreams& stripe);
+    ~DoubleColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char* notNull) override;
+
+  private:
+    std::unique_ptr<SeekableInputStream> inputStream;
+    TypeKind columnKind;
+    const uint64_t bytesPerValue ;
+    const char *bufferPointer;
+    const char *bufferEnd;
+
+    unsigned char readByte() {
+      if (bufferPointer == bufferEnd) {
+        int length;
+        if (!inputStream->Next
+            (reinterpret_cast<const void**>(&bufferPointer), &length)) {
+          throw ParseError("bad read in DoubleColumnReader::next()");
+        }
+        bufferEnd = bufferPointer + length;
+      }
+      return static_cast<unsigned char>(*(bufferPointer++));
+    }
+
+    double readDouble() {
+      int64_t bits = 0;
+      for (uint64_t i=0; i < 8; i++) {
+        bits |= static_cast<int64_t>(readByte()) << (i*8);
+      }
+      double *result = reinterpret_cast<double*>(&bits);
+      return *result;
+    }
+
+    double readFloat() {
+      int32_t bits = 0;
+      for (uint64_t i=0; i < 4; i++) {
+        bits |= readByte() << (i*8);
+      }
+      float *result = reinterpret_cast<float*>(&bits);
+      return *result;
+    }
+  };
+
+  DoubleColumnReader::DoubleColumnReader(const Type& type,
+                                         StripeStreams& stripe
+                                         ): ColumnReader(type, stripe),
+                                            inputStream
+                                               (stripe.getStream
+                                                (columnId,
+                                                 proto::Stream_Kind_DATA,
+                                                 true)),
+                                            columnKind(type.getKind()),
+                                            bytesPerValue((type.getKind() ==
+                                                           FLOAT) ? 4 : 8),
+                                            bufferPointer(NULL),
+                                            bufferEnd(NULL) {
+    // PASS
+  }
+
+  DoubleColumnReader::~DoubleColumnReader() {
+    // PASS
+  }
+
+  uint64_t DoubleColumnReader::skip(uint64_t numValues) {
+    numValues = ColumnReader::skip(numValues);
+
+    if (static_cast<size_t>(bufferEnd - bufferPointer) >=
+        bytesPerValue * numValues) {
+      bufferPointer+= bytesPerValue*numValues;
+    } else {
+      inputStream->Skip(static_cast<int>(bytesPerValue*numValues -
+                                         static_cast<size_t>(bufferEnd -
+                                                             bufferPointer)));
+      bufferEnd = NULL;
+      bufferPointer = NULL;
+    }
+
+    return numValues;
+  }
+
+  void DoubleColumnReader::next(ColumnVectorBatch& rowBatch,
+                                uint64_t numValues,
+                                char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    // update the notNull from the parent class
+    notNull = rowBatch.hasNulls ? rowBatch.notNull.data() : 0;
+    double* outArray = dynamic_cast<DoubleVectorBatch&>(rowBatch).data.data();
+
+    if (columnKind == FLOAT) {
+      if (notNull) {
+        for(size_t i=0; i < numValues; ++i) {
+          if (notNull[i]) {
+            outArray[i] = readFloat();
+          }
+        }
+      } else {
+        for(size_t i=0; i < numValues; ++i) {
+          outArray[i] = readFloat();
+        }
+      }
+    } else {
+      if (notNull) {
+        for(size_t i=0; i < numValues; ++i) {
+          if (notNull[i]) {
+            outArray[i] = readDouble();
+          }
+        }
+      } else {
+        for(size_t i=0; i < numValues; ++i) {
+          outArray[i] = readDouble();
+        }
+      }
+    }
+  }
+
+  void readFully(char* buffer, int64_t bufferSize, SeekableInputStream* stream) {
+    int64_t posn = 0;
+    while (posn < bufferSize) {
+      const void* chunk;
+      int length;
+      if (!stream->Next(&chunk, &length)) {
+        throw ParseError("bad read in readFully");
+      }
+      memcpy(buffer + posn, chunk, static_cast<size_t>(length));
+      posn += length;
+    }
+  }
+
+  class StringDictionaryColumnReader: public ColumnReader {
+  private:
+    DataBuffer<char> dictionaryBlob;
+    DataBuffer<int64_t> dictionaryOffset;
+    std::unique_ptr<RleDecoder> rle;
+    uint64_t dictionaryCount;
+
+  public:
+    StringDictionaryColumnReader(const Type& type, StripeStreams& stipe);
+    ~StringDictionaryColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char *notNull) override;
+  };
+
+  StringDictionaryColumnReader::StringDictionaryColumnReader
+             (const Type& type,
+              StripeStreams& stripe
+              ): ColumnReader(type, stripe),
+                 dictionaryBlob(stripe.getMemoryPool()),
+                 dictionaryOffset(stripe.getMemoryPool()) {
+    RleVersion rleVersion = convertRleVersion(stripe.getEncoding(columnId)
+                                                .kind());
+    dictionaryCount = stripe.getEncoding(columnId).dictionarysize();
+    rle = createRleDecoder(stripe.getStream(columnId,
+                                            proto::Stream_Kind_DATA,
+                                            true),
+                           false, rleVersion, memoryPool);
+    std::unique_ptr<RleDecoder> lengthDecoder =
+      createRleDecoder(stripe.getStream(columnId,
+                                        proto::Stream_Kind_LENGTH,
+                                        false),
+                       false, rleVersion, memoryPool);
+    dictionaryOffset.resize(dictionaryCount+1);
+    int64_t* lengthArray = dictionaryOffset.data();
+    lengthDecoder->next(lengthArray + 1, dictionaryCount, 0);
+    lengthArray[0] = 0;
+    for(uint64_t i=1; i < dictionaryCount + 1; ++i) {
+      lengthArray[i] += lengthArray[i-1];
+    }
+    int64_t blobSize = lengthArray[dictionaryCount];
+    dictionaryBlob.resize(static_cast<uint64_t>(blobSize));
+    std::unique_ptr<SeekableInputStream> blobStream =
+      stripe.getStream(columnId, proto::Stream_Kind_DICTIONARY_DATA, false);
+    readFully(dictionaryBlob.data(), blobSize, blobStream.get());
+  }
+
+  StringDictionaryColumnReader::~StringDictionaryColumnReader() {
+    // PASS
+  }
+
+  uint64_t StringDictionaryColumnReader::skip(uint64_t numValues) {
+    numValues = ColumnReader::skip(numValues);
+    rle->skip(numValues);
+    return numValues;
+  }
+
+  void StringDictionaryColumnReader::next(ColumnVectorBatch& rowBatch,
+                                          uint64_t numValues,
+                                          char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    // update the notNull from the parent class
+    notNull = rowBatch.hasNulls ? rowBatch.notNull.data() : 0;
+    StringVectorBatch& byteBatch = dynamic_cast<StringVectorBatch&>(rowBatch);
+    char *blob = dictionaryBlob.data();
+    int64_t *dictionaryOffsets = dictionaryOffset.data();
+    char **outputStarts = byteBatch.data.data();
+    int64_t *outputLengths = byteBatch.length.data();
+    rle->next(outputLengths, numValues, notNull);
+    if (notNull) {
+      for(uint64_t i=0; i < numValues; ++i) {
+        if (notNull[i]) {
+          int64_t entry = outputLengths[i];
+          outputStarts[i] = blob + dictionaryOffsets[entry];
+          outputLengths[i] = dictionaryOffsets[entry+1] -
+            dictionaryOffsets[entry];
+        }
+      }
+    } else {
+      for(uint64_t i=0; i < numValues; ++i) {
+        int64_t entry = outputLengths[i];
+        outputStarts[i] = blob + dictionaryOffsets[entry];
+        outputLengths[i] = dictionaryOffsets[entry+1] -
+          dictionaryOffsets[entry];
+      }
+    }
+  }
+
+  class StringDirectColumnReader: public ColumnReader {
+  private:
+    DataBuffer<char> blobBuffer;
+    std::unique_ptr<RleDecoder> lengthRle;
+    std::unique_ptr<SeekableInputStream> blobStream;
+    const char *lastBuffer;
+    size_t lastBufferLength;
+
+    /**
+     * Compute the total length of the values.
+     * @param lengths the array of lengths
+     * @param notNull the array of notNull flags
+     * @param numValues the lengths of the arrays
+     * @return the total number of bytes for the non-null values
+     */
+    size_t computeSize(const int64_t *lengths, const char *notNull,
+                       uint64_t numValues);
+
+  public:
+    StringDirectColumnReader(const Type& type, StripeStreams& stipe);
+    ~StringDirectColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char *notNull) override;
+  };
+
+  StringDirectColumnReader::StringDirectColumnReader
+                 (const Type& type,
+                  StripeStreams& stripe
+                  ): ColumnReader(type, stripe),
+                     blobBuffer(stripe.getMemoryPool()) {
+    RleVersion rleVersion = convertRleVersion(stripe.getEncoding(columnId)
+                                                .kind());
+    lengthRle = createRleDecoder(stripe.getStream(columnId,
+                                                  proto::Stream_Kind_LENGTH,
+                                                  true),
+                                 false, rleVersion, memoryPool);
+    blobStream = stripe.getStream(columnId, proto::Stream_Kind_DATA, true);
+    lastBuffer = 0;
+    lastBufferLength = 0;
+  }
+
+  StringDirectColumnReader::~StringDirectColumnReader() {
+    // PASS
+  }
+
+  uint64_t StringDirectColumnReader::skip(uint64_t numValues) {
+    const size_t BUFFER_SIZE = 1024;
+    numValues = ColumnReader::skip(numValues);
+    int64_t buffer[BUFFER_SIZE];
+    uint64_t done = 0;
+    size_t totalBytes = 0;
+    // read the lengths, so we know haw many bytes to skip
+    while (done < numValues) {
+      uint64_t step = std::min(BUFFER_SIZE,
+                                    static_cast<size_t>(numValues - done));
+      lengthRle->next(buffer, step, 0);
+      totalBytes += computeSize(buffer, 0, step);
+      done += step;
+    }
+    if (totalBytes <= lastBufferLength) {
+      // subtract the needed bytes from the ones left over
+      lastBufferLength -= totalBytes;
+      lastBuffer += totalBytes;
+    } else {
+      // move the stream forward after accounting for the buffered bytes
+      totalBytes -= lastBufferLength;
+      blobStream->Skip(static_cast<int>(totalBytes));
+      lastBufferLength = 0;
+      lastBuffer = 0;
+    }
+    return numValues;
+  }
+
+  size_t StringDirectColumnReader::computeSize(const int64_t* lengths,
+                                               const char* notNull,
+                                               uint64_t numValues) {
+    size_t totalLength = 0;
+    if (notNull) {
+      for(size_t i=0; i < numValues; ++i) {
+        if (notNull[i]) {
+          totalLength += static_cast<size_t>(lengths[i]);
+        }
+      }
+    } else {
+      for(size_t i=0; i < numValues; ++i) {
+        totalLength += static_cast<size_t>(lengths[i]);
+      }
+    }
+    return totalLength;
+  }
+
+  void StringDirectColumnReader::next(ColumnVectorBatch& rowBatch,
+                                      uint64_t numValues,
+                                      char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    // update the notNull from the parent class
+    notNull = rowBatch.hasNulls ? rowBatch.notNull.data() : 0;
+    StringVectorBatch& byteBatch = dynamic_cast<StringVectorBatch&>(rowBatch);
+    char **startPtr = byteBatch.data.data();
+    int64_t *lengthPtr = byteBatch.length.data();
+
+    // read the length vector
+    lengthRle->next(lengthPtr, numValues, notNull);
+
+    // figure out the total length of data we need from the blob stream
+    const size_t totalLength = computeSize(lengthPtr, notNull, numValues);
+
+    // Load data from the blob stream into our buffer until we have enough
+    // to get the rest directly out of the stream's buffer.
+    size_t bytesBuffered = 0;
+    blobBuffer.resize(totalLength);
+    char *ptr= blobBuffer.data();
+    while (bytesBuffered + lastBufferLength < totalLength) {
+      blobBuffer.resize(bytesBuffered + lastBufferLength);
+      memcpy(ptr + bytesBuffered, lastBuffer, lastBufferLength);
+      bytesBuffered += lastBufferLength;
+      const void* readBuffer;
+      int readLength;
+      if (!blobStream->Next(&readBuffer, &readLength)) {
+        throw ParseError("failed to read in StringDirectColumnReader.next");
+      }
+      lastBuffer = static_cast<const char*>(readBuffer);
+      lastBufferLength = static_cast<size_t>(readLength);
+    }
+
+    // Set up the start pointers for the ones that will come out of the buffer.
+    size_t filledSlots = 0;
+    size_t usedBytes = 0;
+    ptr = blobBuffer.data();
+    if (notNull) {
+      while (filledSlots < numValues &&
+             (usedBytes + static_cast<size_t>(lengthPtr[filledSlots]) <=
+              bytesBuffered)) {
+        if (notNull[filledSlots]) {
+          startPtr[filledSlots] = ptr + usedBytes;
+          usedBytes += static_cast<size_t>(lengthPtr[filledSlots]);
+        }
+        filledSlots += 1;
+      }
+    } else {
+      while (filledSlots < numValues &&
+             (usedBytes + static_cast<size_t>(lengthPtr[filledSlots]) <=
+              bytesBuffered)) {
+        startPtr[filledSlots] = ptr + usedBytes;
+        usedBytes += static_cast<size_t>(lengthPtr[filledSlots]);
+        filledSlots += 1;
+      }
+    }
+
+    // do we need to complete the last value in the blob buffer?
+    if (usedBytes < bytesBuffered) {
+      size_t moreBytes = static_cast<size_t>(lengthPtr[filledSlots]) -
+        (bytesBuffered - usedBytes);
+      blobBuffer.resize(bytesBuffered + moreBytes);
+      ptr = blobBuffer.data();
+      memcpy(ptr + bytesBuffered, lastBuffer, moreBytes);
+      lastBuffer += moreBytes;
+      lastBufferLength -= moreBytes;
+      startPtr[filledSlots++] = ptr + usedBytes;
+    }
+
+    // Finally, set up any remaining entries into the stream buffer
+    if (notNull) {
+      while (filledSlots < numValues) {
+        if (notNull[filledSlots]) {
+          startPtr[filledSlots] = const_cast<char*>(lastBuffer);
+          lastBuffer += lengthPtr[filledSlots];
+          lastBufferLength -= static_cast<size_t>(lengthPtr[filledSlots]);
+        }
+        filledSlots += 1;
+      }
+    } else {
+      while (filledSlots < numValues) {
+        startPtr[filledSlots] = const_cast<char*>(lastBuffer);
+        lastBuffer += lengthPtr[filledSlots];
+        lastBufferLength -= static_cast<size_t>(lengthPtr[filledSlots]);
+        filledSlots += 1;
+      }
+    }
+  }
+
+  class StructColumnReader: public ColumnReader {
+  private:
+    std::vector<ColumnReader*> children;
+
+  public:
+    StructColumnReader(const Type& type, StripeStreams& stipe);
+    ~StructColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char *notNull) override;
+  };
+
+  StructColumnReader::StructColumnReader(const Type& type,
+                                         StripeStreams& stripe
+                                         ): ColumnReader(type, stripe) {
+    // count the number of selected sub-columns
+    const std::vector<bool> selectedColumns = stripe.getSelectedColumns();
+    switch (static_cast<int64_t>(stripe.getEncoding(columnId).kind())) {
+    case proto::ColumnEncoding_Kind_DIRECT:
+      for(unsigned int i=0; i < type.getSubtypeCount(); ++i) {
+        const Type& child = type.getSubtype(i);
+        if (selectedColumns[static_cast<uint64_t>(child.getColumnId())]) {
+          children.push_back(buildReader(child, stripe).release());
+        }
+      }
+      break;
+    case proto::ColumnEncoding_Kind_DIRECT_V2:
+    case proto::ColumnEncoding_Kind_DICTIONARY:
+    case proto::ColumnEncoding_Kind_DICTIONARY_V2:
+    default:
+      throw ParseError("Unknown encoding for StructColumnReader");
+    }
+  }
+
+  StructColumnReader::~StructColumnReader() {
+    for (size_t i=0; i<children.size(); i++) {
+      delete children[i];
+    }
+  }
+
+  uint64_t StructColumnReader::skip(uint64_t numValues) {
+    numValues = ColumnReader::skip(numValues);
+    for(std::vector<ColumnReader*>::iterator ptr=children.begin(); ptr != children.end(); ++ptr) {
+      (*ptr)->skip(numValues);
+    }
+    return numValues;
+  }
+
+  void StructColumnReader::next(ColumnVectorBatch& rowBatch,
+                                uint64_t numValues,
+                                char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    uint64_t i=0;
+    notNull = rowBatch.hasNulls? rowBatch.notNull.data() : 0;
+    for(std::vector<ColumnReader*>::iterator ptr=children.begin();
+        ptr != children.end(); ++ptr, ++i) {
+      (*ptr)->next(*(dynamic_cast<StructVectorBatch&>(rowBatch).fields[i]),
+                   numValues, notNull);
+    }
+  }
+
+  class ListColumnReader: public ColumnReader {
+  private:
+    std::unique_ptr<ColumnReader> child;
+    std::unique_ptr<RleDecoder> rle;
+
+  public:
+    ListColumnReader(const Type& type, StripeStreams& stipe);
+    ~ListColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char *notNull) override;
+  };
+
+  ListColumnReader::ListColumnReader(const Type& type,
+                                     StripeStreams& stripe
+                                     ): ColumnReader(type, stripe) {
+    // count the number of selected sub-columns
+    const std::vector<bool> selectedColumns = stripe.getSelectedColumns();
+    RleVersion vers = convertRleVersion(stripe.getEncoding(columnId).kind());
+    rle = createRleDecoder(stripe.getStream(columnId,
+                                            proto::Stream_Kind_LENGTH,
+                                            true),
+                           false, vers, memoryPool);
+    const Type& childType = type.getSubtype(0);
+    if (selectedColumns[static_cast<uint64_t>(childType.getColumnId())]) {
+      child = buildReader(childType, stripe);
+    }
+  }
+
+  ListColumnReader::~ListColumnReader() {
+    // PASS
+  }
+
+  uint64_t ListColumnReader::skip(uint64_t numValues) {
+    numValues = ColumnReader::skip(numValues);
+    ColumnReader *childReader = child.get();
+    if (childReader) {
+      const uint64_t BUFFER_SIZE = 1024;
+      int64_t buffer[BUFFER_SIZE];
+      uint64_t childrenElements = 0;
+      uint64_t lengthsRead = 0;
+      while (lengthsRead < numValues) {
+        uint64_t chunk = std::min(numValues - lengthsRead, BUFFER_SIZE);
+        rle->next(buffer, chunk, 0);
+        for(size_t i=0; i < chunk; ++i) {
+          childrenElements += static_cast<size_t>(buffer[i]);
+        }
+        lengthsRead += chunk;
+      }
+      childReader->skip(childrenElements);
+    } else {
+      rle->skip(numValues);
+    }
+    return numValues;
+  }
+
+  void ListColumnReader::next(ColumnVectorBatch& rowBatch,
+                              uint64_t numValues,
+                              char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    ListVectorBatch &listBatch = dynamic_cast<ListVectorBatch&>(rowBatch);
+    int64_t* offsets = listBatch.offsets.data();
+    notNull = listBatch.hasNulls ? listBatch.notNull.data() : 0;
+    rle->next(offsets, numValues, notNull);
+    uint64_t totalChildren = 0;
+    if (notNull) {
+      for(size_t i=0; i < numValues; ++i) {
+        if (notNull[i]) {
+          uint64_t tmp = static_cast<uint64_t>(offsets[i]);
+          offsets[i] = static_cast<int64_t>(totalChildren);
+          totalChildren += tmp;
+        } else {
+          offsets[i] = static_cast<int64_t>(totalChildren);
+        }
+      }
+    } else {
+      for(size_t i=0; i < numValues; ++i) {
+        uint64_t tmp = static_cast<uint64_t>(offsets[i]);
+        offsets[i] = static_cast<int64_t>(totalChildren);
+        totalChildren += tmp;
+      }
+    }
+    offsets[numValues] = static_cast<int64_t>(totalChildren);
+    ColumnReader *childReader = child.get();
+    if (childReader) {
+      childReader->next(*(listBatch.elements.get()), totalChildren, 0);
+    }
+  }
+
+  class MapColumnReader: public ColumnReader {
+  private:
+    std::unique_ptr<ColumnReader> keyReader;
+    std::unique_ptr<ColumnReader> elementReader;
+    std::unique_ptr<RleDecoder> rle;
+
+  public:
+    MapColumnReader(const Type& type, StripeStreams& stipe);
+    ~MapColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char *notNull) override;
+  };
+
+  MapColumnReader::MapColumnReader(const Type& type,
+                                   StripeStreams& stripe
+                                   ): ColumnReader(type, stripe) {
+    // Determine if the key and/or value columns are selected
+    const std::vector<bool> selectedColumns = stripe.getSelectedColumns();
+    RleVersion vers = convertRleVersion(stripe.getEncoding(columnId).kind());
+    rle = createRleDecoder(stripe.getStream(columnId,
+                                            proto::Stream_Kind_LENGTH,
+                                            true),
+                           false, vers, memoryPool);
+    const Type& keyType = type.getSubtype(0);
+    if (selectedColumns[static_cast<uint64_t>(keyType.getColumnId())]) {
+      keyReader = buildReader(keyType, stripe);
+    }
+    const Type& elementType = type.getSubtype(1);
+    if (selectedColumns[static_cast<uint64_t>(elementType.getColumnId())]) {
+      elementReader = buildReader(elementType, stripe);
+    }
+  }
+
+  MapColumnReader::~MapColumnReader() {
+    // PASS
+  }
+
+  uint64_t MapColumnReader::skip(uint64_t numValues) {
+    numValues = ColumnReader::skip(numValues);
+    ColumnReader *rawKeyReader = keyReader.get();
+    ColumnReader *rawElementReader = elementReader.get();
+    if (rawKeyReader || rawElementReader) {
+      const uint64_t BUFFER_SIZE = 1024;
+      int64_t buffer[BUFFER_SIZE];
+      uint64_t childrenElements = 0;
+      uint64_t lengthsRead = 0;
+      while (lengthsRead < numValues) {
+        uint64_t chunk = std::min(numValues - lengthsRead, BUFFER_SIZE);
+        rle->next(buffer, chunk, 0);
+        for(size_t i=0; i < chunk; ++i) {
+          childrenElements += static_cast<size_t>(buffer[i]);
+        }
+        lengthsRead += chunk;
+      }
+      if (rawKeyReader) {
+        rawKeyReader->skip(childrenElements);
+      }
+      if (rawElementReader) {
+        rawElementReader->skip(childrenElements);
+      }
+    } else {
+      rle->skip(numValues);
+    }
+    return numValues;
+  }
+
+  void MapColumnReader::next(ColumnVectorBatch& rowBatch,
+                             uint64_t numValues,
+                             char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    MapVectorBatch &mapBatch = dynamic_cast<MapVectorBatch&>(rowBatch);
+    int64_t* offsets = mapBatch.offsets.data();
+    notNull = mapBatch.hasNulls ? mapBatch.notNull.data() : 0;
+    rle->next(offsets, numValues, notNull);
+    uint64_t totalChildren = 0;
+    if (notNull) {
+      for(size_t i=0; i < numValues; ++i) {
+        if (notNull[i]) {
+          uint64_t tmp = static_cast<uint64_t>(offsets[i]);
+          offsets[i] = static_cast<int64_t>(totalChildren);
+          totalChildren += tmp;
+        } else {
+          offsets[i] = static_cast<int64_t>(totalChildren);
+        }
+      }
+    } else {
+      for(size_t i=0; i < numValues; ++i) {
+        uint64_t tmp = static_cast<uint64_t>(offsets[i]);
+        offsets[i] = static_cast<int64_t>(totalChildren);
+        totalChildren += tmp;
+      }
+    }
+    offsets[numValues] = static_cast<int64_t>(totalChildren);
+    ColumnReader *rawKeyReader = keyReader.get();
+    if (rawKeyReader) {
+      rawKeyReader->next(*(mapBatch.keys.get()), totalChildren, 0);
+    }
+    ColumnReader *rawElementReader = elementReader.get();
+    if (rawElementReader) {
+      rawElementReader->next(*(mapBatch.elements.get()), totalChildren, 0);
+    }
+  }
+
+  class UnionColumnReader: public ColumnReader {
+  private:
+    std::unique_ptr<ByteRleDecoder> rle;
+    std::vector<ColumnReader*> childrenReader;
+    std::vector<int64_t> childrenCounts;
+    uint64_t numChildren;
+
+  public:
+    UnionColumnReader(const Type& type, StripeStreams& stipe);
+    ~UnionColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char *notNull) override;
+  };
+
+  UnionColumnReader::UnionColumnReader(const Type& type,
+                                       StripeStreams& stripe
+                                       ): ColumnReader(type, stripe) {
+    numChildren = type.getSubtypeCount();
+    childrenReader.resize(numChildren);
+    childrenCounts.resize(numChildren);
+
+    rle = createByteRleDecoder(stripe.getStream(columnId,
+                                                proto::Stream_Kind_DATA,
+                                                true));
+    // figure out which types are selected
+    const std::vector<bool> selectedColumns = stripe.getSelectedColumns();
+    for(unsigned int i=0; i < numChildren; ++i) {
+      const Type &child = type.getSubtype(i);
+      if (selectedColumns[static_cast<size_t>(child.getColumnId())]) {
+        childrenReader[i] = buildReader(child, stripe).release();
+      }
+    }
+  }
+
+  UnionColumnReader::~UnionColumnReader() {
+    for(std::vector<ColumnReader*>::iterator itr = childrenReader.begin();
+        itr != childrenReader.end(); ++itr) {
+      delete *itr;
+    }
+  }
+
+  uint64_t UnionColumnReader::skip(uint64_t numValues) {
+    numValues = ColumnReader::skip(numValues);
+    const uint64_t BUFFER_SIZE = 1024;
+    char buffer[BUFFER_SIZE];
+    uint64_t lengthsRead = 0;
+    int64_t *counts = childrenCounts.data();
+    memset(counts, 0, sizeof(int64_t) * numChildren);
+    while (lengthsRead < numValues) {
+      uint64_t chunk = std::min(numValues - lengthsRead, BUFFER_SIZE);
+      rle->next(buffer, chunk, 0);
+      for(size_t i=0; i < chunk; ++i) {
+        counts[static_cast<size_t>(buffer[i])] += 1;
+      }
+      lengthsRead += chunk;
+    }
+    for(size_t i=0; i < numChildren; ++i) {
+      if (counts[i] != 0 && childrenReader[i] != NULL) {
+        childrenReader[i]->skip(static_cast<uint64_t>(counts[i]));
+      }
+    }
+    return numValues;
+  }
+
+  void UnionColumnReader::next(ColumnVectorBatch& rowBatch,
+                               uint64_t numValues,
+                               char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    UnionVectorBatch &unionBatch = dynamic_cast<UnionVectorBatch&>(rowBatch);
+    uint64_t* offsets = unionBatch.offsets.data();
+    int64_t* counts = childrenCounts.data();
+    memset(counts, 0, sizeof(int64_t) * numChildren);
+    unsigned char* tags = unionBatch.tags.data();
+    notNull = unionBatch.hasNulls ? unionBatch.notNull.data() : 0;
+    rle->next(reinterpret_cast<char *>(tags), numValues, notNull);
+    // set the offsets for each row
+    if (notNull) {
+      for(size_t i=0; i < numValues; ++i) {
+        if (notNull[i]) {
+          offsets[i] =
+            static_cast<uint64_t>(counts[static_cast<size_t>(tags[i])]++);
+        }
+      }
+    } else {
+      for(size_t i=0; i < numValues; ++i) {
+        offsets[i] =
+          static_cast<uint64_t>(counts[static_cast<size_t>(tags[i])]++);
+      }
+    }
+    // read the right number of each child column
+    for(size_t i=0; i < numChildren; ++i) {
+      if (childrenReader[i] != nullptr) {
+        childrenReader[i]->next(*(unionBatch.children[i]),
+                                static_cast<uint64_t>(counts[i]), nullptr);
+      }
+    }
+  }
+
+  /**
+   * Destructively convert the number from zigzag encoding to the
+   * natural signed representation.
+   */
+  void unZigZagInt128(Int128& value) {
+    bool needsNegate = value.getLowBits() & 1;
+    value >>= 1;
+    if (needsNegate) {
+      value.negate();
+      value -= 1;
+    }
+  }
+
+  class Decimal64ColumnReader: public ColumnReader {
+  public:
+    static const uint32_t MAX_PRECISION_64 = 18;
+    static const uint32_t MAX_PRECISION_128 = 38;
+    static const int64_t POWERS_OF_TEN[MAX_PRECISION_64 + 1];
+
+  protected:
+    std::unique_ptr<SeekableInputStream> valueStream;
+    int32_t precision;
+    int32_t scale;
+    const char* buffer;
+    const char* bufferEnd;
+
+    std::unique_ptr<RleDecoder> scaleDecoder;
+
+    /**
+     * Read the valueStream for more bytes.
+     */
+    void readBuffer() {
+      while (buffer == bufferEnd) {
+        int length;
+        if (!valueStream->Next(reinterpret_cast<const void**>(&buffer),
+                               &length)) {
+          throw ParseError("Read past end of stream in Decimal64ColumnReader "+
+                           valueStream->getName());
+        }
+        bufferEnd = buffer + length;
+      }
+    }
+
+    void readInt64(int64_t& value, int32_t currentScale) {
+      value = 0;
+      size_t offset = 0;
+      while (true) {
+        readBuffer();
+        unsigned char ch = static_cast<unsigned char>(*(buffer++));
+        value |= static_cast<uint64_t>(ch & 0x7f) << offset;
+        offset += 7;
+        if (!(ch & 0x80)) {
+          break;
+        }
+      }
+      value = unZigZag(static_cast<uint64_t>(value));
+      if (scale > currentScale) {
+        value *= POWERS_OF_TEN[scale - currentScale];
+      } else if (scale < currentScale) {
+        value /= POWERS_OF_TEN[currentScale - scale];
+      }
+    }
+
+  public:
+    Decimal64ColumnReader(const Type& type, StripeStreams& stipe);
+    ~Decimal64ColumnReader();
+
+    uint64_t skip(uint64_t numValues) override;
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char *notNull) override;
+  };
+  const uint32_t Decimal64ColumnReader::MAX_PRECISION_64;
+  const uint32_t Decimal64ColumnReader::MAX_PRECISION_128;
+  const int64_t Decimal64ColumnReader::POWERS_OF_TEN[MAX_PRECISION_64 + 1]=
+    {1,
+     10,
+     100,
+     1000,
+     10000,
+     100000,
+     1000000,
+     10000000,
+     100000000,
+     1000000000,
+     10000000000,
+     100000000000,
+     1000000000000,
+     10000000000000,
+     100000000000000,
+     1000000000000000,
+     10000000000000000,
+     100000000000000000,
+     1000000000000000000};
+
+  Decimal64ColumnReader::Decimal64ColumnReader(const Type& type,
+                                               StripeStreams& stripe
+                                               ): ColumnReader(type, stripe) {
+    scale = static_cast<int32_t>(type.getScale());
+    precision = static_cast<int32_t>(type.getPrecision());
+    valueStream = stripe.getStream(columnId, proto::Stream_Kind_DATA, true);
+    buffer = nullptr;
+    bufferEnd = nullptr;
+    RleVersion vers = convertRleVersion(stripe.getEncoding(columnId).kind());
+    scaleDecoder = createRleDecoder(stripe.getStream
+                                    (columnId,
+                                     proto::Stream_Kind_SECONDARY,
+                                     true),
+                                    true, vers, memoryPool);
+  }
+
+  Decimal64ColumnReader::~Decimal64ColumnReader() {
+    // PASS
+  }
+
+  uint64_t Decimal64ColumnReader::skip(uint64_t numValues) {
+    numValues = ColumnReader::skip(numValues);
+    uint64_t skipped = 0;
+    while (skipped < numValues) {
+      readBuffer();
+      if (!(0x80 & *(buffer++))) {
+        skipped += 1;
+      }
+    }
+    scaleDecoder->skip(numValues);
+    return numValues;
+  }
+
+  void Decimal64ColumnReader::next(ColumnVectorBatch& rowBatch,
+                                   uint64_t numValues,
+                                   char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    notNull = rowBatch.hasNulls ? rowBatch.notNull.data() : 0;
+    Decimal64VectorBatch &batch =
+      dynamic_cast<Decimal64VectorBatch&>(rowBatch);
+    int64_t* values = batch.values.data();
+    // read the next group of scales
+    int64_t* scaleBuffer = batch.readScales.data();
+    scaleDecoder->next(scaleBuffer, numValues, notNull);
+    batch.precision = precision;
+    batch.scale = scale;
+    if (notNull) {
+      for(size_t i=0; i < numValues; ++i) {
+        if (notNull[i]) {
+          readInt64(values[i], static_cast<int32_t>(scaleBuffer[i]));
+        }
+      }
+    } else {
+      for(size_t i=0; i < numValues; ++i) {
+        readInt64(values[i], static_cast<int32_t>(scaleBuffer[i]));
+      }
+    }
+  }
+
+  void scaleInt128(Int128& value, uint32_t scale, uint32_t currentScale) {
+    if (scale > currentScale) {
+      while(scale > currentScale) {
+        uint32_t scaleAdjust =
+          std::min(Decimal64ColumnReader::MAX_PRECISION_64,
+                   scale - currentScale);
+        value *= Decimal64ColumnReader::POWERS_OF_TEN[scaleAdjust];
+        currentScale += scaleAdjust;
+      }
+    } else if (scale < currentScale) {
+      Int128 remainder;
+      while(currentScale > scale) {
+        uint32_t scaleAdjust =
+          std::min(Decimal64ColumnReader::MAX_PRECISION_64,
+                   currentScale - scale);
+        value = value.divide(Decimal64ColumnReader::POWERS_OF_TEN[scaleAdjust],
+                             remainder);
+        currentScale -= scaleAdjust;
+      }
+    }
+  }
+
+  class Decimal128ColumnReader: public Decimal64ColumnReader {
+  public:
+    Decimal128ColumnReader(const Type& type, StripeStreams& stipe);
+    ~Decimal128ColumnReader();
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char *notNull) override;
+
+  private:
+    void readInt128(Int128& value, int32_t currentScale) {
+      value = 0;
+      Int128 work;
+      uint32_t offset = 0;
+      while (true) {
+        readBuffer();
+        unsigned char ch = static_cast<unsigned char>(*(buffer++));
+        work = ch & 0x7f;
+        work <<= offset;
+        value |=  work;
+        offset += 7;
+        if (!(ch & 0x80)) {
+          break;
+        }
+      }
+      unZigZagInt128(value);
+      scaleInt128(value, static_cast<uint32_t>(scale), 
+                  static_cast<uint32_t>(currentScale));
+    }
+  };
+
+  Decimal128ColumnReader::Decimal128ColumnReader
+                (const Type& type,
+                 StripeStreams& stripe
+                 ): Decimal64ColumnReader(type, stripe) {
+    // PASS
+  }
+
+  Decimal128ColumnReader::~Decimal128ColumnReader() {
+    // PASS
+  }
+
+  void Decimal128ColumnReader::next(ColumnVectorBatch& rowBatch,
+                                   uint64_t numValues,
+                                   char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    notNull = rowBatch.hasNulls ? rowBatch.notNull.data() : 0;
+    Decimal128VectorBatch &batch =
+      dynamic_cast<Decimal128VectorBatch&>(rowBatch);
+    Int128* values = batch.values.data();
+    // read the next group of scales
+    int64_t* scaleBuffer = batch.readScales.data();
+    scaleDecoder->next(scaleBuffer, numValues, notNull);
+    batch.precision = precision;
+    batch.scale = scale;
+    if (notNull) {
+      for(size_t i=0; i < numValues; ++i) {
+        if (notNull[i]) {
+          readInt128(values[i], static_cast<int32_t>(scaleBuffer[i]));
+        }
+      }
+    } else {
+      for(size_t i=0; i < numValues; ++i) {
+        readInt128(values[i], static_cast<int32_t>(scaleBuffer[i]));
+      }
+    }
+  }
+
+  class DecimalHive11ColumnReader: public Decimal64ColumnReader {
+  private:
+    bool throwOnOverflow;
+    std::ostream* errorStream;
+
+    /**
+     * Read an Int128 from the stream and correct it to the desired scale.
+     */
+    bool readInt128(Int128& value, int32_t currentScale) {
+      // -/+ 99999999999999999999999999999999999999
+      static const Int128 MIN_VALUE(-0x4b3b4ca85a86c47b, 0xf675ddc000000001);
+      static const Int128 MAX_VALUE( 0x4b3b4ca85a86c47a, 0x098a223fffffffff);
+
+      value = 0;
+      Int128 work;
+      uint32_t offset = 0;
+      bool result = true;
+      while (true) {
+        readBuffer();
+        unsigned char ch = static_cast<unsigned char>(*(buffer++));
+        work = ch & 0x7f;
+        // If we have read more than 128 bits, we flag the error, but keep
+        // reading bytes so the stream isn't thrown off.
+        if (offset > 128 || (offset == 126 && work > 3)) {
+          result = false;
+        }
+        work <<= offset;
+        value |=  work;
+        offset += 7;
+        if (!(ch & 0x80)) {
+          break;
+        }
+      }
+
+      if (!result) {
+        return result;
+      }
+      unZigZagInt128(value);
+      scaleInt128(value, static_cast<uint32_t>(scale),
+                  static_cast<uint32_t>(currentScale));
+      return value >= MIN_VALUE && value <= MAX_VALUE;
+    }
+
+  public:
+    DecimalHive11ColumnReader(const Type& type, StripeStreams& stipe);
+    ~DecimalHive11ColumnReader();
+
+    void next(ColumnVectorBatch& rowBatch,
+              uint64_t numValues,
+              char *notNull) override;
+  };
+
+  DecimalHive11ColumnReader::DecimalHive11ColumnReader
+                    (const Type& type,
+                     StripeStreams& stripe
+                     ): Decimal64ColumnReader(type, stripe) {
+    const ReaderOptions options = stripe.getReaderOptions();
+    scale = options.getForcedScaleOnHive11Decimal();
+    throwOnOverflow = options.getThrowOnHive11DecimalOverflow();
+    errorStream = options.getErrorStream();
+  }
+
+  DecimalHive11ColumnReader::~DecimalHive11ColumnReader() {
+    // PASS
+  }
+
+  void DecimalHive11ColumnReader::next(ColumnVectorBatch& rowBatch,
+                                       uint64_t numValues,
+                                       char *notNull) {
+    ColumnReader::next(rowBatch, numValues, notNull);
+    notNull = rowBatch.hasNulls ? rowBatch.notNull.data() : 0;
+    Decimal128VectorBatch &batch =
+      dynamic_cast<Decimal128VectorBatch&>(rowBatch);
+    Int128* values = batch.values.data();
+    // read the next group of scales
+    int64_t* scaleBuffer = batch.readScales.data();
+
+    scaleDecoder->next(scaleBuffer, numValues, notNull);
+
+    batch.precision = precision;
+    batch.scale = scale;
+    if (notNull) {
+      for(size_t i=0; i < numValues; ++i) {
+        if (notNull[i]) {
+          if (!readInt128(values[i],
+                          static_cast<int32_t>(scaleBuffer[i]))) {
+            if (throwOnOverflow) {
+              throw ParseError("Hive 0.11 decimal was more than 38 digits.");
+            } else {
+              *errorStream << "Warning: "
+                           << "Hive 0.11 decimal with more than 38 digits "
+                           << "replaced by NULL.\n";
+              notNull[i] = false;
+            }
+          }
+        }
+      }
+    } else {
+      for(size_t i=0; i < numValues; ++i) {
+        if (!readInt128(values[i],
+                        static_cast<int32_t>(scaleBuffer[i]))) {
+          if (throwOnOverflow) {
+            throw ParseError("Hive 0.11 decimal was more than 38 digits.");
+          } else {
+            *errorStream << "Warning: "
+                         << "Hive 0.11 decimal with more than 38 digits "
+                         << "replaced by NULL.\n";
+            batch.hasNulls = true;
+            batch.notNull[i] = false;
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Create a reader for the given stripe.
+   */
+  std::unique_ptr<ColumnReader> buildReader(const Type& type,
+                                            StripeStreams& stripe) {
+    switch (static_cast<int64_t>(type.getKind())) {
+    case DATE:
+    case INT:
+    case LONG:
+    case SHORT:
+      return std::unique_ptr<ColumnReader>(
+          new IntegerColumnReader(type, stripe));
+    case BINARY:
+    case CHAR:
+    case STRING:
+    case VARCHAR:
+      switch (static_cast<int64_t>(stripe.getEncoding(type.getColumnId()).kind())){
+      case proto::ColumnEncoding_Kind_DICTIONARY:
+      case proto::ColumnEncoding_Kind_DICTIONARY_V2:
+        return std::unique_ptr<ColumnReader>(
+            new StringDictionaryColumnReader(type, stripe));
+      case proto::ColumnEncoding_Kind_DIRECT:
+      case proto::ColumnEncoding_Kind_DIRECT_V2:
+        return std::unique_ptr<ColumnReader>(
+            new StringDirectColumnReader(type, stripe));
+      default:
+        throw NotImplementedYet("buildReader unhandled string encoding");
+      }
+
+    case BOOLEAN:
+      return std::unique_ptr<ColumnReader>(
+          new BooleanColumnReader(type, stripe));
+
+    case BYTE:
+      return std::unique_ptr<ColumnReader>(
+          new ByteColumnReader(type, stripe));
+
+    case LIST:
+      return std::unique_ptr<ColumnReader>(
+          new ListColumnReader(type, stripe));
+
+    case MAP:
+      return std::unique_ptr<ColumnReader>(
+          new MapColumnReader(type, stripe));
+
+    case UNION:
+      return std::unique_ptr<ColumnReader>(
+          new UnionColumnReader(type, stripe));
+
+    case STRUCT:
+      return std::unique_ptr<ColumnReader>(
+          new StructColumnReader(type, stripe));
+
+    case FLOAT:
+    case DOUBLE:
+      return std::unique_ptr<ColumnReader>(
+          new DoubleColumnReader(type, stripe));
+
+    case TIMESTAMP:
+      return std::unique_ptr<ColumnReader>
+        (new TimestampColumnReader(type, stripe));
+
+    case DECIMAL:
+      // is this a Hive 0.11 or 0.12 file?
+      if (type.getPrecision() == 0) {
+        return std::unique_ptr<ColumnReader>
+          (new DecimalHive11ColumnReader(type, stripe));
+
+      // can we represent the values using int64_t?
+      } else if (type.getPrecision() <=
+                 Decimal64ColumnReader::MAX_PRECISION_64) {
+        return std::unique_ptr<ColumnReader>
+          (new Decimal64ColumnReader(type, stripe));
+
+      // otherwise we use the Int128 implementation
+      } else {
+        return std::unique_ptr<ColumnReader>
+          (new Decimal128ColumnReader(type, stripe));
+      }
+
+    default:
+      throw NotImplementedYet("buildReader unhandled type");
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/orc/blob/7f55b453/c++/src/ColumnReader.hh
----------------------------------------------------------------------
diff --git a/c++/src/ColumnReader.hh b/c++/src/ColumnReader.hh
new file mode 100644
index 0000000..b90c942
--- /dev/null
+++ b/c++/src/ColumnReader.hh
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ORC_COLUMN_READER_HH
+#define ORC_COLUMN_READER_HH
+
+#include "orc/Vector.hh"
+#include "ByteRLE.hh"
+#include "Compression.hh"
+#include "wrap/orc-proto-wrapper.hh"
+
+namespace orc {
+
+  class StripeStreams {
+  public:
+    virtual ~StripeStreams();
+
+    /**
+     * Get the reader options.
+     */
+    virtual const ReaderOptions& getReaderOptions() const = 0;
+
+    /**
+     * Get the array of booleans for which columns are selected.
+     * @return the address of an array which contains true at the index of
+     *    each columnId is selected.
+     */
+    virtual const std::vector<bool> getSelectedColumns() const = 0;
+
+    /**
+     * Get the encoding for the given column for this stripe.
+     */
+    virtual proto::ColumnEncoding getEncoding(int64_t columnId) const = 0;
+
+    /**
+     * Get the stream for the given column/kind in this stripe.
+     * @param columnId the id of the column
+     * @param kind the kind of the stream
+     * @param shouldStream should the reading page the stream in
+     * @return the new stream
+     */
+    virtual std::unique_ptr<SeekableInputStream>
+                    getStream(int64_t columnId,
+                              proto::Stream_Kind kind,
+                              bool shouldStream) const = 0;
+
+    /**
+     * Get the memory pool for this reader.
+     */
+    virtual MemoryPool& getMemoryPool() const = 0;
+  };
+
+  /**
+   * The interface for reading ORC data types.
+   */
+  class ColumnReader {
+  protected:
+    std::unique_ptr<ByteRleDecoder> notNullDecoder;
+    int64_t columnId;
+    MemoryPool& memoryPool;
+
+  public:
+    ColumnReader(const Type& type, StripeStreams& stipe);
+
+    virtual ~ColumnReader();
+
+    /**
+     * Skip number of specified rows.
+     * @param numValues the number of values to skip
+     * @return the number of non-null values skipped
+     */
+    virtual uint64_t skip(uint64_t numValues);
+
+    /**
+     * Read the next group of values into this rowBatch.
+     * @param rowBatch the memory to read into.
+     * @param numValues the number of values to read
+     * @param notNull if null, all values are not null. Otherwise, it is
+     *           a mask (with at least numValues bytes) for which values to
+     *           set.
+     */
+    virtual void next(ColumnVectorBatch& rowBatch,
+                      uint64_t numValues,
+                      char* notNull);
+  };
+
+  /**
+   * Create a reader for the given stripe.
+   */
+  std::unique_ptr<ColumnReader> buildReader(const Type& type,
+                                            StripeStreams& stripe);
+}
+
+#endif

http://git-wip-us.apache.org/repos/asf/orc/blob/7f55b453/c++/src/Compression.cc
----------------------------------------------------------------------
diff --git a/c++/src/Compression.cc b/c++/src/Compression.cc
new file mode 100644
index 0000000..81b2c1c
--- /dev/null
+++ b/c++/src/Compression.cc
@@ -0,0 +1,750 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Compression.hh"
+#include "Exceptions.hh"
+
+#include <algorithm>
+#include <iomanip>
+#include <iostream>
+#include <sstream>
+
+#include "zlib.h"
+
+#include "wrap/snappy-wrapper.h"
+
+namespace orc {
+
+  void printBuffer(std::ostream& out,
+                   const char *buffer,
+                   uint64_t length) {
+    const uint64_t width = 24;
+    out << std::hex;
+    for(uint64_t line = 0; line < (length + width - 1) / width; ++line) {
+      out << std::setfill('0') << std::setw(7) << (line * width);
+      for(uint64_t byte = 0;
+          byte < width && line * width + byte < length; ++byte) {
+        out << " " << std::setfill('0') << std::setw(2)
+                  << static_cast<uint64_t>(0xff & buffer[line * width +
+                                                             byte]);
+      }
+      out << "\n";
+    }
+    out << std::dec;
+  }
+
+  PositionProvider::PositionProvider(const std::list<uint64_t>& posns) {
+    position = posns.begin();
+  }
+
+  uint64_t PositionProvider::next() {
+    uint64_t result = *position;
+    ++position;
+    return result;
+  }
+
+  SeekableInputStream::~SeekableInputStream() {
+    // PASS
+  }
+
+  SeekableArrayInputStream::~SeekableArrayInputStream() {
+    // PASS
+  }
+
+  SeekableArrayInputStream::SeekableArrayInputStream
+               (const unsigned char* values,
+                uint64_t size,
+                int64_t blkSize
+                ): data(reinterpret_cast<const char*>(values)) {
+    length = size;
+    position = 0;
+    blockSize = blkSize == -1 ? length : static_cast<uint64_t>(blkSize);
+  }
+
+  SeekableArrayInputStream::SeekableArrayInputStream(const char* values,
+                                                     uint64_t size,
+                                                     int64_t blkSize
+                                                     ): data(values) {
+    length = size;
+    position = 0;
+    blockSize = blkSize == -1 ? length : static_cast<uint64_t>(blkSize);
+  }
+
+  bool SeekableArrayInputStream::Next(const void** buffer, int*size) {
+    uint64_t currentSize = std::min(length - position, blockSize);
+    if (currentSize > 0) {
+      *buffer = data + position;
+      *size = static_cast<int>(currentSize);
+      position += currentSize;
+      return true;
+    }
+    *size = 0;
+    return false;
+  }
+
+  void SeekableArrayInputStream::BackUp(int count) {
+    if (count >= 0) {
+      uint64_t unsignedCount = static_cast<uint64_t>(count);
+      if (unsignedCount <= blockSize && unsignedCount <= position) {
+        position -= unsignedCount;
+      } else {
+        throw std::logic_error("Can't backup that much!");
+      }
+    }
+  }
+
+  bool SeekableArrayInputStream::Skip(int count) {
+    if (count >= 0) {
+      uint64_t unsignedCount = static_cast<uint64_t>(count);
+      if (unsignedCount + position <= length) {
+        position += unsignedCount;
+        return true;
+      } else {
+        position = length;
+      }
+    }
+    return false;
+  }
+
+  google::protobuf::int64 SeekableArrayInputStream::ByteCount() const {
+    return static_cast<google::protobuf::int64>(position);
+  }
+
+  void SeekableArrayInputStream::seek(PositionProvider& seekPosition) {
+    position = seekPosition.next();
+  }
+
+  std::string SeekableArrayInputStream::getName() const {
+    std::ostringstream result;
+    result << "SeekableArrayInputStream " << position << " of " << length;
+    return result.str();
+  }
+
+  static uint64_t computeBlock(int64_t request, uint64_t length) {
+    return std::min(length,
+                    static_cast<uint64_t>(request < 0 ?
+                                          256 * 1024 : request));
+  }
+
+  SeekableFileInputStream::SeekableFileInputStream(InputStream* stream,
+                                                   uint64_t offset,
+                                                   uint64_t byteCount,
+                                                   MemoryPool& _pool,
+                                                   int64_t _blockSize
+                                                   ):pool(_pool),
+                                                     input(stream),
+                                                     start(offset),
+                                                     length(byteCount),
+                                                     blockSize(computeBlock
+                                                               (_blockSize,
+                                                                length)) {
+
+    position = 0;
+    buffer.reset(new DataBuffer<char>(pool));
+    pushBack = 0;
+  }
+
+  SeekableFileInputStream::~SeekableFileInputStream() {
+    // PASS
+  }
+
+  bool SeekableFileInputStream::Next(const void** data, int*size) {
+    uint64_t bytesRead;
+    if (pushBack != 0) {
+      *data = buffer->data() + (buffer->size() - pushBack);
+      bytesRead = pushBack;
+    } else {
+      bytesRead = std::min(length - position, blockSize);
+      buffer->resize(bytesRead);
+      if (bytesRead > 0) {
+        input->read(buffer->data(), bytesRead, start+position);
+        *data = static_cast<void*>(buffer->data());
+      }
+    }
+    position += bytesRead;
+    pushBack = 0;
+    *size = static_cast<int>(bytesRead);
+    return bytesRead != 0;
+  }
+
+  void SeekableFileInputStream::BackUp(int signedCount) {
+    if (signedCount < 0) {
+      throw std::logic_error("can't backup negative distances");
+    }
+    uint64_t count = static_cast<uint64_t>(signedCount);
+    if (pushBack > 0) {
+      throw std::logic_error("can't backup unless we just called Next");
+    }
+    if (count > blockSize || count > position) {
+      throw std::logic_error("can't backup that far");
+    }
+    pushBack = static_cast<uint64_t>(count);
+    position -= pushBack;
+  }
+
+  bool SeekableFileInputStream::Skip(int signedCount) {
+    if (signedCount < 0) {
+      return false;
+    }
+    uint64_t count = static_cast<uint64_t>(signedCount);
+    position = std::min(position + count, length);
+    pushBack = 0;
+    return position < length;
+  }
+
+  int64_t SeekableFileInputStream::ByteCount() const {
+    return static_cast<int64_t>(position);
+  }
+
+  void SeekableFileInputStream::seek(PositionProvider& location) {
+    position = location.next();
+    if (position > length) {
+      position = length;
+      throw std::logic_error("seek too far");
+    }
+    pushBack = 0;
+  }
+
+  std::string SeekableFileInputStream::getName() const {
+    std::ostringstream result;
+    result << input->getName() << " from " << start << " for "
+           << length;
+    return result.str();
+  }
+
+  enum DecompressState { DECOMPRESS_HEADER,
+                         DECOMPRESS_START,
+                         DECOMPRESS_CONTINUE,
+                         DECOMPRESS_ORIGINAL,
+                         DECOMPRESS_EOF};
+
+  class ZlibDecompressionStream: public SeekableInputStream {
+  public:
+    ZlibDecompressionStream(std::unique_ptr<SeekableInputStream> inStream,
+                            size_t blockSize,
+                            MemoryPool& pool);
+    virtual ~ZlibDecompressionStream();
+    virtual bool Next(const void** data, int*size) override;
+    virtual void BackUp(int count) override;
+    virtual bool Skip(int count) override;
+    virtual int64_t ByteCount() const override;
+    virtual void seek(PositionProvider& position) override;
+    virtual std::string getName() const override;
+
+  private:
+    void readBuffer(bool failOnEof) {
+      int length;
+      if (!input->Next(reinterpret_cast<const void**>(&inputBuffer),
+                       &length)) {
+        if (failOnEof) {
+          throw ParseError("Read past EOF in "
+                           "ZlibDecompressionStream::readBuffer");
+        }
+        state = DECOMPRESS_EOF;
+        inputBuffer = nullptr;
+        inputBufferEnd = nullptr;
+      } else {
+        inputBufferEnd = inputBuffer + length;
+      }
+    }
+
+    uint32_t readByte(bool failOnEof) {
+      if (inputBuffer == inputBufferEnd) {
+        readBuffer(failOnEof);
+        if (state == DECOMPRESS_EOF) {
+          return 0;
+        }
+      }
+      return static_cast<unsigned char>(*(inputBuffer++));
+    }
+
+    void readHeader() {
+      uint32_t header = readByte(false);
+      if (state != DECOMPRESS_EOF) {
+        header |= readByte(true) << 8;
+        header |= readByte(true) << 16;
+        if (header & 1) {
+          state = DECOMPRESS_ORIGINAL;
+        } else {
+          state = DECOMPRESS_START;
+        }
+        remainingLength = header >> 1;
+      } else {
+        remainingLength = 0;
+      }
+    }
+
+    MemoryPool& pool;
+    const size_t blockSize;
+    std::unique_ptr<SeekableInputStream> input;
+    z_stream zstream;
+    DataBuffer<char> buffer;
+
+    // the current state
+    DecompressState state;
+
+    // the start of the current buffer
+    // This pointer is not owned by us. It is either owned by zstream or
+    // the underlying stream.
+    const char* outputBuffer;
+    // the size of the current buffer
+    size_t outputBufferLength;
+    // the size of the current chunk
+    size_t remainingLength;
+
+    // the last buffer returned from the input
+    const char *inputBuffer;
+    const char *inputBufferEnd;
+
+    // roughly the number of bytes returned
+    off_t bytesReturned;
+  };
+
+DIAGNOSTIC_PUSH
+DIAGNOSTIC_IGNORE("-Wold-style-cast")
+
+  ZlibDecompressionStream::ZlibDecompressionStream
+                   (std::unique_ptr<SeekableInputStream> inStream,
+                    size_t _blockSize,
+                    MemoryPool& _pool
+                    ): pool(_pool),
+                       blockSize(_blockSize),
+                       buffer(pool, _blockSize) {
+    input.reset(inStream.release());
+    zstream.next_in = Z_NULL;
+    zstream.avail_in = 0;
+    zstream.zalloc = Z_NULL;
+    zstream.zfree = Z_NULL;
+    zstream.opaque = Z_NULL;
+    zstream.next_out = reinterpret_cast<Bytef*>(buffer.data());
+    zstream.avail_out = static_cast<uInt>(blockSize);
+    int64_t result = inflateInit2(&zstream, -15);
+    switch (result) {
+    case Z_OK:
+      break;
+    case Z_MEM_ERROR:
+      throw std::logic_error("Memory error from inflateInit2");
+    case Z_VERSION_ERROR:
+      throw std::logic_error("Version error from inflateInit2");
+    case Z_STREAM_ERROR:
+      throw std::logic_error("Stream error from inflateInit2");
+    default:
+      throw std::logic_error("Unknown error from inflateInit2");
+    }
+    outputBuffer = nullptr;
+    outputBufferLength = 0;
+    remainingLength = 0;
+    state = DECOMPRESS_HEADER;
+    inputBuffer = nullptr;
+    inputBufferEnd = nullptr;
+    bytesReturned = 0;
+  }
+
+DIAGNOSTIC_POP
+
+  ZlibDecompressionStream::~ZlibDecompressionStream() {
+    int64_t result = inflateEnd(&zstream);
+    if (result != Z_OK) {
+      // really can't throw in destructors
+      std::cout << "Error in ~ZlibDecompressionStream() " << result << "\n";
+    }
+  }
+
+  bool ZlibDecompressionStream::Next(const void** data, int*size) {
+    // if the user pushed back, return them the partial buffer
+    if (outputBufferLength) {
+      *data = outputBuffer;
+      *size = static_cast<int>(outputBufferLength);
+      outputBuffer += outputBufferLength;
+      outputBufferLength = 0;
+      return true;
+    }
+    if (state == DECOMPRESS_HEADER || remainingLength == 0) {
+      readHeader();
+    }
+    if (state == DECOMPRESS_EOF) {
+      return false;
+    }
+    if (inputBuffer == inputBufferEnd) {
+      readBuffer(true);
+    }
+    size_t availSize =
+      std::min(static_cast<size_t>(inputBufferEnd - inputBuffer),
+               remainingLength);
+    if (state == DECOMPRESS_ORIGINAL) {
+      *data = inputBuffer;
+      *size = static_cast<int>(availSize);
+      outputBuffer = inputBuffer + availSize;
+      outputBufferLength = 0;
+    } else if (state == DECOMPRESS_START) {
+      zstream.next_in =
+        reinterpret_cast<Bytef*>(const_cast<char*>(inputBuffer));
+      zstream.avail_in = static_cast<uInt>(availSize);
+      outputBuffer = buffer.data();
+      zstream.next_out =
+        reinterpret_cast<Bytef*>(const_cast<char*>(outputBuffer));
+      zstream.avail_out = static_cast<uInt>(blockSize);
+      if (inflateReset(&zstream) != Z_OK) {
+        throw std::logic_error("Bad inflateReset in "
+                               "ZlibDecompressionStream::Next");
+      }
+      int64_t result;
+      do {
+        result = inflate(&zstream, availSize == remainingLength ? Z_FINISH :
+                         Z_SYNC_FLUSH);
+        switch (result) {
+        case Z_OK:
+          remainingLength -= availSize;
+          inputBuffer += availSize;
+          readBuffer(true);
+          availSize =
+            std::min(static_cast<size_t>(inputBufferEnd - inputBuffer),
+                     remainingLength);
+          zstream.next_in =
+            reinterpret_cast<Bytef*>(const_cast<char*>(inputBuffer));
+          zstream.avail_in = static_cast<uInt>(availSize);
+          break;
+        case Z_STREAM_END:
+          break;
+        case Z_BUF_ERROR:
+          throw std::logic_error("Buffer error in "
+                                 "ZlibDecompressionStream::Next");
+        case Z_DATA_ERROR:
+          throw std::logic_error("Data error in "
+                                 "ZlibDecompressionStream::Next");
+        case Z_STREAM_ERROR:
+          throw std::logic_error("Stream error in "
+                                 "ZlibDecompressionStream::Next");
+        default:
+          throw std::logic_error("Unknown error in "
+                                 "ZlibDecompressionStream::Next");
+        }
+      } while (result != Z_STREAM_END);
+      *size = static_cast<int>(blockSize - zstream.avail_out);
+      *data = outputBuffer;
+      outputBufferLength = 0;
+      outputBuffer += *size;
+    } else {
+      throw std::logic_error("Unknown compression state in "
+                             "ZlibDecompressionStream::Next");
+    }
+    inputBuffer += availSize;
+    remainingLength -= availSize;
+    bytesReturned += *size;
+    return true;
+  }
+
+  void ZlibDecompressionStream::BackUp(int count) {
+    if (outputBuffer == nullptr || outputBufferLength != 0) {
+      throw std::logic_error("Backup without previous Next in "
+                             "ZlibDecompressionStream");
+    }
+    outputBuffer -= static_cast<size_t>(count);
+    outputBufferLength = static_cast<size_t>(count);
+    bytesReturned -= count;
+  }
+
+  bool ZlibDecompressionStream::Skip(int count) {
+    bytesReturned += count;
+    // this is a stupid implementation for now.
+    // should skip entire blocks without decompressing
+    while (count > 0) {
+      const void *ptr;
+      int len;
+      if (!Next(&ptr, &len)) {
+        return false;
+      }
+      if (len > count) {
+        BackUp(len - count);
+        count = 0;
+      } else {
+        count -= len;
+      }
+    }
+    return true;
+  }
+
+  int64_t ZlibDecompressionStream::ByteCount() const {
+    return bytesReturned;
+  }
+
+  void ZlibDecompressionStream::seek(PositionProvider& position) {
+    input->seek(position);
+    bytesReturned = input->ByteCount();
+    if (!Skip(static_cast<int>(position.next()))) {
+      throw ParseError("Bad skip in ZlibDecompressionStream::seek");
+    }
+  }
+
+  std::string ZlibDecompressionStream::getName() const {
+    std::ostringstream result;
+    result << "zlib(" << input->getName() << ")";
+    return result.str();
+  }
+
+  class SnappyDecompressionStream: public SeekableInputStream {
+  public:
+    SnappyDecompressionStream(std::unique_ptr<SeekableInputStream> inStream,
+                              size_t blockSize,
+                              MemoryPool& pool);
+
+    virtual ~SnappyDecompressionStream() {}
+    virtual bool Next(const void** data, int*size) override;
+    virtual void BackUp(int count) override;
+    virtual bool Skip(int count) override;
+    virtual int64_t ByteCount() const override;
+    virtual void seek(PositionProvider& position) override;
+    virtual std::string getName() const override;
+
+  private:
+    void readBuffer(bool failOnEof) {
+      int length;
+      if (!input->Next(reinterpret_cast<const void**>(&inputBufferPtr),
+                       &length)) {
+        if (failOnEof) {
+          throw ParseError("SnappyDecompressionStream read past EOF");
+        }
+        state = DECOMPRESS_EOF;
+        inputBufferPtr = nullptr;
+        inputBufferPtrEnd = nullptr;
+      } else {
+        inputBufferPtrEnd = inputBufferPtr + length;
+      }
+    }
+
+    uint32_t readByte(bool failOnEof) {
+      if (inputBufferPtr == inputBufferPtrEnd) {
+        readBuffer(failOnEof);
+        if (state == DECOMPRESS_EOF) {
+          return 0;
+        }
+      }
+      return static_cast<unsigned char>(*(inputBufferPtr++));
+    }
+
+    void readHeader() {
+      uint32_t header = readByte(false);
+      if (state != DECOMPRESS_EOF) {
+        header |= readByte(true) << 8;
+        header |= readByte(true) << 16;
+        if (header & 1) {
+          state = DECOMPRESS_ORIGINAL;
+        } else {
+          state = DECOMPRESS_START;
+        }
+        remainingLength = header >> 1;
+      } else {
+        remainingLength = 0;
+      }
+    }
+
+    std::unique_ptr<SeekableInputStream> input;
+    MemoryPool& pool;
+
+    // may need to stitch together multiple input buffers;
+    // to give snappy a contiguous block
+    DataBuffer<char> inputBuffer;
+
+    // uncompressed output
+    DataBuffer<char> outputBuffer;
+
+    // the current state
+    DecompressState state;
+
+    // the start of the current output buffer
+    const char* outputBufferPtr;
+    // the size of the current output buffer
+    size_t outputBufferLength;
+
+    // the size of the current chunk
+    size_t remainingLength;
+
+    // the last buffer returned from the input
+    const char *inputBufferPtr;
+    const char *inputBufferPtrEnd;
+
+    // bytes returned by this stream
+    off_t bytesReturned;
+  };
+
+  SnappyDecompressionStream::SnappyDecompressionStream
+                   (std::unique_ptr<SeekableInputStream> inStream,
+                    size_t bufferSize,
+                    MemoryPool& _pool
+                    ) : pool(_pool),
+                        inputBuffer(pool, bufferSize),
+                        outputBuffer(pool, bufferSize),
+                        state(DECOMPRESS_HEADER),
+                        outputBufferPtr(0),
+                        outputBufferLength(0),
+                        remainingLength(0),
+                        inputBufferPtr(0),
+                        inputBufferPtrEnd(0),
+                        bytesReturned(0) {
+    input.reset(inStream.release());
+  }
+
+  bool SnappyDecompressionStream::Next(const void** data, int*size) {
+    // if the user pushed back, return them the partial buffer
+    if (outputBufferLength) {
+      *data = outputBufferPtr;
+      *size = static_cast<int>(outputBufferLength);
+      outputBufferPtr += outputBufferLength;
+      bytesReturned += outputBufferLength;
+      outputBufferLength = 0;
+      return true;
+    }
+    if (state == DECOMPRESS_HEADER || remainingLength == 0) {
+      readHeader();
+    }
+    if (state == DECOMPRESS_EOF) {
+      return false;
+    }
+    if (inputBufferPtr == inputBufferPtrEnd) {
+      readBuffer(true);
+    }
+
+    size_t availSize =
+      std::min(static_cast<size_t>(inputBufferPtrEnd - inputBufferPtr),
+               remainingLength);
+    if (state == DECOMPRESS_ORIGINAL) {
+      *data = inputBufferPtr;
+      *size = static_cast<int>(availSize);
+      outputBufferPtr = inputBufferPtr + availSize;
+      outputBufferLength = 0;
+      inputBufferPtr += availSize;
+      remainingLength -= availSize;
+    } else if (state == DECOMPRESS_START) {
+      // Get contiguous bytes of compressed block.
+      const char *compressed = inputBufferPtr;
+      if (remainingLength == availSize) {
+          inputBufferPtr += availSize;
+      } else {
+        // Did not read enough from input.
+        if (inputBuffer.capacity() < remainingLength) {
+          inputBuffer.resize(remainingLength);
+        }
+        ::memcpy(inputBuffer.data(), inputBufferPtr, availSize);
+        inputBufferPtr += availSize;
+        compressed = inputBuffer.data();
+
+        for (size_t pos = availSize; pos < remainingLength; ) {
+          readBuffer(true);
+          size_t avail =
+              std::min(static_cast<size_t>(inputBufferPtrEnd - inputBufferPtr),
+                       remainingLength - pos);
+          ::memcpy(inputBuffer.data() + pos, inputBufferPtr, avail);
+          pos += avail;
+          inputBufferPtr += avail;
+        }
+      }
+
+      if (!snappy::GetUncompressedLength(compressed, remainingLength,
+                                         &outputBufferLength)) {
+        throw ParseError("SnappyDecompressionStream choked on corrupt input");
+      }
+
+      if (outputBufferLength > outputBuffer.capacity()) {
+        throw std::logic_error("uncompressed length exceeds block size");
+      }
+
+      if (!snappy::RawUncompress(compressed, remainingLength,
+                                 outputBuffer.data())) {
+        throw ParseError("SnappyDecompressionStream choked on corrupt input");
+      }
+
+      remainingLength = 0;
+      state = DECOMPRESS_HEADER;
+      *data = outputBuffer.data();
+      *size = static_cast<int>(outputBufferLength);
+      outputBufferPtr = outputBuffer.data() + outputBufferLength;
+      outputBufferLength = 0;
+    }
+
+    bytesReturned += *size;
+    return true;
+  }
+
+  void SnappyDecompressionStream::BackUp(int count) {
+    if (outputBufferPtr == nullptr || outputBufferLength != 0) {
+      throw std::logic_error("Backup without previous Next in "
+                             "SnappyDecompressionStream");
+    }
+    outputBufferPtr -= static_cast<size_t>(count);
+    outputBufferLength = static_cast<size_t>(count);
+    bytesReturned -= count;
+  }
+
+  bool SnappyDecompressionStream::Skip(int count) {
+    bytesReturned += count;
+    // this is a stupid implementation for now.
+    // should skip entire blocks without decompressing
+    while (count > 0) {
+      const void *ptr;
+      int len;
+      if (!Next(&ptr, &len)) {
+        return false;
+      }
+      if (len > count) {
+        BackUp(len - count);
+        count = 0;
+      } else {
+        count -= len;
+      }
+    }
+    return true;
+  }
+
+  int64_t SnappyDecompressionStream::ByteCount() const {
+    return bytesReturned;
+  }
+
+  void SnappyDecompressionStream::seek(PositionProvider& position) {
+    input->seek(position);
+    if (!Skip(static_cast<int>(position.next()))) {
+      throw ParseError("Bad skip in SnappyDecompressionStream::seek");
+    }
+  }
+
+  std::string SnappyDecompressionStream::getName() const {
+    std::ostringstream result;
+    result << "snappy(" << input->getName() << ")";
+    return result.str();
+  }
+
+  std::unique_ptr<SeekableInputStream>
+     createDecompressor(CompressionKind kind,
+                        std::unique_ptr<SeekableInputStream> input,
+                        uint64_t blockSize,
+                        MemoryPool& pool) {
+    switch (static_cast<int64_t>(kind)) {
+    case CompressionKind_NONE:
+      return std::move(input);
+    case CompressionKind_ZLIB:
+      return std::unique_ptr<SeekableInputStream>
+        (new ZlibDecompressionStream(std::move(input), blockSize, pool));
+    case CompressionKind_SNAPPY:
+      return std::unique_ptr<SeekableInputStream>
+        (new SnappyDecompressionStream(std::move(input), blockSize, pool));
+    case CompressionKind_LZO:
+    default:
+      throw NotImplementedYet("compression codec");
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/orc/blob/7f55b453/c++/src/Compression.hh
----------------------------------------------------------------------
diff --git a/c++/src/Compression.hh b/c++/src/Compression.hh
new file mode 100644
index 0000000..2c02584
--- /dev/null
+++ b/c++/src/Compression.hh
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ORC_COMPRESSION_HH
+#define ORC_COMPRESSION_HH
+
+#include "orc/OrcFile.hh"
+
+#include "Adaptor.hh"
+#include "wrap/zero-copy-stream-wrapper.h"
+
+#include <list>
+#include <vector>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <memory>
+
+namespace orc {
+
+  void printBuffer(std::ostream& out,
+                   const char *buffer,
+                   uint64_t length);
+
+  class PositionProvider {
+  private:
+    std::list<uint64_t>::const_iterator position;
+  public:
+    PositionProvider(const std::list<uint64_t>& positions);
+    uint64_t next();
+  };
+
+  /**
+   * A subclass of Google's ZeroCopyInputStream that supports seek.
+   * By extending Google's class, we get the ability to pass it directly
+   * to the protobuf readers.
+   */
+  class SeekableInputStream: public google::protobuf::io::ZeroCopyInputStream {
+  public:
+    virtual ~SeekableInputStream();
+    virtual void seek(PositionProvider& position) = 0;
+    virtual std::string getName() const = 0;
+  };
+
+  /**
+   * Create a seekable input stream based on a memory range.
+   */
+  class SeekableArrayInputStream: public SeekableInputStream {
+  private:
+    const char* data;
+    uint64_t length;
+    uint64_t position;
+    uint64_t blockSize;
+
+  public:
+    SeekableArrayInputStream(const unsigned char* list,
+                             uint64_t length,
+                             int64_t block_size = -1);
+    SeekableArrayInputStream(const char* list,
+                             uint64_t length,
+                             int64_t block_size = -1);
+    virtual ~SeekableArrayInputStream();
+    virtual bool Next(const void** data, int*size) override;
+    virtual void BackUp(int count) override;
+    virtual bool Skip(int count) override;
+    virtual google::protobuf::int64 ByteCount() const override;
+    virtual void seek(PositionProvider& position) override;
+    virtual std::string getName() const override;
+  };
+
+  /**
+   * Create a seekable input stream based on an input stream.
+   */
+  class SeekableFileInputStream: public SeekableInputStream {
+  private:
+    MemoryPool& pool;
+    InputStream* const input;
+    const uint64_t start;
+    const uint64_t length;
+    const uint64_t blockSize;
+    std::unique_ptr<DataBuffer<char> > buffer;
+    uint64_t position;
+    uint64_t pushBack;
+
+  public:
+    SeekableFileInputStream(InputStream* input,
+                            uint64_t offset,
+                            uint64_t byteCount,
+                            MemoryPool& pool,
+                            int64_t blockSize = -1);
+    virtual ~SeekableFileInputStream();
+
+    virtual bool Next(const void** data, int*size) override;
+    virtual void BackUp(int count) override;
+    virtual bool Skip(int count) override;
+    virtual int64_t ByteCount() const override;
+    virtual void seek(PositionProvider& position) override;
+    virtual std::string getName() const override;
+  };
+
+  /**
+   * Create a decompressor for the given compression kind.
+   * @param kind the compression type to implement
+   * @param input the input stream that is the underlying source
+   * @param bufferSize the maximum size of the buffer
+   * @param pool the memory pool
+   */
+  std::unique_ptr<SeekableInputStream>
+     createDecompressor(CompressionKind kind,
+                        std::unique_ptr<SeekableInputStream> input,
+                        uint64_t bufferSize,
+                        MemoryPool& pool);
+}
+
+#endif