You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/09/04 05:14:32 UTC

[12/13] git commit: DRILL-91/DRILL-160: Update to Parquet Record Reader to support nullable fields (int, long, float, double, bit, varbinary).

DRILL-91/DRILL-160: Update to Parquet Record Reader to support nullable fields (int, long, float, double, bit, varbinary).


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/63a90fe9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/63a90fe9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/63a90fe9

Branch: refs/heads/master
Commit: 63a90fe9ae7f4b911ee624759dfb32ed1b069de7
Parents: 5642dee
Author: Jason Altekruse <al...@gmial.com>
Authored: Mon Sep 2 09:44:46 2013 -0500
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Sep 3 20:11:13 2013 -0700

----------------------------------------------------------------------
 sandbox/prototype/exec/java-exec/pom.xml        |   5 +
 .../codegen/templates/NullableValueVectors.java |  20 +-
 .../codegen/templates/RepeatedValueVectors.java |   4 +
 .../drill/exec/store/parquet/ColumnReader.java  |   8 +-
 .../exec/store/parquet/NullableBitReader.java   |  61 ++
 .../store/parquet/NullableColumnReader.java     | 130 ++++
 .../parquet/NullableFixedByteAlignedReader.java |  48 ++
 .../exec/store/parquet/PageReadStatus.java      |  31 +-
 .../exec/store/parquet/ParquetGroupScan.java    |  22 +-
 .../exec/store/parquet/ParquetRecordReader.java | 232 +++++--
 .../exec/store/parquet/ParquetRowGroupScan.java |  48 +-
 .../store/parquet/ParquetScanBatchCreator.java  |  14 +-
 .../store/parquet/ParquetStorageEngine.java     |   2 +-
 .../exec/store/parquet/VarLenBinaryReader.java  |  92 ++-
 .../drill/exec/vector/BaseValueVector.java      |   6 +-
 .../vector/NullableVectorDefinitionSetter.java  |  23 +
 .../exec/store/ParquetRecordReaderTest.java     | 657 +++++++++++++++++++
 .../src/test/resources/parquet_nullable.json    |  41 ++
 .../src/test/resources/parquet_scan_screen.json |  41 ++
 .../parquet_scan_screen_read_entry_replace.json |  39 ++
 .../parquet_scan_union_screen_physical.json     |  35 +
 21 files changed, 1408 insertions(+), 151 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml
index 2b35534..039c866 100644
--- a/sandbox/prototype/exec/java-exec/pom.xml
+++ b/sandbox/prototype/exec/java-exec/pom.xml
@@ -184,6 +184,11 @@
       <artifactId>jetty-util</artifactId>
       <version>6.1.26</version>
     </dependency>
+    <dependency>
+      <groupId>com.thoughtworks.paranamer</groupId>
+      <artifactId>paranamer</artifactId>
+      <version>2.3</version>
+    </dependency>
   </dependencies>
 
   <profiles>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
index f528ce1..25249ff 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -1,3 +1,7 @@
+import org.apache.drill.exec.vector.NullableVectorDefinitionSetter;
+
+import java.lang.UnsupportedOperationException;
+
 <@pp.dropOutputFile />
 <#list vv.types as type>
 <#list type.minor as minor>
@@ -56,6 +60,10 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
     return values.getBufferSize() + bits.getBufferSize();
   }
 
+  public ByteBuf getData(){
+    return values.getData();
+  }
+
   <#if type.major == "VarLen">
   @Override
   public FieldMetadata getMetadata() {
@@ -242,13 +250,21 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
     public void reset(){}
   }
   
-  public final class Mutator implements ValueVector.Mutator{
+  public final class Mutator implements ValueVector.Mutator, NullableVectorDefinitionSetter{
     
     private int setCount;
     
     private Mutator(){
     }
 
+    public ${valuesName} getVectorWithValues(){
+      return values;
+    }
+
+    public void setIndexDefined(int index){
+      bits.getMutator().set(index, 1);
+    }
+
     /**
      * Set the variable length element at the specified index to the supplied byte array.
      *
@@ -315,4 +331,4 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
   }
 }
 </#list>
-</#list>
\ No newline at end of file
+</#list>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index d86e449..faf9772 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -45,6 +45,10 @@ package org.apache.drill.exec.vector;
   public int getBufferSize(){
     return offsets.getBufferSize() + values.getBufferSize();
   }
+
+  public ByteBuf getData(){
+      return values.getData();
+  }
   
   public TransferPair getTransferPair(){
     return new TransferImpl(getField());

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
index c62613a..e026fa7 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.store.parquet;
 
 import io.netty.buffer.ByteBuf;
 import org.apache.drill.exec.store.VectorHolder;
-import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.BaseValueVector;
 import org.apache.drill.exec.vector.ValueVector;
 import parquet.column.ColumnDescriptor;
 import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -61,7 +61,7 @@ public abstract class ColumnReader {
                boolean fixedLength, ValueVector v){
     this.parentReader = parentReader;
     if (allocateSize > 1) valueVecHolder = new VectorHolder(allocateSize, v);
-    else valueVecHolder = new VectorHolder(5000, (BaseDataValueVector) v);
+    else valueVecHolder = new VectorHolder(5000, v);
 
     columnDescriptor = descriptor;
     this.columnChunkMetaData = columnChunkMetaData;
@@ -82,7 +82,7 @@ public abstract class ColumnReader {
     readLength = 0;
     readLengthInBits = 0;
     recordsReadInThisIteration = 0;
-    vectorData = ((BaseDataValueVector) valueVecHolder.getValueVector()).getData();
+    vectorData = ((BaseValueVector)valueVecHolder.getValueVector()).getData();
     do {
       // if no page has been read, or all of the records have been read out of a page, read the next one
       if (pageReadStatus.currentPage == null
@@ -109,4 +109,4 @@ public abstract class ColumnReader {
   }
 
   protected abstract void readField(long recordsToRead, ColumnReader firstColumnStatus);
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
new file mode 100644
index 0000000..8a9565c
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
@@ -0,0 +1,61 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.NullableBitVector;
+import org.apache.drill.exec.vector.ValueVector;
+import parquet.column.ColumnDescriptor;
+import parquet.hadoop.metadata.ColumnChunkMetaData;
+
+import java.io.IOException;
+
+/**
+ * This class is used in conjunction with its superclass to read nullable bit columns in a parquet file.
+ * It currently is using an inefficient value-by-value approach.
+ * TODO - make this more efficient by copying runs of values like in NullableFixedByteAlignedReader
+ * This will also involve incorporating the ideas from the BitReader (the reader for non-nullable bits)
+ * because page/batch boundaries that do not land on byte boundaries require shifting of all of the values
+ * in the next batch.
+ */
+public final class NullableBitReader extends ColumnReader {
+
+  NullableBitReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
+                    boolean fixedLength, ValueVector v) {
+    super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
+  }
+
+  @Override
+  public void readField(long recordsToReadInThisPass, ColumnReader firstColumnStatus) {
+
+    recordsReadInThisIteration = Math.min(pageReadStatus.currentPage.getValueCount()
+        - pageReadStatus.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
+    int defLevel;
+    for (int i = 0; i < recordsReadInThisIteration; i++){
+      defLevel = pageReadStatus.definitionLevels.readInteger();
+      // if the value is defined
+      if (defLevel == columnDescriptor.getMaxDefinitionLevel()){
+        ((NullableBitVector)valueVecHolder.getValueVector()).getMutator().set(i + valuesReadInCurrentPass,
+            pageReadStatus.valueReader.readBoolean() ? 1 : 0 );
+      }
+      // otherwise the value is skipped, because the bit vector indicating nullability is zero filled
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
new file mode 100644
index 0000000..e5e49b6
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
@@ -0,0 +1,130 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.NullableVectorDefinitionSetter;
+import org.apache.drill.exec.vector.ValueVector;
+import parquet.column.ColumnDescriptor;
+import parquet.hadoop.metadata.ColumnChunkMetaData;
+
+import java.io.IOException;
+
+public abstract class NullableColumnReader extends ColumnReader{
+
+  int nullsFound;
+  // used to skip nulls found
+  int rightBitShift;
+  // used when copying less than a byte worth of data at a time, to indicate the number of used bits in the current byte
+  int bitsUsed;
+
+  NullableColumnReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
+               boolean fixedLength, ValueVector v){
+    super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
+  }
+
+  public void readAllFixedFields(long recordsToReadInThisPass, ColumnReader firstColumnStatus) throws IOException {
+    readStartInBytes = 0;
+    readLength = 0;
+    readLengthInBits = 0;
+    recordsReadInThisIteration = 0;
+    vectorData = ((BaseValueVector)valueVecHolder.getValueVector()).getData();
+
+    do {
+      // if no page has been read, or all of the records have been read out of a page, read the next one
+      if (pageReadStatus.currentPage == null
+          || pageReadStatus.valuesRead == pageReadStatus.currentPage.getValueCount()) {
+        if (!pageReadStatus.next()) {
+          break;
+        }
+      }
+
+      // values need to be spaced out where nulls appear in the column
+      // leaving blank space for nulls allows for random access to values
+      // to optimize copying data out of the buffered disk stream, runs of defined values
+      // are located and copied together, rather than copying individual values
+
+      long runStart = pageReadStatus.readPosInBytes;
+      int runLength = 0;
+      int currentDefinitionLevel = 0;
+      int currentValueIndexInVector = totalValuesRead;
+      boolean lastValueWasNull = true;
+      int definitionLevelsRead;
+      // loop to find the longest run of defined values available, can be preceded by several nulls
+      while (true){
+        definitionLevelsRead = 0;
+        lastValueWasNull = true;
+        nullsFound = 0;
+        if (currentValueIndexInVector - totalValuesRead == recordsToReadInThisPass
+            || currentValueIndexInVector >= valueVecHolder.getValueVector().getValueCapacity()){
+          break;
+        }
+        while(currentValueIndexInVector - totalValuesRead < recordsToReadInThisPass
+            && currentValueIndexInVector < valueVecHolder.getValueVector().getValueCapacity()
+            && definitionLevelsRead < pageReadStatus.currentPage.getValueCount()){
+          currentDefinitionLevel = pageReadStatus.definitionLevels.readInteger();
+          definitionLevelsRead++;
+          if ( currentDefinitionLevel < columnDescriptor.getMaxDefinitionLevel()){
+            // a run of non-null values was found, break out of this loop to do a read in the outer loop
+            nullsFound++;
+            if ( ! lastValueWasNull ){
+              currentValueIndexInVector++;
+              break;
+            }
+            lastValueWasNull = true;
+          }
+          else{
+            if (lastValueWasNull){
+              runStart = pageReadStatus.readPosInBytes;
+              runLength = 0;
+              lastValueWasNull = false;
+            }
+            runLength++;
+            ((NullableVectorDefinitionSetter)valueVecHolder.getValueVector().getMutator()).setIndexDefined(currentValueIndexInVector);
+          }
+          currentValueIndexInVector++;
+        }
+        pageReadStatus.readPosInBytes = runStart;
+        recordsReadInThisIteration = runLength;
+
+        readField( runLength, firstColumnStatus);
+        int writerIndex = ((BaseValueVector) valueVecHolder.getValueVector()).getData().writerIndex();
+        if ( dataTypeLengthInBits > 8  || (dataTypeLengthInBits < 8 && totalValuesRead + runLength % 8 == 0)){
+          ((BaseValueVector) valueVecHolder.getValueVector()).getData().setIndex(0, writerIndex + (int) Math.ceil( nullsFound * dataTypeLengthInBits / 8.0));
+        }
+        else if (dataTypeLengthInBits < 8){
+          rightBitShift += dataTypeLengthInBits * nullsFound;
+        }
+        recordsReadInThisIteration += nullsFound;
+        valuesReadInCurrentPass += recordsReadInThisIteration;
+        totalValuesRead += recordsReadInThisIteration;
+        pageReadStatus.valuesRead += recordsReadInThisIteration;
+        if (readStartInBytes + readLength >= pageReadStatus.byteLength && bitsUsed == 0) {
+          pageReadStatus.next();
+        } else {
+          pageReadStatus.readPosInBytes = readStartInBytes + readLength;
+        }
+      }
+    }
+    while (valuesReadInCurrentPass < recordsToReadInThisPass && pageReadStatus.currentPage != null);
+    valueVecHolder.getValueVector().getMutator().setValueCount(
+        valuesReadInCurrentPass);
+  }
+
+  protected abstract void readField(long recordsToRead, ColumnReader firstColumnStatus);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
new file mode 100644
index 0000000..3d4a29d
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
@@ -0,0 +1,48 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.NullableVectorDefinitionSetter;
+import org.apache.drill.exec.vector.ValueVector;
+import parquet.column.ColumnDescriptor;
+import parquet.hadoop.metadata.ColumnChunkMetaData;
+
+public class NullableFixedByteAlignedReader extends NullableColumnReader {
+
+  NullableFixedByteAlignedReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
+                         boolean fixedLength, ValueVector v) {
+    super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
+  }
+
+  // this method is called by its superclass during a read loop
+  @Override
+  protected void readField(long recordsToReadInThisPass, ColumnReader firstColumnStatus) {
+
+    recordsReadInThisIteration = recordsToReadInThisPass;
+
+    readStartInBytes = pageReadStatus.readPosInBytes;
+    readLengthInBits = recordsReadInThisIteration * dataTypeLengthInBits;
+    readLength = (int) Math.ceil(readLengthInBits / 8.0);
+
+    bytes = pageReadStatus.pageDataByteArray;
+    // vectorData is assigned by the superclass read loop method
+    vectorData.writeBytes(bytes,
+        (int) readStartInBytes, (int) readLength);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
index c5c0d87..0d70138 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
@@ -20,8 +20,10 @@ package org.apache.drill.exec.store.parquet;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.ByteBufInputStream;
 import parquet.bytes.BytesInput;
+import parquet.column.ValuesType;
 import parquet.column.page.Page;
 import parquet.column.page.PageReader;
+import parquet.column.values.ValuesReader;
 import parquet.format.PageHeader;
 
 import java.io.IOException;
@@ -36,11 +38,8 @@ public final class PageReadStatus {
   // store references to the pages that have been uncompressed, but not copied to ValueVectors yet
   Page currentPage;
   // buffer to store bytes of current page, set to max size of parquet page
-  // TODO: add this back once toByteArray accepts an input.  byte[] pageDataByteArray = new byte[ParquetRecordReader.PARQUET_PAGE_MAX_SIZE];
-  byte[] pageDataByteArray;
-  
-  
-  PageReader pageReader;
+  byte[] pageDataByteArray = new byte[ParquetRecordReader.PARQUET_PAGE_MAX_SIZE];
+
   // read position in the current page, stored in the ByteBuf in ParquetRecordReader called bufferWithAllData
   long readPosInBytes;
   // bit shift needed for the next page if the last one did not line up with a byte boundary
@@ -52,8 +51,8 @@ public final class PageReadStatus {
   int valuesRead;
   int byteLength;
   int rowGroupIndex;
-  // buffer with the compressed data of an entire row group
-  ByteBuf bufferWithAllData;
+  ValuesReader definitionLevels;
+  ValuesReader valueReader;
 
   PageReadStatus(ColumnReader parentStatus, int rowGroupIndex, ByteBuf bufferWithAllData){
     this.parentColumnReader = parentStatus;
@@ -101,20 +100,26 @@ public final class PageReadStatus {
 
     parentColumnReader.readPositionInBuffer += pageHeader.compressed_page_size + length;
     byteLength = pageHeader.uncompressed_page_size;
+
     if (currentPage == null) {
       return false;
     }
 
     // if the buffer holding each page's data is not large enough to hold the current page, re-allocate, with a little extra space
-//    if (pageHeader.getUncompressed_page_size() > pageDataByteArray.length) {
-//      pageDataByteArray = new byte[pageHeader.getUncompressed_page_size() + 100];
-//    }
+    if (pageHeader.getUncompressed_page_size() > pageDataByteArray.length) {
+      pageDataByteArray = new byte[pageHeader.getUncompressed_page_size() + 100];
+    }
     // TODO - would like to get this into the mainline, hopefully before alpha
     pageDataByteArray = currentPage.getBytes().toByteArray();
-    //TODO: Fix once parquet supports buffer work or at least passing in array.
-    //pageDataByteArray = currentPage.getBytes().toByteArray(pageDataByteArray, 0, byteLength);
 
-    readPosInBytes = 0;
+    if (parentColumnReader.columnDescriptor.getMaxDefinitionLevel() != 0){
+      definitionLevels = currentPage.getDlEncoding().getValuesReader(parentColumnReader.columnDescriptor, ValuesType.DEFINITION_LEVEL);
+      valueReader = currentPage.getValueEncoding().getValuesReader(parentColumnReader.columnDescriptor, ValuesType.VALUES);
+      int endOfDefinitionLevels = definitionLevels.initFromPage(currentPage.getValueCount(), pageDataByteArray, 0);
+      valueReader.initFromPage(currentPage.getValueCount(), pageDataByteArray, endOfDefinitionLevels);
+      readPosInBytes = endOfDefinitionLevels;
+    }
+
     valuesRead = 0;
     return true;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 209a620..91605a1 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -33,6 +33,7 @@ import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.exec.exception.SetupException;
 import org.apache.drill.exec.physical.EndpointAffinity;
 import org.apache.drill.exec.physical.OperatorCost;
 import org.apache.drill.exec.physical.ReadEntryFromHDFS;
@@ -97,7 +98,7 @@ public class ParquetGroupScan extends AbstractGroupScan {
                           @JsonProperty("storageengine") ParquetStorageEngineConfig storageEngineConfig,
                           @JacksonInject StorageEngineRegistry engineRegistry,
                           @JsonProperty("ref") FieldReference ref
-                           )throws IOException, ExecutionSetupException {
+  )throws IOException, ExecutionSetupException {
     engineRegistry.init(DrillConfig.create());
     this.storageEngine = (ParquetStorageEngine) engineRegistry.getEngine(storageEngineConfig);
     this.availableEndpoints = storageEngine.getContext().getBits();
@@ -316,10 +317,10 @@ public class ParquetGroupScan extends AbstractGroupScan {
         boolean haveAffinity = bytesPerEndpoint.containsKey(currentEndpoint) ;
 
         if (assignAll ||
-                (!bytesPerEndpoint.isEmpty() &&
-                        (!requireAffinity || haveAffinity) &&
-                        (!endpointAssignments.containsKey(minorFragmentId) || endpointAssignments.get(minorFragmentId).size() < maxAssignments) &&
-                        bytesPerEndpoint.get(currentEndpoint) >= rowGroupInfo.getMaxBytes() * requiredPercentage)) {
+            (!bytesPerEndpoint.isEmpty() &&
+                (!requireAffinity || haveAffinity) &&
+                (!endpointAssignments.containsKey(minorFragmentId) || endpointAssignments.get(minorFragmentId).size() < maxAssignments) &&
+                bytesPerEndpoint.get(currentEndpoint) >= rowGroupInfo.getMaxBytes() * requiredPercentage)) {
 
           endpointAssignments.put(minorFragmentId, rowGroupInfo.getRowGroupReadEntry());
           logger.debug("Assigned rowGroup {} to minorFragmentId {} endpoint {}", rowGroupInfo.getRowGroupIndex(), minorFragmentId, endpoints.get(minorFragmentId).getAddress());
@@ -339,10 +340,15 @@ public class ParquetGroupScan extends AbstractGroupScan {
       logger.debug("minorFragmentId: {} Path: {} RowGroupIndex: {}",minorFragmentId, rg.getPath(),rg.getRowGroupIndex());
     }
     Preconditions.checkArgument(!mappings.get(minorFragmentId).isEmpty(), String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId));
-    return new ParquetRowGroupScan(storageEngine, engineConfig, mappings.get(minorFragmentId), ref);
+    try {
+      return new ParquetRowGroupScan(storageEngine, engineConfig, mappings.get(minorFragmentId));
+    } catch (SetupException e) {
+      // TODO - not sure if we want to change interface above to throw exception
+      throw new RuntimeException(e);
+    }
   }
 
-  
+
   public FieldReference getRef() {
     return ref;
   }
@@ -368,7 +374,7 @@ public class ParquetGroupScan extends AbstractGroupScan {
   public StorageEngineConfig getStorageEngineConfig(){
     return this.engineConfig;
   }
-  
+
   @Override
   @JsonIgnore
   public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
index b62ba27..3b2786b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
@@ -17,17 +17,10 @@
  ******************************************************************************/
 package org.apache.drill.exec.store.parquet;
 
-import static com.google.common.base.Preconditions.checkArgument;
 import io.netty.buffer.ByteBuf;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
@@ -36,21 +29,25 @@ import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.proto.SchemaDefProtos;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.store.RecordReader;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-
 import parquet.column.ColumnDescriptor;
 import parquet.hadoop.CodecFactoryExposer;
 import parquet.hadoop.metadata.ColumnChunkMetaData;
 import parquet.hadoop.metadata.ParquetMetadata;
-import parquet.schema.MessageType;
 import parquet.schema.PrimitiveType;
 
-import com.google.common.base.Joiner;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 public class ParquetRecordReader implements RecordReader {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReader.class);
@@ -60,7 +57,7 @@ public class ParquetRecordReader implements RecordReader {
   private static final long DEFAULT_BATCH_LENGTH = 256 * 1024 * NUMBER_OF_VECTORS; // 256kb
   private static final long DEFAULT_BATCH_LENGTH_IN_BITS = DEFAULT_BATCH_LENGTH * 8; // 256kb
 
-  // TODO - should probably find a smarter way to set this, currently 2 megabytes
+  // TODO - should probably find a smarter way to set this, currently 1 megabyte
   private static final int VAR_LEN_FIELD_LENGTH = 1024 * 1024 * 1;
   public static final int PARQUET_PAGE_MAX_SIZE = 1024 * 1024 * 1;
   private static final String SEPERATOR = System.getProperty("file.separator");
@@ -75,7 +72,6 @@ public class ParquetRecordReader implements RecordReader {
   private boolean allFieldsFixedLength;
   private int recordsPerBatch;
   private ByteBuf bufferWithAllData;
-  private final FieldReference ref;
   long totalRecords;
   long rowGroupOffset;
 
@@ -96,19 +92,18 @@ public class ParquetRecordReader implements RecordReader {
 
   public ParquetRecordReader(FragmentContext fragmentContext,
                              String path, int rowGroupIndex, FileSystem fs,
-                             CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref) throws ExecutionSetupException {
-    this(fragmentContext, DEFAULT_BATCH_LENGTH_IN_BITS, path, rowGroupIndex, fs, codecFactoryExposer, footer, ref);
+                             CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer) throws ExecutionSetupException {
+    this(fragmentContext, DEFAULT_BATCH_LENGTH_IN_BITS, path, rowGroupIndex, fs, codecFactoryExposer, footer);
   }
 
 
   public ParquetRecordReader(FragmentContext fragmentContext, long batchSize,
                              String path, int rowGroupIndex, FileSystem fs,
-                             CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref) throws ExecutionSetupException {
+                             CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer) throws ExecutionSetupException {
     this.allocator = fragmentContext.getAllocator();
 
     hadoopPath = new Path(path);
     fileSystem = fs;
-    this.ref = ref;
     this.codecFactoryExposer = codecFactoryExposer;
     this.rowGroupIndex = rowGroupIndex;
     this.batchSize = batchSize;
@@ -125,7 +120,6 @@ public class ParquetRecordReader implements RecordReader {
     // loop to add up the length of the fixed width columns and build the schema
     for (int i = 0; i < columns.size(); ++i) {
       column = columns.get(i);
-      logger.debug("Found Parquet column {} of type {}", column.getPath(), column.getType());
       // sum the lengths of all of the fixed length fields
       if (column.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
         // There is not support for the fixed binary type yet in parquet, leaving a task here as a reminder
@@ -148,23 +142,29 @@ public class ParquetRecordReader implements RecordReader {
     }
     try {
       ArrayList<VarLenBinaryReader.VarLengthColumn> varLengthColumns = new ArrayList<>();
+      ArrayList<VarLenBinaryReader.NullableVarLengthColumn> nullableVarLengthColumns = new ArrayList<>();
       // initialize all of the column read status objects
       boolean fieldFixedLength = false;
       MaterializedField field;
       for (int i = 0; i < columns.size(); ++i) {
         column = columns.get(i);
         columnChunkMetaData = footer.getBlocks().get(0).getColumns().get(i);
-        field = MaterializedField.create(toFieldName(column.getPath()),
-            toMajorType(column.getType(), getDataMode(column, footer.getFileMetaData().getSchema())));
+        field = MaterializedField.create(new SchemaPath(toFieldName(column.getPath()), ExpressionPosition.UNKNOWN),
+            toMajorType(column.getType(), getDataMode(column)));
         fieldFixedLength = column.getType() != PrimitiveType.PrimitiveTypeName.BINARY;
         ValueVector v = TypeHelper.getNewVector(field, allocator);
         if (column.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
-          createFixedColumnReader(fieldFixedLength, field, column, columnChunkMetaData, recordsPerBatch, v);
+          createFixedColumnReader(fieldFixedLength, column, columnChunkMetaData, recordsPerBatch, v);
         } else {
-          varLengthColumns.add(new VarLenBinaryReader.VarLengthColumn(this, -1, column, columnChunkMetaData, false, v));
+          if (column.getMaxDefinitionLevel() == 0){// column is required
+            varLengthColumns.add(new VarLenBinaryReader.VarLengthColumn(this, -1, column, columnChunkMetaData, false, v));
+          }
+          else{
+            nullableVarLengthColumns.add(new VarLenBinaryReader.NullableVarLengthColumn(this, -1, column, columnChunkMetaData, false, v));
+          }
         }
       }
-      varLengthReader = new VarLenBinaryReader(this, varLengthColumns);
+      varLengthReader = new VarLenBinaryReader(this, varLengthColumns, nullableVarLengthColumns);
     } catch (SchemaChangeException e) {
       throw new ExecutionSetupException(e);
     }
@@ -215,6 +215,9 @@ public class ParquetRecordReader implements RecordReader {
       for (VarLenBinaryReader.VarLengthColumn r : varLengthReader.columns) {
         output.addField(r.valueVecHolder.getValueVector());
       }
+      for (VarLenBinaryReader.NullableVarLengthColumn r : varLengthReader.nullableColumns) {
+        output.addField(r.valueVecHolder.getValueVector());
+      }
       output.setNewSchema();
     }catch(SchemaChangeException e) {
       throw new ExecutionSetupException("Error setting up output mutator.", e);
@@ -242,6 +245,9 @@ public class ParquetRecordReader implements RecordReader {
     for (VarLenBinaryReader.VarLengthColumn r : varLengthReader.columns){
       totalByteLength += r.columnChunkMetaData.getTotalSize();
     }
+    for (VarLenBinaryReader.NullableVarLengthColumn r : varLengthReader.nullableColumns){
+      totalByteLength += r.columnChunkMetaData.getTotalSize();
+    }
     int bufferSize = 64*1024;
     long totalBytesWritten = 0;
     int validBytesInCurrentBuffer;
@@ -262,17 +268,22 @@ public class ParquetRecordReader implements RecordReader {
     }
   }
 
-  private SchemaPath toFieldName(String[] paths) {
-    if(this.ref == null){
-      return new SchemaPath(Joiner.on('/').join(paths), ExpressionPosition.UNKNOWN);
-    }else{
-      return ref.getChild(paths);
+  private static String toFieldName(String[] paths) {
+    StringBuilder sb = new StringBuilder();
+    boolean first = true;
+    for(String np : paths){
+      if(first){
+        first = false;
+      }else{
+        sb.append(".");
+      }
+      sb.append(np);
     }
-    
+    return sb.toString();
   }
 
-  private TypeProtos.DataMode getDataMode(ColumnDescriptor column, MessageType schema) {
-    if (schema.getColumnDescription(column.getPath()).getMaxDefinitionLevel() == 0) {
+  private TypeProtos.DataMode getDataMode(ColumnDescriptor column) {
+    if (column.getMaxDefinitionLevel() == 0) {
       return TypeProtos.DataMode.REQUIRED;
     } else {
       return TypeProtos.DataMode.OPTIONAL;
@@ -288,30 +299,46 @@ public class ParquetRecordReader implements RecordReader {
       r.valueVecHolder.reset();
       r.valuesReadInCurrentPass = 0;
     }
+    for (VarLenBinaryReader.NullableVarLengthColumn r : varLengthReader.nullableColumns){
+      r.valueVecHolder.reset();
+      r.valuesReadInCurrentPass = 0;
+    }
   }
 
   /**
    * @param fixedLength
-   * @param field
    * @param descriptor
    * @param columnChunkMetaData
    * @param allocateSize - the size of the vector to create
    * @return
    * @throws SchemaChangeException
    */
-  private boolean createFixedColumnReader(boolean fixedLength, MaterializedField field, ColumnDescriptor descriptor,
+  private boolean createFixedColumnReader(boolean fixedLength, ColumnDescriptor descriptor,
                                           ColumnChunkMetaData columnChunkMetaData, int allocateSize, ValueVector v)
       throws SchemaChangeException {
-    TypeProtos.MajorType type = field.getType();
-    if (columnChunkMetaData.getType() == PrimitiveType.PrimitiveTypeName.BOOLEAN){
-      columnStatuses.add(new BitReader(this, allocateSize, descriptor, columnChunkMetaData,
-          fixedLength, v));
+    // if the column is required
+    if (descriptor.getMaxDefinitionLevel() == 0){
+      if (columnChunkMetaData.getType() == PrimitiveType.PrimitiveTypeName.BOOLEAN){
+        columnStatuses.add(new BitReader(this, allocateSize, descriptor, columnChunkMetaData,
+            fixedLength, v));
+      }
+      else{
+        columnStatuses.add(new FixedByteAlignedReader(this, allocateSize, descriptor, columnChunkMetaData,
+            fixedLength, v));
+      }
+      return true;
     }
-    else{
-      columnStatuses.add(new FixedByteAlignedReader(this, allocateSize, descriptor, columnChunkMetaData,
-          fixedLength, v));
+    else { // if the column is nullable
+      if (columnChunkMetaData.getType() == PrimitiveType.PrimitiveTypeName.BOOLEAN){
+        columnStatuses.add(new NullableBitReader(this, allocateSize, descriptor, columnChunkMetaData,
+            fixedLength, v));
+      }
+      else{
+        columnStatuses.add(new NullableFixedByteAlignedReader(this, allocateSize, descriptor, columnChunkMetaData,
+            fixedLength, v));
+      }
+      return true;
     }
-    return true;
   }
 
  public void readAllFixedFields(long recordsToRead, ColumnReader firstColumnStatus) throws IOException {
@@ -326,12 +353,24 @@ public class ParquetRecordReader implements RecordReader {
     resetBatch();
     long recordsToRead = 0;
     try {
-      ColumnReader firstColumnStatus = columnStatuses.iterator().next();
+      ColumnReader firstColumnStatus;
+      if (columnStatuses.size() > 0){
+        firstColumnStatus = columnStatuses.iterator().next();
+      }
+      else{
+        if (varLengthReader.columns.size() > 0){
+          firstColumnStatus = varLengthReader.columns.iterator().next();
+        }
+        else{
+         firstColumnStatus = varLengthReader.nullableColumns.iterator().next();
+        }
+      }
+
       if (allFieldsFixedLength) {
         recordsToRead = Math.min(recordsPerBatch, firstColumnStatus.columnChunkMetaData.getValueCount() - firstColumnStatus.totalValuesRead);
       } else {
         // arbitrary
-        recordsToRead = 8000;
+        recordsToRead = 4000;
 
         // going to incorporate looking at length of values and copying the data into a single loop, hopefully it won't
         // get too complicated
@@ -343,6 +382,7 @@ public class ParquetRecordReader implements RecordReader {
         // cannot find more information on this right now, will keep looking
       }
 
+      logger.debug("records to read in this pass: {}", recordsToRead);
       if (allFieldsFixedLength) {
         readAllFixedFields(recordsToRead, firstColumnStatus);
       } else { // variable length columns
@@ -363,31 +403,88 @@ public class ParquetRecordReader implements RecordReader {
 
   static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName primitiveTypeName, int length,
                                                TypeProtos.DataMode mode) {
-    switch (primitiveTypeName) {
-      case BINARY:
-        return Types.required(TypeProtos.MinorType.VARBINARY);
-      case INT64:
-        return Types.required(TypeProtos.MinorType.BIGINT);
-      case INT32:
-        return Types.required(TypeProtos.MinorType.INT);
-      case BOOLEAN:
-        return Types.required(TypeProtos.MinorType.BIT);
-      case FLOAT:
-        return Types.required(TypeProtos.MinorType.FLOAT4);
-      case DOUBLE:
-        return Types.required(TypeProtos.MinorType.FLOAT8);
-      // Both of these are not supported by the parquet library yet (7/3/13),
-      // but they are declared here for when they are implemented
-      case INT96:
-        return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
-            .setMode(mode).build();
-      case FIXED_LEN_BYTE_ARRAY:
-        checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
-        return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY)
-            .setWidth(length).setMode(mode).build();
-      default:
-        throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+    switch (mode) {
+
+      case OPTIONAL:
+        switch (primitiveTypeName) {
+          case BINARY:
+            return Types.optional(TypeProtos.MinorType.VARBINARY);
+          case INT64:
+            return Types.optional(TypeProtos.MinorType.BIGINT);
+          case INT32:
+            return Types.optional(TypeProtos.MinorType.INT);
+          case BOOLEAN:
+            return Types.optional(TypeProtos.MinorType.BIT);
+          case FLOAT:
+            return Types.optional(TypeProtos.MinorType.FLOAT4);
+          case DOUBLE:
+            return Types.optional(TypeProtos.MinorType.FLOAT8);
+          // TODO - Both of these are not supported by the parquet library yet (7/3/13),
+          // but they are declared here for when they are implemented
+          case INT96:
+            return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
+                .setMode(mode).build();
+          case FIXED_LEN_BYTE_ARRAY:
+            checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
+            return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY)
+                .setWidth(length).setMode(mode).build();
+          default:
+            throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+        }
+      case REQUIRED:
+        switch (primitiveTypeName) {
+          case BINARY:
+            return Types.required(TypeProtos.MinorType.VARBINARY);
+          case INT64:
+            return Types.required(TypeProtos.MinorType.BIGINT);
+          case INT32:
+            return Types.required(TypeProtos.MinorType.INT);
+          case BOOLEAN:
+            return Types.required(TypeProtos.MinorType.BIT);
+          case FLOAT:
+            return Types.required(TypeProtos.MinorType.FLOAT4);
+          case DOUBLE:
+            return Types.required(TypeProtos.MinorType.FLOAT8);
+          // Both of these are not supported by the parquet library yet (7/3/13),
+          // but they are declared here for when they are implemented
+          case INT96:
+            return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
+                .setMode(mode).build();
+          case FIXED_LEN_BYTE_ARRAY:
+            checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
+            return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY)
+                .setWidth(length).setMode(mode).build();
+          default:
+            throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+        }
+      case REPEATED:
+        switch (primitiveTypeName) {
+          case BINARY:
+            return Types.repeated(TypeProtos.MinorType.VARBINARY);
+          case INT64:
+            return Types.repeated(TypeProtos.MinorType.BIGINT);
+          case INT32:
+            return Types.repeated(TypeProtos.MinorType.INT);
+          case BOOLEAN:
+            return Types.repeated(TypeProtos.MinorType.BIT);
+          case FLOAT:
+            return Types.repeated(TypeProtos.MinorType.FLOAT4);
+          case DOUBLE:
+            return Types.repeated(TypeProtos.MinorType.FLOAT8);
+          // Both of these are not supported by the parquet library yet (7/3/13),
+          // but they are declared here for when they are implemented
+          case INT96:
+            return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
+                .setMode(mode).build();
+          case FIXED_LEN_BYTE_ARRAY:
+            checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
+            return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY)
+                .setWidth(length).setMode(mode).build();
+          default:
+            throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+        }
     }
+    throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName + " Mode: " + mode);
   }
 
   static String join(String delimiter, String... str) {
@@ -406,7 +503,8 @@ public class ParquetRecordReader implements RecordReader {
   @Override
   public void cleanup() {
     columnStatuses.clear();
-    this.varLengthReader.columns.clear();
     bufferWithAllData.release();
+    varLengthReader.columns.clear();
+    varLengthReader.nullableColumns.clear();
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
index 7cefcf4..6553736 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
@@ -17,56 +17,40 @@
  ******************************************************************************/
 package org.apache.drill.exec.store.parquet;
 
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
+import com.fasterxml.jackson.annotation.*;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.exec.exception.SetupException;
 import org.apache.drill.exec.physical.OperatorCost;
 import org.apache.drill.exec.physical.ReadEntryFromHDFS;
-import org.apache.drill.exec.physical.base.AbstractBase;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
-import org.apache.drill.exec.physical.base.SubScan;
+import org.apache.drill.exec.physical.base.*;
 import org.apache.drill.exec.store.StorageEngineRegistry;
 
-import com.fasterxml.jackson.annotation.JacksonInject;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
+import java.util.*;
 
 // Class containing information for reading a single parquet row group form HDFS
 @JsonTypeName("parquet-row-group-scan")
 public class ParquetRowGroupScan extends AbstractBase implements SubScan {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRowGroupScan.class);
 
-  public final StorageEngineConfig engineConfig;
-  private final ParquetStorageEngine parquetStorageEngine;
-  private final List<RowGroupReadEntry> rowGroupReadEntries;
-  private final FieldReference ref;
+  public StorageEngineConfig engineConfig;
+  private ParquetStorageEngine parquetStorageEngine;
+  private List<RowGroupReadEntry> rowGroupReadEntries;
 
   @JsonCreator
   public ParquetRowGroupScan(@JacksonInject StorageEngineRegistry registry, @JsonProperty("engineConfig") StorageEngineConfig engineConfig,
-                             @JsonProperty("rowGroupReadEntries") LinkedList<RowGroupReadEntry> rowGroupReadEntries, @JsonProperty("ref") FieldReference ref) throws ExecutionSetupException {
+                             @JsonProperty("rowGroupReadEntries") LinkedList<RowGroupReadEntry> rowGroupReadEntries) throws ExecutionSetupException {
     parquetStorageEngine = (ParquetStorageEngine) registry.getEngine(engineConfig);
     this.rowGroupReadEntries = rowGroupReadEntries;
-    this.engineConfig = engineConfig;
-    this.ref = ref;
   }
 
   public ParquetRowGroupScan(ParquetStorageEngine engine, ParquetStorageEngineConfig config,
-                              List<RowGroupReadEntry> rowGroupReadEntries, FieldReference ref) {
+                              List<RowGroupReadEntry> rowGroupReadEntries) throws SetupException {
     parquetStorageEngine = engine;
     engineConfig = config;
     this.rowGroupReadEntries = rowGroupReadEntries;
-    this.ref = ref;
   }
 
   public List<RowGroupReadEntry> getRowGroupReadEntries() {
@@ -82,11 +66,6 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
     return null;
   }
 
-  
-  public FieldReference getRef() {
-    return ref;
-  }
-
   @Override
   public Size getSize() {
     return null;
@@ -110,7 +89,12 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
   @Override
   public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
     Preconditions.checkArgument(children.isEmpty());
-    return new ParquetRowGroupScan(parquetStorageEngine, (ParquetStorageEngineConfig) engineConfig, rowGroupReadEntries, ref);
+    try {
+      return new ParquetRowGroupScan(parquetStorageEngine, (ParquetStorageEngineConfig) engineConfig, rowGroupReadEntries);
+    } catch (SetupException e) {
+      // TODO - handle this
+      throw new RuntimeException(e);
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index a09f1ac..b11eadc 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -18,26 +18,20 @@
 package org.apache.drill.exec.store.parquet;
 
 import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
 import java.util.List;
-import java.util.concurrent.TimeUnit;
 
-import com.google.common.base.Stopwatch;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.exec.ops.FragmentContext;
+
 import org.apache.drill.exec.physical.impl.BatchCreator;
 import org.apache.drill.exec.physical.impl.ScanBatch;
 import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.store.RecordReader;
-import org.apache.drill.exec.store.mock.MockScanBatchCreator;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-
 import parquet.hadoop.ParquetFileReader;
 import parquet.hadoop.metadata.ParquetMetadata;
 
@@ -46,8 +40,6 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
 
   @Override
   public RecordBatch getBatch(FragmentContext context, ParquetRowGroupScan rowGroupScan, List<RecordBatch> children) throws ExecutionSetupException {
-    Stopwatch watch = new Stopwatch();
-    watch.start();
     Preconditions.checkArgument(children.isEmpty());
     List<RecordReader> readers = Lists.newArrayList();
     for(ParquetRowGroupScan.RowGroupReadEntry e : rowGroupScan.getRowGroupReadEntries()){
@@ -63,15 +55,13 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
             new ParquetRecordReader(
                 context, e.getPath(), e.getRowGroupIndex(), rowGroupScan.getStorageEngine().getFileSystem(),
                 rowGroupScan.getStorageEngine().getCodecFactoryExposer(),
-                ParquetFileReader.readFooter( rowGroupScan.getStorageEngine().getFileSystem().getConf(), new Path(e.getPath())),
-                rowGroupScan.getRef()
+                ParquetFileReader.readFooter( rowGroupScan.getStorageEngine().getFileSystem().getConf(), new Path(e.getPath()))
             )
         );
       } catch (IOException e1) {
         throw new ExecutionSetupException(e1);
       }
     }
-    logger.debug("total time in ScanBatchCreator.getBatch: {} ms", watch.elapsed(TimeUnit.MILLISECONDS));
     return new ScanBatch(context, readers.iterator());
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
index 939545e..e6e2f03 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
@@ -121,4 +121,4 @@ public class ParquetStorageEngine extends AbstractStorageEngine{
   public ParquetSchemaProvider getSchemaProvider() {
     return schemaProvider;
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
index 08e1023..f20a2f3 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
@@ -17,6 +17,7 @@
  ******************************************************************************/
 package org.apache.drill.exec.store.parquet;
 
+import org.apache.drill.exec.vector.NullableVarBinaryVector;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.VarBinaryVector;
 import parquet.bytes.BytesUtils;
@@ -24,15 +25,19 @@ import parquet.column.ColumnDescriptor;
 import parquet.hadoop.metadata.ColumnChunkMetaData;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 
 public class VarLenBinaryReader {
 
   ParquetRecordReader parentReader;
   final List<VarLengthColumn> columns;
+  final List<NullableVarLengthColumn> nullableColumns;
 
-  public VarLenBinaryReader(ParquetRecordReader parentReader, List<VarLengthColumn> columns){
+  public VarLenBinaryReader(ParquetRecordReader parentReader, List<VarLengthColumn> columns,
+                            List<NullableVarLengthColumn> nullableColumns){
     this.parentReader = parentReader;
+    this.nullableColumns = nullableColumns;
     this.columns = columns;
   }
 
@@ -48,6 +53,21 @@ public class VarLenBinaryReader {
     }
   }
 
+  public static class NullableVarLengthColumn extends ColumnReader {
+
+    int nullsRead;
+    boolean currentValNull = false;
+
+    NullableVarLengthColumn(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v) {
+      super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
+    }
+
+    @Override
+    protected void readField(long recordsToRead, ColumnReader firstColumnStatus) {
+      throw new UnsupportedOperationException();
+    }
+  }
+
   /**
    * Reads as many variable length values as possible.
    *
@@ -63,22 +83,42 @@ public class VarLenBinaryReader {
     boolean rowGroupFinished = false;
     byte[] bytes;
     VarBinaryVector currVec;
+    NullableVarBinaryVector currNullVec;
     // write the first 0 offset
     for (ColumnReader columnReader : columns) {
-      if (columnReader.isFixedLength) {
-        continue;
-      }
       currVec = (VarBinaryVector) columnReader.valueVecHolder.getValueVector();
       currVec.getAccessor().getOffsetVector().getData().writeInt(0);
       columnReader.bytesReadInCurrentPass = 0;
       columnReader.valuesReadInCurrentPass = 0;
     }
+    // same for the nullable columns
+    for (NullableVarLengthColumn columnReader : nullableColumns) {
+      currNullVec = (NullableVarBinaryVector) columnReader.valueVecHolder.getValueVector();
+      currNullVec.getMutator().getVectorWithValues().getAccessor().getOffsetVector().getData().writeInt(0);
+      columnReader.bytesReadInCurrentPass = 0;
+      columnReader.valuesReadInCurrentPass = 0;
+      columnReader.nullsRead = 0;
+    }
     do {
       lengthVarFieldsInCurrentRecord = 0;
       for (ColumnReader columnReader : columns) {
-        if (columnReader.isFixedLength) {
-          continue;
+        if (columnReader.pageReadStatus.currentPage == null
+            || columnReader.pageReadStatus.valuesRead == columnReader.pageReadStatus.currentPage.getValueCount()) {
+          columnReader.totalValuesRead += columnReader.pageReadStatus.valuesRead;
+          if (!columnReader.pageReadStatus.next()) {
+            rowGroupFinished = true;
+            break;
+          }
         }
+        bytes = columnReader.pageReadStatus.pageDataByteArray;
+
+        // re-purposing this field here for length in BYTES to prevent repetitive multiplication/division
+        columnReader.dataTypeLengthInBits = BytesUtils.readIntLittleEndian(bytes,
+            (int) columnReader.pageReadStatus.readPosInBytes);
+        lengthVarFieldsInCurrentRecord += columnReader.dataTypeLengthInBits;
+
+      }
+      for (NullableVarLengthColumn columnReader : nullableColumns) {
         if (columnReader.pageReadStatus.currentPage == null
             || columnReader.pageReadStatus.valuesRead == columnReader.pageReadStatus.currentPage.getValueCount()) {
           columnReader.totalValuesRead += columnReader.pageReadStatus.valuesRead;
@@ -88,6 +128,12 @@ public class VarLenBinaryReader {
           }
         }
         bytes = columnReader.pageReadStatus.pageDataByteArray;
+        if ( columnReader.columnDescriptor.getMaxDefinitionLevel() > columnReader.pageReadStatus.definitionLevels.readInteger()){
+          columnReader.currentValNull = true;
+          columnReader.dataTypeLengthInBits = 0;
+          columnReader.nullsRead++;
+          continue;// field is null, no length to add to data vector
+        }
 
         // re-purposing  this field here for length in BYTES to prevent repetitive multiplication/division
         columnReader.dataTypeLengthInBits = BytesUtils.readIntLittleEndian(bytes,
@@ -96,7 +142,7 @@ public class VarLenBinaryReader {
 
       }
       // check that the next record will fit in the batch
-      if (rowGroupFinished || (recordsReadInCurrentPass + 1) * parentReader.getBitWidthAllFixedFields() + lengthVarFieldsInCurrentRecord * 8
+      if (rowGroupFinished || (recordsReadInCurrentPass + 1) * parentReader.getBitWidthAllFixedFields() + lengthVarFieldsInCurrentRecord
           > parentReader.getBatchSize()){
         break;
       }
@@ -104,9 +150,6 @@ public class VarLenBinaryReader {
         recordsReadInCurrentPass++;
       }
       for (ColumnReader columnReader : columns) {
-        if (columnReader.isFixedLength) {
-          continue;
-        }
         bytes = columnReader.pageReadStatus.pageDataByteArray;
         currVec = (VarBinaryVector) columnReader.valueVecHolder.getValueVector();
         // again, I am re-purposing the unused field here, it is a length n BYTES, not bits
@@ -124,7 +167,34 @@ public class VarLenBinaryReader {
           columnReader.pageReadStatus.next();
         }
       }
+      for (NullableVarLengthColumn columnReader : nullableColumns) {
+        bytes = columnReader.pageReadStatus.pageDataByteArray;
+        currNullVec = (NullableVarBinaryVector) columnReader.valueVecHolder.getValueVector();
+        // again, I am re-purposing the unused field here, it is a length n BYTES, not bits
+        currNullVec.getMutator().getVectorWithValues().getAccessor().getOffsetVector().getData()
+            .writeInt(
+                (int) columnReader.bytesReadInCurrentPass  +
+                columnReader.dataTypeLengthInBits - 4 * (columnReader.valuesReadInCurrentPass -
+                    (columnReader.currentValNull ? Math.max (0, columnReader.nullsRead - 1) : columnReader.nullsRead)));
+        columnReader.currentValNull = false;
+        if (columnReader.dataTypeLengthInBits > 0){
+          currNullVec.getData().writeBytes(bytes, (int) columnReader.pageReadStatus.readPosInBytes + 4,
+              columnReader.dataTypeLengthInBits);
+          ((NullableVarBinaryVector)columnReader.valueVecHolder.getValueVector()).getMutator().setIndexDefined(columnReader.valuesReadInCurrentPass);
+        }
+        if (columnReader.dataTypeLengthInBits > 0){
+          columnReader.pageReadStatus.readPosInBytes += columnReader.dataTypeLengthInBits + 4;
+          columnReader.bytesReadInCurrentPass += columnReader.dataTypeLengthInBits + 4;
+        }
+        columnReader.pageReadStatus.valuesRead++;
+        columnReader.valuesReadInCurrentPass++;
+        currNullVec.getMutator().setValueCount((int)recordsReadInCurrentPass);
+        // reached the end of a page
+        if ( columnReader.pageReadStatus.valuesRead == columnReader.pageReadStatus.currentPage.getValueCount()) {
+          columnReader.pageReadStatus.next();
+        }
+      }
     } while (recordsReadInCurrentPass < recordsToReadInThisPass);
     return recordsReadInCurrentPass;
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index 358334e..c6515d6 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -1,10 +1,11 @@
 package org.apache.drill.exec.vector;
 
 import org.apache.drill.common.expression.FieldReference;
+import io.netty.buffer.ByteBuf;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.record.MaterializedField;
 
-abstract class BaseValueVector implements ValueVector{
+public abstract class BaseValueVector implements ValueVector{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseValueVector.class);
   
   protected final BufferAllocator allocator;
@@ -29,6 +30,9 @@ abstract class BaseValueVector implements ValueVector{
     return getField().clone(ref);
   }
   
+
+  abstract public ByteBuf getData();
+
   abstract class BaseAccessor implements ValueVector.Accessor{
     public abstract int getValueCount();
     public void reset(){}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
new file mode 100644
index 0000000..cdccf4e
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
@@ -0,0 +1,23 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.vector;
+
+public interface NullableVectorDefinitionSetter {
+
+  public void setIndexDefined(int index);
+}