You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/09/04 05:14:32 UTC
[12/13] git commit: DRILL-91/DRILL-160: Update to Parquet Record
Reader to support nullable fields (int, long, float, double, bit, varbinary).
DRILL-91/DRILL-160: Update to Parquet Record Reader to support nullable fields (int, long, float, double, bit, varbinary).
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/63a90fe9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/63a90fe9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/63a90fe9
Branch: refs/heads/master
Commit: 63a90fe9ae7f4b911ee624759dfb32ed1b069de7
Parents: 5642dee
Author: Jason Altekruse <al...@gmial.com>
Authored: Mon Sep 2 09:44:46 2013 -0500
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Sep 3 20:11:13 2013 -0700
----------------------------------------------------------------------
sandbox/prototype/exec/java-exec/pom.xml | 5 +
.../codegen/templates/NullableValueVectors.java | 20 +-
.../codegen/templates/RepeatedValueVectors.java | 4 +
.../drill/exec/store/parquet/ColumnReader.java | 8 +-
.../exec/store/parquet/NullableBitReader.java | 61 ++
.../store/parquet/NullableColumnReader.java | 130 ++++
.../parquet/NullableFixedByteAlignedReader.java | 48 ++
.../exec/store/parquet/PageReadStatus.java | 31 +-
.../exec/store/parquet/ParquetGroupScan.java | 22 +-
.../exec/store/parquet/ParquetRecordReader.java | 232 +++++--
.../exec/store/parquet/ParquetRowGroupScan.java | 48 +-
.../store/parquet/ParquetScanBatchCreator.java | 14 +-
.../store/parquet/ParquetStorageEngine.java | 2 +-
.../exec/store/parquet/VarLenBinaryReader.java | 92 ++-
.../drill/exec/vector/BaseValueVector.java | 6 +-
.../vector/NullableVectorDefinitionSetter.java | 23 +
.../exec/store/ParquetRecordReaderTest.java | 657 +++++++++++++++++++
.../src/test/resources/parquet_nullable.json | 41 ++
.../src/test/resources/parquet_scan_screen.json | 41 ++
.../parquet_scan_screen_read_entry_replace.json | 39 ++
.../parquet_scan_union_screen_physical.json | 35 +
21 files changed, 1408 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml
index 2b35534..039c866 100644
--- a/sandbox/prototype/exec/java-exec/pom.xml
+++ b/sandbox/prototype/exec/java-exec/pom.xml
@@ -184,6 +184,11 @@
<artifactId>jetty-util</artifactId>
<version>6.1.26</version>
</dependency>
+ <dependency>
+ <groupId>com.thoughtworks.paranamer</groupId>
+ <artifactId>paranamer</artifactId>
+ <version>2.3</version>
+ </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
index f528ce1..25249ff 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -1,3 +1,7 @@
+import org.apache.drill.exec.vector.NullableVectorDefinitionSetter;
+
+import java.lang.UnsupportedOperationException;
+
<@pp.dropOutputFile />
<#list vv.types as type>
<#list type.minor as minor>
@@ -56,6 +60,10 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
return values.getBufferSize() + bits.getBufferSize();
}
+ public ByteBuf getData(){
+ return values.getData();
+ }
+
<#if type.major == "VarLen">
@Override
public FieldMetadata getMetadata() {
@@ -242,13 +250,21 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public void reset(){}
}
- public final class Mutator implements ValueVector.Mutator{
+ public final class Mutator implements ValueVector.Mutator, NullableVectorDefinitionSetter{
private int setCount;
private Mutator(){
}
+ public ${valuesName} getVectorWithValues(){
+ return values;
+ }
+
+ public void setIndexDefined(int index){
+ bits.getMutator().set(index, 1);
+ }
+
/**
* Set the variable length element at the specified index to the supplied byte array.
*
@@ -315,4 +331,4 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
}
}
</#list>
-</#list>
\ No newline at end of file
+</#list>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index d86e449..faf9772 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -45,6 +45,10 @@ package org.apache.drill.exec.vector;
public int getBufferSize(){
return offsets.getBufferSize() + values.getBufferSize();
}
+
+ public ByteBuf getData(){
+ return values.getData();
+ }
public TransferPair getTransferPair(){
return new TransferImpl(getField());
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
index c62613a..e026fa7 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ColumnReader.java
@@ -19,7 +19,7 @@ package org.apache.drill.exec.store.parquet;
import io.netty.buffer.ByteBuf;
import org.apache.drill.exec.store.VectorHolder;
-import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.ValueVector;
import parquet.column.ColumnDescriptor;
import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -61,7 +61,7 @@ public abstract class ColumnReader {
boolean fixedLength, ValueVector v){
this.parentReader = parentReader;
if (allocateSize > 1) valueVecHolder = new VectorHolder(allocateSize, v);
- else valueVecHolder = new VectorHolder(5000, (BaseDataValueVector) v);
+ else valueVecHolder = new VectorHolder(5000, v);
columnDescriptor = descriptor;
this.columnChunkMetaData = columnChunkMetaData;
@@ -82,7 +82,7 @@ public abstract class ColumnReader {
readLength = 0;
readLengthInBits = 0;
recordsReadInThisIteration = 0;
- vectorData = ((BaseDataValueVector) valueVecHolder.getValueVector()).getData();
+ vectorData = ((BaseValueVector)valueVecHolder.getValueVector()).getData();
do {
// if no page has been read, or all of the records have been read out of a page, read the next one
if (pageReadStatus.currentPage == null
@@ -109,4 +109,4 @@ public abstract class ColumnReader {
}
protected abstract void readField(long recordsToRead, ColumnReader firstColumnStatus);
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
new file mode 100644
index 0000000..8a9565c
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableBitReader.java
@@ -0,0 +1,61 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.NullableBitVector;
+import org.apache.drill.exec.vector.ValueVector;
+import parquet.column.ColumnDescriptor;
+import parquet.hadoop.metadata.ColumnChunkMetaData;
+
+import java.io.IOException;
+
+/**
+ * This class is used in conjunction with its superclass to read nullable bit columns in a parquet file.
+ * It currently is using an inefficient value-by-value approach.
+ * TODO - make this more efficient by copying runs of values like in NullableFixedByteAlignedReader
+ * This will also involve incorporating the ideas from the BitReader (the reader for non-nullable bits)
+ * because page/batch boundaries that do not land on byte boundaries require shifting of all of the values
+ * in the next batch.
+ */
+public final class NullableBitReader extends ColumnReader {
+
+ NullableBitReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
+ boolean fixedLength, ValueVector v) {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
+ }
+
+ @Override
+ public void readField(long recordsToReadInThisPass, ColumnReader firstColumnStatus) {
+
+ recordsReadInThisIteration = Math.min(pageReadStatus.currentPage.getValueCount()
+ - pageReadStatus.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
+ int defLevel;
+ for (int i = 0; i < recordsReadInThisIteration; i++){
+ defLevel = pageReadStatus.definitionLevels.readInteger();
+ // if the value is defined
+ if (defLevel == columnDescriptor.getMaxDefinitionLevel()){
+ ((NullableBitVector)valueVecHolder.getValueVector()).getMutator().set(i + valuesReadInCurrentPass,
+ pageReadStatus.valueReader.readBoolean() ? 1 : 0 );
+ }
+ // otherwise the value is skipped, because the bit vector indicating nullability is zero filled
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
new file mode 100644
index 0000000..e5e49b6
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableColumnReader.java
@@ -0,0 +1,130 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.NullableVectorDefinitionSetter;
+import org.apache.drill.exec.vector.ValueVector;
+import parquet.column.ColumnDescriptor;
+import parquet.hadoop.metadata.ColumnChunkMetaData;
+
+import java.io.IOException;
+
+public abstract class NullableColumnReader extends ColumnReader{
+
+ int nullsFound;
+ // used to skip nulls found
+ int rightBitShift;
+ // used when copying less than a byte worth of data at a time, to indicate the number of used bits in the current byte
+ int bitsUsed;
+
+ NullableColumnReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
+ boolean fixedLength, ValueVector v){
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
+ }
+
+ public void readAllFixedFields(long recordsToReadInThisPass, ColumnReader firstColumnStatus) throws IOException {
+ readStartInBytes = 0;
+ readLength = 0;
+ readLengthInBits = 0;
+ recordsReadInThisIteration = 0;
+ vectorData = ((BaseValueVector)valueVecHolder.getValueVector()).getData();
+
+ do {
+ // if no page has been read, or all of the records have been read out of a page, read the next one
+ if (pageReadStatus.currentPage == null
+ || pageReadStatus.valuesRead == pageReadStatus.currentPage.getValueCount()) {
+ if (!pageReadStatus.next()) {
+ break;
+ }
+ }
+
+ // values need to be spaced out where nulls appear in the column
+ // leaving blank space for nulls allows for random access to values
+ // to optimize copying data out of the buffered disk stream, runs of defined values
+ // are located and copied together, rather than copying individual values
+
+ long runStart = pageReadStatus.readPosInBytes;
+ int runLength = 0;
+ int currentDefinitionLevel = 0;
+ int currentValueIndexInVector = totalValuesRead;
+ boolean lastValueWasNull = true;
+ int definitionLevelsRead;
+ // loop to find the longest run of defined values available, can be preceded by several nulls
+ while (true){
+ definitionLevelsRead = 0;
+ lastValueWasNull = true;
+ nullsFound = 0;
+ if (currentValueIndexInVector - totalValuesRead == recordsToReadInThisPass
+ || currentValueIndexInVector >= valueVecHolder.getValueVector().getValueCapacity()){
+ break;
+ }
+ while(currentValueIndexInVector - totalValuesRead < recordsToReadInThisPass
+ && currentValueIndexInVector < valueVecHolder.getValueVector().getValueCapacity()
+ && definitionLevelsRead < pageReadStatus.currentPage.getValueCount()){
+ currentDefinitionLevel = pageReadStatus.definitionLevels.readInteger();
+ definitionLevelsRead++;
+ if ( currentDefinitionLevel < columnDescriptor.getMaxDefinitionLevel()){
+ // a run of non-null values was found, break out of this loop to do a read in the outer loop
+ nullsFound++;
+ if ( ! lastValueWasNull ){
+ currentValueIndexInVector++;
+ break;
+ }
+ lastValueWasNull = true;
+ }
+ else{
+ if (lastValueWasNull){
+ runStart = pageReadStatus.readPosInBytes;
+ runLength = 0;
+ lastValueWasNull = false;
+ }
+ runLength++;
+ ((NullableVectorDefinitionSetter)valueVecHolder.getValueVector().getMutator()).setIndexDefined(currentValueIndexInVector);
+ }
+ currentValueIndexInVector++;
+ }
+ pageReadStatus.readPosInBytes = runStart;
+ recordsReadInThisIteration = runLength;
+
+ readField( runLength, firstColumnStatus);
+ int writerIndex = ((BaseValueVector) valueVecHolder.getValueVector()).getData().writerIndex();
+ if ( dataTypeLengthInBits > 8 || (dataTypeLengthInBits < 8 && totalValuesRead + runLength % 8 == 0)){
+ ((BaseValueVector) valueVecHolder.getValueVector()).getData().setIndex(0, writerIndex + (int) Math.ceil( nullsFound * dataTypeLengthInBits / 8.0));
+ }
+ else if (dataTypeLengthInBits < 8){
+ rightBitShift += dataTypeLengthInBits * nullsFound;
+ }
+ recordsReadInThisIteration += nullsFound;
+ valuesReadInCurrentPass += recordsReadInThisIteration;
+ totalValuesRead += recordsReadInThisIteration;
+ pageReadStatus.valuesRead += recordsReadInThisIteration;
+ if (readStartInBytes + readLength >= pageReadStatus.byteLength && bitsUsed == 0) {
+ pageReadStatus.next();
+ } else {
+ pageReadStatus.readPosInBytes = readStartInBytes + readLength;
+ }
+ }
+ }
+ while (valuesReadInCurrentPass < recordsToReadInThisPass && pageReadStatus.currentPage != null);
+ valueVecHolder.getValueVector().getMutator().setValueCount(
+ valuesReadInCurrentPass);
+ }
+
+ protected abstract void readField(long recordsToRead, ColumnReader firstColumnStatus);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
new file mode 100644
index 0000000..3d4a29d
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/NullableFixedByteAlignedReader.java
@@ -0,0 +1,48 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.NullableVectorDefinitionSetter;
+import org.apache.drill.exec.vector.ValueVector;
+import parquet.column.ColumnDescriptor;
+import parquet.hadoop.metadata.ColumnChunkMetaData;
+
+public class NullableFixedByteAlignedReader extends NullableColumnReader {
+
+ NullableFixedByteAlignedReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
+ boolean fixedLength, ValueVector v) {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
+ }
+
+ // this method is called by its superclass during a read loop
+ @Override
+ protected void readField(long recordsToReadInThisPass, ColumnReader firstColumnStatus) {
+
+ recordsReadInThisIteration = recordsToReadInThisPass;
+
+ readStartInBytes = pageReadStatus.readPosInBytes;
+ readLengthInBits = recordsReadInThisIteration * dataTypeLengthInBits;
+ readLength = (int) Math.ceil(readLengthInBits / 8.0);
+
+ bytes = pageReadStatus.pageDataByteArray;
+ // vectorData is assigned by the superclass read loop method
+ vectorData.writeBytes(bytes,
+ (int) readStartInBytes, (int) readLength);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
index c5c0d87..0d70138 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/PageReadStatus.java
@@ -20,8 +20,10 @@ package org.apache.drill.exec.store.parquet;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufInputStream;
import parquet.bytes.BytesInput;
+import parquet.column.ValuesType;
import parquet.column.page.Page;
import parquet.column.page.PageReader;
+import parquet.column.values.ValuesReader;
import parquet.format.PageHeader;
import java.io.IOException;
@@ -36,11 +38,8 @@ public final class PageReadStatus {
// store references to the pages that have been uncompressed, but not copied to ValueVectors yet
Page currentPage;
// buffer to store bytes of current page, set to max size of parquet page
- // TODO: add this back once toByteArray accepts an input. byte[] pageDataByteArray = new byte[ParquetRecordReader.PARQUET_PAGE_MAX_SIZE];
- byte[] pageDataByteArray;
-
-
- PageReader pageReader;
+ byte[] pageDataByteArray = new byte[ParquetRecordReader.PARQUET_PAGE_MAX_SIZE];
+
// read position in the current page, stored in the ByteBuf in ParquetRecordReader called bufferWithAllData
long readPosInBytes;
// bit shift needed for the next page if the last one did not line up with a byte boundary
@@ -52,8 +51,8 @@ public final class PageReadStatus {
int valuesRead;
int byteLength;
int rowGroupIndex;
- // buffer with the compressed data of an entire row group
- ByteBuf bufferWithAllData;
+ ValuesReader definitionLevels;
+ ValuesReader valueReader;
PageReadStatus(ColumnReader parentStatus, int rowGroupIndex, ByteBuf bufferWithAllData){
this.parentColumnReader = parentStatus;
@@ -101,20 +100,26 @@ public final class PageReadStatus {
parentColumnReader.readPositionInBuffer += pageHeader.compressed_page_size + length;
byteLength = pageHeader.uncompressed_page_size;
+
if (currentPage == null) {
return false;
}
// if the buffer holding each page's data is not large enough to hold the current page, re-allocate, with a little extra space
-// if (pageHeader.getUncompressed_page_size() > pageDataByteArray.length) {
-// pageDataByteArray = new byte[pageHeader.getUncompressed_page_size() + 100];
-// }
+ if (pageHeader.getUncompressed_page_size() > pageDataByteArray.length) {
+ pageDataByteArray = new byte[pageHeader.getUncompressed_page_size() + 100];
+ }
// TODO - would like to get this into the mainline, hopefully before alpha
pageDataByteArray = currentPage.getBytes().toByteArray();
- //TODO: Fix once parquet supports buffer work or at least passing in array.
- //pageDataByteArray = currentPage.getBytes().toByteArray(pageDataByteArray, 0, byteLength);
- readPosInBytes = 0;
+ if (parentColumnReader.columnDescriptor.getMaxDefinitionLevel() != 0){
+ definitionLevels = currentPage.getDlEncoding().getValuesReader(parentColumnReader.columnDescriptor, ValuesType.DEFINITION_LEVEL);
+ valueReader = currentPage.getValueEncoding().getValuesReader(parentColumnReader.columnDescriptor, ValuesType.VALUES);
+ int endOfDefinitionLevels = definitionLevels.initFromPage(currentPage.getValueCount(), pageDataByteArray, 0);
+ valueReader.initFromPage(currentPage.getValueCount(), pageDataByteArray, endOfDefinitionLevels);
+ readPosInBytes = endOfDefinitionLevels;
+ }
+
valuesRead = 0;
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 209a620..91605a1 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -33,6 +33,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.exec.exception.SetupException;
import org.apache.drill.exec.physical.EndpointAffinity;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.ReadEntryFromHDFS;
@@ -97,7 +98,7 @@ public class ParquetGroupScan extends AbstractGroupScan {
@JsonProperty("storageengine") ParquetStorageEngineConfig storageEngineConfig,
@JacksonInject StorageEngineRegistry engineRegistry,
@JsonProperty("ref") FieldReference ref
- )throws IOException, ExecutionSetupException {
+ )throws IOException, ExecutionSetupException {
engineRegistry.init(DrillConfig.create());
this.storageEngine = (ParquetStorageEngine) engineRegistry.getEngine(storageEngineConfig);
this.availableEndpoints = storageEngine.getContext().getBits();
@@ -316,10 +317,10 @@ public class ParquetGroupScan extends AbstractGroupScan {
boolean haveAffinity = bytesPerEndpoint.containsKey(currentEndpoint) ;
if (assignAll ||
- (!bytesPerEndpoint.isEmpty() &&
- (!requireAffinity || haveAffinity) &&
- (!endpointAssignments.containsKey(minorFragmentId) || endpointAssignments.get(minorFragmentId).size() < maxAssignments) &&
- bytesPerEndpoint.get(currentEndpoint) >= rowGroupInfo.getMaxBytes() * requiredPercentage)) {
+ (!bytesPerEndpoint.isEmpty() &&
+ (!requireAffinity || haveAffinity) &&
+ (!endpointAssignments.containsKey(minorFragmentId) || endpointAssignments.get(minorFragmentId).size() < maxAssignments) &&
+ bytesPerEndpoint.get(currentEndpoint) >= rowGroupInfo.getMaxBytes() * requiredPercentage)) {
endpointAssignments.put(minorFragmentId, rowGroupInfo.getRowGroupReadEntry());
logger.debug("Assigned rowGroup {} to minorFragmentId {} endpoint {}", rowGroupInfo.getRowGroupIndex(), minorFragmentId, endpoints.get(minorFragmentId).getAddress());
@@ -339,10 +340,15 @@ public class ParquetGroupScan extends AbstractGroupScan {
logger.debug("minorFragmentId: {} Path: {} RowGroupIndex: {}",minorFragmentId, rg.getPath(),rg.getRowGroupIndex());
}
Preconditions.checkArgument(!mappings.get(minorFragmentId).isEmpty(), String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId));
- return new ParquetRowGroupScan(storageEngine, engineConfig, mappings.get(minorFragmentId), ref);
+ try {
+ return new ParquetRowGroupScan(storageEngine, engineConfig, mappings.get(minorFragmentId));
+ } catch (SetupException e) {
+ // TODO - not sure if we want to change interface above to throw exception
+ throw new RuntimeException(e);
+ }
}
-
+
public FieldReference getRef() {
return ref;
}
@@ -368,7 +374,7 @@ public class ParquetGroupScan extends AbstractGroupScan {
public StorageEngineConfig getStorageEngineConfig(){
return this.engineConfig;
}
-
+
@Override
@JsonIgnore
public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
index b62ba27..3b2786b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
@@ -17,17 +17,10 @@
******************************************************************************/
package org.apache.drill.exec.store.parquet;
-import static com.google.common.base.Preconditions.checkArgument;
import io.netty.buffer.ByteBuf;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
@@ -36,21 +29,25 @@ import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-
import parquet.column.ColumnDescriptor;
import parquet.hadoop.CodecFactoryExposer;
import parquet.hadoop.metadata.ColumnChunkMetaData;
import parquet.hadoop.metadata.ParquetMetadata;
-import parquet.schema.MessageType;
import parquet.schema.PrimitiveType;
-import com.google.common.base.Joiner;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
public class ParquetRecordReader implements RecordReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReader.class);
@@ -60,7 +57,7 @@ public class ParquetRecordReader implements RecordReader {
private static final long DEFAULT_BATCH_LENGTH = 256 * 1024 * NUMBER_OF_VECTORS; // 256kb
private static final long DEFAULT_BATCH_LENGTH_IN_BITS = DEFAULT_BATCH_LENGTH * 8; // 256kb
- // TODO - should probably find a smarter way to set this, currently 2 megabytes
+ // TODO - should probably find a smarter way to set this, currently 1 megabyte
private static final int VAR_LEN_FIELD_LENGTH = 1024 * 1024 * 1;
public static final int PARQUET_PAGE_MAX_SIZE = 1024 * 1024 * 1;
private static final String SEPERATOR = System.getProperty("file.separator");
@@ -75,7 +72,6 @@ public class ParquetRecordReader implements RecordReader {
private boolean allFieldsFixedLength;
private int recordsPerBatch;
private ByteBuf bufferWithAllData;
- private final FieldReference ref;
long totalRecords;
long rowGroupOffset;
@@ -96,19 +92,18 @@ public class ParquetRecordReader implements RecordReader {
public ParquetRecordReader(FragmentContext fragmentContext,
String path, int rowGroupIndex, FileSystem fs,
- CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref) throws ExecutionSetupException {
- this(fragmentContext, DEFAULT_BATCH_LENGTH_IN_BITS, path, rowGroupIndex, fs, codecFactoryExposer, footer, ref);
+ CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer) throws ExecutionSetupException {
+ this(fragmentContext, DEFAULT_BATCH_LENGTH_IN_BITS, path, rowGroupIndex, fs, codecFactoryExposer, footer);
}
public ParquetRecordReader(FragmentContext fragmentContext, long batchSize,
String path, int rowGroupIndex, FileSystem fs,
- CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref) throws ExecutionSetupException {
+ CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer) throws ExecutionSetupException {
this.allocator = fragmentContext.getAllocator();
hadoopPath = new Path(path);
fileSystem = fs;
- this.ref = ref;
this.codecFactoryExposer = codecFactoryExposer;
this.rowGroupIndex = rowGroupIndex;
this.batchSize = batchSize;
@@ -125,7 +120,6 @@ public class ParquetRecordReader implements RecordReader {
// loop to add up the length of the fixed width columns and build the schema
for (int i = 0; i < columns.size(); ++i) {
column = columns.get(i);
- logger.debug("Found Parquet column {} of type {}", column.getPath(), column.getType());
// sum the lengths of all of the fixed length fields
if (column.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
// There is not support for the fixed binary type yet in parquet, leaving a task here as a reminder
@@ -148,23 +142,29 @@ public class ParquetRecordReader implements RecordReader {
}
try {
ArrayList<VarLenBinaryReader.VarLengthColumn> varLengthColumns = new ArrayList<>();
+ ArrayList<VarLenBinaryReader.NullableVarLengthColumn> nullableVarLengthColumns = new ArrayList<>();
// initialize all of the column read status objects
boolean fieldFixedLength = false;
MaterializedField field;
for (int i = 0; i < columns.size(); ++i) {
column = columns.get(i);
columnChunkMetaData = footer.getBlocks().get(0).getColumns().get(i);
- field = MaterializedField.create(toFieldName(column.getPath()),
- toMajorType(column.getType(), getDataMode(column, footer.getFileMetaData().getSchema())));
+ field = MaterializedField.create(new SchemaPath(toFieldName(column.getPath()), ExpressionPosition.UNKNOWN),
+ toMajorType(column.getType(), getDataMode(column)));
fieldFixedLength = column.getType() != PrimitiveType.PrimitiveTypeName.BINARY;
ValueVector v = TypeHelper.getNewVector(field, allocator);
if (column.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
- createFixedColumnReader(fieldFixedLength, field, column, columnChunkMetaData, recordsPerBatch, v);
+ createFixedColumnReader(fieldFixedLength, column, columnChunkMetaData, recordsPerBatch, v);
} else {
- varLengthColumns.add(new VarLenBinaryReader.VarLengthColumn(this, -1, column, columnChunkMetaData, false, v));
+ if (column.getMaxDefinitionLevel() == 0){// column is required
+ varLengthColumns.add(new VarLenBinaryReader.VarLengthColumn(this, -1, column, columnChunkMetaData, false, v));
+ }
+ else{
+ nullableVarLengthColumns.add(new VarLenBinaryReader.NullableVarLengthColumn(this, -1, column, columnChunkMetaData, false, v));
+ }
}
}
- varLengthReader = new VarLenBinaryReader(this, varLengthColumns);
+ varLengthReader = new VarLenBinaryReader(this, varLengthColumns, nullableVarLengthColumns);
} catch (SchemaChangeException e) {
throw new ExecutionSetupException(e);
}
@@ -215,6 +215,9 @@ public class ParquetRecordReader implements RecordReader {
for (VarLenBinaryReader.VarLengthColumn r : varLengthReader.columns) {
output.addField(r.valueVecHolder.getValueVector());
}
+ for (VarLenBinaryReader.NullableVarLengthColumn r : varLengthReader.nullableColumns) {
+ output.addField(r.valueVecHolder.getValueVector());
+ }
output.setNewSchema();
}catch(SchemaChangeException e) {
throw new ExecutionSetupException("Error setting up output mutator.", e);
@@ -242,6 +245,9 @@ public class ParquetRecordReader implements RecordReader {
for (VarLenBinaryReader.VarLengthColumn r : varLengthReader.columns){
totalByteLength += r.columnChunkMetaData.getTotalSize();
}
+ for (VarLenBinaryReader.NullableVarLengthColumn r : varLengthReader.nullableColumns){
+ totalByteLength += r.columnChunkMetaData.getTotalSize();
+ }
int bufferSize = 64*1024;
long totalBytesWritten = 0;
int validBytesInCurrentBuffer;
@@ -262,17 +268,22 @@ public class ParquetRecordReader implements RecordReader {
}
}
- private SchemaPath toFieldName(String[] paths) {
- if(this.ref == null){
- return new SchemaPath(Joiner.on('/').join(paths), ExpressionPosition.UNKNOWN);
- }else{
- return ref.getChild(paths);
+ private static String toFieldName(String[] paths) {
+ StringBuilder sb = new StringBuilder();
+ boolean first = true;
+ for(String np : paths){
+ if(first){
+ first = false;
+ }else{
+ sb.append(".");
+ }
+ sb.append(np);
}
-
+ return sb.toString();
}
- private TypeProtos.DataMode getDataMode(ColumnDescriptor column, MessageType schema) {
- if (schema.getColumnDescription(column.getPath()).getMaxDefinitionLevel() == 0) {
+ private TypeProtos.DataMode getDataMode(ColumnDescriptor column) {
+ if (column.getMaxDefinitionLevel() == 0) {
return TypeProtos.DataMode.REQUIRED;
} else {
return TypeProtos.DataMode.OPTIONAL;
@@ -288,30 +299,46 @@ public class ParquetRecordReader implements RecordReader {
r.valueVecHolder.reset();
r.valuesReadInCurrentPass = 0;
}
+ for (VarLenBinaryReader.NullableVarLengthColumn r : varLengthReader.nullableColumns){
+ r.valueVecHolder.reset();
+ r.valuesReadInCurrentPass = 0;
+ }
}
/**
* @param fixedLength
- * @param field
* @param descriptor
* @param columnChunkMetaData
* @param allocateSize - the size of the vector to create
* @return
* @throws SchemaChangeException
*/
- private boolean createFixedColumnReader(boolean fixedLength, MaterializedField field, ColumnDescriptor descriptor,
+ private boolean createFixedColumnReader(boolean fixedLength, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, int allocateSize, ValueVector v)
throws SchemaChangeException {
- TypeProtos.MajorType type = field.getType();
- if (columnChunkMetaData.getType() == PrimitiveType.PrimitiveTypeName.BOOLEAN){
- columnStatuses.add(new BitReader(this, allocateSize, descriptor, columnChunkMetaData,
- fixedLength, v));
+ // if the column is required
+ if (descriptor.getMaxDefinitionLevel() == 0){
+ if (columnChunkMetaData.getType() == PrimitiveType.PrimitiveTypeName.BOOLEAN){
+ columnStatuses.add(new BitReader(this, allocateSize, descriptor, columnChunkMetaData,
+ fixedLength, v));
+ }
+ else{
+ columnStatuses.add(new FixedByteAlignedReader(this, allocateSize, descriptor, columnChunkMetaData,
+ fixedLength, v));
+ }
+ return true;
}
- else{
- columnStatuses.add(new FixedByteAlignedReader(this, allocateSize, descriptor, columnChunkMetaData,
- fixedLength, v));
+ else { // if the column is nullable
+ if (columnChunkMetaData.getType() == PrimitiveType.PrimitiveTypeName.BOOLEAN){
+ columnStatuses.add(new NullableBitReader(this, allocateSize, descriptor, columnChunkMetaData,
+ fixedLength, v));
+ }
+ else{
+ columnStatuses.add(new NullableFixedByteAlignedReader(this, allocateSize, descriptor, columnChunkMetaData,
+ fixedLength, v));
+ }
+ return true;
}
- return true;
}
public void readAllFixedFields(long recordsToRead, ColumnReader firstColumnStatus) throws IOException {
@@ -326,12 +353,24 @@ public class ParquetRecordReader implements RecordReader {
resetBatch();
long recordsToRead = 0;
try {
- ColumnReader firstColumnStatus = columnStatuses.iterator().next();
+ ColumnReader firstColumnStatus;
+ if (columnStatuses.size() > 0){
+ firstColumnStatus = columnStatuses.iterator().next();
+ }
+ else{
+ if (varLengthReader.columns.size() > 0){
+ firstColumnStatus = varLengthReader.columns.iterator().next();
+ }
+ else{
+ firstColumnStatus = varLengthReader.nullableColumns.iterator().next();
+ }
+ }
+
if (allFieldsFixedLength) {
recordsToRead = Math.min(recordsPerBatch, firstColumnStatus.columnChunkMetaData.getValueCount() - firstColumnStatus.totalValuesRead);
} else {
// arbitrary
- recordsToRead = 8000;
+ recordsToRead = 4000;
// going to incorporate looking at length of values and copying the data into a single loop, hopefully it won't
// get too complicated
@@ -343,6 +382,7 @@ public class ParquetRecordReader implements RecordReader {
// cannot find more information on this right now, will keep looking
}
+ logger.debug("records to read in this pass: {}", recordsToRead);
if (allFieldsFixedLength) {
readAllFixedFields(recordsToRead, firstColumnStatus);
} else { // variable length columns
@@ -363,31 +403,88 @@ public class ParquetRecordReader implements RecordReader {
static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName primitiveTypeName, int length,
TypeProtos.DataMode mode) {
- switch (primitiveTypeName) {
- case BINARY:
- return Types.required(TypeProtos.MinorType.VARBINARY);
- case INT64:
- return Types.required(TypeProtos.MinorType.BIGINT);
- case INT32:
- return Types.required(TypeProtos.MinorType.INT);
- case BOOLEAN:
- return Types.required(TypeProtos.MinorType.BIT);
- case FLOAT:
- return Types.required(TypeProtos.MinorType.FLOAT4);
- case DOUBLE:
- return Types.required(TypeProtos.MinorType.FLOAT8);
- // Both of these are not supported by the parquet library yet (7/3/13),
- // but they are declared here for when they are implemented
- case INT96:
- return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
- .setMode(mode).build();
- case FIXED_LEN_BYTE_ARRAY:
- checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
- return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY)
- .setWidth(length).setMode(mode).build();
- default:
- throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+ switch (mode) {
+
+ case OPTIONAL:
+ switch (primitiveTypeName) {
+ case BINARY:
+ return Types.optional(TypeProtos.MinorType.VARBINARY);
+ case INT64:
+ return Types.optional(TypeProtos.MinorType.BIGINT);
+ case INT32:
+ return Types.optional(TypeProtos.MinorType.INT);
+ case BOOLEAN:
+ return Types.optional(TypeProtos.MinorType.BIT);
+ case FLOAT:
+ return Types.optional(TypeProtos.MinorType.FLOAT4);
+ case DOUBLE:
+ return Types.optional(TypeProtos.MinorType.FLOAT8);
+ // TODO - Both of these are not supported by the parquet library yet (7/3/13),
+ // but they are declared here for when they are implemented
+ case INT96:
+ return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
+ .setMode(mode).build();
+ case FIXED_LEN_BYTE_ARRAY:
+ checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
+ return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY)
+ .setWidth(length).setMode(mode).build();
+ default:
+ throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+ }
+ case REQUIRED:
+ switch (primitiveTypeName) {
+ case BINARY:
+ return Types.required(TypeProtos.MinorType.VARBINARY);
+ case INT64:
+ return Types.required(TypeProtos.MinorType.BIGINT);
+ case INT32:
+ return Types.required(TypeProtos.MinorType.INT);
+ case BOOLEAN:
+ return Types.required(TypeProtos.MinorType.BIT);
+ case FLOAT:
+ return Types.required(TypeProtos.MinorType.FLOAT4);
+ case DOUBLE:
+ return Types.required(TypeProtos.MinorType.FLOAT8);
+ // Both of these are not supported by the parquet library yet (7/3/13),
+ // but they are declared here for when they are implemented
+ case INT96:
+ return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
+ .setMode(mode).build();
+ case FIXED_LEN_BYTE_ARRAY:
+ checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
+ return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY)
+ .setWidth(length).setMode(mode).build();
+ default:
+ throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+ }
+ case REPEATED:
+ switch (primitiveTypeName) {
+ case BINARY:
+ return Types.repeated(TypeProtos.MinorType.VARBINARY);
+ case INT64:
+ return Types.repeated(TypeProtos.MinorType.BIGINT);
+ case INT32:
+ return Types.repeated(TypeProtos.MinorType.INT);
+ case BOOLEAN:
+ return Types.repeated(TypeProtos.MinorType.BIT);
+ case FLOAT:
+ return Types.repeated(TypeProtos.MinorType.FLOAT4);
+ case DOUBLE:
+ return Types.repeated(TypeProtos.MinorType.FLOAT8);
+ // Both of these are not supported by the parquet library yet (7/3/13),
+ // but they are declared here for when they are implemented
+ case INT96:
+ return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
+ .setMode(mode).build();
+ case FIXED_LEN_BYTE_ARRAY:
+ checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
+ return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY)
+ .setWidth(length).setMode(mode).build();
+ default:
+ throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+ }
}
+ throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName + " Mode: " + mode);
}
static String join(String delimiter, String... str) {
@@ -406,7 +503,8 @@ public class ParquetRecordReader implements RecordReader {
@Override
public void cleanup() {
columnStatuses.clear();
- this.varLengthReader.columns.clear();
bufferWithAllData.release();
+ varLengthReader.columns.clear();
+ varLengthReader.nullableColumns.clear();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
index 7cefcf4..6553736 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
@@ -17,56 +17,40 @@
******************************************************************************/
package org.apache.drill.exec.store.parquet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
+import com.fasterxml.jackson.annotation.*;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.logical.StorageEngineConfig;
import org.apache.drill.exec.exception.SetupException;
import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.physical.ReadEntryFromHDFS;
-import org.apache.drill.exec.physical.base.AbstractBase;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.base.PhysicalVisitor;
-import org.apache.drill.exec.physical.base.Size;
-import org.apache.drill.exec.physical.base.SubScan;
+import org.apache.drill.exec.physical.base.*;
import org.apache.drill.exec.store.StorageEngineRegistry;
-import com.fasterxml.jackson.annotation.JacksonInject;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
+import java.util.*;
// Class containing information for reading a single parquet row group form HDFS
@JsonTypeName("parquet-row-group-scan")
public class ParquetRowGroupScan extends AbstractBase implements SubScan {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRowGroupScan.class);
- public final StorageEngineConfig engineConfig;
- private final ParquetStorageEngine parquetStorageEngine;
- private final List<RowGroupReadEntry> rowGroupReadEntries;
- private final FieldReference ref;
+ public StorageEngineConfig engineConfig;
+ private ParquetStorageEngine parquetStorageEngine;
+ private List<RowGroupReadEntry> rowGroupReadEntries;
@JsonCreator
public ParquetRowGroupScan(@JacksonInject StorageEngineRegistry registry, @JsonProperty("engineConfig") StorageEngineConfig engineConfig,
- @JsonProperty("rowGroupReadEntries") LinkedList<RowGroupReadEntry> rowGroupReadEntries, @JsonProperty("ref") FieldReference ref) throws ExecutionSetupException {
+ @JsonProperty("rowGroupReadEntries") LinkedList<RowGroupReadEntry> rowGroupReadEntries) throws ExecutionSetupException {
parquetStorageEngine = (ParquetStorageEngine) registry.getEngine(engineConfig);
this.rowGroupReadEntries = rowGroupReadEntries;
- this.engineConfig = engineConfig;
- this.ref = ref;
}
public ParquetRowGroupScan(ParquetStorageEngine engine, ParquetStorageEngineConfig config,
- List<RowGroupReadEntry> rowGroupReadEntries, FieldReference ref) {
+ List<RowGroupReadEntry> rowGroupReadEntries) throws SetupException {
parquetStorageEngine = engine;
engineConfig = config;
this.rowGroupReadEntries = rowGroupReadEntries;
- this.ref = ref;
}
public List<RowGroupReadEntry> getRowGroupReadEntries() {
@@ -82,11 +66,6 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
return null;
}
-
- public FieldReference getRef() {
- return ref;
- }
-
@Override
public Size getSize() {
return null;
@@ -110,7 +89,12 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
@Override
public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
Preconditions.checkArgument(children.isEmpty());
- return new ParquetRowGroupScan(parquetStorageEngine, (ParquetStorageEngineConfig) engineConfig, rowGroupReadEntries, ref);
+ try {
+ return new ParquetRowGroupScan(parquetStorageEngine, (ParquetStorageEngineConfig) engineConfig, rowGroupReadEntries);
+ } catch (SetupException e) {
+ // TODO - handle this
+ throw new RuntimeException(e);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index a09f1ac..b11eadc 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -18,26 +18,20 @@
package org.apache.drill.exec.store.parquet;
import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
import java.util.List;
-import java.util.concurrent.TimeUnit;
-import com.google.common.base.Stopwatch;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.ops.FragmentContext;
+
import org.apache.drill.exec.physical.impl.BatchCreator;
import org.apache.drill.exec.physical.impl.ScanBatch;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.store.RecordReader;
-import org.apache.drill.exec.store.mock.MockScanBatchCreator;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-
import parquet.hadoop.ParquetFileReader;
import parquet.hadoop.metadata.ParquetMetadata;
@@ -46,8 +40,6 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
@Override
public RecordBatch getBatch(FragmentContext context, ParquetRowGroupScan rowGroupScan, List<RecordBatch> children) throws ExecutionSetupException {
- Stopwatch watch = new Stopwatch();
- watch.start();
Preconditions.checkArgument(children.isEmpty());
List<RecordReader> readers = Lists.newArrayList();
for(ParquetRowGroupScan.RowGroupReadEntry e : rowGroupScan.getRowGroupReadEntries()){
@@ -63,15 +55,13 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
new ParquetRecordReader(
context, e.getPath(), e.getRowGroupIndex(), rowGroupScan.getStorageEngine().getFileSystem(),
rowGroupScan.getStorageEngine().getCodecFactoryExposer(),
- ParquetFileReader.readFooter( rowGroupScan.getStorageEngine().getFileSystem().getConf(), new Path(e.getPath())),
- rowGroupScan.getRef()
+ ParquetFileReader.readFooter( rowGroupScan.getStorageEngine().getFileSystem().getConf(), new Path(e.getPath()))
)
);
} catch (IOException e1) {
throw new ExecutionSetupException(e1);
}
}
- logger.debug("total time in ScanBatchCreator.getBatch: {} ms", watch.elapsed(TimeUnit.MILLISECONDS));
return new ScanBatch(context, readers.iterator());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
index 939545e..e6e2f03 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
@@ -121,4 +121,4 @@ public class ParquetStorageEngine extends AbstractStorageEngine{
public ParquetSchemaProvider getSchemaProvider() {
return schemaProvider;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
index 08e1023..f20a2f3 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/VarLenBinaryReader.java
@@ -17,6 +17,7 @@
******************************************************************************/
package org.apache.drill.exec.store.parquet;
+import org.apache.drill.exec.vector.NullableVarBinaryVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import parquet.bytes.BytesUtils;
@@ -24,15 +25,19 @@ import parquet.column.ColumnDescriptor;
import parquet.hadoop.metadata.ColumnChunkMetaData;
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
public class VarLenBinaryReader {
ParquetRecordReader parentReader;
final List<VarLengthColumn> columns;
+ final List<NullableVarLengthColumn> nullableColumns;
- public VarLenBinaryReader(ParquetRecordReader parentReader, List<VarLengthColumn> columns){
+ public VarLenBinaryReader(ParquetRecordReader parentReader, List<VarLengthColumn> columns,
+ List<NullableVarLengthColumn> nullableColumns){
this.parentReader = parentReader;
+ this.nullableColumns = nullableColumns;
this.columns = columns;
}
@@ -48,6 +53,21 @@ public class VarLenBinaryReader {
}
}
+ public static class NullableVarLengthColumn extends ColumnReader {
+
+ int nullsRead;
+ boolean currentValNull = false;
+
+ NullableVarLengthColumn(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, ValueVector v) {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v);
+ }
+
+ @Override
+ protected void readField(long recordsToRead, ColumnReader firstColumnStatus) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
/**
* Reads as many variable length values as possible.
*
@@ -63,22 +83,42 @@ public class VarLenBinaryReader {
boolean rowGroupFinished = false;
byte[] bytes;
VarBinaryVector currVec;
+ NullableVarBinaryVector currNullVec;
// write the first 0 offset
for (ColumnReader columnReader : columns) {
- if (columnReader.isFixedLength) {
- continue;
- }
currVec = (VarBinaryVector) columnReader.valueVecHolder.getValueVector();
currVec.getAccessor().getOffsetVector().getData().writeInt(0);
columnReader.bytesReadInCurrentPass = 0;
columnReader.valuesReadInCurrentPass = 0;
}
+ // same for the nullable columns
+ for (NullableVarLengthColumn columnReader : nullableColumns) {
+ currNullVec = (NullableVarBinaryVector) columnReader.valueVecHolder.getValueVector();
+ currNullVec.getMutator().getVectorWithValues().getAccessor().getOffsetVector().getData().writeInt(0);
+ columnReader.bytesReadInCurrentPass = 0;
+ columnReader.valuesReadInCurrentPass = 0;
+ columnReader.nullsRead = 0;
+ }
do {
lengthVarFieldsInCurrentRecord = 0;
for (ColumnReader columnReader : columns) {
- if (columnReader.isFixedLength) {
- continue;
+ if (columnReader.pageReadStatus.currentPage == null
+ || columnReader.pageReadStatus.valuesRead == columnReader.pageReadStatus.currentPage.getValueCount()) {
+ columnReader.totalValuesRead += columnReader.pageReadStatus.valuesRead;
+ if (!columnReader.pageReadStatus.next()) {
+ rowGroupFinished = true;
+ break;
+ }
}
+ bytes = columnReader.pageReadStatus.pageDataByteArray;
+
+ // re-purposing this field here for length in BYTES to prevent repetitive multiplication/division
+ columnReader.dataTypeLengthInBits = BytesUtils.readIntLittleEndian(bytes,
+ (int) columnReader.pageReadStatus.readPosInBytes);
+ lengthVarFieldsInCurrentRecord += columnReader.dataTypeLengthInBits;
+
+ }
+ for (NullableVarLengthColumn columnReader : nullableColumns) {
if (columnReader.pageReadStatus.currentPage == null
|| columnReader.pageReadStatus.valuesRead == columnReader.pageReadStatus.currentPage.getValueCount()) {
columnReader.totalValuesRead += columnReader.pageReadStatus.valuesRead;
@@ -88,6 +128,12 @@ public class VarLenBinaryReader {
}
}
bytes = columnReader.pageReadStatus.pageDataByteArray;
+ if ( columnReader.columnDescriptor.getMaxDefinitionLevel() > columnReader.pageReadStatus.definitionLevels.readInteger()){
+ columnReader.currentValNull = true;
+ columnReader.dataTypeLengthInBits = 0;
+ columnReader.nullsRead++;
+ continue;// field is null, no length to add to data vector
+ }
// re-purposing this field here for length in BYTES to prevent repetitive multiplication/division
columnReader.dataTypeLengthInBits = BytesUtils.readIntLittleEndian(bytes,
@@ -96,7 +142,7 @@ public class VarLenBinaryReader {
}
// check that the next record will fit in the batch
- if (rowGroupFinished || (recordsReadInCurrentPass + 1) * parentReader.getBitWidthAllFixedFields() + lengthVarFieldsInCurrentRecord * 8
+ if (rowGroupFinished || (recordsReadInCurrentPass + 1) * parentReader.getBitWidthAllFixedFields() + lengthVarFieldsInCurrentRecord
> parentReader.getBatchSize()){
break;
}
@@ -104,9 +150,6 @@ public class VarLenBinaryReader {
recordsReadInCurrentPass++;
}
for (ColumnReader columnReader : columns) {
- if (columnReader.isFixedLength) {
- continue;
- }
bytes = columnReader.pageReadStatus.pageDataByteArray;
currVec = (VarBinaryVector) columnReader.valueVecHolder.getValueVector();
// again, I am re-purposing the unused field here, it is a length n BYTES, not bits
@@ -124,7 +167,34 @@ public class VarLenBinaryReader {
columnReader.pageReadStatus.next();
}
}
+ for (NullableVarLengthColumn columnReader : nullableColumns) {
+ bytes = columnReader.pageReadStatus.pageDataByteArray;
+ currNullVec = (NullableVarBinaryVector) columnReader.valueVecHolder.getValueVector();
+ // again, I am re-purposing the unused field here, it is a length n BYTES, not bits
+ currNullVec.getMutator().getVectorWithValues().getAccessor().getOffsetVector().getData()
+ .writeInt(
+ (int) columnReader.bytesReadInCurrentPass +
+ columnReader.dataTypeLengthInBits - 4 * (columnReader.valuesReadInCurrentPass -
+ (columnReader.currentValNull ? Math.max (0, columnReader.nullsRead - 1) : columnReader.nullsRead)));
+ columnReader.currentValNull = false;
+ if (columnReader.dataTypeLengthInBits > 0){
+ currNullVec.getData().writeBytes(bytes, (int) columnReader.pageReadStatus.readPosInBytes + 4,
+ columnReader.dataTypeLengthInBits);
+ ((NullableVarBinaryVector)columnReader.valueVecHolder.getValueVector()).getMutator().setIndexDefined(columnReader.valuesReadInCurrentPass);
+ }
+ if (columnReader.dataTypeLengthInBits > 0){
+ columnReader.pageReadStatus.readPosInBytes += columnReader.dataTypeLengthInBits + 4;
+ columnReader.bytesReadInCurrentPass += columnReader.dataTypeLengthInBits + 4;
+ }
+ columnReader.pageReadStatus.valuesRead++;
+ columnReader.valuesReadInCurrentPass++;
+ currNullVec.getMutator().setValueCount((int)recordsReadInCurrentPass);
+ // reached the end of a page
+ if ( columnReader.pageReadStatus.valuesRead == columnReader.pageReadStatus.currentPage.getValueCount()) {
+ columnReader.pageReadStatus.next();
+ }
+ }
} while (recordsReadInCurrentPass < recordsToReadInThisPass);
return recordsReadInCurrentPass;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index 358334e..c6515d6 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -1,10 +1,11 @@
package org.apache.drill.exec.vector;
import org.apache.drill.common.expression.FieldReference;
+import io.netty.buffer.ByteBuf;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.record.MaterializedField;
-abstract class BaseValueVector implements ValueVector{
+public abstract class BaseValueVector implements ValueVector{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseValueVector.class);
protected final BufferAllocator allocator;
@@ -29,6 +30,9 @@ abstract class BaseValueVector implements ValueVector{
return getField().clone(ref);
}
+
+ abstract public ByteBuf getData();
+
abstract class BaseAccessor implements ValueVector.Accessor{
public abstract int getValueCount();
public void reset(){}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63a90fe9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
new file mode 100644
index 0000000..cdccf4e
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
@@ -0,0 +1,23 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.vector;
+
+public interface NullableVectorDefinitionSetter {
+
+ public void setIndexDefined(int index);
+}