You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by sm...@apache.org on 2015/03/07 04:18:15 UTC
[1/5] drill git commit: DRILL-2359:
ClassPathFileSystem.getFileStatus() should throw FileNotFoundException when
path doesn't exist
Repository: drill
Updated Branches:
refs/heads/master fb293ba52 -> fe94970e1
DRILL-2359: ClassPathFileSystem.getFileStatus() should throw FileNotFoundException when path doesn't exist
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2dd8a2ca
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2dd8a2ca
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2dd8a2ca
Branch: refs/heads/master
Commit: 2dd8a2ca91ed7333946b543f2aa297529fb06195
Parents: fb293ba
Author: adeneche <ad...@gmail.com>
Authored: Mon Mar 2 16:03:40 2015 -0800
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Fri Mar 6 17:39:32 2015 -0800
----------------------------------------------------------------------
.../apache/drill/exec/store/ClassPathFileSystem.java | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/2dd8a2ca/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
index cab5339..dc5a4b5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -72,18 +73,15 @@ public class ClassPathFileSystem extends FileSystem{
@Override
public FileStatus getFileStatus(Path arg0) throws IOException {
String file = getFileName(arg0);
+ URL url;
try{
- URL url = Resources.getResource(file);
- if(url == null){
- throw new IOException(String.format("Unable to find path %s.", arg0.toString()));
+ url = Resources.getResource(file);
+ }catch(IllegalArgumentException e){
+ throw new FileNotFoundException(String.format("Unable to find path %s.", arg0.toString()));
}
-
return new FileStatus(Resources.asByteSource(url).size(), false, 1, 8096, System.currentTimeMillis(), arg0);
- }catch(RuntimeException e){
- throw new IOException(String.format("Failure trying to load file %s", arg0), e);
- }
}
@Override
[3/5] drill git commit: DRILL-2267: bumping parquet version to
1.5.1-r7
Posted by sm...@apache.org.
DRILL-2267: bumping parquet version to 1.5.1-r7
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/36fedf99
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/36fedf99
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/36fedf99
Branch: refs/heads/master
Commit: 36fedf99aaad01189479952ccbf8d3158684166f
Parents: 6af2dd4
Author: adeneche <ad...@gmail.com>
Authored: Thu Feb 26 07:55:03 2015 -0800
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Fri Mar 6 17:41:46 2015 -0800
----------------------------------------------------------------------
exec/java-exec/pom.xml | 12 ++++++------
pom.xml | 1 +
2 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/36fedf99/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index a7b6a67..4b9b24f 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -146,7 +146,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-column</artifactId>
- <version>1.5.1-drill-r6</version>
+ <version>${parquet.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -161,7 +161,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop</artifactId>
- <version>1.5.1-drill-r4</version>
+ <version>${parquet.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -191,7 +191,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-common</artifactId>
- <version>1.5.1-drill-r4</version>
+ <version>${parquet.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -206,7 +206,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-jackson</artifactId>
- <version>1.5.1-drill-r4</version>
+ <version>${parquet.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -221,7 +221,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-encoding</artifactId>
- <version>1.5.1-drill-r4</version>
+ <version>${parquet.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
@@ -236,7 +236,7 @@
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-generator</artifactId>
- <version>1.5.1-drill-r4</version>
+ <version>${parquet.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/drill/blob/36fedf99/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 525579a..1f08574 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,6 +33,7 @@
<proto.cas.path>${project.basedir}/src/main/protobuf/</proto.cas.path>
<dep.junit.version>4.11</dep.junit.version>
<dep.slf4j.version>1.7.5</dep.slf4j.version>
+ <parquet.version>1.5.1-drill-r7</parquet.version>
</properties>
<scm>
[2/5] drill git commit: DRILL-2337: disable dictionary encoding by
default
Posted by sm...@apache.org.
DRILL-2337: disable dictionary encoding by default
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/6af2dd4a
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/6af2dd4a
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/6af2dd4a
Branch: refs/heads/master
Commit: 6af2dd4a37987cbc3d2f9efe5b352cf9bc282adf
Parents: 2dd8a2c
Author: adeneche <ad...@gmail.com>
Authored: Fri Feb 27 20:14:00 2015 +0000
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Fri Mar 6 17:41:00 2015 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/drill/exec/ExecConstants.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/6af2dd4a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 40fe588..33b05af 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -91,7 +91,7 @@ public interface ExecConstants {
PARQUET_WRITER_COMPRESSION_TYPE, "snappy", "snappy", "gzip", "none");
public static final String PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING = "store.parquet.enable_dictionary_encoding";
public static final OptionValidator PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING_VALIDATOR = new BooleanValidator(
- PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, true);
+ PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, false);
public static final String PARQUET_VECTOR_FILL_THRESHOLD = "store.parquet.vector_fill_threshold";
public static final OptionValidator PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_THRESHOLD, 99l, 85l);
[5/5] drill git commit: DRILL-2262: selecting columns of certain
datatypes from a dictionary encoded parquet file created by drill fails
Posted by sm...@apache.org.
DRILL-2262: selecting columns of certain datatypes from a dictionary encoded parquet file created by drill fails
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/fe94970e
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/fe94970e
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/fe94970e
Branch: refs/heads/master
Commit: fe94970e1fbfd6ceacbfc7f2187c5495d497e791
Parents: deba946
Author: adeneche <ad...@gmail.com>
Authored: Thu Feb 26 19:55:08 2015 -0800
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Fri Mar 6 18:24:38 2015 -0800
----------------------------------------------------------------------
.../columnreaders/ColumnReaderFactory.java | 88 ++++++++++----
.../NullableFixedByteAlignedReaders.java | 94 +++++++++++++++
.../ParquetFixedWidthDictionaryReaders.java | 114 ++++++++++++++++++-
.../columnreaders/TestColumnReaderFactory.java | 84 ++++++++++++++
.../parquet/decimal_dictionary.parquet | Bin 0 -> 602 bytes
.../parquet/decimal_nodictionary.parquet | Bin 0 -> 558 bytes
.../resources/parquet/time_dictionary.parquet | Bin 0 -> 618 bytes
.../resources/parquet/time_nodictionary.parquet | Bin 0 -> 576 bytes
8 files changed, 356 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
index cbdc0b0..70b2342 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
@@ -19,25 +19,33 @@ package org.apache.drill.exec.store.parquet.columnreaders;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.exception.SchemaChangeException;
+
import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.vector.Decimal18Vector;
import org.apache.drill.exec.vector.Decimal28SparseVector;
import org.apache.drill.exec.vector.Decimal38SparseVector;
+import org.apache.drill.exec.vector.Decimal9Vector;
import org.apache.drill.exec.vector.Float4Vector;
import org.apache.drill.exec.vector.Float8Vector;
import org.apache.drill.exec.vector.IntVector;
import org.apache.drill.exec.vector.NullableBigIntVector;
+import org.apache.drill.exec.vector.NullableDecimal18Vector;
import org.apache.drill.exec.vector.NullableDecimal28SparseVector;
import org.apache.drill.exec.vector.NullableDecimal38SparseVector;
+import org.apache.drill.exec.vector.NullableDecimal9Vector;
import org.apache.drill.exec.vector.NullableFloat4Vector;
import org.apache.drill.exec.vector.NullableFloat8Vector;
import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableTimeStampVector;
+import org.apache.drill.exec.vector.NullableTimeVector;
import org.apache.drill.exec.vector.NullableVarBinaryVector;
import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.TimeStampVector;
+import org.apache.drill.exec.vector.TimeVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
import org.apache.drill.exec.vector.VariableWidthVector;
-
import parquet.column.ColumnDescriptor;
import parquet.column.Encoding;
import parquet.format.ConvertedType;
@@ -83,9 +91,29 @@ public class ColumnReaderFactory {
if (columnChunkMetaData.getEncodings().contains(Encoding.PLAIN_DICTIONARY)) {
switch (columnChunkMetaData.getType()) {
case INT32:
- return new ParquetFixedWidthDictionaryReaders.DictionaryIntReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (IntVector) v, schemaElement);
+ if (convertedType == null) {
+ return new ParquetFixedWidthDictionaryReaders.DictionaryIntReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (IntVector) v, schemaElement);
+ }
+ switch (convertedType) {
+ case DECIMAL:
+ return new ParquetFixedWidthDictionaryReaders.DictionaryDecimal9Reader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Decimal9Vector) v, schemaElement);
+ case TIME_MILLIS:
+ return new ParquetFixedWidthDictionaryReaders.DictionaryTimeReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (TimeVector) v, schemaElement);
+ default:
+ throw new ExecutionSetupException("Unsupported dictionary converted type " + convertedType + " for primitive type INT32");
+ }
case INT64:
- return new ParquetFixedWidthDictionaryReaders.DictionaryBigIntReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (BigIntVector) v, schemaElement);
+ if (convertedType == null) {
+ return new ParquetFixedWidthDictionaryReaders.DictionaryBigIntReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (BigIntVector) v, schemaElement);
+ }
+ switch (convertedType) {
+ case DECIMAL:
+ return new ParquetFixedWidthDictionaryReaders.DictionaryDecimal18Reader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Decimal18Vector) v, schemaElement);
+ case TIMESTAMP_MILLIS:
+ return new ParquetFixedWidthDictionaryReaders.DictionaryTimeStampReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (TimeStampVector) v, schemaElement);
+ default:
+ throw new ExecutionSetupException("Unsupported dictionary converted type " + convertedType + " for primitive type INT64");
+ }
case FLOAT:
return new ParquetFixedWidthDictionaryReaders.DictionaryFloat4Reader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Float4Vector) v, schemaElement);
case DOUBLE:
@@ -169,28 +197,42 @@ public class ColumnReaderFactory {
boolean fixedLength,
ValueVector valueVec,
SchemaElement schemaElement) throws ExecutionSetupException {
+ ConvertedType convertedType = schemaElement.getConverted_type();
+
if (! columnChunkMetaData.getEncodings().contains(Encoding.PLAIN_DICTIONARY)) {
- return new NullableFixedByteAlignedReaders.NullableFixedByteAlignedReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData,
- fixedLength, valueVec, schemaElement);
+ return new NullableFixedByteAlignedReaders.NullableFixedByteAlignedReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, valueVec, schemaElement);
} else {
- if (columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.INT64) {
- return new NullableFixedByteAlignedReaders.NullableDictionaryBigIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData,
- fixedLength, (NullableBigIntVector)valueVec, schemaElement);
- }
- else if (columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.INT32) {
- return new NullableFixedByteAlignedReaders.NullableDictionaryIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData,
- fixedLength, (NullableIntVector)valueVec, schemaElement);
- }
- else if (columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.FLOAT) {
- return new NullableFixedByteAlignedReaders.NullableDictionaryFloat4Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData,
- fixedLength, (NullableFloat4Vector)valueVec, schemaElement);
- }
- else if (columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.DOUBLE) {
- return new NullableFixedByteAlignedReaders.NullableDictionaryFloat8Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData,
- fixedLength, (NullableFloat8Vector)valueVec, schemaElement);
- }
- else{
- throw new ExecutionSetupException("Unsupported nullable column type " + columnDescriptor.getType().name() );
+ switch (columnDescriptor.getType()) {
+ case INT32:
+ if (convertedType == null) {
+ return new NullableFixedByteAlignedReaders.NullableDictionaryIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableIntVector) valueVec, schemaElement);
+ }
+ switch (convertedType) {
+ case DECIMAL:
+ return new NullableFixedByteAlignedReaders.NullableDictionaryDecimal9Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableDecimal9Vector) valueVec, schemaElement);
+ case TIME_MILLIS:
+ return new NullableFixedByteAlignedReaders.NullableDictionaryTimeReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableTimeVector)valueVec, schemaElement);
+ default:
+ throw new ExecutionSetupException("Unsupported nullable converted type " + convertedType + " for primitive type INT32");
+ }
+ case INT64:
+ if (convertedType == null) {
+ return new NullableFixedByteAlignedReaders.NullableDictionaryBigIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableBigIntVector)valueVec, schemaElement);
+ }
+ switch (convertedType) {
+ case DECIMAL:
+ return new NullableFixedByteAlignedReaders.NullableDictionaryDecimal18Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableDecimal18Vector)valueVec, schemaElement);
+ case TIMESTAMP_MILLIS:
+ return new NullableFixedByteAlignedReaders.NullableDictionaryTimeStampReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableTimeStampVector)valueVec, schemaElement);
+ default:
+ throw new ExecutionSetupException("Unsupported nullable converted type " + convertedType + " for primitive type INT64");
+ }
+ case FLOAT:
+ return new NullableFixedByteAlignedReaders.NullableDictionaryFloat4Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableFloat4Vector)valueVec, schemaElement);
+ case DOUBLE:
+ return new NullableFixedByteAlignedReaders.NullableDictionaryFloat8Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableFloat8Vector)valueVec, schemaElement);
+ default:
+ throw new ExecutionSetupException("Unsupported nullable column type " + columnDescriptor.getType().name() );
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
index 707bc9c..8087118 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
@@ -34,6 +34,10 @@ import org.apache.drill.exec.vector.NullableDecimal38SparseVector;
import org.apache.drill.exec.vector.NullableFloat4Vector;
import org.apache.drill.exec.vector.NullableFloat8Vector;
import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableDecimal9Vector;
+import org.apache.drill.exec.vector.NullableDecimal18Vector;
+import org.apache.drill.exec.vector.NullableTimeVector;
+import org.apache.drill.exec.vector.NullableTimeStampVector;
import org.apache.drill.exec.vector.ValueVector;
import org.joda.time.DateTimeUtils;
@@ -84,6 +88,52 @@ public class NullableFixedByteAlignedReaders {
}
}
+ static class NullableDictionaryDecimal9Reader extends NullableColumnReader<NullableDecimal9Vector> {
+
+ NullableDictionaryDecimal9Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+ ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableDecimal9Vector v,
+ SchemaElement schemaElement) throws ExecutionSetupException {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+ }
+
+ // this method is called by its superclass during a read loop
+ @Override
+ protected void readField(long recordsToReadInThisPass) {
+ if (usingDictionary) {
+ for (int i = 0; i < recordsToReadInThisPass; i++){
+ valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readInteger());
+ }
+ } else {
+ for (int i = 0; i < recordsToReadInThisPass; i++){
+ valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.valueReader.readInteger());
+ }
+ }
+ }
+ }
+
+ static class NullableDictionaryTimeReader extends NullableColumnReader<NullableTimeVector> {
+
+ NullableDictionaryTimeReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+ ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableTimeVector v,
+ SchemaElement schemaElement) throws ExecutionSetupException {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+ }
+
+ // this method is called by its superclass during a read loop
+ @Override
+ protected void readField(long recordsToReadInThisPass) {
+ if (usingDictionary) {
+ for (int i = 0; i < recordsToReadInThisPass; i++){
+ valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readInteger());
+ }
+ } else {
+ for (int i = 0; i < recordsToReadInThisPass; i++){
+ valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.valueReader.readInteger());
+ }
+ }
+ }
+ }
+
static class NullableDictionaryBigIntReader extends NullableColumnReader<NullableBigIntVector> {
NullableDictionaryBigIntReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
@@ -107,6 +157,50 @@ public class NullableFixedByteAlignedReaders {
}
}
+ static class NullableDictionaryTimeStampReader extends NullableColumnReader<NullableTimeStampVector> {
+
+ NullableDictionaryTimeStampReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+ ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableTimeStampVector v,
+ SchemaElement schemaElement) throws ExecutionSetupException {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+ }
+
+ // this method is called by its superclass during a read loop
+ @Override
+ protected void readField(long recordsToReadInThisPass) {
+ if (usingDictionary) {
+ for (int i = 0; i < recordsToReadInThisPass; i++){
+ valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readLong());
+ }
+ } else {
+ for (int i = 0; i < recordsToReadInThisPass; i++){
+ valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.valueReader.readLong());
+ }
+ }
+ }
+ }
+ static class NullableDictionaryDecimal18Reader extends NullableColumnReader<NullableDecimal18Vector> {
+
+ NullableDictionaryDecimal18Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+ ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableDecimal18Vector v,
+ SchemaElement schemaElement) throws ExecutionSetupException {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+ }
+
+ // this method is called by its superclass during a read loop
+ @Override
+ protected void readField(long recordsToReadInThisPass) {
+ if (usingDictionary) {
+ for (int i = 0; i < recordsToReadInThisPass; i++){
+ valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readLong());
+ }
+ } else {
+ for (int i = 0; i < recordsToReadInThisPass; i++){
+ valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.valueReader.readLong());
+ }
+ }
+ }
+ }
static class NullableDictionaryFloat4Reader extends NullableColumnReader<NullableFloat4Vector> {
NullableDictionaryFloat4Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java
index 02b583a..15f4d28 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java
@@ -18,11 +18,15 @@
package org.apache.drill.exec.store.parquet.columnreaders;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+
import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.vector.Decimal18Vector;
+import org.apache.drill.exec.vector.Decimal9Vector;
import org.apache.drill.exec.vector.Float4Vector;
import org.apache.drill.exec.vector.Float8Vector;
import org.apache.drill.exec.vector.IntVector;
-
+import org.apache.drill.exec.vector.TimeStampVector;
+import org.apache.drill.exec.vector.TimeVector;
import parquet.column.ColumnDescriptor;
import parquet.format.SchemaElement;
import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -55,6 +59,58 @@ public class ParquetFixedWidthDictionaryReaders {
}
}
+ static class DictionaryDecimal9Reader extends FixedByteAlignedReader {
+
+ Decimal9Vector castedVector;
+
+ DictionaryDecimal9Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+ ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, Decimal9Vector v,
+ SchemaElement schemaElement) throws ExecutionSetupException {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+ castedVector = v;
+ }
+
+ // this method is called by its superclass during a read loop
+ @Override
+ protected void readField(long recordsToReadInThisPass) {
+
+ recordsReadInThisIteration = Math.min(pageReader.currentPage.getValueCount()
+ - pageReader.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
+
+ if (usingDictionary) {
+ for (int i = 0; i < recordsReadInThisIteration; i++){
+ castedVector.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readInteger());
+ }
+ }
+ }
+ }
+
+ static class DictionaryTimeReader extends FixedByteAlignedReader {
+
+ TimeVector castedVector;
+
+ DictionaryTimeReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+ ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, TimeVector v,
+ SchemaElement schemaElement) throws ExecutionSetupException {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+ castedVector = v;
+ }
+
+ // this method is called by its superclass during a read loop
+ @Override
+ protected void readField(long recordsToReadInThisPass) {
+
+ recordsReadInThisIteration = Math.min(pageReader.currentPage.getValueCount()
+ - pageReader.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
+
+ if (usingDictionary) {
+ for (int i = 0; i < recordsReadInThisIteration; i++){
+ castedVector.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readInteger());
+ }
+ }
+ }
+ }
+
static class DictionaryBigIntReader extends FixedByteAlignedReader {
BigIntVector castedVector;
@@ -83,6 +139,62 @@ public class ParquetFixedWidthDictionaryReaders {
}
}
+ static class DictionaryDecimal18Reader extends FixedByteAlignedReader {
+
+ Decimal18Vector castedVector;
+
+ DictionaryDecimal18Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+ ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, Decimal18Vector v,
+ SchemaElement schemaElement) throws ExecutionSetupException {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+ castedVector = v;
+ }
+
+ // this method is called by its superclass during a read loop
+ @Override
+ protected void readField(long recordsToReadInThisPass) {
+
+ recordsReadInThisIteration = Math.min(pageReader.currentPage.getValueCount()
+ - pageReader.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
+
+ for (int i = 0; i < recordsReadInThisIteration; i++){
+ try {
+ castedVector.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readLong());
+ } catch ( Exception ex) {
+ throw ex;
+ }
+ }
+ }
+ }
+
+ static class DictionaryTimeStampReader extends FixedByteAlignedReader {
+
+ TimeStampVector castedVector;
+
+ DictionaryTimeStampReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+ ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, TimeStampVector v,
+ SchemaElement schemaElement) throws ExecutionSetupException {
+ super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+ castedVector = v;
+ }
+
+ // this method is called by its superclass during a read loop
+ @Override
+ protected void readField(long recordsToReadInThisPass) {
+
+ recordsReadInThisIteration = Math.min(pageReader.currentPage.getValueCount()
+ - pageReader.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
+
+ for (int i = 0; i < recordsReadInThisIteration; i++){
+ try {
+ castedVector.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readLong());
+ } catch ( Exception ex) {
+ throw ex;
+ }
+ }
+ }
+ }
+
static class DictionaryFloat4Reader extends FixedByteAlignedReader {
Float4Vector castedVector;
http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
new file mode 100644
index 0000000..9ae6b78
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet.columnreaders;
+
+import org.apache.drill.BaseTestQuery;
+import org.junit.Test;
+
+public class TestColumnReaderFactory extends BaseTestQuery {
+
+ /**
+ * check if Time and TimeStamp are read correctly with dictionary encoding.
+ */
+ @Test
+ public void testTimeAndTimeStampWithDictionary() throws Exception {
+ // the file 'time_dictionary.parquet' uses a PLAIN_DICTIONARY encoding and contains 4 columns:
+ // time_opt: INT32/TIME_MILLIS/OPTIONAL
+ // time_req: INT32/TIME_MILLIS/REQUIRED
+ // timestampt_opt: INT64/TIMESTAMP_MILLIS/OPTIONAL
+ // timestampt_req: INT64/TIMESTAMP_MILLIS/REQUIRED
+
+ // query parquet file. We shouldn't get any exception
+ testNoResult("SELECT * FROM cp.`parquet/time_dictionary.parquet`");
+ }
+
+ /**
+ * check if Time and TimeStamp are read correctly with plain encoding.
+ */
+ @Test
+ public void testTimeAndTimeStampWithNoDictionary() throws Exception {
+ // the file 'time_dictionary.parquet' uses a PLAIN encoding and contains 4 columns:
+ // time_opt: INT32/TIME_MILLIS/OPTIONAL
+ // time_req: INT32/TIME_MILLIS/REQUIRED
+ // timestampt_opt: INT64/TIMESTAMP_MILLIS/OPTIONAL
+ // timestampt_req: INT64/TIMESTAMP_MILLIS/REQUIRED
+
+ // query parquet file. We shouldn't get any exception
+ testNoResult("SELECT * FROM cp.`parquet/time_nodictionary.parquet`");
+ }
+
+ /**
+ * check if Decimal9 and Decimal18 are read correctly with dictionary encoding.
+ */
+ @Test
+ public void testDecimal9AndDecimal18WithDictionary() throws Exception {
+ // the file 'decimal_dictionary.parquet' uses a PLAIN_DICTIONARY encoding and contains 4 columns:
+ // d9_opt: INT32/DECIMAL9/OPTIONAL
+ // d9_req: INT32/DECIMAL9/REQUIRED
+ // d18_opt: INT64/DECIMAL18/OPTIONAL
+ // d18_req: INT64/DECIMAL18/REQUIRED
+
+ // query parquet file. We shouldn't get any exception
+ testNoResult("SELECT * FROM cp.`parquet/decimal_dictionary.parquet`");
+ }
+
+ /**
+ * check if Decimal9 and Decimal18 are read correctly with plain encoding.
+ */
+ @Test
+ public void testDecimal9AndDecimal18WithNoDictionary() throws Exception {
+ // the file 'decimal_dictionary.parquet' uses a PLAIN encoding and contains 4 columns:
+ // d9_opt: INT32/DECIMAL9/OPTIONAL
+ // d9_req: INT32/DECIMAL9/REQUIRED
+ // d18_opt: INT64/DECIMAL18/OPTIONAL
+ // d18_req: INT64/DECIMAL18/REQUIRED
+
+ // query parquet file. We shouldn't get any exception
+ testNoResult("SELECT * FROM cp.`parquet/decimal_nodictionary.parquet`");
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/test/resources/parquet/decimal_dictionary.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/decimal_dictionary.parquet b/exec/java-exec/src/test/resources/parquet/decimal_dictionary.parquet
new file mode 100644
index 0000000..bf1bd9c
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/decimal_dictionary.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/test/resources/parquet/decimal_nodictionary.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/decimal_nodictionary.parquet b/exec/java-exec/src/test/resources/parquet/decimal_nodictionary.parquet
new file mode 100644
index 0000000..b9d24c5
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/decimal_nodictionary.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/test/resources/parquet/time_dictionary.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/time_dictionary.parquet b/exec/java-exec/src/test/resources/parquet/time_dictionary.parquet
new file mode 100644
index 0000000..5b75154
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/time_dictionary.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/test/resources/parquet/time_nodictionary.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/time_nodictionary.parquet b/exec/java-exec/src/test/resources/parquet/time_nodictionary.parquet
new file mode 100644
index 0000000..f78ff17
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/time_nodictionary.parquet differ
[4/5] drill git commit: DRILL-2266: Complex parquet reader fails on
reading timestamp datatype
Posted by sm...@apache.org.
DRILL-2266: Complex parquet reader fails on reading timestamp datatype
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/deba9463
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/deba9463
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/deba9463
Branch: refs/heads/master
Commit: deba94630b88afefd644f17cfe5d4122fb9eb456
Parents: 36fedf9
Author: adeneche <ad...@gmail.com>
Authored: Thu Feb 26 08:56:09 2015 -0800
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Fri Mar 6 17:42:16 2015 -0800
----------------------------------------------------------------------
.../drill/exec/store/parquet2/DrillParquetGroupConverter.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/deba9463/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
index d219927..1ec24e0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
@@ -340,7 +340,7 @@ public class DrillParquetGroupConverter extends GroupConverter {
}
@Override
- public void addInt(int value) {
+ public void addLong(long value) {
holder.value = value;
writer.write(holder);
}