You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by sm...@apache.org on 2015/03/07 04:18:15 UTC

[1/5] drill git commit: DRILL-2359: ClassPathFileSystem.getFileStatus() should throw FileNotFoundException when path doesn't exist

Repository: drill
Updated Branches:
  refs/heads/master fb293ba52 -> fe94970e1


DRILL-2359: ClassPathFileSystem.getFileStatus() should throw FileNotFoundException when path doesn't exist


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2dd8a2ca
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2dd8a2ca
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2dd8a2ca

Branch: refs/heads/master
Commit: 2dd8a2ca91ed7333946b543f2aa297529fb06195
Parents: fb293ba
Author: adeneche <ad...@gmail.com>
Authored: Mon Mar 2 16:03:40 2015 -0800
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Fri Mar 6 17:39:32 2015 -0800

----------------------------------------------------------------------
 .../apache/drill/exec/store/ClassPathFileSystem.java    | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/2dd8a2ca/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
index cab5339..dc5a4b5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
@@ -17,6 +17,7 @@
  */
 package org.apache.drill.exec.store;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -72,18 +73,15 @@ public class ClassPathFileSystem extends FileSystem{
   @Override
   public FileStatus getFileStatus(Path arg0) throws IOException {
     String file = getFileName(arg0);
+    URL url;
 
     try{
-    URL url = Resources.getResource(file);
-    if(url == null){
-      throw new IOException(String.format("Unable to find path %s.", arg0.toString()));
+      url = Resources.getResource(file);
+    }catch(IllegalArgumentException e){
+      throw new FileNotFoundException(String.format("Unable to find path %s.", arg0.toString()));
     }
 
-
     return new FileStatus(Resources.asByteSource(url).size(), false, 1, 8096, System.currentTimeMillis(), arg0);
-    }catch(RuntimeException e){
-      throw new IOException(String.format("Failure trying to load file %s", arg0), e);
-    }
   }
 
   @Override


[3/5] drill git commit: DRILL-2267: bumping parquet version to 1.5.1-r7

Posted by sm...@apache.org.
DRILL-2267: bumping parquet version to 1.5.1-r7


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/36fedf99
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/36fedf99
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/36fedf99

Branch: refs/heads/master
Commit: 36fedf99aaad01189479952ccbf8d3158684166f
Parents: 6af2dd4
Author: adeneche <ad...@gmail.com>
Authored: Thu Feb 26 07:55:03 2015 -0800
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Fri Mar 6 17:41:46 2015 -0800

----------------------------------------------------------------------
 exec/java-exec/pom.xml | 12 ++++++------
 pom.xml                |  1 +
 2 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/36fedf99/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index a7b6a67..4b9b24f 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -146,7 +146,7 @@
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>parquet-column</artifactId>
-      <version>1.5.1-drill-r6</version>
+      <version>${parquet.version}</version>
       <exclusions>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>
@@ -161,7 +161,7 @@
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>parquet-hadoop</artifactId>
-      <version>1.5.1-drill-r4</version>
+      <version>${parquet.version}</version>
       <exclusions>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>
@@ -191,7 +191,7 @@
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>parquet-common</artifactId>
-      <version>1.5.1-drill-r4</version>
+      <version>${parquet.version}</version>
       <exclusions>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>
@@ -206,7 +206,7 @@
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>parquet-jackson</artifactId>
-      <version>1.5.1-drill-r4</version>
+      <version>${parquet.version}</version>
       <exclusions>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>
@@ -221,7 +221,7 @@
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>parquet-encoding</artifactId>
-      <version>1.5.1-drill-r4</version>
+      <version>${parquet.version}</version>
       <exclusions>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>
@@ -236,7 +236,7 @@
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>parquet-generator</artifactId>
-      <version>1.5.1-drill-r4</version>
+      <version>${parquet.version}</version>
       <exclusions>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>

http://git-wip-us.apache.org/repos/asf/drill/blob/36fedf99/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 525579a..1f08574 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,6 +33,7 @@
     <proto.cas.path>${project.basedir}/src/main/protobuf/</proto.cas.path>
     <dep.junit.version>4.11</dep.junit.version>
     <dep.slf4j.version>1.7.5</dep.slf4j.version>
+    <parquet.version>1.5.1-drill-r7</parquet.version>
   </properties>
 
   <scm>


[2/5] drill git commit: DRILL-2337: disable dictionary encoding by default

Posted by sm...@apache.org.
DRILL-2337: disable dictionary encoding by default


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/6af2dd4a
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/6af2dd4a
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/6af2dd4a

Branch: refs/heads/master
Commit: 6af2dd4a37987cbc3d2f9efe5b352cf9bc282adf
Parents: 2dd8a2c
Author: adeneche <ad...@gmail.com>
Authored: Fri Feb 27 20:14:00 2015 +0000
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Fri Mar 6 17:41:00 2015 -0800

----------------------------------------------------------------------
 .../src/main/java/org/apache/drill/exec/ExecConstants.java         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/6af2dd4a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 40fe588..33b05af 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -91,7 +91,7 @@ public interface ExecConstants {
       PARQUET_WRITER_COMPRESSION_TYPE, "snappy", "snappy", "gzip", "none");
   public static final String PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING = "store.parquet.enable_dictionary_encoding";
   public static final OptionValidator PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING_VALIDATOR = new BooleanValidator(
-      PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, true);
+      PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING, false);
 
   public static final String PARQUET_VECTOR_FILL_THRESHOLD = "store.parquet.vector_fill_threshold";
   public static final OptionValidator PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_THRESHOLD, 99l, 85l);


[5/5] drill git commit: DRILL-2262: selecting columns of certain datatypes from a dictionary encoded parquet file created by drill fails

Posted by sm...@apache.org.
DRILL-2262: selecting columns of certain datatypes from a dictionary encoded parquet file created by drill fails


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/fe94970e
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/fe94970e
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/fe94970e

Branch: refs/heads/master
Commit: fe94970e1fbfd6ceacbfc7f2187c5495d497e791
Parents: deba946
Author: adeneche <ad...@gmail.com>
Authored: Thu Feb 26 19:55:08 2015 -0800
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Fri Mar 6 18:24:38 2015 -0800

----------------------------------------------------------------------
 .../columnreaders/ColumnReaderFactory.java      |  88 ++++++++++----
 .../NullableFixedByteAlignedReaders.java        |  94 +++++++++++++++
 .../ParquetFixedWidthDictionaryReaders.java     | 114 ++++++++++++++++++-
 .../columnreaders/TestColumnReaderFactory.java  |  84 ++++++++++++++
 .../parquet/decimal_dictionary.parquet          | Bin 0 -> 602 bytes
 .../parquet/decimal_nodictionary.parquet        | Bin 0 -> 558 bytes
 .../resources/parquet/time_dictionary.parquet   | Bin 0 -> 618 bytes
 .../resources/parquet/time_nodictionary.parquet | Bin 0 -> 576 bytes
 8 files changed, 356 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
index cbdc0b0..70b2342 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
@@ -19,25 +19,33 @@ package org.apache.drill.exec.store.parquet.columnreaders;
 
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.exec.exception.SchemaChangeException;
+
 import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.vector.Decimal18Vector;
 import org.apache.drill.exec.vector.Decimal28SparseVector;
 import org.apache.drill.exec.vector.Decimal38SparseVector;
+import org.apache.drill.exec.vector.Decimal9Vector;
 import org.apache.drill.exec.vector.Float4Vector;
 import org.apache.drill.exec.vector.Float8Vector;
 import org.apache.drill.exec.vector.IntVector;
 import org.apache.drill.exec.vector.NullableBigIntVector;
+import org.apache.drill.exec.vector.NullableDecimal18Vector;
 import org.apache.drill.exec.vector.NullableDecimal28SparseVector;
 import org.apache.drill.exec.vector.NullableDecimal38SparseVector;
+import org.apache.drill.exec.vector.NullableDecimal9Vector;
 import org.apache.drill.exec.vector.NullableFloat4Vector;
 import org.apache.drill.exec.vector.NullableFloat8Vector;
 import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableTimeStampVector;
+import org.apache.drill.exec.vector.NullableTimeVector;
 import org.apache.drill.exec.vector.NullableVarBinaryVector;
 import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.TimeStampVector;
+import org.apache.drill.exec.vector.TimeVector;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.VarBinaryVector;
 import org.apache.drill.exec.vector.VarCharVector;
 import org.apache.drill.exec.vector.VariableWidthVector;
-
 import parquet.column.ColumnDescriptor;
 import parquet.column.Encoding;
 import parquet.format.ConvertedType;
@@ -83,9 +91,29 @@ public class ColumnReaderFactory {
         if (columnChunkMetaData.getEncodings().contains(Encoding.PLAIN_DICTIONARY)) {
           switch (columnChunkMetaData.getType()) {
             case INT32:
-              return new ParquetFixedWidthDictionaryReaders.DictionaryIntReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (IntVector) v, schemaElement);
+              if (convertedType == null) {
+                return new ParquetFixedWidthDictionaryReaders.DictionaryIntReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (IntVector) v, schemaElement);
+              }
+              switch (convertedType) {
+                case DECIMAL:
+                  return new ParquetFixedWidthDictionaryReaders.DictionaryDecimal9Reader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Decimal9Vector) v, schemaElement);
+                case TIME_MILLIS:
+                  return new ParquetFixedWidthDictionaryReaders.DictionaryTimeReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (TimeVector) v, schemaElement);
+                default:
+                  throw new ExecutionSetupException("Unsupported dictionary converted type " + convertedType + " for primitive type INT32");
+              }
             case INT64:
-              return new ParquetFixedWidthDictionaryReaders.DictionaryBigIntReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (BigIntVector) v, schemaElement);
+              if (convertedType == null) {
+                return new ParquetFixedWidthDictionaryReaders.DictionaryBigIntReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (BigIntVector) v, schemaElement);
+              }
+              switch (convertedType) {
+                case DECIMAL:
+                  return new ParquetFixedWidthDictionaryReaders.DictionaryDecimal18Reader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Decimal18Vector) v, schemaElement);
+                case TIMESTAMP_MILLIS:
+                  return new ParquetFixedWidthDictionaryReaders.DictionaryTimeStampReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (TimeStampVector) v, schemaElement);
+                default:
+                  throw new ExecutionSetupException("Unsupported dictionary converted type " + convertedType + " for primitive type INT64");
+              }
             case FLOAT:
               return new ParquetFixedWidthDictionaryReaders.DictionaryFloat4Reader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Float4Vector) v, schemaElement);
             case DOUBLE:
@@ -169,28 +197,42 @@ public class ColumnReaderFactory {
                                                              boolean fixedLength,
                                                              ValueVector valueVec,
                                                              SchemaElement schemaElement) throws ExecutionSetupException {
+    ConvertedType convertedType = schemaElement.getConverted_type();
+
     if (! columnChunkMetaData.getEncodings().contains(Encoding.PLAIN_DICTIONARY)) {
-      return new NullableFixedByteAlignedReaders.NullableFixedByteAlignedReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData,
-          fixedLength, valueVec, schemaElement);
+      return new NullableFixedByteAlignedReaders.NullableFixedByteAlignedReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, valueVec, schemaElement);
     } else {
-      if (columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.INT64) {
-        return new NullableFixedByteAlignedReaders.NullableDictionaryBigIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData,
-            fixedLength, (NullableBigIntVector)valueVec, schemaElement);
-      }
-      else if (columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.INT32) {
-        return new NullableFixedByteAlignedReaders.NullableDictionaryIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData,
-            fixedLength, (NullableIntVector)valueVec, schemaElement);
-      }
-      else if (columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.FLOAT) {
-        return new NullableFixedByteAlignedReaders.NullableDictionaryFloat4Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData,
-            fixedLength, (NullableFloat4Vector)valueVec, schemaElement);
-      }
-      else if (columnDescriptor.getType() == PrimitiveType.PrimitiveTypeName.DOUBLE) {
-        return new NullableFixedByteAlignedReaders.NullableDictionaryFloat8Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData,
-            fixedLength, (NullableFloat8Vector)valueVec, schemaElement);
-      }
-      else{
-        throw new ExecutionSetupException("Unsupported nullable column type " + columnDescriptor.getType().name() );
+      switch (columnDescriptor.getType()) {
+        case INT32:
+          if (convertedType == null) {
+            return new NullableFixedByteAlignedReaders.NullableDictionaryIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableIntVector) valueVec, schemaElement);
+          }
+          switch (convertedType) {
+            case DECIMAL:
+              return new NullableFixedByteAlignedReaders.NullableDictionaryDecimal9Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableDecimal9Vector) valueVec, schemaElement);
+            case TIME_MILLIS:
+              return new NullableFixedByteAlignedReaders.NullableDictionaryTimeReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableTimeVector)valueVec, schemaElement);
+            default:
+              throw new ExecutionSetupException("Unsupported nullable converted type " + convertedType + " for primitive type INT32");
+          }
+        case INT64:
+          if (convertedType == null) {
+            return new NullableFixedByteAlignedReaders.NullableDictionaryBigIntReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableBigIntVector)valueVec, schemaElement);
+          }
+          switch (convertedType) {
+            case DECIMAL:
+              return new NullableFixedByteAlignedReaders.NullableDictionaryDecimal18Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableDecimal18Vector)valueVec, schemaElement);
+            case TIMESTAMP_MILLIS:
+              return new NullableFixedByteAlignedReaders.NullableDictionaryTimeStampReader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableTimeStampVector)valueVec, schemaElement);
+            default:
+              throw new ExecutionSetupException("Unsupported nullable converted type " + convertedType + " for primitive type INT64");
+          }
+        case FLOAT:
+          return new NullableFixedByteAlignedReaders.NullableDictionaryFloat4Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableFloat4Vector)valueVec, schemaElement);
+        case DOUBLE:
+          return new NullableFixedByteAlignedReaders.NullableDictionaryFloat8Reader(parentReader, allocateSize, columnDescriptor, columnChunkMetaData, fixedLength, (NullableFloat8Vector)valueVec, schemaElement);
+        default:
+          throw new ExecutionSetupException("Unsupported nullable column type " + columnDescriptor.getType().name() );
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
index 707bc9c..8087118 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
@@ -34,6 +34,10 @@ import org.apache.drill.exec.vector.NullableDecimal38SparseVector;
 import org.apache.drill.exec.vector.NullableFloat4Vector;
 import org.apache.drill.exec.vector.NullableFloat8Vector;
 import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableDecimal9Vector;
+import org.apache.drill.exec.vector.NullableDecimal18Vector;
+import org.apache.drill.exec.vector.NullableTimeVector;
+import org.apache.drill.exec.vector.NullableTimeStampVector;
 import org.apache.drill.exec.vector.ValueVector;
 import org.joda.time.DateTimeUtils;
 
@@ -84,6 +88,52 @@ public class NullableFixedByteAlignedReaders {
     }
   }
 
+  static class NullableDictionaryDecimal9Reader extends NullableColumnReader<NullableDecimal9Vector> {
+
+    NullableDictionaryDecimal9Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+                                ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableDecimal9Vector v,
+                                SchemaElement schemaElement) throws ExecutionSetupException {
+      super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+    }
+
+    // this method is called by its superclass during a read loop
+    @Override
+    protected void readField(long recordsToReadInThisPass) {
+      if (usingDictionary) {
+        for (int i = 0; i < recordsToReadInThisPass; i++){
+          valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readInteger());
+        }
+      } else {
+        for (int i = 0; i < recordsToReadInThisPass; i++){
+          valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.valueReader.readInteger());
+        }
+      }
+    }
+  }
+
+  static class NullableDictionaryTimeReader extends NullableColumnReader<NullableTimeVector> {
+
+    NullableDictionaryTimeReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+                                     ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableTimeVector v,
+                                     SchemaElement schemaElement) throws ExecutionSetupException {
+      super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+    }
+
+    // this method is called by its superclass during a read loop
+    @Override
+    protected void readField(long recordsToReadInThisPass) {
+      if (usingDictionary) {
+        for (int i = 0; i < recordsToReadInThisPass; i++){
+          valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readInteger());
+        }
+      } else {
+        for (int i = 0; i < recordsToReadInThisPass; i++){
+          valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.valueReader.readInteger());
+        }
+      }
+    }
+  }
+
   static class NullableDictionaryBigIntReader extends NullableColumnReader<NullableBigIntVector> {
 
     NullableDictionaryBigIntReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
@@ -107,6 +157,50 @@ public class NullableFixedByteAlignedReaders {
     }
   }
 
+  static class NullableDictionaryTimeStampReader extends NullableColumnReader<NullableTimeStampVector> {
+
+    NullableDictionaryTimeStampReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+                                   ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableTimeStampVector v,
+                                   SchemaElement schemaElement) throws ExecutionSetupException {
+      super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+    }
+
+    // this method is called by its superclass during a read loop
+    @Override
+    protected void readField(long recordsToReadInThisPass) {
+      if (usingDictionary) {
+        for (int i = 0; i < recordsToReadInThisPass; i++){
+          valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readLong());
+        }
+      } else {
+        for (int i = 0; i < recordsToReadInThisPass; i++){
+          valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.valueReader.readLong());
+        }
+      }
+    }
+  }
+  static class NullableDictionaryDecimal18Reader extends NullableColumnReader<NullableDecimal18Vector> {
+
+    NullableDictionaryDecimal18Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+                                   ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableDecimal18Vector v,
+                                   SchemaElement schemaElement) throws ExecutionSetupException {
+      super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+    }
+
+    // this method is called by its superclass during a read loop
+    @Override
+    protected void readField(long recordsToReadInThisPass) {
+      if (usingDictionary) {
+        for (int i = 0; i < recordsToReadInThisPass; i++){
+          valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readLong());
+        }
+      } else {
+        for (int i = 0; i < recordsToReadInThisPass; i++){
+          valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.valueReader.readLong());
+        }
+      }
+    }
+  }
   static class NullableDictionaryFloat4Reader extends NullableColumnReader<NullableFloat4Vector> {
 
     NullableDictionaryFloat4Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,

http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java
index 02b583a..15f4d28 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java
@@ -18,11 +18,15 @@
 package org.apache.drill.exec.store.parquet.columnreaders;
 
 import org.apache.drill.common.exceptions.ExecutionSetupException;
+
 import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.vector.Decimal18Vector;
+import org.apache.drill.exec.vector.Decimal9Vector;
 import org.apache.drill.exec.vector.Float4Vector;
 import org.apache.drill.exec.vector.Float8Vector;
 import org.apache.drill.exec.vector.IntVector;
-
+import org.apache.drill.exec.vector.TimeStampVector;
+import org.apache.drill.exec.vector.TimeVector;
 import parquet.column.ColumnDescriptor;
 import parquet.format.SchemaElement;
 import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -55,6 +59,58 @@ public class ParquetFixedWidthDictionaryReaders {
     }
   }
 
+  static class DictionaryDecimal9Reader extends FixedByteAlignedReader {
+
+    Decimal9Vector castedVector;
+
+    DictionaryDecimal9Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+                        ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, Decimal9Vector v,
+                        SchemaElement schemaElement) throws ExecutionSetupException {
+      super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+      castedVector = v;
+    }
+
+    // this method is called by its superclass during a read loop
+    @Override
+    protected void readField(long recordsToReadInThisPass) {
+
+      recordsReadInThisIteration = Math.min(pageReader.currentPage.getValueCount()
+        - pageReader.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
+
+      if (usingDictionary) {
+        for (int i = 0; i < recordsReadInThisIteration; i++){
+          castedVector.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readInteger());
+        }
+      }
+    }
+  }
+
+  static class DictionaryTimeReader extends FixedByteAlignedReader {
+
+    TimeVector castedVector;
+
+    DictionaryTimeReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+                        ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, TimeVector v,
+                        SchemaElement schemaElement) throws ExecutionSetupException {
+      super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+      castedVector = v;
+    }
+
+    // this method is called by its superclass during a read loop
+    @Override
+    protected void readField(long recordsToReadInThisPass) {
+
+      recordsReadInThisIteration = Math.min(pageReader.currentPage.getValueCount()
+        - pageReader.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
+
+      if (usingDictionary) {
+        for (int i = 0; i < recordsReadInThisIteration; i++){
+          castedVector.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readInteger());
+        }
+      }
+    }
+  }
+
   static class DictionaryBigIntReader extends FixedByteAlignedReader {
 
     BigIntVector castedVector;
@@ -83,6 +139,62 @@ public class ParquetFixedWidthDictionaryReaders {
     }
   }
 
+  static class DictionaryDecimal18Reader extends FixedByteAlignedReader {
+
+    Decimal18Vector castedVector;
+
+    DictionaryDecimal18Reader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+                           ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, Decimal18Vector v,
+                           SchemaElement schemaElement) throws ExecutionSetupException {
+      super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+      castedVector = v;
+    }
+
+    // this method is called by its superclass during a read loop
+    @Override
+    protected void readField(long recordsToReadInThisPass) {
+
+      recordsReadInThisIteration = Math.min(pageReader.currentPage.getValueCount()
+        - pageReader.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
+
+      for (int i = 0; i < recordsReadInThisIteration; i++){
+        try {
+          castedVector.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readLong());
+        } catch ( Exception ex) {
+          throw ex;
+        }
+      }
+    }
+  }
+
+  static class DictionaryTimeStampReader extends FixedByteAlignedReader {
+
+    TimeStampVector castedVector;
+
+    DictionaryTimeStampReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
+                           ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, TimeStampVector v,
+                           SchemaElement schemaElement) throws ExecutionSetupException {
+      super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
+      castedVector = v;
+    }
+
+    // this method is called by its superclass during a read loop
+    @Override
+    protected void readField(long recordsToReadInThisPass) {
+
+      recordsReadInThisIteration = Math.min(pageReader.currentPage.getValueCount()
+        - pageReader.valuesRead, recordsToReadInThisPass - valuesReadInCurrentPass);
+
+      for (int i = 0; i < recordsReadInThisIteration; i++){
+        try {
+          castedVector.getMutator().setSafe(valuesReadInCurrentPass + i, pageReader.dictionaryValueReader.readLong());
+        } catch ( Exception ex) {
+          throw ex;
+        }
+      }
+    }
+  }
+
   static class DictionaryFloat4Reader extends FixedByteAlignedReader {
 
     Float4Vector castedVector;

http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
new file mode 100644
index 0000000..9ae6b78
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet.columnreaders;
+
+import org.apache.drill.BaseTestQuery;
+import org.junit.Test;
+
+public class TestColumnReaderFactory extends BaseTestQuery {
+
+  /**
+   * check if Time and TimeStamp are read correctly with dictionary encoding.
+   */
+  @Test
+  public void testTimeAndTimeStampWithDictionary() throws Exception {
+    // the file 'time_dictionary.parquet' uses a PLAIN_DICTIONARY encoding and contains 4 columns:
+    // time_opt: INT32/TIME_MILLIS/OPTIONAL
+    // time_req: INT32/TIME_MILLIS/REQUIRED
+    // timestampt_opt: INT64/TIMESTAMP_MILLIS/OPTIONAL
+    // timestampt_req: INT64/TIMESTAMP_MILLIS/REQUIRED
+
+    // query parquet file. We shouldn't get any exception
+    testNoResult("SELECT * FROM cp.`parquet/time_dictionary.parquet`");
+  }
+
+  /**
+   * check if Time and TimeStamp are read correctly with plain encoding.
+   */
+  @Test
+  public void testTimeAndTimeStampWithNoDictionary() throws Exception {
+    // the file 'time_dictionary.parquet' uses a PLAIN encoding and contains 4 columns:
+    // time_opt: INT32/TIME_MILLIS/OPTIONAL
+    // time_req: INT32/TIME_MILLIS/REQUIRED
+    // timestampt_opt: INT64/TIMESTAMP_MILLIS/OPTIONAL
+    // timestampt_req: INT64/TIMESTAMP_MILLIS/REQUIRED
+
+    // query parquet file. We shouldn't get any exception
+    testNoResult("SELECT * FROM cp.`parquet/time_nodictionary.parquet`");
+  }
+
+  /**
+   * check if Decimal9 and Decimal18 are read correctly with dictionary encoding.
+   */
+  @Test
+  public void testDecimal9AndDecimal18WithDictionary() throws Exception {
+    // the file 'decimal_dictionary.parquet' uses a PLAIN_DICTIONARY encoding and contains 4 columns:
+    // d9_opt: INT32/DECIMAL9/OPTIONAL
+    // d9_req: INT32/DECIMAL9/REQUIRED
+    // d18_opt: INT64/DECIMAL18/OPTIONAL
+    // d18_req: INT64/DECIMAL18/REQUIRED
+
+    // query parquet file. We shouldn't get any exception
+    testNoResult("SELECT * FROM cp.`parquet/decimal_dictionary.parquet`");
+  }
+
+  /**
+   * check if Decimal9 and Decimal18 are read correctly with plain encoding.
+   */
+  @Test
+  public void testDecimal9AndDecimal18WithNoDictionary() throws Exception {
+    // the file 'decimal_dictionary.parquet' uses a PLAIN encoding and contains 4 columns:
+    // d9_opt: INT32/DECIMAL9/OPTIONAL
+    // d9_req: INT32/DECIMAL9/REQUIRED
+    // d18_opt: INT64/DECIMAL18/OPTIONAL
+    // d18_req: INT64/DECIMAL18/REQUIRED
+
+    // query parquet file. We shouldn't get any exception
+    testNoResult("SELECT * FROM cp.`parquet/decimal_nodictionary.parquet`");
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/test/resources/parquet/decimal_dictionary.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/decimal_dictionary.parquet b/exec/java-exec/src/test/resources/parquet/decimal_dictionary.parquet
new file mode 100644
index 0000000..bf1bd9c
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/decimal_dictionary.parquet differ

http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/test/resources/parquet/decimal_nodictionary.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/decimal_nodictionary.parquet b/exec/java-exec/src/test/resources/parquet/decimal_nodictionary.parquet
new file mode 100644
index 0000000..b9d24c5
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/decimal_nodictionary.parquet differ

http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/test/resources/parquet/time_dictionary.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/time_dictionary.parquet b/exec/java-exec/src/test/resources/parquet/time_dictionary.parquet
new file mode 100644
index 0000000..5b75154
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/time_dictionary.parquet differ

http://git-wip-us.apache.org/repos/asf/drill/blob/fe94970e/exec/java-exec/src/test/resources/parquet/time_nodictionary.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/time_nodictionary.parquet b/exec/java-exec/src/test/resources/parquet/time_nodictionary.parquet
new file mode 100644
index 0000000..f78ff17
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/time_nodictionary.parquet differ


[4/5] drill git commit: DRILL-2266: Complex parquet reader fails on reading timestamp datatype

Posted by sm...@apache.org.
DRILL-2266: Complex parquet reader fails on reading timestamp datatype


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/deba9463
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/deba9463
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/deba9463

Branch: refs/heads/master
Commit: deba94630b88afefd644f17cfe5d4122fb9eb456
Parents: 36fedf9
Author: adeneche <ad...@gmail.com>
Authored: Thu Feb 26 08:56:09 2015 -0800
Committer: Steven Phillips <sp...@maprtech.com>
Committed: Fri Mar 6 17:42:16 2015 -0800

----------------------------------------------------------------------
 .../drill/exec/store/parquet2/DrillParquetGroupConverter.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/deba9463/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
index d219927..1ec24e0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
@@ -340,7 +340,7 @@ public class DrillParquetGroupConverter extends GroupConverter {
     }
 
     @Override
-    public void addInt(int value) {
+    public void addLong(long value) {
       holder.value = value;
       writer.write(holder);
     }