You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2017/01/24 06:15:23 UTC
[3/5] drill git commit: DRILL-5097: Using
store.parquet.reader.int96_as_timestamp gives IOOB whereas convert_from works
DRILL-5097: Using store.parquet.reader.int96_as_timestamp gives IOOB whereas convert_from works
close apache/drill#697
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/08ca5e09
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/08ca5e09
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/08ca5e09
Branch: refs/heads/master
Commit: 08ca5e0923436ebeb3e190c60892167d9beab0c2
Parents: bb29f19
Author: Vitalii Diravka <vi...@gmail.com>
Authored: Wed Dec 14 16:24:08 2016 +0000
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Mon Jan 23 17:08:20 2017 -0800
----------------------------------------------------------------------
.../NullableFixedByteAlignedReaders.java | 10 +++++++++-
.../physical/impl/writer/TestParquetWriter.java | 19 +++++++++++++++++++
.../test/resources/parquet/data.snappy.parquet | Bin 0 -> 59200 bytes
3 files changed, 28 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/08ca5e09/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
index e20504f..b233a65 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java
@@ -110,9 +110,14 @@ public class NullableFixedByteAlignedReaders {
/**
* Class for reading parquet fixed binary type INT96, which is used for storing hive,
- * impala timestamp values with nanoseconds precision. So it reads such values as a drill timestamp.
+ * impala timestamp values with nanoseconds precision (12 bytes). So it reads such values as a drill timestamp (8 bytes).
*/
static class NullableFixedBinaryAsTimeStampReader extends NullableFixedByteAlignedReader<NullableTimeStampVector> {
+ /**
+ * The width of each element of the TimeStampVector is 8 byte(s).
+ */
+ private static final int TIMESTAMP_LENGTH_IN_BITS = 64;
+
NullableFixedBinaryAsTimeStampReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor,
ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableTimeStampVector v, SchemaElement schemaElement) throws ExecutionSetupException {
super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
@@ -132,6 +137,9 @@ public class NullableFixedByteAlignedReaders {
valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, getDateTimeValueFromBinary(binaryTimeStampValue));
}
}
+ // The nanos precision is cut to millis. Therefore the length of single timestamp value is 8 bytes(s)
+ // instead of 12 byte(s).
+ dataTypeLengthInBits = TIMESTAMP_LENGTH_IN_BITS;
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/08ca5e09/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index ae0e699..362c943 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -25,6 +25,7 @@ import java.io.File;
import java.io.FileWriter;
import java.math.BigDecimal;
import java.sql.Date;
+import java.sql.Timestamp;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collection;
@@ -963,5 +964,23 @@ public class TestParquetWriter extends BaseTestQuery {
}
}
+ @Test // DRILL-5097
+ public void testInt96TimeStampValueWidth() throws Exception {
+ try {
+ testBuilder()
+ .ordered()
+ .sqlQuery("select c, d from cp.`parquet/data.snappy.parquet` where d = '2015-07-18 13:52:51'")
+ .optionSettingQueriesForTestQuery(
+ "alter session set `%s` = true", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP)
+ .baselineColumns("c", "d")
+ .baselineValues(new DateTime(Date.valueOf("2011-04-11").getTime()),
+ new DateTime(Timestamp.valueOf("2015-07-18 13:52:51").getTime()))
+ .build()
+ .run();
+ } finally {
+ test("alter system reset `%s`", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/08ca5e09/exec/java-exec/src/test/resources/parquet/data.snappy.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/data.snappy.parquet b/exec/java-exec/src/test/resources/parquet/data.snappy.parquet
new file mode 100644
index 0000000..1ce3d75
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/data.snappy.parquet differ