You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by GitBox <gi...@apache.org> on 2018/08/18 14:04:10 UTC
[GitHub] asfgit closed pull request #1428: DRILL-6670: align Parquet
TIMESTAMP_MICROS logical type handling with earlier versions + minor fixes
asfgit closed pull request #1428: DRILL-6670: align Parquet TIMESTAMP_MICROS logical type handling with earlier versions + minor fixes
URL: https://github.com/apache/drill/pull/1428
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetScanBatchCreator.java
index 47f2e18a9a7..e0ff85d4c3c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetScanBatchCreator.java
@@ -106,11 +106,21 @@ protected ScanBatch getBatch(ExecutorFragmentContext context, AbstractParquetRow
ParquetReaderUtility.detectCorruptDates(footer, rowGroupScan.getColumns(), autoCorrectCorruptDates);
logger.debug("Contains corrupt dates: {}", containsCorruptDates);
- if (!context.getOptions().getBoolean(ExecConstants.PARQUET_NEW_RECORD_READER)
- && !ParquetReaderUtility.containsComplexColumn(footer, rowGroupScan.getColumns())) {
- logger.debug("Query {} qualifies for new Parquet reader",
- QueryIdHelper.getQueryId(oContext.getFragmentContext().getHandle().getQueryId()));
- readers.add(new ParquetRecordReader(context,
+ boolean useNewReader = context.getOptions().getBoolean(ExecConstants.PARQUET_NEW_RECORD_READER);
+ boolean containsComplexColumn = ParquetReaderUtility.containsComplexColumn(footer, rowGroupScan.getColumns());
+ logger.debug("PARQUET_NEW_RECORD_READER is {}. Complex columns {}.", useNewReader ? "enabled" : "disabled",
+ containsComplexColumn ? "found." : "not found.");
+ RecordReader reader;
+
+ if (useNewReader || containsComplexColumn) {
+ reader = new DrillParquetReader(context,
+ footer,
+ rowGroup,
+ columnExplorer.getTableColumns(),
+ fs,
+ containsCorruptDates);
+ } else {
+ reader = new ParquetRecordReader(context,
rowGroup.getPath(),
rowGroup.getRowGroupIndex(),
rowGroup.getNumRecordsToRead(),
@@ -118,18 +128,14 @@ protected ScanBatch getBatch(ExecutorFragmentContext context, AbstractParquetRow
CodecFactory.createDirectCodecFactory(fs.getConf(), new ParquetDirectByteBufferAllocator(oContext.getAllocator()), 0),
footer,
rowGroupScan.getColumns(),
- containsCorruptDates));
- } else {
- logger.debug("Query {} doesn't qualify for new reader, using old one",
- QueryIdHelper.getQueryId(oContext.getFragmentContext().getHandle().getQueryId()));
- readers.add(new DrillParquetReader(context,
- footer,
- rowGroup,
- columnExplorer.getTableColumns(),
- fs,
- containsCorruptDates));
+ containsCorruptDates);
}
+ logger.debug("Query {} uses {}",
+ QueryIdHelper.getQueryId(oContext.getFragmentContext().getHandle().getQueryId()),
+ reader.getClass().getSimpleName());
+ readers.add(reader);
+
List<String> partitionValues = rowGroupScan.getPartitionValues(rowGroup);
Map<String, String> implicitValues = columnExplorer.populateImplicitColumns(rowGroup.getPath(), partitionValues, rowGroupScan.supportsFileImplicitColumns());
implicitColumns.add(implicitValues);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
index 798d3c27306..03d5382fcda 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java
@@ -77,6 +77,10 @@
} else if (!columnChunkMetaData.getEncodings().contains(Encoding.PLAIN_DICTIONARY) && (
columnChunkMetaData.getType() == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY
|| columnChunkMetaData.getType() == PrimitiveType.PrimitiveTypeName.INT96)) {
+ if (convertedType == null) {
+ return new FixedByteAlignedReader.FixedBinaryReader(recordReader, descriptor,
+ columnChunkMetaData, (VariableWidthVector) v, schemaElement);
+ }
switch (convertedType) {
case DECIMAL:
return new FixedByteAlignedReader.VarDecimalReader(recordReader, descriptor,
@@ -131,7 +135,9 @@
return new ParquetFixedWidthDictionaryReaders.DictionaryBigIntReader(recordReader, descriptor, columnChunkMetaData, fixedLength, (BigIntVector) v, schemaElement);
}
switch (convertedType) {
+ // DRILL-6670: handle TIMESTAMP_MICROS as INT64 with no logical type
case INT_64:
+ case TIMESTAMP_MICROS:
return new ParquetFixedWidthDictionaryReaders.DictionaryBigIntReader(recordReader, descriptor, columnChunkMetaData, fixedLength, (BigIntVector) v, schemaElement);
case UINT_64:
return new ParquetFixedWidthDictionaryReaders.DictionaryUInt8Reader(recordReader, descriptor, columnChunkMetaData, fixedLength, (UInt8Vector) v, schemaElement);
@@ -291,6 +297,9 @@
columnDescriptor, columnChunkMetaData, fixedLength, (NullableVarDecimalVector) valueVec, schemaElement);
case TIMESTAMP_MILLIS:
return new NullableFixedByteAlignedReaders.NullableDictionaryTimeStampReader(parentReader, columnDescriptor, columnChunkMetaData, fixedLength, (NullableTimeStampVector)valueVec, schemaElement);
+ // DRILL-6670: handle TIMESTAMP_MICROS as INT64 with no logical type
+ case TIMESTAMP_MICROS:
+ return new NullableFixedByteAlignedReaders.NullableDictionaryBigIntReader(parentReader, columnDescriptor, columnChunkMetaData, fixedLength, (NullableBigIntVector)valueVec, schemaElement);
default:
throw new ExecutionSetupException("Unsupported nullable converted type " + convertedType + " for primitive type INT64");
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
index cb688fdc891..3a8c254a4f3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
@@ -54,16 +54,15 @@
return (TypeProtos.MinorType.BIGINT);
}
switch(convertedType) {
+ // DRILL-6670: handle TIMESTAMP_MICROS as INT64 with no logical type
case INT_64:
+ case TIMESTAMP_MICROS:
return TypeProtos.MinorType.BIGINT;
case UINT_64:
return TypeProtos.MinorType.UINT8;
case DECIMAL:
ParquetReaderUtility.checkDecimalTypeEnabled(options);
return TypeProtos.MinorType.VARDECIMAL;
- // TODO - add this back if it is decided to be added upstream, was removed form our pull request July 2014
-// case TIME_MICROS:
-// throw new UnsupportedOperationException();
case TIMESTAMP_MILLIS:
return TypeProtos.MinorType.TIMESTAMP;
default:
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
index 59302d067a8..d1acd8098f9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
@@ -219,8 +219,10 @@ private PrimitiveConverter getConverterForType(String name, PrimitiveType type)
return new DrillBigIntConverter(writer);
}
switch(type.getOriginalType()) {
+ // DRILL-6670: handle TIMESTAMP_MICROS as INT64 with no logical type
case UINT_64:
- case INT_64 : {
+ case INT_64 :
+ case TIMESTAMP_MICROS: {
BigIntWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).bigInt() : mapWriter.bigInt(name);
return new DrillBigIntConverter(writer);
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetSimpleTestFileGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetSimpleTestFileGenerator.java
index 2e6923a622f..e399ee75982 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetSimpleTestFileGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetSimpleTestFileGenerator.java
@@ -84,7 +84,7 @@
" required int32 _TIME_MILLIS_int32 ( TIME_MILLIS ) ; \n" +
// " required int64 _TIME_MICROS_int64 ( TIME_MICROS ) ; \n" +
" required int64 _TIMESTAMP_MILLIS_int64 ( TIMESTAMP_MILLIS ) ; \n" +
- // " required int64 _TIMESTAMP_MICROS_int64 ( TIMESTAMP_MICROS ) ; \n" +
+ " required int64 _TIMESTAMP_MICROS_int64 ( TIMESTAMP_MICROS ) ; \n" +
" required fixed_len_byte_array(12) _INTERVAL_fixed_len_byte_array_12 ( INTERVAL ) ; \n" +
" required int96 _INT96_RAW ; \n" +
"} \n";
@@ -112,7 +112,7 @@
" optional int32 _TIME_MILLIS_int32 ( TIME_MILLIS ) ; \n" +
// " optional int64 _TIME_MICROS_int64 ( TIME_MICROS ) ; \n" +
" optional int64 _TIMESTAMP_MILLIS_int64 ( TIMESTAMP_MILLIS ) ; \n" +
- // " optional int64 _TIMESTAMP_MICROS_int64 ( TIMESTAMP_MICROS ) ; \n" +
+ " optional int64 _TIMESTAMP_MICROS_int64 ( TIMESTAMP_MICROS ) ; \n" +
" optional fixed_len_byte_array(12) _INTERVAL_fixed_len_byte_array_12 ( INTERVAL ) ; \n" +
" optional int96 _INT96_RAW ; \n" +
"} \n";
@@ -153,7 +153,7 @@
" required int32 _TIME_MILLIS_int32 ( TIME_MILLIS ) ; \n" +
// " required int64 _TIME_MICROS_int64 ( TIME_MICROS ) ; \n" +
" required int64 _TIMESTAMP_MILLIS_int64 ( TIMESTAMP_MILLIS ) ; \n" +
- // " required int64 _TIMESTAMP_MICROS_int64 ( TIMESTAMP_MICROS ) ; \n" +
+ " required int64 _TIMESTAMP_MICROS_int64 ( TIMESTAMP_MICROS ) ; \n" +
" required fixed_len_byte_array(12) _INTERVAL_fixed_len_byte_array_12 ( INTERVAL ) ; \n" +
" } \n" +
" required group Int96 { \n" +
@@ -197,7 +197,7 @@
" optional int32 _TIME_MILLIS_int32 ( TIME_MILLIS ) ; \n" +
// " optional int64 _TIME_MICROS_int64 ( TIME_MICROS ) ; \n" +
" optional int64 _TIMESTAMP_MILLIS_int64 ( TIMESTAMP_MILLIS ) ; \n" +
- // " optional int64 _TIMESTAMP_MICROS_int64 ( TIMESTAMP_MICROS ) ; \n" +
+ " optional int64 _TIMESTAMP_MICROS_int64 ( TIMESTAMP_MICROS ) ; \n" +
" optional fixed_len_byte_array(12) _INTERVAL_fixed_len_byte_array_12 ( INTERVAL ) ; \n" +
" } \n" +
" optional group Int96 { \n" +
@@ -216,7 +216,7 @@ public static Path initFile(String fileName) {
return new Path(root, fileName);
}
- public static ParquetWriter<Group> initWriter(MessageType schema, String fileName) throws IOException {
+ public static ParquetWriter<Group> initWriter(MessageType schema, String fileName, boolean dictEncoding) throws IOException {
GroupWriteSupport.setSchema(schema, conf);
@@ -228,7 +228,7 @@ public static Path initFile(String fileName) {
1024,
1024,
512,
- true, // enable dictionary encoding,
+ dictEncoding, // enable dictionary encoding,
false,
ParquetProperties.WriterVersion.PARQUET_1_0, conf
);
@@ -267,6 +267,7 @@ public static void writeComplexValues(GroupFactory gf, ParquetWriter<Group> comp
.append("_DATE_int32", 1234567)
.append("_TIME_MILLIS_int32", 1234567)
.append("_TIMESTAMP_MILLIS_int64", 123456789012L)
+ .append("_TIMESTAMP_MICROS_int64", 123456789012L)
.append("_INTERVAL_fixed_len_byte_array_12", Binary.fromConstantByteArray(bytes12, 0, 12));
numeric.addGroup("Int96").append("_INT96_RAW", Binary.fromConstantByteArray(bytes12, 0, 12));
complexWriter.write(complexGroup);
@@ -300,6 +301,7 @@ public static void writeComplexValues(GroupFactory gf, ParquetWriter<Group> comp
.append("_DATE_int32", 0xFFFFFFFF)
.append("_TIME_MILLIS_int32", 0xFFFFFFFF)
.append("_TIMESTAMP_MILLIS_int64", 0x1F3FFFFFFFFL)
+ .append("_TIMESTAMP_MICROS_int64", 0x7FFFFFFFFFFFFFFFL)
.append("_INTERVAL_fixed_len_byte_array_12", Binary.fromConstantByteArray(bytes, 0, 12));
numeric.addGroup("Int96").append("_INT96_RAW", Binary.fromConstantByteArray(bytes, 0, 12));
complexWriter.write(complexGroup);
@@ -331,6 +333,7 @@ public static void writeComplexValues(GroupFactory gf, ParquetWriter<Group> comp
.append("_DATE_int32", 0x0)
.append("_TIME_MILLIS_int32", 0x0)
.append("_TIMESTAMP_MILLIS_int64", 0x0L)
+ .append("_TIMESTAMP_MICROS_int64", 0x0L)
.append("_INTERVAL_fixed_len_byte_array_12", Binary.fromConstantByteArray( new byte[12], 0, 12));
numeric.addGroup("Int96").append("_INT96_RAW", Binary.fromConstantByteArray( new byte[12], 0, 12));
complexWriter.write(complexGroup);
@@ -370,6 +373,7 @@ public static void writeSimpleValues(SimpleGroupFactory sgf, ParquetWriter<Group
.append("_DATE_int32", 1234567)
.append("_TIME_MILLIS_int32", 1234567)
.append("_TIMESTAMP_MILLIS_int64", 123456789012L)
+ .append("_TIMESTAMP_MICROS_int64", 123456789012L)
.append("_INTERVAL_fixed_len_byte_array_12", Binary.fromConstantByteArray(bytes12, 0, 12))
.append("_INT96_RAW", Binary.fromConstantByteArray(bytes12, 0, 12));
simpleWriter.write(simpleGroup);
@@ -399,6 +403,7 @@ public static void writeSimpleValues(SimpleGroupFactory sgf, ParquetWriter<Group
.append("_DATE_int32", 0xFFFFFFFF)
.append("_TIME_MILLIS_int32", 0xFFFFFFFF)
.append("_TIMESTAMP_MILLIS_int64", 0x1F3FFFFFFFFL)
+ .append("_TIMESTAMP_MICROS_int64", 0x7FFFFFFFFFFFFFFFL)
.append("_INTERVAL_fixed_len_byte_array_12", Binary.fromConstantByteArray(bytes, 0, 12))
.append("_INT96_RAW", Binary.fromConstantByteArray(bytes, 0, 12));
simpleWriter.write(simpleGroup);
@@ -425,6 +430,7 @@ public static void writeSimpleValues(SimpleGroupFactory sgf, ParquetWriter<Group
.append("_DATE_int32", 0x0)
.append("_TIME_MILLIS_int32", 0x0)
.append("_TIMESTAMP_MILLIS_int64", 0x0L)
+ .append("_TIMESTAMP_MICROS_int64", 0x0L)
.append("_INTERVAL_fixed_len_byte_array_12", Binary.fromConstantByteArray( new byte[12], 0, 12))
.append("_INT96_RAW", Binary.fromConstantByteArray( new byte[12], 0, 12));
simpleWriter.write(simpleGroup);
@@ -443,20 +449,33 @@ public static void main(String[] args) throws IOException {
SimpleGroupFactory sngf = new SimpleGroupFactory(simpleNullableSchema);
GroupFactory ngf = new SimpleGroupFactory(complexNullableSchema);
- ParquetWriter<Group> simpleWriter = initWriter(simpleSchema, "drill/parquet_test_file_simple");
- ParquetWriter<Group> complexWriter = initWriter(complexSchema, "drill/parquet_test_file_complex");
- ParquetWriter<Group> simpleNullableWriter = initWriter(simpleNullableSchema, "drill/parquet_test_file_simple_nullable");
- ParquetWriter<Group> complexNullableWriter = initWriter(complexNullableSchema, "drill/parquet_test_file_complex_nullable");
+ // Generate files with dictionary encoding enabled and disabled
+ ParquetWriter<Group> simpleWriter = initWriter(simpleSchema, "drill/parquet_test_file_simple", true);
+ ParquetWriter<Group> complexWriter = initWriter(complexSchema, "drill/parquet_test_file_complex", true);
+ ParquetWriter<Group> simpleNullableWriter = initWriter(simpleNullableSchema, "drill/parquet_test_file_simple_nullable", true);
+ ParquetWriter<Group> complexNullableWriter = initWriter(complexNullableSchema, "drill/parquet_test_file_complex_nullable", true);
+ ParquetWriter<Group> simpleNoDictWriter = initWriter(simpleSchema, "drill/parquet_test_file_simple_nodict", false);
+ ParquetWriter<Group> complexNoDictWriter = initWriter(complexSchema, "drill/parquet_test_file_complex_nodict", false);
+ ParquetWriter<Group> simpleNullableNoDictWriter = initWriter(simpleNullableSchema, "drill/parquet_test_file_simple_nullable_nodict", false);
+ ParquetWriter<Group> complexNullableNoDictWriter = initWriter(complexNullableSchema, "drill/parquet_test_file_complex_nullable_nodict", false);
ParquetSimpleTestFileGenerator.writeSimpleValues(sgf, simpleWriter, false);
ParquetSimpleTestFileGenerator.writeSimpleValues(sngf, simpleNullableWriter, true);
ParquetSimpleTestFileGenerator.writeComplexValues(gf, complexWriter, false);
ParquetSimpleTestFileGenerator.writeComplexValues(ngf, complexNullableWriter, true);
+ ParquetSimpleTestFileGenerator.writeSimpleValues(sgf, simpleNoDictWriter, false);
+ ParquetSimpleTestFileGenerator.writeSimpleValues(sngf, simpleNullableNoDictWriter, true);
+ ParquetSimpleTestFileGenerator.writeComplexValues(gf, complexNoDictWriter, false);
+ ParquetSimpleTestFileGenerator.writeComplexValues(ngf, complexNullableNoDictWriter, true);
simpleWriter.close();
complexWriter.close();
simpleNullableWriter.close();
complexNullableWriter.close();
+ simpleNoDictWriter.close();
+ complexNoDictWriter.close();
+ simpleNullableNoDictWriter.close();
+ complexNullableNoDictWriter.close();
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
index c309cf3152f..32ffb68c3d3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
@@ -17,18 +17,10 @@
*/
package org.apache.drill.exec.store.parquet;
-import static org.apache.drill.test.TestBuilder.mapOf;
-
import java.math.BigDecimal;
-import java.time.Instant;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.ZoneOffset;
-import java.util.Arrays;
import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.test.BaseTestQuery;
-import org.joda.time.Period;
import org.junit.Test;
public class TestParquetComplex extends BaseTestQuery {
@@ -204,175 +196,6 @@ public void notxistsField() throws Exception {
.run();
}
- @Test //DRILL-5971
- public void testComplexLogicalIntTypes() throws Exception {
- String query = "select t.complextype as complextype, " +
- "t.uint_64 as uint_64, t.uint_32 as uint_32, t.uint_16 as uint_16, t.uint_8 as uint_8, " +
- "t.int_64 as int_64, t.int_32 as int_32, t.int_16 as int_16, t.int_8 as int_8 " +
- "from cp.`store/parquet/complex/logical_int_complex.parquet` t";
- String[] columns = {"complextype", "uint_64", "uint_32", "uint_16", "uint_8", "int_64", "int_32", "int_16", "int_8" };
- testBuilder()
- .sqlQuery(query)
- .unOrdered()
- .baselineColumns(columns)
- .baselineValues(mapOf("a","a","b","b"), 0L, 0, 0, 0, 0L, 0, 0, 0)
- .baselineValues(mapOf("a","a","b","b"), -1L, -1, -1, -1, -1L, -1, -1, -1)
- .baselineValues(mapOf("a","a","b","b"), 1L, 1, 1, 1, -9223372036854775808L, 1, 1, 1)
- .baselineValues(mapOf("a","a","b","b"), 9223372036854775807L, 2147483647, 65535, 255, 9223372036854775807L, -2147483648, -32768, -128)
- .build()
- .run();
- }
-
- @Test //DRILL-5971
- public void testComplexLogicalIntTypes2() throws Exception {
- byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
- byte[] bytesOnes = new byte[12];
- byte[] bytesZeros = new byte[12];
- Arrays.fill(bytesOnes, (byte) 1);
- String query =
- " select " +
- " t.rowKey as rowKey, " +
- " t.StringTypes._UTF8 as _UTF8, " +
- " t.StringTypes._Enum as _Enum, " +
- " t.NumericTypes.Int32._INT32_RAW as _INT32_RAW, " +
- " t.NumericTypes.Int32._INT_8 as _INT_8, " +
- " t.NumericTypes.Int32._INT_16 as _INT_16, " +
- " t.NumericTypes.Int32._INT_32 as _INT_32, " +
- " t.NumericTypes.Int32._UINT_8 as _UINT_8, " +
- " t.NumericTypes.Int32._UINT_16 as _UINT_16, " +
- " t.NumericTypes.Int32._UINT_32 as _UINT_32, " +
- " t.NumericTypes.Int64._INT64_RAW as _INT64_RAW, " +
- " t.NumericTypes.Int64._INT_64 as _INT_64, " +
- " t.NumericTypes.Int64._UINT_64 as _UINT_64, " +
- " t.NumericTypes.DateTimeTypes._DATE_int32 as _DATE_int32, " +
- " t.NumericTypes.DateTimeTypes._TIME_MILLIS_int32 as _TIME_MILLIS_int32, " +
- " t.NumericTypes.DateTimeTypes._TIMESTAMP_MILLIS_int64 as _TIMESTAMP_MILLIS_int64, " +
- " t.NumericTypes.DateTimeTypes._INTERVAL_fixed_len_byte_array_12 as _INTERVAL_fixed_len_byte_array_12, " +
- " t.NumericTypes.Int96._INT96_RAW as _INT96_RAW " +
- " from " +
- " cp.`store/parquet/complex/parquet_logical_types_complex.parquet` t " +
- " order by t.rowKey ";
- String[] columns = {
- "rowKey ",
- "_UTF8",
- "_Enum",
- "_INT32_RAW",
- "_INT_8",
- "_INT_16",
- "_INT_32",
- "_UINT_8",
- "_UINT_16",
- "_UINT_32",
- "_INT64_RAW",
- "_INT_64",
- "_UINT_64",
- "_DATE_int32",
- "_TIME_MILLIS_int32",
- "_TIMESTAMP_MILLIS_int64",
- "_INTERVAL_fixed_len_byte_array_12",
- "_INT96_RAW"
-
- };
- testBuilder()
- .sqlQuery(query)
- .ordered()
- .baselineColumns(columns)
- .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
- 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
- LocalDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("1973-11-29T21:33:09.012"),
- new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
- bytes12)
- .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
- 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
- LocalDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("2038-01-19T03:14:07.999"),
- new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
- bytesOnes)
- .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
- -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
- LocalDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("1970-01-01T00:00:00.0"), new Period("PT0S"), bytesZeros)
- .build()
- .run();
- }
-
- @Test //DRILL-5971
- public void testComplexLogicalIntTypes3() throws Exception {
- byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
- byte[] bytesOnes = new byte[12];
- byte[] bytesZeros = new byte[12];
- Arrays.fill(bytesOnes, (byte) 1);
- String query =
- " select " +
- " t.rowKey as rowKey, " +
- " t.StringTypes._UTF8 as _UTF8, " +
- " t.StringTypes._Enum as _Enum, " +
- " t.NumericTypes.Int32._INT32_RAW as _INT32_RAW, " +
- " t.NumericTypes.Int32._INT_8 as _INT_8, " +
- " t.NumericTypes.Int32._INT_16 as _INT_16, " +
- " t.NumericTypes.Int32._INT_32 as _INT_32, " +
- " t.NumericTypes.Int32._UINT_8 as _UINT_8, " +
- " t.NumericTypes.Int32._UINT_16 as _UINT_16, " +
- " t.NumericTypes.Int32._UINT_32 as _UINT_32, " +
- " t.NumericTypes.Int64._INT64_RAW as _INT64_RAW, " +
- " t.NumericTypes.Int64._INT_64 as _INT_64, " +
- " t.NumericTypes.Int64._UINT_64 as _UINT_64, " +
- " t.NumericTypes.DateTimeTypes._DATE_int32 as _DATE_int32, " +
- " t.NumericTypes.DateTimeTypes._TIME_MILLIS_int32 as _TIME_MILLIS_int32, " +
- " t.NumericTypes.DateTimeTypes._TIMESTAMP_MILLIS_int64 as _TIMESTAMP_MILLIS_int64, " +
- " t.NumericTypes.DateTimeTypes._INTERVAL_fixed_len_byte_array_12 as _INTERVAL_fixed_len_byte_array_12, " +
- " t.NumericTypes.Int96._INT96_RAW as _INT96_RAW " +
- " from " +
- " cp.`store/parquet/complex/parquet_logical_types_complex_nullable.parquet` t " +
- " order by t.rowKey ";
- String[] columns = {
- "rowKey ",
- "_UTF8",
- "_Enum",
- "_INT32_RAW",
- "_INT_8",
- "_INT_16",
- "_INT_32",
- "_UINT_8",
- "_UINT_16",
- "_UINT_32",
- "_INT64_RAW",
- "_INT_64",
- "_UINT_64",
- "_DATE_int32",
- "_TIME_MILLIS_int32",
- "_TIMESTAMP_MILLIS_int64",
- "_INTERVAL_fixed_len_byte_array_12",
- "_INT96_RAW"
-
- };
- testBuilder()
- .sqlQuery(query)
- .ordered()
- .baselineColumns(columns)
- .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
- 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
- LocalDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("1973-11-29T21:33:09.012"),
- new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
- bytes12)
- .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
- 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
- LocalDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("2038-01-19T03:14:07.999"),
- new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
- bytesOnes)
- .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
- -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
- LocalDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("1970-01-01T00:00:00.0"), new Period("PT0S"), bytesZeros)
- .baselineValues(4, null, null, null, null, null, null, null, null, null, null, null, null, null,
- null, null, null, null)
- .build().run();
- }
-
- @Test
public void testReadRepeatedDecimals() throws Exception {
JsonStringArrayList<BigDecimal> ints = new JsonStringArrayList<>();
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetLogicalTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetLogicalTypes.java
new file mode 100644
index 00000000000..74cec6e6c68
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetLogicalTypes.java
@@ -0,0 +1,665 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet;
+
+import static org.apache.drill.test.TestBuilder.mapOf;
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.Arrays;
+
+import org.apache.drill.test.BaseTestQuery;
+import org.joda.time.Period;
+import org.junit.Test;
+
+/**
+ * Test Parquet logical type handling with default Parquet reader.
+ * Tests are executed on data files with dictionary encoding enabled and disabled.
+ * parquet_logical_types* files are generated with ParquetSimpleTestFileGenerator class.
+ */
+public class TestParquetLogicalTypes extends BaseTestQuery {
+
+ @Test //DRILL-5971
+ public void testComplexLogicalIntTypes() throws Exception {
+ String query = "select t.complextype as complextype, " +
+ "t.uint_64 as uint_64, t.uint_32 as uint_32, t.uint_16 as uint_16, t.uint_8 as uint_8, " +
+ "t.int_64 as int_64, t.int_32 as int_32, t.int_16 as int_16, t.int_8 as int_8 " +
+ "from cp.`store/parquet/complex/logical_int_complex.parquet` t";
+ String[] columns = {"complextype", "uint_64", "uint_32", "uint_16", "uint_8", "int_64", "int_32", "int_16", "int_8" };
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns(columns)
+ .baselineValues(mapOf("a","a","b","b"), 0L, 0, 0, 0, 0L, 0, 0, 0)
+ .baselineValues(mapOf("a","a","b","b"), -1L, -1, -1, -1, -1L, -1, -1, -1)
+ .baselineValues(mapOf("a","a","b","b"), 1L, 1, 1, 1, -9223372036854775808L, 1, 1, 1)
+ .baselineValues(mapOf("a","a","b","b"), 9223372036854775807L, 2147483647, 65535, 255, 9223372036854775807L, -2147483648, -32768, -128)
+ .build()
+ .run();
+ }
+
+ @Test //DRILL-5971
+ public void testComplexLogicalIntTypes2() throws Exception {
+ byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
+ byte[] bytesOnes = new byte[12];
+ byte[] bytesZeros = new byte[12];
+ Arrays.fill(bytesOnes, (byte) 1);
+ String query =
+ " select " +
+ " t.rowKey as rowKey, " +
+ " t.StringTypes._UTF8 as _UTF8, " +
+ " t.StringTypes._Enum as _Enum, " +
+ " t.NumericTypes.Int32._INT32_RAW as _INT32_RAW, " +
+ " t.NumericTypes.Int32._INT_8 as _INT_8, " +
+ " t.NumericTypes.Int32._INT_16 as _INT_16, " +
+ " t.NumericTypes.Int32._INT_32 as _INT_32, " +
+ " t.NumericTypes.Int32._UINT_8 as _UINT_8, " +
+ " t.NumericTypes.Int32._UINT_16 as _UINT_16, " +
+ " t.NumericTypes.Int32._UINT_32 as _UINT_32, " +
+ " t.NumericTypes.Int64._INT64_RAW as _INT64_RAW, " +
+ " t.NumericTypes.Int64._INT_64 as _INT_64, " +
+ " t.NumericTypes.Int64._UINT_64 as _UINT_64, " +
+ " t.NumericTypes.DateTimeTypes._DATE_int32 as _DATE_int32, " +
+ " t.NumericTypes.DateTimeTypes._TIME_MILLIS_int32 as _TIME_MILLIS_int32, " +
+ " t.NumericTypes.DateTimeTypes._TIMESTAMP_MILLIS_int64 as _TIMESTAMP_MILLIS_int64, " +
+ " t.NumericTypes.DateTimeTypes._TIMESTAMP_MICROS_int64 as _TIMESTAMP_MICROS_int64, " +
+ " t.NumericTypes.DateTimeTypes._INTERVAL_fixed_len_byte_array_12 as _INTERVAL_fixed_len_byte_array_12, " +
+ " t.NumericTypes.Int96._INT96_RAW as _INT96_RAW " +
+ " from " +
+ " cp.`store/parquet/complex/parquet_logical_types_complex.parquet` t " +
+ " order by t.rowKey ";
+ String[] columns = {
+ "rowKey ",
+ "_UTF8",
+ "_Enum",
+ "_INT32_RAW",
+ "_INT_8",
+ "_INT_16",
+ "_INT_32",
+ "_UINT_8",
+ "_UINT_16",
+ "_UINT_32",
+ "_INT64_RAW",
+ "_INT_64",
+ "_UINT_64",
+ "_DATE_int32",
+ "_TIME_MILLIS_int32",
+ "_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
+ "_INTERVAL_fixed_len_byte_array_12",
+ "_INT96_RAW"
+
+ };
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns(columns)
+ .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
+ 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
+ new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
+ bytes12)
+ .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
+ 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
+ new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
+ bytesOnes)
+ .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
+ -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
+ .build()
+ .run();
+ }
+
+ @Test //DRILL-5971
+ public void testComplexLogicalIntTypes3() throws Exception {
+ byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
+ byte[] bytesOnes = new byte[12];
+ byte[] bytesZeros = new byte[12];
+ Arrays.fill(bytesOnes, (byte) 1);
+ String query =
+ " select " +
+ " t.rowKey as rowKey, " +
+ " t.StringTypes._UTF8 as _UTF8, " +
+ " t.StringTypes._Enum as _Enum, " +
+ " t.NumericTypes.Int32._INT32_RAW as _INT32_RAW, " +
+ " t.NumericTypes.Int32._INT_8 as _INT_8, " +
+ " t.NumericTypes.Int32._INT_16 as _INT_16, " +
+ " t.NumericTypes.Int32._INT_32 as _INT_32, " +
+ " t.NumericTypes.Int32._UINT_8 as _UINT_8, " +
+ " t.NumericTypes.Int32._UINT_16 as _UINT_16, " +
+ " t.NumericTypes.Int32._UINT_32 as _UINT_32, " +
+ " t.NumericTypes.Int64._INT64_RAW as _INT64_RAW, " +
+ " t.NumericTypes.Int64._INT_64 as _INT_64, " +
+ " t.NumericTypes.Int64._UINT_64 as _UINT_64, " +
+ " t.NumericTypes.DateTimeTypes._DATE_int32 as _DATE_int32, " +
+ " t.NumericTypes.DateTimeTypes._TIME_MILLIS_int32 as _TIME_MILLIS_int32, " +
+ " t.NumericTypes.DateTimeTypes._TIMESTAMP_MILLIS_int64 as _TIMESTAMP_MILLIS_int64, " +
+ " t.NumericTypes.DateTimeTypes._TIMESTAMP_MICROS_int64 as _TIMESTAMP_MICROS_int64, " +
+ " t.NumericTypes.DateTimeTypes._INTERVAL_fixed_len_byte_array_12 as _INTERVAL_fixed_len_byte_array_12, " +
+ " t.NumericTypes.Int96._INT96_RAW as _INT96_RAW " +
+ " from " +
+ " cp.`store/parquet/complex/parquet_logical_types_complex_nullable.parquet` t " +
+ " order by t.rowKey ";
+ String[] columns = {
+ "rowKey ",
+ "_UTF8",
+ "_Enum",
+ "_INT32_RAW",
+ "_INT_8",
+ "_INT_16",
+ "_INT_32",
+ "_UINT_8",
+ "_UINT_16",
+ "_UINT_32",
+ "_INT64_RAW",
+ "_INT_64",
+ "_UINT_64",
+ "_DATE_int32",
+ "_TIME_MILLIS_int32",
+ "_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
+ "_INTERVAL_fixed_len_byte_array_12",
+ "_INT96_RAW"
+ };
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns(columns)
+ .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
+ 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
+ new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
+ bytes12)
+ .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
+ 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
+ new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
+ bytesOnes)
+ .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
+ -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
+ .baselineValues(4, null, null, null, null, null, null, null, null, null, null, null, null, null,
+ null, null, null, null, null)
+ .build().run();
+ }
+
+ @Test //DRILL-6670: include tests on data with dictionary encoding disabled
+ public void testComplexLogicalIntTypes4() throws Exception {
+ byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
+ byte[] bytesOnes = new byte[12];
+ byte[] bytesZeros = new byte[12];
+ Arrays.fill(bytesOnes, (byte) 1);
+ String query =
+ " select " +
+ " t.rowKey as rowKey, " +
+ " t.StringTypes._UTF8 as _UTF8, " +
+ " t.StringTypes._Enum as _Enum, " +
+ " t.NumericTypes.Int32._INT32_RAW as _INT32_RAW, " +
+ " t.NumericTypes.Int32._INT_8 as _INT_8, " +
+ " t.NumericTypes.Int32._INT_16 as _INT_16, " +
+ " t.NumericTypes.Int32._INT_32 as _INT_32, " +
+ " t.NumericTypes.Int32._UINT_8 as _UINT_8, " +
+ " t.NumericTypes.Int32._UINT_16 as _UINT_16, " +
+ " t.NumericTypes.Int32._UINT_32 as _UINT_32, " +
+ " t.NumericTypes.Int64._INT64_RAW as _INT64_RAW, " +
+ " t.NumericTypes.Int64._INT_64 as _INT_64, " +
+ " t.NumericTypes.Int64._UINT_64 as _UINT_64, " +
+ " t.NumericTypes.DateTimeTypes._DATE_int32 as _DATE_int32, " +
+ " t.NumericTypes.DateTimeTypes._TIME_MILLIS_int32 as _TIME_MILLIS_int32, " +
+ " t.NumericTypes.DateTimeTypes._TIMESTAMP_MILLIS_int64 as _TIMESTAMP_MILLIS_int64, " +
+ " t.NumericTypes.DateTimeTypes._TIMESTAMP_MICROS_int64 as _TIMESTAMP_MICROS_int64, " +
+ " t.NumericTypes.DateTimeTypes._INTERVAL_fixed_len_byte_array_12 as _INTERVAL_fixed_len_byte_array_12, " +
+ " t.NumericTypes.Int96._INT96_RAW as _INT96_RAW " +
+ " from " +
+ " cp.`store/parquet/complex/parquet_logical_types_complex_nodict.parquet` t " +
+ " order by t.rowKey ";
+ String[] columns = {
+ "rowKey ",
+ "_UTF8",
+ "_Enum",
+ "_INT32_RAW",
+ "_INT_8",
+ "_INT_16",
+ "_INT_32",
+ "_UINT_8",
+ "_UINT_16",
+ "_UINT_32",
+ "_INT64_RAW",
+ "_INT_64",
+ "_UINT_64",
+ "_DATE_int32",
+ "_TIME_MILLIS_int32",
+ "_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
+ "_INTERVAL_fixed_len_byte_array_12",
+ "_INT96_RAW"
+ };
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns(columns)
+ .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
+ 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
+ new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
+ bytes12)
+ .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
+ 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
+ new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
+ bytesOnes)
+ .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
+ -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
+ .build()
+ .run();
+ }
+
+ @Test //DRILL-6670: include tests on data with dictionary encoding disabled
+ public void testComplexLogicalIntTypes5() throws Exception {
+ byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
+ byte[] bytesOnes = new byte[12];
+ byte[] bytesZeros = new byte[12];
+ Arrays.fill(bytesOnes, (byte) 1);
+ String query =
+ " select " +
+ " t.rowKey as rowKey, " +
+ " t.StringTypes._UTF8 as _UTF8, " +
+ " t.StringTypes._Enum as _Enum, " +
+ " t.NumericTypes.Int32._INT32_RAW as _INT32_RAW, " +
+ " t.NumericTypes.Int32._INT_8 as _INT_8, " +
+ " t.NumericTypes.Int32._INT_16 as _INT_16, " +
+ " t.NumericTypes.Int32._INT_32 as _INT_32, " +
+ " t.NumericTypes.Int32._UINT_8 as _UINT_8, " +
+ " t.NumericTypes.Int32._UINT_16 as _UINT_16, " +
+ " t.NumericTypes.Int32._UINT_32 as _UINT_32, " +
+ " t.NumericTypes.Int64._INT64_RAW as _INT64_RAW, " +
+ " t.NumericTypes.Int64._INT_64 as _INT_64, " +
+ " t.NumericTypes.Int64._UINT_64 as _UINT_64, " +
+ " t.NumericTypes.DateTimeTypes._DATE_int32 as _DATE_int32, " +
+ " t.NumericTypes.DateTimeTypes._TIME_MILLIS_int32 as _TIME_MILLIS_int32, " +
+ " t.NumericTypes.DateTimeTypes._TIMESTAMP_MILLIS_int64 as _TIMESTAMP_MILLIS_int64, " +
+ " t.NumericTypes.DateTimeTypes._TIMESTAMP_MICROS_int64 as _TIMESTAMP_MICROS_int64, " +
+ " t.NumericTypes.DateTimeTypes._INTERVAL_fixed_len_byte_array_12 as _INTERVAL_fixed_len_byte_array_12, " +
+ " t.NumericTypes.Int96._INT96_RAW as _INT96_RAW " +
+ " from " +
+ " cp.`store/parquet/complex/parquet_logical_types_complex_nullable.parquet` t " +
+ " order by t.rowKey ";
+ String[] columns = {
+ "rowKey ",
+ "_UTF8",
+ "_Enum",
+ "_INT32_RAW",
+ "_INT_8",
+ "_INT_16",
+ "_INT_32",
+ "_UINT_8",
+ "_UINT_16",
+ "_UINT_32",
+ "_INT64_RAW",
+ "_INT_64",
+ "_UINT_64",
+ "_DATE_int32",
+ "_TIME_MILLIS_int32",
+ "_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
+ "_INTERVAL_fixed_len_byte_array_12",
+ "_INT96_RAW"
+
+ };
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns(columns)
+ .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
+ 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
+ new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
+ bytes12)
+ .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
+ 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
+ new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
+ bytesOnes)
+ .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
+ -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
+ .baselineValues(4, null, null, null, null, null, null, null, null, null, null, null, null, null,
+ null, null, null, null, null)
+ .build().run();
+ }
+
+ @Test //DRILL-6670
+ public void testSimpleLogicalIntTypes1() throws Exception {
+ byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
+ byte[] bytesOnes = new byte[12];
+ byte[] bytesZeros = new byte[12];
+ Arrays.fill(bytesOnes, (byte) 1);
+ String query =
+ " select " +
+ " rowKey, " +
+ " _UTF8, " +
+ " _Enum, " +
+ " _INT32_RAW, " +
+ " _INT_8, " +
+ " _INT_16, " +
+ " _INT_32, " +
+ " _UINT_8, " +
+ " _UINT_16, " +
+ " _UINT_32, " +
+ " _INT64_RAW, " +
+ " _INT_64, " +
+ " _UINT_64, " +
+ " _DATE_int32, " +
+ " _TIME_MILLIS_int32, " +
+ " _TIMESTAMP_MILLIS_int64, " +
+ " _TIMESTAMP_MICROS_int64, " +
+ " _INTERVAL_fixed_len_byte_array_12, " +
+ " _INT96_RAW " +
+ " from " +
+ " cp.`parquet/parquet_logical_types_simple.parquet` t " +
+ " order by t.rowKey ";
+ String[] columns = {
+ "rowKey ",
+ "_UTF8",
+ "_Enum",
+ "_INT32_RAW",
+ "_INT_8",
+ "_INT_16",
+ "_INT_32",
+ "_UINT_8",
+ "_UINT_16",
+ "_UINT_32",
+ "_INT64_RAW",
+ "_INT_64",
+ "_UINT_64",
+ "_DATE_int32",
+ "_TIME_MILLIS_int32",
+ "_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
+ "_INTERVAL_fixed_len_byte_array_12",
+ "_INT96_RAW"
+ };
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns(columns)
+ .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
+ 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
+ new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
+ bytes12)
+ .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
+ 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
+ new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
+ bytesOnes)
+ .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
+ -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
+ .build()
+ .run();
+ }
+
+ @Test //DRILL-6670
+ public void testSimpleLogicalIntTypes2() throws Exception {
+ byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
+ byte[] bytesOnes = new byte[12];
+ byte[] bytesZeros = new byte[12];
+ Arrays.fill(bytesOnes, (byte) 1);
+ String query =
+ " select " +
+ " rowKey, " +
+ " _UTF8, " +
+ " _Enum, " +
+ " _INT32_RAW, " +
+ " _INT_8, " +
+ " _INT_16, " +
+ " _INT_32, " +
+ " _UINT_8, " +
+ " _UINT_16, " +
+ " _UINT_32, " +
+ " _INT64_RAW, " +
+ " _INT_64, " +
+ " _UINT_64, " +
+ " _DATE_int32, " +
+ " _TIME_MILLIS_int32, " +
+ " _TIMESTAMP_MILLIS_int64, " +
+ " _TIMESTAMP_MICROS_int64, " +
+ " _INTERVAL_fixed_len_byte_array_12, " +
+ " _INT96_RAW " +
+ " from " +
+ " cp.`parquet/parquet_logical_types_simple_nullable.parquet` t " +
+ " order by t.rowKey ";
+ String[] columns = {
+ "rowKey ",
+ "_UTF8",
+ "_Enum",
+ "_INT32_RAW",
+ "_INT_8",
+ "_INT_16",
+ "_INT_32",
+ "_UINT_8",
+ "_UINT_16",
+ "_UINT_32",
+ "_INT64_RAW",
+ "_INT_64",
+ "_UINT_64",
+ "_DATE_int32",
+ "_TIME_MILLIS_int32",
+ "_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
+ "_INTERVAL_fixed_len_byte_array_12",
+ "_INT96_RAW"
+ };
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns(columns)
+ .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
+ 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
+ new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
+ bytes12)
+ .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
+ 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
+ new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
+ bytesOnes)
+ .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
+ -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
+ .baselineValues(4, null, null, null, null, null, null, null, null, null, null, null, null, null,
+ null, null, null, null, null)
+ .build().run();
+ }
+
+ @Test //DRILL-6670
+ public void testSimpleLogicalIntTypes3() throws Exception {
+ byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
+ byte[] bytesOnes = new byte[12];
+ byte[] bytesZeros = new byte[12];
+ Arrays.fill(bytesOnes, (byte) 1);
+ String query =
+ " select " +
+ " rowKey, " +
+ " _UTF8, " +
+ " _Enum, " +
+ " _INT32_RAW, " +
+ " _INT_8, " +
+ " _INT_16, " +
+ " _INT_32, " +
+ " _UINT_8, " +
+ " _UINT_16, " +
+ " _UINT_32, " +
+ " _INT64_RAW, " +
+ " _INT_64, " +
+ " _UINT_64, " +
+ " _DATE_int32, " +
+ " _TIME_MILLIS_int32, " +
+ " _TIMESTAMP_MILLIS_int64, " +
+ " _TIMESTAMP_MICROS_int64, " +
+ " _INTERVAL_fixed_len_byte_array_12, " +
+ " _INT96_RAW " +
+ " from " +
+ " cp.`parquet/parquet_logical_types_simple_nodict.parquet` t " +
+ " order by t.rowKey ";
+ String[] columns = {
+ "rowKey ",
+ "_UTF8",
+ "_Enum",
+ "_INT32_RAW",
+ "_INT_8",
+ "_INT_16",
+ "_INT_32",
+ "_UINT_8",
+ "_UINT_16",
+ "_UINT_32",
+ "_INT64_RAW",
+ "_INT_64",
+ "_UINT_64",
+ "_DATE_int32",
+ "_TIME_MILLIS_int32",
+ "_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
+ "_INTERVAL_fixed_len_byte_array_12",
+ "_INT96_RAW"
+
+ };
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns(columns)
+ .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
+ 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
+ new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
+ bytes12)
+ .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
+ 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
+ new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
+ bytesOnes)
+ .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
+ -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
+ .build()
+ .run();
+ }
+
+ @Test //DRILL-6670
+ public void testSimpleLogicalIntTypes4() throws Exception {
+ byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
+ byte[] bytesOnes = new byte[12];
+ byte[] bytesZeros = new byte[12];
+ Arrays.fill(bytesOnes, (byte) 1);
+ String query =
+ " select " +
+ " rowKey, " +
+ " _UTF8, " +
+ " _Enum, " +
+ " _INT32_RAW, " +
+ " _INT_8, " +
+ " _INT_16, " +
+ " _INT_32, " +
+ " _UINT_8, " +
+ " _UINT_16, " +
+ " _UINT_32, " +
+ " _INT64_RAW, " +
+ " _INT_64, " +
+ " _UINT_64, " +
+ " _DATE_int32, " +
+ " _TIME_MILLIS_int32, " +
+ " _TIMESTAMP_MILLIS_int64, " +
+ " _TIMESTAMP_MICROS_int64, " +
+ " _INTERVAL_fixed_len_byte_array_12, " +
+ " _INT96_RAW " +
+ " from " +
+ " cp.`parquet/parquet_logical_types_simple_nullable_nodict.parquet` t " +
+ " order by t.rowKey ";
+ String[] columns = {
+ "rowKey ",
+ "_UTF8",
+ "_Enum",
+ "_INT32_RAW",
+ "_INT_8",
+ "_INT_16",
+ "_INT_32",
+ "_UINT_8",
+ "_UINT_16",
+ "_UINT_32",
+ "_INT64_RAW",
+ "_INT_64",
+ "_UINT_64",
+ "_DATE_int32",
+ "_TIME_MILLIS_int32",
+ "_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
+ "_INTERVAL_fixed_len_byte_array_12",
+ "_INT96_RAW"
+ };
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns(columns)
+ .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
+ 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
+ new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
+ bytes12)
+ .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
+ 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
+ new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
+ bytesOnes)
+ .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
+ -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
+ .baselineValues(4, null, null, null, null, null, null, null, null, null, null, null, null, null,
+ null, null, null, null, null)
+ .build().run();
+ }
+
+}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
index 941f50f3900..80fb495ba0e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
@@ -25,6 +25,7 @@
import java.time.ZonedDateTime;
import java.util.Arrays;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.test.BaseTestQuery;
import org.joda.time.Period;
@@ -33,20 +34,20 @@
import org.junit.Test;
public class TestDrillParquetReader extends BaseTestQuery {
- // enable decimal data type
+ // enable decimal data type and make sure DrillParquetReader is used to handle test queries
@BeforeClass
- public static void enableDecimalDataType() throws Exception {
- test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ public static void setup() throws Exception {
+ alterSession(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY, true);
+ alterSession(ExecConstants.PARQUET_NEW_RECORD_READER, true);
}
@AfterClass
- public static void disableDecimalDataType() throws Exception {
- test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ public static void cleanup() throws Exception {
+ resetSessionOption(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY);
+ resetSessionOption(ExecConstants.PARQUET_NEW_RECORD_READER);
}
private void testColumn(String columnName) throws Exception {
- testNoResult("alter session set `store.parquet.use_new_reader` = true");
-
BigDecimal result = new BigDecimal("1.20000000");
testBuilder()
@@ -55,8 +56,6 @@ private void testColumn(String columnName) throws Exception {
.baselineColumns(columnName)
.baselineValues(result)
.go();
-
- testNoResult("alter session set `store.parquet.use_new_reader` = false");
}
@Test
@@ -149,6 +148,7 @@ public void testLogicalIntTypes2() throws Exception {
" t._DATE_int32 as _DATE_int32, " +
" t._TIME_MILLIS_int32 as _TIME_MILLIS_int32, " +
" t._TIMESTAMP_MILLIS_int64 as _TIMESTAMP_MILLIS_int64, " +
+ " t._TIMESTAMP_MICROS_int64 as _TIMESTAMP_MICROS_int64, " +
" t._INTERVAL_fixed_len_byte_array_12 as _INTERVAL_fixed_len_byte_array_12, " +
" t._INT96_RAW as _INT96_RAW " +
" from " +
@@ -172,6 +172,7 @@ public void testLogicalIntTypes2() throws Exception {
"_DATE_int32",
"_TIME_MILLIS_int32",
"_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
"_INTERVAL_fixed_len_byte_array_12",
"_INT96_RAW"
@@ -183,19 +184,19 @@ public void testLogicalIntTypes2() throws Exception {
.baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
ZonedDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("1973-11-29T21:33:09.012"),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
bytes12)
.baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
ZonedDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("2038-01-19T03:14:07.999"),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
bytesOnes)
.baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
-9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
ZonedDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("1970-01-01T00:00:00.0"), new Period("PT0S"), bytesZeros)
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
.build()
.run();
}
@@ -224,6 +225,7 @@ public void testLogicalIntTypes3() throws Exception {
" t._DATE_int32 as _DATE_int32, " +
" t._TIME_MILLIS_int32 as _TIME_MILLIS_int32, " +
" t._TIMESTAMP_MILLIS_int64 as _TIMESTAMP_MILLIS_int64, " +
+ " t._TIMESTAMP_MICROS_int64 as _TIMESTAMP_MICROS_int64, " +
" t._INTERVAL_fixed_len_byte_array_12 as _INTERVAL_fixed_len_byte_array_12, " +
" t._INT96_RAW as _INT96_RAW " +
" from " +
@@ -247,6 +249,162 @@ public void testLogicalIntTypes3() throws Exception {
"_DATE_int32",
"_TIME_MILLIS_int32",
"_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
+ "_INTERVAL_fixed_len_byte_array_12",
+ "_INT96_RAW"
+
+ };
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns(columns)
+ .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
+ 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
+ ZonedDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
+ new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
+ bytes12)
+ .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
+ 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
+ ZonedDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
+ new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
+ bytesOnes)
+ .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
+ -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
+ ZonedDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
+ .baselineValues(4, null, null, null, null, null, null, null, null, null, null, null, null, null,
+ null, null, null, null, null)
+ .build().run();
+ }
+
+ @Test //DRILL-6670: include tests on data with dictionary encoding disabled
+ public void testLogicalIntTypes4() throws Exception {
+ byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
+ byte[] bytesOnes = new byte[12];
+ Arrays.fill(bytesOnes, (byte)1);
+ byte[] bytesZeros = new byte[12];
+ String query = String.format(
+ " select " +
+ " t.rowKey as rowKey, " +
+ " t._UTF8 as _UTF8, " +
+ " t._Enum as _Enum, " +
+ " t._INT32_RAW as _INT32_RAW, " +
+ " t._INT_8 as _INT_8, " +
+ " t._INT_16 as _INT_16, " +
+ " t._INT_32 as _INT_32, " +
+ " t._UINT_8 as _UINT_8, " +
+ " t._UINT_16 as _UINT_16, " +
+ " t._UINT_32 as _UINT_32, " +
+ " t._INT64_RAW as _INT64_RAW, " +
+ " t._INT_64 as _INT_64, " +
+ " t._UINT_64 as _UINT_64, " +
+ " t._DATE_int32 as _DATE_int32, " +
+ " t._TIME_MILLIS_int32 as _TIME_MILLIS_int32, " +
+ " t._TIMESTAMP_MILLIS_int64 as _TIMESTAMP_MILLIS_int64, " +
+ " t._TIMESTAMP_MICROS_int64 as _TIMESTAMP_MICROS_int64, " +
+ " t._INTERVAL_fixed_len_byte_array_12 as _INTERVAL_fixed_len_byte_array_12, " +
+ " t._INT96_RAW as _INT96_RAW " +
+ " from " +
+ " cp.`parquet/parquet_logical_types_simple_nodict.parquet` t " +
+ " order by t.rowKey "
+ );
+ String[] columns = {
+ "rowKey ",
+ "_UTF8",
+ "_Enum",
+ "_INT32_RAW",
+ "_INT_8",
+ "_INT_16",
+ "_INT_32",
+ "_UINT_8",
+ "_UINT_16",
+ "_UINT_32",
+ "_INT64_RAW",
+ "_INT_64",
+ "_UINT_64",
+ "_DATE_int32",
+ "_TIME_MILLIS_int32",
+ "_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
+ "_INTERVAL_fixed_len_byte_array_12",
+ "_INT96_RAW"
+
+ };
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns(columns)
+ .baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
+ 1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
+ ZonedDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
+ new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
+ bytes12)
+ .baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
+ 9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
+ ZonedDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
+ new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
+ bytesOnes)
+ .baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
+ -9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
+ ZonedDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
+ .build()
+ .run();
+ }
+
+ @Test //DRILL-6670: include tests on data with dictionary encoding disabled
+ public void testLogicalIntTypes5() throws Exception {
+ byte[] bytes12 = {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b' };
+ byte[] bytesOnes = new byte[12];
+ Arrays.fill(bytesOnes, (byte)1);
+ byte[] bytesZeros = new byte[12];
+ String query = String.format(
+ " select " +
+ " t.rowKey as rowKey, " +
+ " t._UTF8 as _UTF8, " +
+ " t._Enum as _Enum, " +
+ " t._INT32_RAW as _INT32_RAW, " +
+ " t._INT_8 as _INT_8, " +
+ " t._INT_16 as _INT_16, " +
+ " t._INT_32 as _INT_32, " +
+ " t._UINT_8 as _UINT_8, " +
+ " t._UINT_16 as _UINT_16, " +
+ " t._UINT_32 as _UINT_32, " +
+ " t._INT64_RAW as _INT64_RAW, " +
+ " t._INT_64 as _INT_64, " +
+ " t._UINT_64 as _UINT_64, " +
+ " t._DATE_int32 as _DATE_int32, " +
+ " t._TIME_MILLIS_int32 as _TIME_MILLIS_int32, " +
+ " t._TIMESTAMP_MILLIS_int64 as _TIMESTAMP_MILLIS_int64, " +
+ " t._TIMESTAMP_MICROS_int64 as _TIMESTAMP_MICROS_int64, " +
+ " t._INTERVAL_fixed_len_byte_array_12 as _INTERVAL_fixed_len_byte_array_12, " +
+ " t._INT96_RAW as _INT96_RAW " +
+ " from " +
+ " cp.`parquet/parquet_logical_types_simple_nullable_nodict.parquet` t " +
+ " order by t.rowKey "
+ );
+ String[] columns = {
+ "rowKey ",
+ "_UTF8",
+ "_Enum",
+ "_INT32_RAW",
+ "_INT_8",
+ "_INT_16",
+ "_INT_32",
+ "_UINT_8",
+ "_UINT_16",
+ "_UINT_32",
+ "_INT64_RAW",
+ "_INT_64",
+ "_UINT_64",
+ "_DATE_int32",
+ "_TIME_MILLIS_int32",
+ "_TIMESTAMP_MILLIS_int64",
+ "_TIMESTAMP_MICROS_int64",
"_INTERVAL_fixed_len_byte_array_12",
"_INT96_RAW"
@@ -258,21 +416,21 @@ public void testLogicalIntTypes3() throws Exception {
.baselineValues(1, "UTF8 string1", "RANDOM_VALUE", 1234567, 123, 12345, 1234567, 123, 1234, 1234567,
1234567890123456L, 1234567890123456L, 1234567890123456L, LocalDate.parse("5350-02-17"),
ZonedDateTime.ofInstant(Instant.ofEpochMilli(1234567), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("1973-11-29T21:33:09.012"),
+ LocalDateTime.parse("1973-11-29T21:33:09.012"), 123456789012L,
new Period().plusMonths(875770417).plusDays(943142453).plusMillis(1650536505),
bytes12)
.baselineValues(2, "UTF8 string2", "MAX_VALUE", 2147483647, 127, 32767, 2147483647, 255, 65535, -1,
9223372036854775807L, 9223372036854775807L, -1L, LocalDate.parse("1969-12-31"),
ZonedDateTime.ofInstant(Instant.ofEpochMilli(0xFFFFFFFF), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("2038-01-19T03:14:07.999"),
+ LocalDateTime.parse("2038-01-19T03:14:07.999"), 9223372036854775807L,
new Period().plusMonths(16843009).plusDays(16843009).plusMillis(16843009),
bytesOnes)
.baselineValues(3, "UTF8 string3", "MIN_VALUE", -2147483648, -128, -32768, -2147483648, 0, 0, 0,
-9223372036854775808L, -9223372036854775808L, 0L, LocalDate.parse("1970-01-01"),
ZonedDateTime.ofInstant(Instant.ofEpochMilli(0), ZoneOffset.UTC).toLocalTime(),
- LocalDateTime.parse("1970-01-01T00:00:00.0"), new Period("PT0S"), bytesZeros)
+ LocalDateTime.parse("1970-01-01T00:00:00.0"), 0L, new Period("PT0S"), bytesZeros)
.baselineValues(4, null, null, null, null, null, null, null, null, null, null, null, null, null,
- null, null, null, null)
+ null, null, null, null, null)
.build().run();
}
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple.parquet b/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple.parquet
index edae0e53475..c24b96b50e8 100644
Binary files a/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple.parquet and b/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple.parquet differ
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple_nodict.parquet b/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple_nodict.parquet
new file mode 100644
index 00000000000..7255e8e83a5
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple_nodict.parquet differ
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple_nullable.parquet b/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple_nullable.parquet
index 4c97da9e30b..6db45276b41 100644
Binary files a/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple_nullable.parquet and b/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple_nullable.parquet differ
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple_nullable_nodict.parquet b/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple_nullable_nodict.parquet
new file mode 100644
index 00000000000..3ac60704bf1
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/parquet_logical_types_simple_nullable_nodict.parquet differ
diff --git a/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex.parquet b/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex.parquet
index ec797e0b30d..4600694d642 100644
Binary files a/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex.parquet and b/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex.parquet differ
diff --git a/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex_nodict.parquet b/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex_nodict.parquet
new file mode 100644
index 00000000000..1b023b2b9ba
Binary files /dev/null and b/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex_nodict.parquet differ
diff --git a/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex_nullable.parquet b/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex_nullable.parquet
index 59b7a75a27e..bc1b59b484c 100644
Binary files a/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex_nullable.parquet and b/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex_nullable.parquet differ
diff --git a/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex_nullable_nodict.parquet b/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex_nullable_nodict.parquet
new file mode 100644
index 00000000000..3d3377d48cc
Binary files /dev/null and b/exec/java-exec/src/test/resources/store/parquet/complex/parquet_logical_types_complex_nullable_nodict.parquet differ
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services