You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by vo...@apache.org on 2020/03/10 18:41:38 UTC
[drill] 09/10: DRILL-7630: Add additional types into SchemaParser
for Parquet
This is an automated email from the ASF dual-hosted git repository.
volodymyr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 87a6725718446b316d85d5dab1d3ad135f5d9475
Author: Arina Ielchiieva <ar...@gmail.com>
AuthorDate: Tue Mar 10 16:06:57 2020 +0200
DRILL-7630: Add additional types into SchemaParser for Parquet
closes #2019
---
.../record/metadata/schema/parser/SchemaLexer.g4 | 8 ++++++
.../record/metadata/schema/parser/SchemaParser.g4 | 6 +++++
.../metadata/schema/parser/SchemaVisitor.java | 30 ++++++++++++++++++++++
.../metadata/schema/parser/TestSchemaParser.java | 15 +++++++++++
4 files changed, 59 insertions(+)
diff --git a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4 b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4
index 62d9f10..c531356 100644
--- a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4
+++ b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaLexer.g4
@@ -58,6 +58,14 @@ MAP: 'MAP';
STRUCT: 'STRUCT';
ARRAY: 'ARRAY';
+// additional data types, primary used for Parquet
+UINT1: 'UINT1';
+UINT2: 'UINT2';
+UINT4: 'UINT4';
+UINT8: 'UINT8';
+TINYINT: 'TINYINT';
+SMALLINT: 'SMALLINT';
+
// symbols
COMMA: ',';
REVERSE_QUOTE: '`';
diff --git a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4 b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4
index 5bd55a7..1ff90ec 100644
--- a/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4
+++ b/exec/vector/src/main/antlr4/org/apache/drill/exec/record/metadata/schema/parser/SchemaParser.g4
@@ -63,6 +63,12 @@ simple_type
| INTERVAL (YEAR | MONTH) # interval_year
| INTERVAL (DAY | HOUR | MINUTE | SECOND) # interval_day
| INTERVAL # interval
+| UINT1 # unit1
+| UINT2 # unit2
+| UINT4 # unit4
+| UINT8 # unit8
+| TINYINT # tinyint
+| SMALLINT # smallint
;
array_type: (simple_array_type | complex_array_type);
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java
index cd5da79..18ed651 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/schema/parser/SchemaVisitor.java
@@ -279,6 +279,36 @@ public class SchemaVisitor extends SchemaParserBaseVisitor<TupleMetadata> {
}
@Override
+ public ColumnMetadata visitUnit1(SchemaParser.Unit1Context ctx) {
+ return constructColumn(Types.withMode(TypeProtos.MinorType.UINT1, mode));
+ }
+
+ @Override
+ public ColumnMetadata visitUnit2(SchemaParser.Unit2Context ctx) {
+ return constructColumn(Types.withMode(TypeProtos.MinorType.UINT2, mode));
+ }
+
+ @Override
+ public ColumnMetadata visitUnit4(SchemaParser.Unit4Context ctx) {
+ return constructColumn(Types.withMode(TypeProtos.MinorType.UINT4, mode));
+ }
+
+ @Override
+ public ColumnMetadata visitUnit8(SchemaParser.Unit8Context ctx) {
+ return constructColumn(Types.withMode(TypeProtos.MinorType.UINT8, mode));
+ }
+
+ @Override
+ public ColumnMetadata visitTinyint(SchemaParser.TinyintContext ctx) {
+ return constructColumn(Types.withMode(TypeProtos.MinorType.TINYINT, mode));
+ }
+
+ @Override
+ public ColumnMetadata visitSmallint(SchemaParser.SmallintContext ctx) {
+ return constructColumn(Types.withMode(TypeProtos.MinorType.SMALLINT, mode));
+ }
+
+ @Override
public ColumnMetadata visitStruct_type(SchemaParser.Struct_typeContext ctx) {
// internally Drill refers to structs as maps
MapBuilder builder = new MapBuilder(null, name, mode);
diff --git a/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java b/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java
index a0acf10..201a2dc 100644
--- a/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java
+++ b/exec/vector/src/test/java/org/apache/drill/exec/record/metadata/schema/parser/TestSchemaParser.java
@@ -185,6 +185,21 @@ public class TestSchemaParser extends BaseTest {
}
@Test
+ public void testAdditionalParquetTypes() throws Exception {
+ TupleMetadata schema = new SchemaBuilder()
+ .addNullable("uint1_col", TypeProtos.MinorType.UINT1)
+ .addNullable("uint2_col", TypeProtos.MinorType.UINT2)
+ .addNullable("uint4_col", TypeProtos.MinorType.UINT4)
+ .addNullable("uint8_col", TypeProtos.MinorType.UINT8)
+ .addNullable("tinyint_col", TypeProtos.MinorType.TINYINT)
+ .addNullable("smallint_col", TypeProtos.MinorType.SMALLINT)
+ .buildSchema();
+
+ checkSchema("uint1_col uint1, uint2_col uint2, uint4_col uint4, uint8_col uint8, " +
+ "tinyint_col tinyint, smallint_col smallint", schema);
+ }
+
+ @Test
public void testArray() throws Exception {
TupleMetadata schema = new SchemaBuilder()
.addArray("simple_array", TypeProtos.MinorType.INT)