You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/07/30 07:27:21 UTC
[08/14] git commit: DRILL 1221: Update for changes in Parquet-12 for
new converted types. Also converted existing binary files in the test-data
directory.
DRILL 1221: Update for changes in Parquet-12 for new converted types. Also converted existing binary files in the test-data directory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c2e32ac0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c2e32ac0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c2e32ac0
Branch: refs/heads/master
Commit: c2e32ac00e850e1ab6bee81e81070dfde4a48d9b
Parents: 5bae9c8
Author: Jason Altekruse <al...@gmail.com>
Authored: Tue Jul 29 17:07:34 2014 -0500
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Jul 29 16:49:47 2014 -0700
----------------------------------------------------------------------
.../codegen/templates/ParquetTypeHelper.java | 6 ++---
.../ParquetToDrillTypeConverter.java | 25 ++++++++++---------
sample-data/nation.parquet | Bin 3109 -> 1210 bytes
sample-data/nationsMF/nation.parquet | Bin 3109 -> 0 bytes
sample-data/nationsMF/nation2.parquet | Bin 3109 -> 0 bytes
sample-data/nationsMF/nationsMF.parquet | Bin 0 -> 1302 bytes
sample-data/nationsSF/nation.parquet | Bin 3109 -> 0 bytes
sample-data/nationsSF/nationsSF.parquet | Bin 0 -> 1210 bytes
sample-data/region.parquet | Bin 819 -> 455 bytes
sample-data/regionsMF/region.parquet | Bin 819 -> 0 bytes
sample-data/regionsMF/region2.parquet | Bin 819 -> 0 bytes
sample-data/regionsMF/regionsMF_Typed.parquet | Bin 0 -> 468 bytes
sample-data/regionsSF/region.parquet | Bin 819 -> 0 bytes
sample-data/regionsSF/regionsSF.parquet | Bin 0 -> 455 bytes
14 files changed, 16 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java b/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java
index 15830f6..5dbc860 100644
--- a/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java
+++ b/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java
@@ -110,9 +110,9 @@ public class ParquetTypeHelper {
</#list>
originalTypeMap.put(MinorType.VARCHAR, OriginalType.UTF8);
originalTypeMap.put(MinorType.DATE, OriginalType.DATE);
- originalTypeMap.put(MinorType.TIME, OriginalType.TIME);
- originalTypeMap.put(MinorType.TIMESTAMP, OriginalType.TIMESTAMP);
- originalTypeMap.put(MinorType.TIMESTAMPTZ, OriginalType.TIMESTAMPTZ);
+ originalTypeMap.put(MinorType.TIME, OriginalType.TIME_MILLIS);
+ originalTypeMap.put(MinorType.TIMESTAMP, OriginalType.TIMESTAMP_MILLIS);
+// originalTypeMap.put(MinorType.TIMESTAMPTZ, OriginalType.TIMESTAMPTZ);
}
public static PrimitiveTypeName getPrimitiveTypeNameForMinorType(MinorType minorType) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
index 7eeeeaa..5bba6be 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
@@ -59,9 +59,10 @@ public class ParquetToDrillTypeConverter {
switch(convertedType) {
case DECIMAL:
return Types.withScaleAndPrecision(TypeProtos.MinorType.DECIMAL18, DataMode.OPTIONAL, schemaElement.getScale(), schemaElement.getPrecision());
- case FINETIME:
- throw new UnsupportedOperationException();
- case TIMESTAMP:
+ // TODO - add this back if it is decided to be added upstream, was removed form our pull request July 2014
+// case TIME_MICROS:
+// throw new UnsupportedOperationException();
+ case TIMESTAMP_MILLIS:
return Types.optional(MinorType.TIMESTAMP);
default:
throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
@@ -75,7 +76,7 @@ public class ParquetToDrillTypeConverter {
return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.OPTIONAL, schemaElement.getScale(), schemaElement.getPrecision());
case DATE:
return Types.optional(MinorType.DATE);
- case TIME:
+ case TIME_MILLIS:
return Types.optional(MinorType.TIME);
default:
throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
@@ -123,9 +124,9 @@ public class ParquetToDrillTypeConverter {
switch(convertedType) {
case DECIMAL:
return Types.withScaleAndPrecision(MinorType.DECIMAL18, DataMode.REQUIRED, schemaElement.getScale(), schemaElement.getPrecision());
- case FINETIME:
- throw new UnsupportedOperationException();
- case TIMESTAMP:
+// case FINETIME:
+// throw new UnsupportedOperationException();
+ case TIMESTAMP_MILLIS:
return Types.required(MinorType.TIMESTAMP);
default:
throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
@@ -139,7 +140,7 @@ public class ParquetToDrillTypeConverter {
return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.REQUIRED, schemaElement.getScale(), schemaElement.getPrecision());
case DATE:
return Types.required(MinorType.DATE);
- case TIME:
+ case TIME_MILLIS:
return Types.required(MinorType.TIME);
default:
throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
@@ -187,9 +188,9 @@ public class ParquetToDrillTypeConverter {
switch(convertedType) {
case DECIMAL:
return Types.withScaleAndPrecision(MinorType.DECIMAL18, DataMode.REPEATED, schemaElement.getScale(), schemaElement.getPrecision());
- case FINETIME:
- throw new UnsupportedOperationException();
- case TIMESTAMP:
+// case FINETIME:
+// throw new UnsupportedOperationException();
+ case TIMESTAMP_MILLIS:
return Types.repeated(MinorType.TIMESTAMP);
default:
throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
@@ -203,7 +204,7 @@ public class ParquetToDrillTypeConverter {
return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.REPEATED, schemaElement.getScale(), schemaElement.getPrecision());
case DATE:
return Types.repeated(MinorType.DATE);
- case TIME:
+ case TIME_MILLIS:
return Types.repeated(MinorType.TIME);
default:
throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nation.parquet
----------------------------------------------------------------------
diff --git a/sample-data/nation.parquet b/sample-data/nation.parquet
old mode 100644
new mode 100755
index db65492..9109d8b
Binary files a/sample-data/nation.parquet and b/sample-data/nation.parquet differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nationsMF/nation.parquet
----------------------------------------------------------------------
diff --git a/sample-data/nationsMF/nation.parquet b/sample-data/nationsMF/nation.parquet
deleted file mode 100644
index db65492..0000000
Binary files a/sample-data/nationsMF/nation.parquet and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nationsMF/nation2.parquet
----------------------------------------------------------------------
diff --git a/sample-data/nationsMF/nation2.parquet b/sample-data/nationsMF/nation2.parquet
deleted file mode 100644
index db65492..0000000
Binary files a/sample-data/nationsMF/nation2.parquet and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nationsMF/nationsMF.parquet
----------------------------------------------------------------------
diff --git a/sample-data/nationsMF/nationsMF.parquet b/sample-data/nationsMF/nationsMF.parquet
new file mode 100755
index 0000000..614487b
Binary files /dev/null and b/sample-data/nationsMF/nationsMF.parquet differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nationsSF/nation.parquet
----------------------------------------------------------------------
diff --git a/sample-data/nationsSF/nation.parquet b/sample-data/nationsSF/nation.parquet
deleted file mode 100644
index db65492..0000000
Binary files a/sample-data/nationsSF/nation.parquet and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nationsSF/nationsSF.parquet
----------------------------------------------------------------------
diff --git a/sample-data/nationsSF/nationsSF.parquet b/sample-data/nationsSF/nationsSF.parquet
new file mode 100755
index 0000000..9109d8b
Binary files /dev/null and b/sample-data/nationsSF/nationsSF.parquet differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/region.parquet
----------------------------------------------------------------------
diff --git a/sample-data/region.parquet b/sample-data/region.parquet
old mode 100644
new mode 100755
index ab8122c..fc72fcf
Binary files a/sample-data/region.parquet and b/sample-data/region.parquet differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/regionsMF/region.parquet
----------------------------------------------------------------------
diff --git a/sample-data/regionsMF/region.parquet b/sample-data/regionsMF/region.parquet
deleted file mode 100644
index ab8122c..0000000
Binary files a/sample-data/regionsMF/region.parquet and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/regionsMF/region2.parquet
----------------------------------------------------------------------
diff --git a/sample-data/regionsMF/region2.parquet b/sample-data/regionsMF/region2.parquet
deleted file mode 100644
index ab8122c..0000000
Binary files a/sample-data/regionsMF/region2.parquet and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/regionsMF/regionsMF_Typed.parquet
----------------------------------------------------------------------
diff --git a/sample-data/regionsMF/regionsMF_Typed.parquet b/sample-data/regionsMF/regionsMF_Typed.parquet
new file mode 100755
index 0000000..eda97a2
Binary files /dev/null and b/sample-data/regionsMF/regionsMF_Typed.parquet differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/regionsSF/region.parquet
----------------------------------------------------------------------
diff --git a/sample-data/regionsSF/region.parquet b/sample-data/regionsSF/region.parquet
deleted file mode 100644
index ab8122c..0000000
Binary files a/sample-data/regionsSF/region.parquet and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/regionsSF/regionsSF.parquet
----------------------------------------------------------------------
diff --git a/sample-data/regionsSF/regionsSF.parquet b/sample-data/regionsSF/regionsSF.parquet
new file mode 100755
index 0000000..fc72fcf
Binary files /dev/null and b/sample-data/regionsSF/regionsSF.parquet differ