You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by me...@apache.org on 2015/05/08 05:27:36 UTC
[1/2] drill git commit: DRILL-2848: Hygiene - ParquetToDrillConverter
cleanup
Repository: drill
Updated Branches:
refs/heads/master 79a712aae -> 7abd7cf4e
DRILL-2848: Hygiene - ParquetToDrillConverter cleanup
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a8fa4023
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a8fa4023
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a8fa4023
Branch: refs/heads/master
Commit: a8fa402365831c89f1a5b7fbd9e68f862104a026
Parents: 79a712a
Author: Mehant Baid <me...@gmail.com>
Authored: Fri May 1 14:12:09 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Thu May 7 16:31:01 2015 -0700
----------------------------------------------------------------------
.../drill/common/util/CoreDecimalUtility.java | 5 +-
.../ParquetToDrillTypeConverter.java | 258 +++++--------------
2 files changed, 73 insertions(+), 190 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a8fa4023/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
index 302652e..9f61ccf 100644
--- a/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
+++ b/common/src/main/java/org/apache/drill/common/util/CoreDecimalUtility.java
@@ -78,7 +78,10 @@ public class CoreDecimalUtility {
* Helper function to detect if the given data type is Decimal
*/
public static boolean isDecimalType(TypeProtos.MajorType type) {
- TypeProtos.MinorType minorType = type.getMinorType();
+ return isDecimalType(type.getMinorType());
+ }
+
+ public static boolean isDecimalType(TypeProtos.MinorType minorType) {
if (minorType == TypeProtos.MinorType.DECIMAL9 || minorType == TypeProtos.MinorType.DECIMAL18 ||
minorType == TypeProtos.MinorType.DECIMAL28SPARSE || minorType == TypeProtos.MinorType.DECIMAL38SPARSE) {
return true;
http://git-wip-us.apache.org/repos/asf/drill/blob/a8fa4023/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
index 7c3eeb8..53f059e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
@@ -20,10 +20,9 @@ package org.apache.drill.exec.store.parquet.columnreaders;
import static parquet.Preconditions.checkArgument;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
+import org.apache.drill.common.util.CoreDecimalUtility;
import parquet.format.ConvertedType;
import parquet.format.SchemaElement;
import parquet.schema.PrimitiveType;
@@ -34,202 +33,83 @@ public class ParquetToDrillTypeConverter {
return schemaElement.getPrecision() <= 28 ? TypeProtos.MinorType.DECIMAL28SPARSE : MinorType.DECIMAL38SPARSE;
}
- public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName primitiveTypeName, int length,
- TypeProtos.DataMode mode, SchemaElement schemaElement) {
+ private static TypeProtos.MinorType getMinorType(PrimitiveType.PrimitiveTypeName primitiveTypeName, int length,
+ SchemaElement schemaElement) {
+
ConvertedType convertedType = schemaElement.getConverted_type();
- switch (mode) {
- case OPTIONAL:
- switch (primitiveTypeName) {
- case BINARY:
- if (convertedType == null) {
- return Types.optional(TypeProtos.MinorType.VARBINARY);
- }
- switch (convertedType) {
- case UTF8:
- return Types.optional(TypeProtos.MinorType.VARCHAR);
- case DECIMAL:
- return Types.withScaleAndPrecision(getDecimalType(schemaElement), TypeProtos.DataMode.OPTIONAL, schemaElement.getScale(), schemaElement.getPrecision());
- default:
- return Types.optional(TypeProtos.MinorType.VARBINARY);
- }
- case INT64:
- if (convertedType == null) {
- return Types.optional(TypeProtos.MinorType.BIGINT);
- }
- switch(convertedType) {
- case DECIMAL:
- return Types.withScaleAndPrecision(TypeProtos.MinorType.DECIMAL18, DataMode.OPTIONAL, schemaElement.getScale(), schemaElement.getPrecision());
- // TODO - add this back if it is decided to be added upstream, was removed form our pull request July 2014
-// case TIME_MICROS:
-// throw new UnsupportedOperationException();
- case TIMESTAMP_MILLIS:
- return Types.optional(MinorType.TIMESTAMP);
- default:
- throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
- }
- case INT32:
- if (convertedType == null) {
- return Types.optional(TypeProtos.MinorType.INT);
- }
- switch(convertedType) {
- case DECIMAL:
- return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.OPTIONAL, schemaElement.getScale(), schemaElement.getPrecision());
- case DATE:
- return Types.optional(MinorType.DATE);
- case TIME_MILLIS:
- return Types.optional(MinorType.TIME);
- default:
- throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
- }
- case BOOLEAN:
- return Types.optional(TypeProtos.MinorType.BIT);
- case FLOAT:
- return Types.optional(TypeProtos.MinorType.FLOAT4);
- case DOUBLE:
- return Types.optional(TypeProtos.MinorType.FLOAT8);
- // TODO - Both of these are not supported by the parquet library yet (7/3/13),
- // but they are declared here for when they are implemented
- case INT96:
- return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
- .setMode(mode).build();
- case FIXED_LEN_BYTE_ARRAY:
- if (convertedType == null) {
- checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
- return TypeProtos.MajorType.newBuilder().setMinorType(MinorType.VARBINARY).setMode(mode).build();
- } else if (convertedType == ConvertedType.DECIMAL) {
- return Types.withScaleAndPrecision(getDecimalType(schemaElement), DataMode.OPTIONAL, schemaElement.getScale(), schemaElement.getPrecision());
- }
+ switch (primitiveTypeName) {
+ case BINARY:
+ if (convertedType == null) {
+ return (TypeProtos.MinorType.VARBINARY);
+ }
+ switch (convertedType) {
+ case UTF8:
+ return (TypeProtos.MinorType.VARCHAR);
+ case DECIMAL:
+ return (getDecimalType(schemaElement));
default:
- throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+ return (TypeProtos.MinorType.VARBINARY);
+ }
+ case INT64:
+ if (convertedType == null) {
+ return (TypeProtos.MinorType.BIGINT);
}
- case REQUIRED:
- switch (primitiveTypeName) {
- case BINARY:
- if (convertedType == null) {
- return Types.required(TypeProtos.MinorType.VARBINARY);
- }
- switch (convertedType) {
- case UTF8:
- return Types.required(MinorType.VARCHAR);
- case DECIMAL:
- return Types.withScaleAndPrecision(getDecimalType(schemaElement), DataMode.REQUIRED, schemaElement.getScale(), schemaElement.getPrecision());
- default:
- return Types.required(TypeProtos.MinorType.VARBINARY);
- }
- case INT64:
- if (convertedType == null) {
- return Types.required(MinorType.BIGINT);
- }
- switch(convertedType) {
- case DECIMAL:
- return Types.withScaleAndPrecision(MinorType.DECIMAL18, DataMode.REQUIRED, schemaElement.getScale(), schemaElement.getPrecision());
-// case FINETIME:
+ switch(convertedType) {
+ case DECIMAL:
+ return TypeProtos.MinorType.DECIMAL18;
+ // TODO - add this back if it is decided to be added upstream, was removed form our pull request July 2014
+// case TIME_MICROS:
// throw new UnsupportedOperationException();
- case TIMESTAMP_MILLIS:
- return Types.required(MinorType.TIMESTAMP);
- default:
- throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
- }
- case INT32:
- if (convertedType == null) {
- return Types.required(MinorType.INT);
- }
- switch(convertedType) {
- case DECIMAL:
- return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.REQUIRED, schemaElement.getScale(), schemaElement.getPrecision());
- case DATE:
- return Types.required(MinorType.DATE);
- case TIME_MILLIS:
- return Types.required(MinorType.TIME);
- default:
- throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
- }
- case BOOLEAN:
- return Types.required(TypeProtos.MinorType.BIT);
- case FLOAT:
- return Types.required(TypeProtos.MinorType.FLOAT4);
- case DOUBLE:
- return Types.required(TypeProtos.MinorType.FLOAT8);
- // Both of these are not supported by the parquet library yet (7/3/13),
- // but they are declared here for when they are implemented
- case INT96:
- return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
- .setMode(mode).build();
- case FIXED_LEN_BYTE_ARRAY:
- if (convertedType == null) {
- checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
- return TypeProtos.MajorType.newBuilder().setMinorType(MinorType.VARBINARY).setMode(mode).build();
- } else if (convertedType == ConvertedType.DECIMAL) {
- return Types.withScaleAndPrecision(getDecimalType(schemaElement), DataMode.REQUIRED, schemaElement.getScale(), schemaElement.getPrecision());
- }
+ case TIMESTAMP_MILLIS:
+ return TypeProtos.MinorType.TIMESTAMP;
default:
- throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+ throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
}
- case REPEATED:
- switch (primitiveTypeName) {
- case BINARY:
- if (convertedType == null) {
- return Types.repeated(TypeProtos.MinorType.VARBINARY);
- }
- switch (schemaElement.getConverted_type()) {
- case UTF8:
- return Types.repeated(MinorType.VARCHAR);
- case DECIMAL:
- return Types.withScaleAndPrecision(getDecimalType(schemaElement), DataMode.REPEATED, schemaElement.getScale(), schemaElement.getPrecision());
- default:
- return Types.repeated(TypeProtos.MinorType.VARBINARY);
- }
- case INT64:
- if (convertedType == null) {
- return Types.repeated(MinorType.BIGINT);
- }
- switch(convertedType) {
- case DECIMAL:
- return Types.withScaleAndPrecision(MinorType.DECIMAL18, DataMode.REPEATED, schemaElement.getScale(), schemaElement.getPrecision());
-// case FINETIME:
-// throw new UnsupportedOperationException();
- case TIMESTAMP_MILLIS:
- return Types.repeated(MinorType.TIMESTAMP);
- default:
- throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
- }
- case INT32:
- if (convertedType == null) {
- return Types.repeated(MinorType.INT);
- }
- switch(convertedType) {
- case DECIMAL:
- return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.REPEATED, schemaElement.getScale(), schemaElement.getPrecision());
- case DATE:
- return Types.repeated(MinorType.DATE);
- case TIME_MILLIS:
- return Types.repeated(MinorType.TIME);
- default:
- throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
- }
- case BOOLEAN:
- return Types.repeated(TypeProtos.MinorType.BIT);
- case FLOAT:
- return Types.repeated(TypeProtos.MinorType.FLOAT4);
- case DOUBLE:
- return Types.repeated(TypeProtos.MinorType.FLOAT8);
- // Both of these are not supported by the parquet library yet (7/3/13),
- // but they are declared here for when they are implemented
- case INT96:
- return TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.FIXEDBINARY).setWidth(12)
- .setMode(mode).build();
- case FIXED_LEN_BYTE_ARRAY:
- if (convertedType == null) {
- checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
- return TypeProtos.MajorType.newBuilder().setMinorType(MinorType.VARBINARY).setMode(mode).build();
- } else if (convertedType == ConvertedType.DECIMAL) {
- return Types.withScaleAndPrecision(getDecimalType(schemaElement), DataMode.REPEATED, schemaElement.getScale(), schemaElement.getPrecision());
- }
+ case INT32:
+ if (convertedType == null) {
+ return TypeProtos.MinorType.INT;
+ }
+ switch(convertedType) {
+ case DECIMAL:
+ return TypeProtos.MinorType.DECIMAL9;
+ case DATE:
+ return TypeProtos.MinorType.DATE;
+ case TIME_MILLIS:
+ return TypeProtos.MinorType.TIME;
default:
- throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+ throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType));
+ }
+ case BOOLEAN:
+ return TypeProtos.MinorType.BIT;
+ case FLOAT:
+ return TypeProtos.MinorType.FLOAT4;
+ case DOUBLE:
+ return TypeProtos.MinorType.FLOAT8;
+ // TODO - Both of these are not supported by the parquet library yet (7/3/13),
+ // but they are declared here for when they are implemented
+ case INT96:
+ return TypeProtos.MinorType.FIXEDBINARY;
+ case FIXED_LEN_BYTE_ARRAY:
+ if (convertedType == null) {
+ checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
+ return TypeProtos.MinorType.VARBINARY;
+ } else if (convertedType == ConvertedType.DECIMAL) {
+ return getDecimalType(schemaElement);
}
+ default:
+ throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName);
+ }
+ }
+
+ public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName primitiveTypeName, int length,
+ TypeProtos.DataMode mode, SchemaElement schemaElement) {
+ MinorType minorType = getMinorType(primitiveTypeName, length, schemaElement);
+ TypeProtos.MajorType.Builder typeBuilder = TypeProtos.MajorType.newBuilder().setMinorType(minorType).setMode(mode);
+
+ if (CoreDecimalUtility.isDecimalType(minorType)) {
+ typeBuilder.setPrecision(schemaElement.getPrecision()).setScale(schemaElement.getScale());
}
- throw new UnsupportedOperationException("Type not supported: " + primitiveTypeName + " Mode: " + mode);
+ return typeBuilder.build();
}
}
[2/2] drill git commit: DRILL-2848: Provide option to disable decimal
data type Disable casting to decimal Disable reading decimal from Parquet
Disable reading decimal from Hive Add unit tests Modify existing tests to
enable decimal data type
Posted by me...@apache.org.
DRILL-2848: Provide option to disable decimal data type
Disable casting to decimal
Disable reading decimal from Parquet
Disable reading decimal from Hive
Add unit tests
Modify existing tests to enable decimal data type
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7abd7cf4
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7abd7cf4
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7abd7cf4
Branch: refs/heads/master
Commit: 7abd7cf4e3a6e67b4168d3d598ee01eb62e346e6
Parents: a8fa402
Author: Mehant Baid <me...@gmail.com>
Authored: Thu Apr 30 17:02:18 2015 -0700
Committer: Mehant Baid <me...@gmail.com>
Committed: Thu May 7 16:31:43 2015 -0700
----------------------------------------------------------------------
.../store/hbase/HBasePushFilterIntoScan.java | 3 +-
.../drill/exec/store/hive/HiveRecordReader.java | 13 +-
.../drill/exec/TestHiveProjectPushDown.java | 15 ++
.../drill/exec/fn/hive/TestSampleHiveUDFs.java | 18 +-
.../apache/drill/exec/hive/TestHiveStorage.java | 199 ++++++++++---------
.../store/mongo/MongoPushDownFilterForScan.java | 3 +-
.../planner/logical/DrillConstExecutor.java | 8 +-
.../drill/exec/planner/logical/DrillOptiq.java | 42 ++--
.../exec/planner/logical/DrillParseContext.java | 12 +-
.../logical/partition/PruneScanRule.java | 2 +-
.../drill/exec/planner/physical/FilterPrel.java | 2 +-
.../exec/planner/physical/FlattenPrel.java | 2 +-
.../exec/planner/physical/PlannerSettings.java | 2 +
.../planner/physical/ProjectAllowDupPrel.java | 2 +-
.../exec/planner/physical/ProjectPrel.java | 3 +-
.../drill/exec/planner/sql/DrillSqlWorker.java | 2 +-
.../planner/sql/handlers/ExplainHandler.java | 2 +-
.../server/options/SystemOptionManager.java | 1 +
...InfoSchemaPushFilterIntoRecordGenerator.java | 3 +-
.../store/parquet/ParquetReaderUtility.java | 37 ++++
.../columnreaders/ParquetRecordReader.java | 8 +-
.../ParquetToDrillTypeConverter.java | 17 +-
.../parquet2/DrillParquetGroupConverter.java | 21 +-
.../exec/store/parquet2/DrillParquetReader.java | 4 +-
.../DrillParquetRecordMaterializer.java | 6 +-
.../drill/exec/work/ExecErrorConstants.java | 28 +++
.../java/org/apache/drill/TestBugFixes.java | 10 +-
.../apache/drill/TestDisabledFunctionality.java | 10 +
.../org/apache/drill/TestFrameworkTest.java | 54 ++---
.../org/apache/drill/TestFunctionsQuery.java | 14 ++
.../exec/fn/impl/TestCastEmptyStrings.java | 14 ++
.../exec/fn/interp/TestConstantFolding.java | 6 +
.../physical/impl/writer/TestParquetWriter.java | 10 +
.../exec/physical/impl/writer/TestWriter.java | 18 +-
.../columnreaders/TestColumnReaderFactory.java | 13 ++
.../store/parquet2/TestDrillParquetReader.java | 13 ++
.../test/JdbcNullOrderingAndGroupingTest.java | 11 +-
.../drill/jdbc/test/JdbcTestQueryBase.java | 2 +-
.../jdbc/test/TestAggregateFunctionsQuery.java | 17 +-
39 files changed, 465 insertions(+), 182 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java
index f1f3a0b..cd97b59 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBasePushFilterIntoScan.java
@@ -23,6 +23,7 @@ import org.apache.drill.exec.planner.logical.DrillOptiq;
import org.apache.drill.exec.planner.logical.DrillParseContext;
import org.apache.drill.exec.planner.logical.RelOptHelper;
import org.apache.drill.exec.planner.physical.FilterPrel;
+import org.apache.drill.exec.planner.physical.PrelUtil;
import org.apache.drill.exec.planner.physical.ScanPrel;
import org.apache.drill.exec.store.StoragePluginOptimizerRule;
import org.apache.calcite.rel.RelNode;
@@ -55,7 +56,7 @@ public class HBasePushFilterIntoScan extends StoragePluginOptimizerRule {
return;
}
- LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(), scan, condition);
+ LogicalExpression conditionExp = DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
HBaseFilterBuilder hbaseFilterBuilder = new HBaseFilterBuilder(groupScan, conditionExp);
HBaseScanSpec newScanSpec = hbaseFilterBuilder.parseTree();
if (newScanSpec == null) {
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
index a4ad0c4..5394ee3 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
@@ -27,6 +27,7 @@ import java.util.Properties;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.DataMode;
@@ -41,6 +42,7 @@ import org.apache.drill.exec.expr.holders.Decimal9Holder;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.rpc.ProtobufLengthDecoder;
import org.apache.drill.exec.store.AbstractRecordReader;
@@ -62,6 +64,7 @@ import org.apache.drill.exec.vector.TinyIntVector;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
+import org.apache.drill.exec.work.ExecErrorConstants;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -347,7 +350,7 @@ public class HiveRecordReader extends AbstractRecordReader {
}
}
- public static MinorType getMinorTypeFromHivePrimitiveTypeInfo(PrimitiveTypeInfo primitiveTypeInfo) {
+ private MinorType getMinorTypeFromHivePrimitiveTypeInfo(PrimitiveTypeInfo primitiveTypeInfo) {
switch(primitiveTypeInfo.getPrimitiveCategory()) {
case BINARY:
return TypeProtos.MinorType.VARBINARY;
@@ -356,6 +359,12 @@ public class HiveRecordReader extends AbstractRecordReader {
case BYTE:
return MinorType.TINYINT;
case DECIMAL: {
+
+ if (context.getOptions().getOption(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY).bool_val == false) {
+ throw UserException.unsupportedError()
+ .message(ExecErrorConstants.DECIMAL_DISABLE_ERR_MSG)
+ .build();
+ }
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
return DecimalUtility.getDecimalDataType(decimalTypeInfo.precision());
}
@@ -382,7 +391,7 @@ public class HiveRecordReader extends AbstractRecordReader {
return null;
}
- public static MajorType getMajorTypeFromHiveTypeInfo(TypeInfo typeInfo, boolean nullable) {
+ public MajorType getMajorTypeFromHiveTypeInfo(TypeInfo typeInfo, boolean nullable) {
switch (typeInfo.getCategory()) {
case PRIMITIVE: {
PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
index 2842976..6423a36 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
@@ -20,9 +20,24 @@ package org.apache.drill.exec;
import static org.junit.Assert.assertEquals;
import org.apache.drill.exec.hive.HiveTestBase;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestHiveProjectPushDown extends HiveTestBase {
+
+ // enable decimal data type
+ @BeforeClass
+ public static void enableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
+ @AfterClass
+ public static void disableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
private void testHelper(String query, String expectedColNamesInPlan, int expectedRecordCount)throws Exception {
testPhysicalPlan(query, expectedColNamesInPlan);
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
index f4b6351..ac11752 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue;
import java.util.List;
import org.apache.drill.exec.hive.HiveTestBase;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.junit.Ignore;
import org.junit.Test;
@@ -138,11 +139,16 @@ public class TestSampleHiveUDFs extends HiveTestBase {
@Test
public void decimalInOut() throws Exception{
- String query = "SELECT " +
- "testHiveUDFDecimal(cast('1234567891234567891234567891234567891.4' as decimal(38, 1))) as col1 " +
- "FROM hive.kv LIMIT 1";
-
- String expected = "col1\n" + "1234567891234567891234567891234567891.4\n";
- helper(query, expected);
+ try {
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ String query = "SELECT " +
+ "testHiveUDFDecimal(cast('1234567891234567891234567891234567891.4' as decimal(38, 1))) as col1 " +
+ "FROM hive.kv LIMIT 1";
+
+ String expected = "col1\n" + "1234567891234567891234567891234567891.4\n";
+ helper(query, expected);
+ } finally {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index 03747bf..a25a524 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.hive;
import com.google.common.collect.ImmutableMap;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.hadoop.fs.FileSystem;
import org.joda.time.DateTime;
import org.junit.Test;
@@ -47,103 +48,109 @@ public class TestHiveStorage extends HiveTestBase {
*/
@Test
public void readAllSupportedHiveDataTypes() throws Exception {
+ try {
+ // enable decimal type
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
- testBuilder().sqlQuery("SELECT * FROM hive.readtest")
- .unOrdered()
- .baselineColumns(
- "binary_field",
- "boolean_field",
- "tinyint_field",
- "decimal0_field",
- "decimal9_field",
- "decimal18_field",
- "decimal28_field",
- "decimal38_field",
- "double_field",
- "float_field",
- "int_field",
- "bigint_field",
- "smallint_field",
- "string_field",
- "varchar_field",
- "timestamp_field",
- "date_field",
- "binary_part",
- "boolean_part",
- "tinyint_part",
- "decimal0_part",
- "decimal9_part",
- "decimal18_part",
- "decimal28_part",
- "decimal38_part",
- "double_part",
- "float_part",
- "int_part",
- "bigint_part",
- "smallint_part",
- "string_part",
- "varchar_part",
- "timestamp_part",
- "date_part")
- .baselineValues(
- "binaryfield",
- false,
- (byte) 34,
- new BigDecimal("66"),
- new BigDecimal("2347.92"),
- new BigDecimal("2758725827.99990"),
- new BigDecimal("29375892739852.8"),
- new BigDecimal("89853749534593985.783"),
- 8.345d,
- 4.67f,
- 123456,
- 234235L,
- (short) 3455,
- "stringfield",
- "varcharfield",
- new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
- new DateTime(Date.valueOf("2013-07-05").getTime()),
- "binary",
- true,
- (byte) 64,
- new BigDecimal("37"),
- new BigDecimal("36.90"),
- new BigDecimal("3289379872.94565"),
- new BigDecimal("39579334534534.4"),
- new BigDecimal("363945093845093890.900"),
- 8.345d,
- 4.67f,
- 123456,
- 234235L,
- (short) 3455,
- "string",
- "varchar",
- new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
- new DateTime(Date.valueOf("2013-07-05").getTime()))
- .baselineValues( // All fields are null, but partition fields have non-null values
- "", // For binary (varchar, string too) empty value is considered as empty string instead of "null"
- null, null, null, null, null, null, null, null, null, null, null, null,
- "", // string_field
- "", // varchar_field
- null, null,
- "binary",
- true,
- (byte) 64,
- new BigDecimal("37"),
- new BigDecimal("36.90"),
- new BigDecimal("3289379872.94565"),
- new BigDecimal("39579334534534.4"),
- new BigDecimal("363945093845093890.900"),
- 8.345d,
- 4.67f,
- 123456,
- 234235L,
- (short) 3455,
- "string",
- "varchar",
- new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
- new DateTime(Date.valueOf("2013-07-05").getTime()))
- .go();
+ testBuilder().sqlQuery("SELECT * FROM hive.readtest")
+ .unOrdered()
+ .baselineColumns(
+ "binary_field",
+ "boolean_field",
+ "tinyint_field",
+ "decimal0_field",
+ "decimal9_field",
+ "decimal18_field",
+ "decimal28_field",
+ "decimal38_field",
+ "double_field",
+ "float_field",
+ "int_field",
+ "bigint_field",
+ "smallint_field",
+ "string_field",
+ "varchar_field",
+ "timestamp_field",
+ "date_field",
+ "binary_part",
+ "boolean_part",
+ "tinyint_part",
+ "decimal0_part",
+ "decimal9_part",
+ "decimal18_part",
+ "decimal28_part",
+ "decimal38_part",
+ "double_part",
+ "float_part",
+ "int_part",
+ "bigint_part",
+ "smallint_part",
+ "string_part",
+ "varchar_part",
+ "timestamp_part",
+ "date_part")
+ .baselineValues(
+ "binaryfield",
+ false,
+ (byte) 34,
+ new BigDecimal("66"),
+ new BigDecimal("2347.92"),
+ new BigDecimal("2758725827.99990"),
+ new BigDecimal("29375892739852.8"),
+ new BigDecimal("89853749534593985.783"),
+ 8.345d,
+ 4.67f,
+ 123456,
+ 234235L,
+ (short) 3455,
+ "stringfield",
+ "varcharfield",
+ new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
+ new DateTime(Date.valueOf("2013-07-05").getTime()),
+ "binary",
+ true,
+ (byte) 64,
+ new BigDecimal("37"),
+ new BigDecimal("36.90"),
+ new BigDecimal("3289379872.94565"),
+ new BigDecimal("39579334534534.4"),
+ new BigDecimal("363945093845093890.900"),
+ 8.345d,
+ 4.67f,
+ 123456,
+ 234235L,
+ (short) 3455,
+ "string",
+ "varchar",
+ new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
+ new DateTime(Date.valueOf("2013-07-05").getTime()))
+ .baselineValues( // All fields are null, but partition fields have non-null values
+ "", // For binary (varchar, string too) empty value is considered as empty string instead of "null"
+ null, null, null, null, null, null, null, null, null, null, null, null,
+ "", // string_field
+ "", // varchar_field
+ null, null,
+ "binary",
+ true,
+ (byte) 64,
+ new BigDecimal("37"),
+ new BigDecimal("36.90"),
+ new BigDecimal("3289379872.94565"),
+ new BigDecimal("39579334534534.4"),
+ new BigDecimal("363945093845093890.900"),
+ 8.345d,
+ 4.67f,
+ 123456,
+ 234235L,
+ (short) 3455,
+ "string",
+ "varchar",
+ new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
+ new DateTime(Date.valueOf("2013-07-05").getTime()))
+ .go();
+ } finally {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoPushDownFilterForScan.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoPushDownFilterForScan.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoPushDownFilterForScan.java
index 4fd80bd..5b3fcd8 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoPushDownFilterForScan.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoPushDownFilterForScan.java
@@ -25,6 +25,7 @@ import org.apache.drill.exec.planner.logical.DrillOptiq;
import org.apache.drill.exec.planner.logical.DrillParseContext;
import org.apache.drill.exec.planner.logical.RelOptHelper;
import org.apache.drill.exec.planner.physical.FilterPrel;
+import org.apache.drill.exec.planner.physical.PrelUtil;
import org.apache.drill.exec.planner.physical.ScanPrel;
import org.apache.drill.exec.store.StoragePluginOptimizerRule;
import org.apache.calcite.rel.RelNode;
@@ -58,7 +59,7 @@ public class MongoPushDownFilterForScan extends StoragePluginOptimizerRule {
}
LogicalExpression conditionExp = DrillOptiq.toDrill(
- new DrillParseContext(), scan, condition);
+ new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), scan, condition);
MongoFilterBuilder mongoFilterBuilder = new MongoFilterBuilder(groupScan,
conditionExp);
MongoScanSpec newScanSpec = mongoFilterBuilder.parseTree();
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
index 92e5678..15258ef 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
@@ -57,6 +57,7 @@ import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.NlsString;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
@@ -67,6 +68,8 @@ import java.util.List;
public class DrillConstExecutor implements RelOptPlanner.Executor {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConstExecutor.class);
+ private final PlannerSettings plannerSettings;
+
public static ImmutableMap<TypeProtos.MinorType, SqlTypeName> DRILL_TO_CALCITE_TYPE_MAPPING =
ImmutableMap.<TypeProtos.MinorType, SqlTypeName> builder()
.put(TypeProtos.MinorType.INT, SqlTypeName.INTEGER)
@@ -123,9 +126,10 @@ public class DrillConstExecutor implements RelOptPlanner.Executor {
FunctionImplementationRegistry funcImplReg;
UdfUtilities udfUtilities;
- public DrillConstExecutor(FunctionImplementationRegistry funcImplReg, UdfUtilities udfUtilities) {
+ public DrillConstExecutor(FunctionImplementationRegistry funcImplReg, UdfUtilities udfUtilities, PlannerSettings plannerSettings) {
this.funcImplReg = funcImplReg;
this.udfUtilities = udfUtilities;
+ this.plannerSettings = plannerSettings;
}
private RelDataType createCalciteTypeWithNullability(RelDataTypeFactory typeFactory,
@@ -155,7 +159,7 @@ public class DrillConstExecutor implements RelOptPlanner.Executor {
@Override
public void reduce(RexBuilder rexBuilder, List<RexNode> constExps, List<RexNode> reducedValues) {
for (RexNode newCall : constExps) {
- LogicalExpression logEx = DrillOptiq.toDrill(new DrillParseContext(), null /* input rel */, newCall);
+ LogicalExpression logEx = DrillOptiq.toDrill(new DrillParseContext(plannerSettings), null /* input rel */, newCall);
ErrorCollectorImpl errors = new ErrorCollectorImpl();
LogicalExpression materializedExpr = ExpressionTreeMaterializer.materialize(logEx, null, errors, funcImplReg);
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index 441f2e3..daf2567 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -21,6 +21,7 @@ import java.math.BigDecimal;
import java.util.GregorianCalendar;
import java.util.List;
+import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.FunctionCallFactory;
@@ -55,6 +56,8 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.util.NlsString;
import com.google.common.collect.Lists;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.work.ExecErrorConstants;
/**
* Utilities for Drill's planner.
@@ -252,23 +255,30 @@ public class DrillOptiq {
case "FLOAT": castType = Types.required(MinorType.FLOAT4); break;
case "DOUBLE": castType = Types.required(MinorType.FLOAT8); break;
case "DECIMAL":
+ if (context.getPlannerSettings().getOptions().
+ getOption(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY).bool_val == false ) {
+ throw UserException
+ .unsupportedError()
+ .message(ExecErrorConstants.DECIMAL_DISABLE_ERR_MSG)
+ .build();
+ }
- int precision = call.getType().getPrecision();
- int scale = call.getType().getScale();
-
- if (precision <= 9) {
- castType = TypeProtos.MajorType.newBuilder().setMinorType(MinorType.DECIMAL9).setPrecision(precision).setScale(scale).build();
- } else if (precision <= 18) {
- castType = TypeProtos.MajorType.newBuilder().setMinorType(MinorType.DECIMAL18).setPrecision(precision).setScale(scale).build();
- } else if (precision <= 28) {
- // Inject a cast to SPARSE before casting to the dense type.
- castType = TypeProtos.MajorType.newBuilder().setMinorType(MinorType.DECIMAL28SPARSE).setPrecision(precision).setScale(scale).build();
- } else if (precision <= 38) {
- castType = TypeProtos.MajorType.newBuilder().setMinorType(MinorType.DECIMAL38SPARSE).setPrecision(precision).setScale(scale).build();
- } else {
- throw new UnsupportedOperationException("Only Decimal types with precision range 0 - 38 is supported");
- }
- break;
+ int precision = call.getType().getPrecision();
+ int scale = call.getType().getScale();
+
+ if (precision <= 9) {
+ castType = TypeProtos.MajorType.newBuilder().setMinorType(MinorType.DECIMAL9).setPrecision(precision).setScale(scale).build();
+ } else if (precision <= 18) {
+ castType = TypeProtos.MajorType.newBuilder().setMinorType(MinorType.DECIMAL18).setPrecision(precision).setScale(scale).build();
+ } else if (precision <= 28) {
+ // Inject a cast to SPARSE before casting to the dense type.
+ castType = TypeProtos.MajorType.newBuilder().setMinorType(MinorType.DECIMAL28SPARSE).setPrecision(precision).setScale(scale).build();
+ } else if (precision <= 38) {
+ castType = TypeProtos.MajorType.newBuilder().setMinorType(MinorType.DECIMAL38SPARSE).setPrecision(precision).setScale(scale).build();
+ } else {
+ throw new UnsupportedOperationException("Only Decimal types with precision range 0 - 38 is supported");
+ }
+ break;
case "INTERVAL_YEAR_MONTH": castType = Types.required(MinorType.INTERVALYEAR); break;
case "INTERVAL_DAY_TIME": castType = Types.required(MinorType.INTERVALDAY); break;
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
index be4474f..63b38f6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
@@ -17,12 +17,18 @@
*/
package org.apache.drill.exec.planner.logical;
-// TODO: Can we remove this completely? Or is there any plan to add any props in future for parsing?
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+
public class DrillParseContext {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillParseContext.class);
- public DrillParseContext() {
- super();
+ private final PlannerSettings plannerSettings;
+
+ public DrillParseContext(PlannerSettings plannerSettings) {
+ this.plannerSettings = plannerSettings;
}
+ public PlannerSettings getPlannerSettings() {
+ return plannerSettings;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
index c8be019..2544d34 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
@@ -213,7 +213,7 @@ public abstract class PruneScanRule extends RelOptRule {
// materialize the expression
logger.debug("Attempting to prune {}", pruneCondition);
- LogicalExpression expr = DrillOptiq.toDrill(new DrillParseContext(), scanRel, pruneCondition);
+ LogicalExpression expr = DrillOptiq.toDrill(new DrillParseContext(settings), scanRel, pruneCondition);
ErrorCollectorImpl errors = new ErrorCollectorImpl();
LogicalExpression materializedExpr = ExpressionTreeMaterializer.materialize(expr, container, errors, context.getFunctionRegistry());
if (errors.getErrorCount() != 0) {
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
index b631cdc..233adbe 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FilterPrel.java
@@ -51,7 +51,7 @@ public class FilterPrel extends DrillFilterRelBase implements Prel {
PhysicalOperator childPOP = child.getPhysicalOperator(creator);
- Filter p = new Filter(childPOP, getFilterExpression(new DrillParseContext()), 1.0f);
+ Filter p = new Filter(childPOP, getFilterExpression(new DrillParseContext(PrelUtil.getSettings(getCluster()))), 1.0f);
return creator.addMetadata(this, p);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
index e206951..f6a2b7a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
@@ -61,7 +61,7 @@ public class FlattenPrel extends SinglePrel implements Prel {
Prel child = (Prel) this.getInput();
PhysicalOperator childPOP = child.getPhysicalOperator(creator);
- FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext()));
+ FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext(PrelUtil.getSettings(getCluster()))));
return creator.addMetadata(this, f);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index 2fd8135..7d8dd97 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -61,6 +61,8 @@ public class PlannerSettings implements Context{
public static final OptionValidator HASH_SINGLE_KEY = new BooleanValidator("planner.enable_hash_single_key", true);
public static final OptionValidator HASH_JOIN_SWAP = new BooleanValidator("planner.enable_hashjoin_swap", true);
public static final OptionValidator HASH_JOIN_SWAP_MARGIN_FACTOR = new RangeDoubleValidator("planner.join.hash_join_swap_margin_factor", 0, 100, 10d);
+ public static final String ENABLE_DECIMAL_DATA_TYPE_KEY = "planner.enable_decimal_data_type";
+ public static final OptionValidator ENABLE_DECIMAL_DATA_TYPE = new BooleanValidator(ENABLE_DECIMAL_DATA_TYPE_KEY, false);
public static final OptionValidator IDENTIFIER_MAX_LENGTH =
new RangeLongValidator("planner.identifier_max_length", 128 /* A minimum length is needed because option names are identifiers themselves */,
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectAllowDupPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectAllowDupPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectAllowDupPrel.java
index cc215f8..3c5c045 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectAllowDupPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectAllowDupPrel.java
@@ -55,7 +55,7 @@ public class ProjectAllowDupPrel extends ProjectPrel {
PhysicalOperator childPOP = child.getPhysicalOperator(creator);
- Project p = new Project(this.getProjectExpressions(new DrillParseContext()), childPOP);
+ Project p = new Project(this.getProjectExpressions(new DrillParseContext(PrelUtil.getSettings(getCluster()))), childPOP);
return creator.addMetadata(this, p);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
index 35fa5be..e109dc4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ProjectPrel.java
@@ -54,7 +54,8 @@ public class ProjectPrel extends DrillProjectRelBase implements Prel{
PhysicalOperator childPOP = child.getPhysicalOperator(creator);
- org.apache.drill.exec.physical.config.Project p = new org.apache.drill.exec.physical.config.Project(this.getProjectExpressions(new DrillParseContext()), childPOP);
+ org.apache.drill.exec.physical.config.Project p = new org.apache.drill.exec.physical.config.Project(
+ this.getProjectExpressions(new DrillParseContext(PrelUtil.getSettings(getCluster()))), childPOP);
return creator.addMetadata(this, p);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
index c918723..3c78c08 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
@@ -100,7 +100,7 @@ public class DrillSqlWorker {
.context(context.getPlannerSettings()) //
.ruleSets(getRules(context)) //
.costFactory(costFactory) //
- .executor(new DrillConstExecutor(context.getFunctionRegistry(), context))
+ .executor(new DrillConstExecutor(context.getFunctionRegistry(), context, context.getPlannerSettings()))
.typeSystem(DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM) //
.build();
this.planner = Frameworks.getPlanner(config);
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java
index 1636a25..5924c7e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java
@@ -109,7 +109,7 @@ public class ExplainHandler extends DefaultSqlHandler {
public LogicalExplain(RelNode node, SqlExplainLevel level, QueryContext context) {
this.text = RelOptUtil.toString(node, level);
- DrillImplementor implementor = new DrillImplementor(new DrillParseContext(), ResultMode.LOGICAL);
+ DrillImplementor implementor = new DrillImplementor(new DrillParseContext(context.getPlannerSettings()), ResultMode.LOGICAL);
implementor.go( (DrillRel) node);
LogicalPlan plan = implementor.getPlan();
this.json = plan.unparse(context.getConfig());
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index ea4af43..4d8b034 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -70,6 +70,7 @@ public class SystemOptionManager extends BaseOptionManager {
PlannerSettings.PARTITION_SENDER_THREADS_FACTOR,
PlannerSettings.PARTITION_SENDER_MAX_THREADS,
PlannerSettings.PARTITION_SENDER_SET_THREADS,
+ PlannerSettings.ENABLE_DECIMAL_DATA_TYPE,
ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION,
ExecConstants.OUTPUT_FORMAT_VALIDATOR,
ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaPushFilterIntoRecordGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaPushFilterIntoRecordGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaPushFilterIntoRecordGenerator.java
index 0cf12b4..7148262 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaPushFilterIntoRecordGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaPushFilterIntoRecordGenerator.java
@@ -24,6 +24,7 @@ import org.apache.drill.exec.planner.logical.DrillOptiq;
import org.apache.drill.exec.planner.logical.DrillParseContext;
import org.apache.drill.exec.planner.logical.RelOptHelper;
import org.apache.drill.exec.planner.physical.FilterPrel;
+import org.apache.drill.exec.planner.physical.PrelUtil;
import org.apache.drill.exec.planner.physical.ProjectPrel;
import org.apache.drill.exec.planner.physical.ScanPrel;
import org.apache.drill.exec.store.StoragePluginOptimizerRule;
@@ -89,7 +90,7 @@ public abstract class InfoSchemaPushFilterIntoRecordGenerator extends StoragePlu
}
LogicalExpression conditionExp =
- DrillOptiq.toDrill(new DrillParseContext(), project != null ? project : scan, condition);
+ DrillOptiq.toDrill(new DrillParseContext(PrelUtil.getPlannerSettings(call.getPlanner())), project != null ? project : scan, condition);
InfoSchemaFilterBuilder filterBuilder = new InfoSchemaFilterBuilder(conditionExp);
InfoSchemaFilter infoSchemaFilter = filterBuilder.build();
if (infoSchemaFilter == null) {
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java
new file mode 100644
index 0000000..5291855
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.server.options.OptionManager;
+import org.apache.drill.exec.work.ExecErrorConstants;
+
+/*
+ * Utility class where we can capture common logic between the two parquet readers
+ */
+public class ParquetReaderUtility {
+ public static void checkDecimalTypeEnabled(OptionManager options) {
+ if (options.getOption(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY).bool_val == false) {
+ throw UserException
+ .unsupportedError()
+ .message(ExecErrorConstants.DECIMAL_DISABLE_ERR_MSG)
+ .build();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
index ebfbd54..58cf321 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java
@@ -104,6 +104,7 @@ public class ParquetRecordReader extends AbstractRecordReader {
private final DirectCodecFactory codecFactory;
int rowGroupIndex;
long totalRecordsRead;
+ private final FragmentContext fragmentContext;
public ParquetRecordReader(FragmentContext fragmentContext,
String path,
@@ -131,6 +132,7 @@ public class ParquetRecordReader extends AbstractRecordReader {
this.rowGroupIndex = rowGroupIndex;
this.batchSize = batchSize;
this.footer = footer;
+ this.fragmentContext = fragmentContext;
setColumns(columns);
}
@@ -240,7 +242,8 @@ public class ParquetRecordReader extends AbstractRecordReader {
column = columns.get(i);
logger.debug("name: " + fileMetaData.getSchema().get(i).name);
SchemaElement se = schemaElements.get(column.getPath()[0]);
- MajorType mt = ParquetToDrillTypeConverter.toMajorType(column.getType(), se.getType_length(), getDataMode(column), se);
+ MajorType mt = ParquetToDrillTypeConverter.toMajorType(column.getType(), se.getType_length(),
+ getDataMode(column), se, fragmentContext.getOptions());
field = MaterializedField.create(toFieldName(column.getPath()),mt);
if ( ! fieldSelected(field)) {
continue;
@@ -280,7 +283,8 @@ public class ParquetRecordReader extends AbstractRecordReader {
column = columns.get(i);
columnChunkMetaData = footer.getBlocks().get(rowGroupIndex).getColumns().get(i);
schemaElement = schemaElements.get(column.getPath()[0]);
- MajorType type = ParquetToDrillTypeConverter.toMajorType(column.getType(), schemaElement.getType_length(), getDataMode(column), schemaElement);
+ MajorType type = ParquetToDrillTypeConverter.toMajorType(column.getType(), schemaElement.getType_length(),
+ getDataMode(column), schemaElement, fragmentContext.getOptions());
field = MaterializedField.create(toFieldName(column.getPath()), type);
// the field was not requested to be read
if ( ! fieldSelected(field)) {
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
index 53f059e..8ab5fea 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java
@@ -19,10 +19,16 @@ package org.apache.drill.exec.store.parquet.columnreaders;
import static parquet.Preconditions.checkArgument;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.util.CoreDecimalUtility;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.server.options.OptionManager;
+import org.apache.drill.exec.store.parquet.ParquetReaderUtility;
+import org.apache.drill.exec.work.ExecErrorConstants;
import parquet.format.ConvertedType;
import parquet.format.SchemaElement;
import parquet.schema.PrimitiveType;
@@ -34,7 +40,7 @@ public class ParquetToDrillTypeConverter {
}
private static TypeProtos.MinorType getMinorType(PrimitiveType.PrimitiveTypeName primitiveTypeName, int length,
- SchemaElement schemaElement) {
+ SchemaElement schemaElement, OptionManager options) {
ConvertedType convertedType = schemaElement.getConverted_type();
@@ -47,6 +53,7 @@ public class ParquetToDrillTypeConverter {
case UTF8:
return (TypeProtos.MinorType.VARCHAR);
case DECIMAL:
+ ParquetReaderUtility.checkDecimalTypeEnabled(options);
return (getDecimalType(schemaElement));
default:
return (TypeProtos.MinorType.VARBINARY);
@@ -57,6 +64,7 @@ public class ParquetToDrillTypeConverter {
}
switch(convertedType) {
case DECIMAL:
+ ParquetReaderUtility.checkDecimalTypeEnabled(options);
return TypeProtos.MinorType.DECIMAL18;
// TODO - add this back if it is decided to be added upstream, was removed form our pull request July 2014
// case TIME_MICROS:
@@ -72,6 +80,7 @@ public class ParquetToDrillTypeConverter {
}
switch(convertedType) {
case DECIMAL:
+ ParquetReaderUtility.checkDecimalTypeEnabled(options);
return TypeProtos.MinorType.DECIMAL9;
case DATE:
return TypeProtos.MinorType.DATE;
@@ -95,6 +104,7 @@ public class ParquetToDrillTypeConverter {
checkArgument(length > 0, "A length greater than zero must be provided for a FixedBinary type.");
return TypeProtos.MinorType.VARBINARY;
} else if (convertedType == ConvertedType.DECIMAL) {
+ ParquetReaderUtility.checkDecimalTypeEnabled(options);
return getDecimalType(schemaElement);
}
default:
@@ -103,8 +113,9 @@ public class ParquetToDrillTypeConverter {
}
public static TypeProtos.MajorType toMajorType(PrimitiveType.PrimitiveTypeName primitiveTypeName, int length,
- TypeProtos.DataMode mode, SchemaElement schemaElement) {
- MinorType minorType = getMinorType(primitiveTypeName, length, schemaElement);
+ TypeProtos.DataMode mode, SchemaElement schemaElement,
+ OptionManager options) {
+ MinorType minorType = getMinorType(primitiveTypeName, length, schemaElement, options);
TypeProtos.MajorType.Builder typeBuilder = TypeProtos.MajorType.newBuilder().setMinorType(minorType).setMode(mode);
if (CoreDecimalUtility.isDecimalType(minorType)) {
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
index 389c1f6..c6367ae 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetGroupConverter.java
@@ -25,6 +25,7 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.List;
+import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.expr.holders.BigIntHolder;
@@ -42,7 +43,10 @@ import org.apache.drill.exec.expr.holders.TimeStampHolder;
import org.apache.drill.exec.expr.holders.VarBinaryHolder;
import org.apache.drill.exec.expr.holders.VarCharHolder;
import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.store.ParquetOutputRecordWriter;
+import org.apache.drill.exec.store.parquet.ParquetReaderUtility;
import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.vector.complex.impl.ComplexWriterImpl;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
@@ -60,6 +64,7 @@ import org.apache.drill.exec.vector.complex.writer.TimeStampWriter;
import org.apache.drill.exec.vector.complex.writer.TimeWriter;
import org.apache.drill.exec.vector.complex.writer.VarBinaryWriter;
import org.apache.drill.exec.vector.complex.writer.VarCharWriter;
+import org.apache.drill.exec.work.ExecErrorConstants;
import org.joda.time.DateTimeUtils;
import parquet.io.api.Binary;
@@ -81,17 +86,19 @@ public class DrillParquetGroupConverter extends GroupConverter {
private List<Converter> converters;
private MapWriter mapWriter;
private final OutputMutator mutator;
+ private final OptionManager options;
- public DrillParquetGroupConverter(OutputMutator mutator, ComplexWriterImpl complexWriter, MessageType schema, Collection<SchemaPath> columns) {
- this(mutator, complexWriter.rootAsMap(), schema, columns);
+ public DrillParquetGroupConverter(OutputMutator mutator, ComplexWriterImpl complexWriter, MessageType schema, Collection<SchemaPath> columns, OptionManager options) {
+ this(mutator, complexWriter.rootAsMap(), schema, columns, options);
}
// This function assumes that the fields in the schema parameter are in the same order as the fields in the columns parameter. The
// columns parameter may have fields that are not present in the schema, though.
- public DrillParquetGroupConverter(OutputMutator mutator, MapWriter mapWriter, GroupType schema, Collection<SchemaPath> columns) {
+ public DrillParquetGroupConverter(OutputMutator mutator, MapWriter mapWriter, GroupType schema, Collection<SchemaPath> columns, OptionManager options) {
this.mapWriter = mapWriter;
this.mutator = mutator;
converters = Lists.newArrayList();
+ this.options = options;
Iterator<SchemaPath> colIterator=columns.iterator();
@@ -137,10 +144,10 @@ public class DrillParquetGroupConverter extends GroupConverter {
c.add(s);
}
if (rep != Repetition.REPEATED) {
- DrillParquetGroupConverter converter = new DrillParquetGroupConverter(mutator, mapWriter.map(name), type.asGroupType(), c);
+ DrillParquetGroupConverter converter = new DrillParquetGroupConverter(mutator, mapWriter.map(name), type.asGroupType(), c, options);
converters.add(converter);
} else {
- DrillParquetGroupConverter converter = new DrillParquetGroupConverter(mutator, mapWriter.list(name).map(), type.asGroupType(), c);
+ DrillParquetGroupConverter converter = new DrillParquetGroupConverter(mutator, mapWriter.list(name).map(), type.asGroupType(), c, options);
converters.add(converter);
}
} else {
@@ -160,6 +167,7 @@ public class DrillParquetGroupConverter extends GroupConverter {
}
switch(type.getOriginalType()) {
case DECIMAL: {
+ ParquetReaderUtility.checkDecimalTypeEnabled(options);
Decimal9Writer writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal9() : mapWriter.decimal9(name);
return new DrillDecimal9Converter(writer, type.getDecimalMetadata().getPrecision(), type.getDecimalMetadata().getScale());
}
@@ -183,6 +191,7 @@ public class DrillParquetGroupConverter extends GroupConverter {
}
switch(type.getOriginalType()) {
case DECIMAL: {
+ ParquetReaderUtility.checkDecimalTypeEnabled(options);
Decimal18Writer writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal18() : mapWriter.decimal18(name);
return new DrillDecimal18Converter(writer, type.getDecimalMetadata().getPrecision(), type.getDecimalMetadata().getScale());
}
@@ -219,6 +228,7 @@ public class DrillParquetGroupConverter extends GroupConverter {
}
//TODO not sure if BINARY/DECIMAL is actually supported
case DECIMAL: {
+ ParquetReaderUtility.checkDecimalTypeEnabled(options);
DecimalMetadata metadata = type.getDecimalMetadata();
if (metadata.getPrecision() <= 28) {
Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal28Sparse() : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision());
@@ -235,6 +245,7 @@ public class DrillParquetGroupConverter extends GroupConverter {
}
case FIXED_LEN_BYTE_ARRAY:
if (type.getOriginalType() == OriginalType.DECIMAL) {
+ ParquetReaderUtility.checkDecimalTypeEnabled(options);
DecimalMetadata metadata = type.getDecimalMetadata();
if (metadata.getPrecision() <= 28) {
Decimal28SparseWriter writer = type.getRepetition() == Repetition.REPEATED ? mapWriter.list(name).decimal28Sparse() : mapWriter.decimal28Sparse(name, metadata.getScale(), metadata.getPrecision());
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 8ad0d4e..99ac19c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -259,7 +259,8 @@ public class DrillParquetReader extends AbstractRecordReader {
if(!noColumnsFound) {
writer = new VectorContainerWriter(output);
- recordMaterializer = new DrillParquetRecordMaterializer(output, writer, projection, getColumns());
+ recordMaterializer = new DrillParquetRecordMaterializer(output, writer, projection, getColumns(),
+ fragmentContext.getOptions());
primitiveVectors = writer.getMapVector().getPrimitiveVectors();
recordReader = columnIO.getRecordReader(pageReadStore, recordMaterializer);
}
@@ -269,6 +270,7 @@ public class DrillParquetReader extends AbstractRecordReader {
}
protected void handleAndRaise(String s, Exception e) {
+ cleanup();
String message = "Error in drill parquet reader (complex).\nMessage: " + s +
"\nParquet Metadata: " + footer;
throw new DrillRuntimeException(message, e);
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
index 574df40..a80eb57 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetRecordMaterializer.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.store.parquet2;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
import parquet.io.api.GroupConverter;
@@ -33,9 +34,10 @@ public class DrillParquetRecordMaterializer extends RecordMaterializer<Void> {
public DrillParquetGroupConverter root;
private ComplexWriter complexWriter;
- public DrillParquetRecordMaterializer(OutputMutator mutator, ComplexWriter complexWriter, MessageType schema, Collection<SchemaPath> columns) {
+ public DrillParquetRecordMaterializer(OutputMutator mutator, ComplexWriter complexWriter, MessageType schema,
+ Collection<SchemaPath> columns, OptionManager options) {
this.complexWriter = complexWriter;
- root = new DrillParquetGroupConverter(mutator, complexWriter.rootAsMap(), schema, columns);
+ root = new DrillParquetGroupConverter(mutator, complexWriter.rootAsMap(), schema, columns, options);
}
public void setPosition(int position) {
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/main/java/org/apache/drill/exec/work/ExecErrorConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/ExecErrorConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/ExecErrorConstants.java
new file mode 100644
index 0000000..094f661
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/ExecErrorConstants.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.work;
+
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+
+public interface ExecErrorConstants {
+
+ // Error message when decimal data type is disabled
+ public static final String DECIMAL_DISABLE_ERR_MSG = String.format("Decimal data type is disabled. \n" +
+ "As of this release decimal data type is a beta level feature and should not be used in production \n" +
+ "Use option '%s' to enable decimal data type", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY);
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java b/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
index c627ff2..7aed478 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestBugFixes.java
@@ -18,6 +18,7 @@
package org.apache.drill;
import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.junit.Ignore;
import org.junit.Test;
@@ -68,8 +69,13 @@ public class TestBugFixes extends BaseTestQuery {
@Test
public void DRILL1126() throws Exception {
- String query = "select sum(cast(employee_id as decimal(38, 18))), avg(cast(employee_id as decimal(38, 18))) from cp.`employee.json` group by (department_id)";
- test(query);
+ try {
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ String query = "select sum(cast(employee_id as decimal(38, 18))), avg(cast(employee_id as decimal(38, 18))) from cp.`employee.json` group by (department_id)";
+ test(query);
+ } finally {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java b/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
index fd002c5..9420170 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDisabledFunctionality.java
@@ -354,4 +354,14 @@ public class TestDisabledFunctionality extends BaseTestQuery{
throw ex;
}
}
+
+ @Test(expected = UserException.class) // DRILL-2848
+ public void testDisableDecimalCasts() throws Exception {
+ test("select cast('1.2' as decimal(9, 2)) from cp.`employee.json` limit 1");
+ }
+
+ @Test(expected = UserException.class) // DRILL-2848
+ public void testDisableDecimalFromParquet() throws Exception {
+ test("select * from cp.`parquet/decimal_dictionary.parquet`");
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
index 31a7a64..c6e81a9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
@@ -21,6 +21,7 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.MinorType;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.util.JsonStringHashMap;
import org.apache.hadoop.io.Text;
@@ -71,33 +72,38 @@ public class TestFrameworkTest extends BaseTestQuery{
@Test
public void testDecimalBaseline() throws Exception {
- // type information can be provided explicitly
- testBuilder()
- .sqlQuery("select cast(dec_col as decimal(38,2)) dec_col from cp.`testframework/decimal_test.json`")
- .unOrdered()
- .csvBaselineFile("testframework/decimal_test.tsv")
- .baselineTypes(Types.withScaleAndPrecision(TypeProtos.MinorType.DECIMAL38SPARSE, TypeProtos.DataMode.REQUIRED, 2, 38))
- .baselineColumns("dec_col")
- .build().run();
+ try {
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
- // type information can also be left out, this will prompt the result types of the test query to drive the
- // interpretation of the test file
- testBuilder()
- .sqlQuery("select cast(dec_col as decimal(38,2)) dec_col from cp.`testframework/decimal_test.json`")
- .unOrdered()
- .csvBaselineFile("testframework/decimal_test.tsv")
- .baselineColumns("dec_col")
- .build().run();
+ // type information can be provided explicitly
+ testBuilder()
+ .sqlQuery("select cast(dec_col as decimal(38,2)) dec_col from cp.`testframework/decimal_test.json`")
+ .unOrdered()
+ .csvBaselineFile("testframework/decimal_test.tsv")
+ .baselineTypes(Types.withScaleAndPrecision(TypeProtos.MinorType.DECIMAL38SPARSE, TypeProtos.DataMode.REQUIRED, 2, 38))
+ .baselineColumns("dec_col")
+ .build().run();
- // Or you can provide explicit values to the builder itself to avoid going through the drill engine at all to
- // populate the baseline results
- testBuilder()
- .sqlQuery("select cast(dec_col as decimal(38,2)) dec_col from cp.`testframework/decimal_test.json`")
- .unOrdered()
- .baselineColumns("dec_col")
- .baselineValues(new BigDecimal("3.70"))
- .build().run();
+ // type information can also be left out, this will prompt the result types of the test query to drive the
+ // interpretation of the test file
+ testBuilder()
+ .sqlQuery("select cast(dec_col as decimal(38,2)) dec_col from cp.`testframework/decimal_test.json`")
+ .unOrdered()
+ .csvBaselineFile("testframework/decimal_test.tsv")
+ .baselineColumns("dec_col")
+ .build().run();
+ // Or you can provide explicit values to the builder itself to avoid going through the drill engine at all to
+ // populate the baseline results
+ testBuilder()
+ .sqlQuery("select cast(dec_col as decimal(38,2)) dec_col from cp.`testframework/decimal_test.json`")
+ .unOrdered()
+ .baselineColumns("dec_col")
+ .baselineValues(new BigDecimal("3.70"))
+ .build().run();
+ } finally {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
index 67131c1..a31e8db 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFunctionsQuery.java
@@ -18,7 +18,10 @@
package org.apache.drill;
import org.apache.drill.exec.expr.fn.impl.DateUtility;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.joda.time.DateTime;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -26,6 +29,17 @@ import java.math.BigDecimal;
public class TestFunctionsQuery extends BaseTestQuery {
+ // enable decimal data type
+ @BeforeClass
+ public static void enableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
+ @AfterClass
+ public static void disableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
@Test
public void testAbsDecimalFunction() throws Exception{
String query = "SELECT " +
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
index 3e05c0e..f8a133a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
@@ -20,9 +20,23 @@ package org.apache.drill.exec.fn.impl;
import org.apache.drill.BaseTestQuery;
import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestCastEmptyStrings extends BaseTestQuery {
+ // enable decimal data type
+ @BeforeClass
+ public static void enableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
+ @AfterClass
+ public static void disableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
@Test // see DRILL-1874
public void testCastInputTypeNullableVarCharToNumeric() throws Exception {
String root = FileUtils.getResourceAsFile("/emptyStrings.csv").toURI().toString();
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
index 2c23df4..fdc63bb 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/interp/TestConstantFolding.java
@@ -21,8 +21,11 @@ import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.drill.PlanTestBase;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.hadoop.io.Text;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
@@ -117,6 +120,8 @@ public class TestConstantFolding extends PlanTestBase {
public void testConstantFolding_allTypes() throws Exception {
try {
test("alter session set `store.json.all_text_mode` = true;");
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+
String query2 = "SELECT * " +
"FROM cp.`/parquet/alltypes.json` " +
"WHERE 12 = extract(day from (to_timestamp('2014-02-12 03:18:31:07 AM', 'YYYY-MM-dd HH:mm:ss:SS a'))) " +
@@ -162,6 +167,7 @@ public class TestConstantFolding extends PlanTestBase {
.go();
} finally {
test("alter session set `store.json.all_text_mode` = false;");
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
index 4a41669..958cf1a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java
@@ -23,10 +23,13 @@ import java.sql.Date;
import org.apache.drill.BaseTestQuery;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.fn.interp.TestConstantFolding;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.joda.time.DateTime;
+import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
@@ -46,8 +49,15 @@ public class TestParquetWriter extends BaseTestQuery {
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "local");
fs = FileSystem.get(conf);
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
}
+ @AfterClass
+ public static void disableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
+
@Test
public void testSimple() throws Exception {
String selection = "*";
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
index 5991046..f4d505d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestWriter.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.drill.BaseTestQuery;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
import org.apache.drill.exec.vector.BigIntVector;
@@ -137,10 +138,19 @@ public class TestWriter extends BaseTestQuery {
@Test
public void simpleParquetDecimal() throws Exception {
- final String tableName = "simpleparquetdecimal";
- final String testQuery = String.format("CREATE TABLE dfs_test.tmp.`%s` AS SELECT cast(salary as " +
- "decimal(30,2)) * -1 as salary FROM cp.`employee.json`", tableName);
- testCTASQueryHelper(tableName, testQuery, 1155);
+ try {
+ final String tableName = "simpleparquetdecimal";
+ final String testQuery = String.format("CREATE TABLE dfs_test.tmp.`%s` AS SELECT cast(salary as " +
+ "decimal(30,2)) * -1 as salary FROM cp.`employee.json`", tableName);
+
+ // enable decimal
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ testCTASQueryHelper(tableName, testQuery, 1155);
+
+ // disable decimal
+ } finally {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
}
private void testCTASQueryHelper(String tableName, String testQuery, int expectedOutputCount) throws Exception {
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
index 9ae6b78..4dff928 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/columnreaders/TestColumnReaderFactory.java
@@ -18,9 +18,22 @@
package org.apache.drill.exec.store.parquet.columnreaders;
import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestColumnReaderFactory extends BaseTestQuery {
+ // enable decimal data type
+ @BeforeClass
+ public static void enableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
+ @AfterClass
+ public static void disableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
/**
* check if Time and TimeStamp are read correctly with dictionary encoding.
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
index 782191f..05ca7fc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet2/TestDrillParquetReader.java
@@ -18,11 +18,24 @@
package org.apache.drill.exec.store.parquet2;
import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
import java.math.BigDecimal;
public class TestDrillParquetReader extends BaseTestQuery {
+ // enable decimal data type
+ @BeforeClass
+ public static void enableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
+ @AfterClass
+ public static void disableDecimalDataType() throws Exception {
+ test(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
private void testColumn(String columnName) throws Exception {
testNoResult("alter session set `store.parquet.use_new_reader` = true");
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcNullOrderingAndGroupingTest.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcNullOrderingAndGroupingTest.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcNullOrderingAndGroupingTest.java
index d14b344..c86d28b 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcNullOrderingAndGroupingTest.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcNullOrderingAndGroupingTest.java
@@ -23,6 +23,9 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import static org.junit.Assert.assertThat;
+
+import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -39,10 +42,16 @@ public class JdbcNullOrderingAndGroupingTest extends JdbcTestQueryBase {
// TODO: Move this to where is covers more tests: HACK: Disable Jetty
// status(?) server so unit tests run (without Maven setup).
@BeforeClass
- public static void setUpClass() {
+ public static void setUpClass() throws Exception {
+ testQuery(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
System.setProperty( "drill.exec.http.enabled", "false" );
}
+ @AfterClass
+ public static void resetDefaults() throws Exception {
+ testQuery(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
@Test
public void testNullsOrderInJdbcMetaData() throws SQLException {
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
index d4eec1e..e1137bd 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
@@ -45,7 +45,7 @@ public class JdbcTestQueryBase extends JdbcTestBase {
}
- protected void testQuery(String sql) throws Exception{
+ protected static void testQuery(String sql) throws Exception{
boolean success = false;
try (Connection conn = connect("jdbc:drill:zk=local")) {
for (int x = 0; x < 1; x++) {
http://git-wip-us.apache.org/repos/asf/drill/blob/7abd7cf4/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
index f04c2af..1f973d9 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestAggregateFunctionsQuery.java
@@ -20,13 +20,26 @@ package org.apache.drill.jdbc.test;
import java.nio.file.Paths;
import java.sql.Date;
+import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.jdbc.Driver;
-import org.apache.drill.jdbc.JdbcTestBase;
import org.joda.time.chrono.ISOChronology;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
-public class TestAggregateFunctionsQuery extends JdbcTestBase {
+public class TestAggregateFunctionsQuery extends JdbcTestQueryBase {
+
+ // enable decimal data type
+ @BeforeClass
+ public static void enableDecimalDataType() throws Exception {
+ testQuery(String.format("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
+
+ @AfterClass
+ public static void disableDecimalDataType() throws Exception {
+ testQuery(String.format("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY));
+ }
public static final String WORKING_PATH;
static{