You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by bo...@apache.org on 2019/09/17 14:54:52 UTC
[drill] branch master updated: DRILL-7373: Fix problems involving
reading from DICT type
This is an automated email from the ASF dual-hosted git repository.
bohdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new e32488c DRILL-7373: Fix problems involving reading from DICT type
e32488c is described below
commit e32488c7d5992b1d111e8d6e4bbaf6369e8dd433
Author: Bohdan Kazydub <bo...@gmail.com>
AuthorDate: Thu Sep 12 20:10:25 2019 +0300
DRILL-7373: Fix problems involving reading from DICT type
- Fixed FieldIdUtil to resolve reading from DICT for some complex cases;
- optimized reading from DICT given a key by passing an appropriate Object type to DictReader#find(...) and DictReader#read(...) methods when schema is known (e.g. when reading from Hive tables) instead of generating it on fly based on int or String path and key type;
- fixed error when accessing value by not existing key value in Avro table.
---
.../apache/drill/exec/expr/EvaluationVisitor.java | 141 ++++++++++--
.../drill/exec/planner/logical/DrillOptiq.java | 253 +++++++++++++++------
.../drill/exec/store/avro/AvroRecordReader.java | 2 +-
.../drill/exec/vector/complex/FieldIdUtil.java | 16 +-
.../drill/exec/store/avro/AvroFormatTest.java | 16 ++
.../codegen/templates/AbstractFieldReader.java | 9 +
.../src/main/codegen/templates/BaseReader.java | 30 +++
.../src/main/codegen/templates/NullReader.java | 9 +
.../vector/complex/impl/SingleDictReaderImpl.java | 25 +-
.../drill/common/expression/PathSegment.java | 64 ++++--
.../apache/drill/common/expression/SchemaPath.java | 11 +
11 files changed, 459 insertions(+), 117 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 4d6203f..a24b0bd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -17,6 +17,10 @@
*/
package org.apache.drill.exec.expr;
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -515,10 +519,11 @@ public class EvaluationVisitor {
eval.add(expr.invoke("setPosition").arg(recordIndex));
int listNum = 0;
- JVar valueIndex = eval.decl(generator.getModel().INT, "valueIndex", JExpr.lit(-1));
+ // variable used to store index of key in dict (if there is one)
+ // if entry for the key is not found, will be assigned a value of SingleDictReaderImpl#NOT_FOUND.
+ JVar valueIndex = eval.decl(generator.getModel().INT, "valueIndex", JExpr.lit(-2));
int depth = 0;
- boolean isDict = e.getFieldId().isDict(depth);
while (seg != null) {
if (seg.isArray()) {
@@ -529,15 +534,10 @@ public class EvaluationVisitor {
break;
}
- depth++;
-
- if (isDict) {
- JExpression keyExpr = JExpr.lit(seg.getArraySegment().getIndex());
-
- expr = getDictReaderReadByKeyExpression(generator, eval, expr, keyExpr, valueIndex, isNull);
-
+ if (e.getFieldId().isDict(depth)) {
+ expr = getDictReaderReadByKeyExpression(generator, eval, expr, seg, valueIndex, isNull);
seg = seg.getChild();
- isDict = e.getFieldId().isDict(depth);
+ depth++;
continue;
}
@@ -569,20 +569,17 @@ public class EvaluationVisitor {
} else {
if (e.getFieldId().isDict(depth)) {
- depth++;
- JExpression keyExpr = JExpr.lit(seg.getNameSegment().getPath());
-
MajorType finalType = e.getFieldId().getFinalType();
if (seg.getChild() == null && !(Types.isComplex(finalType) || Types.isRepeated(finalType))) {
// This is the last segment:
- eval.add(expr.invoke("read").arg(keyExpr).arg(out.getHolder()));
+ eval.add(expr.invoke("read").arg(getKeyExpression(seg, generator)).arg(out.getHolder()));
return out;
}
- expr = getDictReaderReadByKeyExpression(generator, eval, expr, keyExpr, valueIndex, isNull);
+ expr = getDictReaderReadByKeyExpression(generator, eval, expr, seg, valueIndex, isNull);
seg = seg.getChild();
- isDict = e.getFieldId().isDict(depth);
+ depth++;
continue;
}
@@ -590,11 +587,12 @@ public class EvaluationVisitor {
expr = expr.invoke("reader").arg(fieldName);
}
seg = seg.getChild();
+ depth++;
}
if (complex || repeated) {
- if (isDict) {
+ if (e.getFieldId().isDict(depth)) {
JVar dictReader = generator.declareClassField("dictReader", generator.getModel()._ref(FieldReader.class));
eval.assign(dictReader, expr);
@@ -625,10 +623,16 @@ public class EvaluationVisitor {
} else {
if (seg != null) {
JExpression holderExpr = out.getHolder();
+ JExpression argExpr;
if (e.getFieldId().isDict(depth)) {
holderExpr = JExpr.cast(generator.getModel()._ref(ValueHolder.class), holderExpr);
+ argExpr = getKeyExpression(seg, generator);
+ } else {
+ argExpr = JExpr.lit(seg.getArraySegment().getIndex());
}
- eval.add(expr.invoke("read").arg(JExpr.lit(seg.getArraySegment().getIndex())).arg(holderExpr));
+ JClass dictReaderClass = generator.getModel().ref(org.apache.drill.exec.vector.complex.impl.SingleDictReaderImpl.class);
+ JConditional jc = eval._if(valueIndex.ne(dictReaderClass.staticRef("NOT_FOUND")));
+ jc._then().add(expr.invoke("read").arg(argExpr).arg(holderExpr));
} else {
eval.add(expr.invoke("read").arg(out.getHolder()));
}
@@ -667,7 +671,7 @@ public class EvaluationVisitor {
* @param generator current class generator
* @param eval evaluation block the code will be added to
* @param expr DICT reader to read values from
- * @param keyExpr key literal
+ * @param segment segment containing original key value
* @param valueIndex current value index (will be reassigned in the method)
* @param isNull variable to indicate whether entry with the key exists in the DICT.
* Will be set to {@literal 1} if the key is not present
@@ -675,9 +679,12 @@ public class EvaluationVisitor {
* reader with its position set to index corresponding to the key
*/
private JExpression getDictReaderReadByKeyExpression(ClassGenerator generator, JBlock eval, JExpression expr,
- JExpression keyExpr, JVar valueIndex, JVar isNull) {
+ PathSegment segment, JVar valueIndex, JVar isNull) {
JVar dictReader = generator.declareClassField("dictReader", generator.getModel()._ref(FieldReader.class));
eval.assign(dictReader, expr);
+
+ JExpression keyExpr = getKeyExpression(segment, generator);
+
eval.assign(valueIndex, expr.invoke("find").arg(keyExpr));
JConditional conditional = eval._if(valueIndex.gt(JExpr.lit(-1)));
@@ -686,12 +693,102 @@ public class EvaluationVisitor {
ifFound.add(expr.invoke("setPosition").arg(valueIndex));
JBlock elseBlock = conditional._else().block();
- elseBlock.add(dictReader.invoke("setPosition").arg(valueIndex));
- elseBlock.assign(isNull, JExpr.lit(1));
+
+ JClass nrClass = generator.getModel().ref(org.apache.drill.exec.vector.complex.impl.NullReader.class);
+ JExpression nullReader = nrClass.staticRef("EMPTY_MAP_INSTANCE");
+
+ elseBlock.assign(dictReader, nullReader);
+ if (isNull != null) {
+ elseBlock.assign(isNull, JExpr.lit(1));
+ }
return expr;
}
+ /**
+ * Transforms a segment to appropriate Java Object representation of key ({@link org.apache.drill.exec.vector.complex.DictVector#FIELD_KEY_NAME})
+ * which is used when retrieving values from dict with key. In case if key vector's Java equivalent is primitive,
+ * i.e. {@code boolean}, {@code int}, {@code double} etc., then primitive type is used.
+ * Otherwise, an {@code Object} instance is created (i.e, {@code BigDecimal} for {@link org.apache.drill.common.types.TypeProtos.MinorType#VARDECIMAL},
+ * {@code LocalDateTime} for {@link org.apache.drill.common.types.TypeProtos.MinorType#TIMESTAMP} etc.).
+ *
+ * @param segment a path segment providing the value
+ * @param generator current class generator
+ * @return Java Object representation of key wrapped into {@link JVar}
+ */
+ private JExpression getKeyExpression(PathSegment segment, ClassGenerator generator) {
+ MajorType valueType = segment.getOriginalValueType();
+ JType keyType;
+ JExpression newKeyObject;
+ JVar dictKey;
+ if (segment.isArray()) {
+ if (valueType == null) {
+ return JExpr.lit(segment.getArraySegment().getIndex());
+ }
+ switch(valueType.getMinorType()) {
+ case INT:
+ return JExpr.cast(generator.getModel().ref(Object.class), JExpr.lit(segment.getArraySegment().getIndex()));
+ case SMALLINT:
+ return JExpr.lit((short) segment.getOriginalValue());
+ case TINYINT:
+ return JExpr.lit((byte) segment.getOriginalValue());
+ default:
+ throw new IllegalArgumentException("ArraySegment!");
+ }
+ } else { // named
+ if (valueType == null) {
+ return JExpr.lit(segment.getNameSegment().getPath());
+ }
+ switch (valueType.getMinorType()) {
+ case VARCHAR:
+ String vcValue = (String) segment.getOriginalValue();
+ keyType = generator.getModel()._ref(org.apache.drill.exec.util.Text.class);
+ newKeyObject = JExpr._new(keyType).arg(vcValue);
+ dictKey = generator.declareClassField("dictKey", keyType);
+ generator.getSetupBlock().assign(dictKey, newKeyObject);
+ return dictKey;
+ case VARDECIMAL:
+ BigDecimal bdValue = (BigDecimal) segment.getOriginalValue();
+ keyType = generator.getModel()._ref(java.math.BigDecimal.class);
+
+ JClass rmClass = generator.getModel().ref(java.math.RoundingMode.class);
+ newKeyObject = JExpr._new(keyType).arg(JExpr.lit(bdValue.doubleValue())).invoke("setScale")
+ .arg(JExpr.lit(bdValue.scale()))
+ .arg(rmClass.staticRef("HALF_UP"));
+ dictKey = generator.declareClassField("dictKey", keyType);
+ generator.getSetupBlock().assign(dictKey, newKeyObject);
+ return dictKey;
+ case BIGINT:
+ return JExpr.lit((long) segment.getOriginalValue());
+ case FLOAT4:
+ return JExpr.lit((float) segment.getOriginalValue());
+ case FLOAT8:
+ return JExpr.lit((double) segment.getOriginalValue());
+ case BIT:
+ return JExpr.lit((boolean) segment.getOriginalValue());
+ case TIMESTAMP:
+ return getDateTimeKey(segment, generator, LocalDateTime.class, "parseBest");
+ case DATE:
+ return getDateTimeKey(segment, generator, LocalDate.class, "parseLocalDate");
+ case TIME:
+ return getDateTimeKey(segment, generator, LocalTime.class, "parseLocalTime");
+ default:
+ throw new IllegalArgumentException("NamedSegment!");
+ }
+ }
+ }
+
+ private JVar getDateTimeKey(PathSegment segment, ClassGenerator generator, Class<?> javaClass, String methodName) {
+ String strValue = (String) segment.getOriginalValue();
+
+ JClass dateUtilityClass = generator.getModel().ref(org.apache.drill.exec.expr.fn.impl.DateUtility.class);
+ JExpression newKeyObject = dateUtilityClass.staticInvoke(methodName).arg(JExpr.lit(strValue));
+ JType keyType = generator.getModel()._ref(javaClass);
+ JVar dictKey = generator.declareClassField("dictKey", keyType);
+ generator.getSetupBlock().assign(dictKey, newKeyObject);
+ return dictKey;
+ }
+
private HoldingContainer visitReturnValueExpression(ReturnValueExpression e, ClassGenerator<?> generator) {
LogicalExpression child = e.getChild();
// Preconditions.checkArgument(child.getMajorType().equals(Types.REQUIRED_BOOLEAN));
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index dabb22b..ba3e72b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -25,7 +25,11 @@ import java.util.List;
import org.apache.calcite.avatica.util.TimeUnit;
import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.commons.lang3.StringUtils;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
@@ -198,7 +202,7 @@ public class DrillOptiq {
return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(),
ExpressionPosition.UNKNOWN, call.getOperands().get(0).accept(this));
}
- throw new AssertionError("todo: implement syntax " + syntax + "(" + call + ")");
+ throw notImplementedException(syntax, call);
case PREFIX:
LogicalExpression arg = call.getOperands().get(0).accept(this);
switch(call.getKind()){
@@ -210,7 +214,7 @@ public class DrillOptiq {
operands.add(call.getOperands().get(0).accept(this));
return FunctionCallFactory.createExpression("u-", operands);
}
- throw new AssertionError("todo: implement syntax " + syntax + "(" + call + ")");
+ throw notImplementedException(syntax, call);
case SPECIAL:
switch(call.getKind()){
case CAST:
@@ -249,26 +253,7 @@ public class DrillOptiq {
}
if (call.getOperator() == SqlStdOperatorTable.ITEM) {
- SchemaPath left = (SchemaPath) call.getOperands().get(0).accept(this);
-
- // Convert expr of item[*, 'abc'] into column expression 'abc'
- String rootSegName = left.getRootSegment().getPath();
- if (StarColumnHelper.isStarColumn(rootSegName)) {
- rootSegName = rootSegName.substring(0, rootSegName.indexOf(SchemaPath.DYNAMIC_STAR));
- final RexLiteral literal = (RexLiteral) call.getOperands().get(1);
- return SchemaPath.getSimplePath(rootSegName + literal.getValue2().toString());
- }
-
- final RexLiteral literal = (RexLiteral) call.getOperands().get(1);
- switch(literal.getTypeName()){
- case DECIMAL:
- case INTEGER:
- return left.getChild(((BigDecimal)literal.getValue()).intValue());
- case CHAR:
- return left.getChild(literal.getValue2().toString());
- default:
- // fall through
- }
+ return handleItemOperator(call, syntax);
}
if (call.getOperator() == SqlStdOperatorTable.DATETIME_PLUS) {
@@ -281,8 +266,146 @@ public class DrillOptiq {
// fall through
default:
- throw new AssertionError("todo: implement syntax " + syntax + "(" + call + ")");
+ throw notImplementedException(syntax, call);
+ }
+ }
+
+ private SchemaPath handleItemOperator(RexCall call, SqlSyntax syntax) {
+ SchemaPath left = (SchemaPath) call.getOperands().get(0).accept(this);
+
+ RelDataType dataType = call.getOperands().get(0).getType();
+ boolean isMap = dataType.getSqlTypeName() == SqlTypeName.MAP;
+
+ // Convert expr of item[*, 'abc'] into column expression 'abc'
+ String rootSegName = left.getRootSegment().getPath();
+ if (StarColumnHelper.isStarColumn(rootSegName)) {
+ rootSegName = rootSegName.substring(0, rootSegName.indexOf(SchemaPath.DYNAMIC_STAR));
+ final RexLiteral literal = (RexLiteral) call.getOperands().get(1);
+ return SchemaPath.getSimplePath(rootSegName + literal.getValue2().toString());
+ }
+
+ final RexLiteral literal;
+ RexNode operand = call.getOperands().get(1);
+ if (operand instanceof RexLiteral) {
+ literal = (RexLiteral) operand;
+ } else if (isMap && operand.getKind() == SqlKind.CAST) {
+ SqlTypeName castType = operand.getType().getSqlTypeName();
+ SqlTypeName keyType = dataType.getKeyType().getSqlTypeName();
+ Preconditions.checkArgument(castType == keyType,
+ String.format("Wrong type CAST: expected '%s' but found '%s'", keyType.getName(), castType.getName()));
+ literal = (RexLiteral) ((RexCall) operand).operands.get(0);
+ } else {
+ throw notImplementedException(syntax, call);
+ }
+
+ switch (literal.getTypeName()) {
+ case DECIMAL:
+ case INTEGER:
+ if (isMap) {
+ return handleMapNumericKey(literal, operand, dataType, left);
+ }
+ return left.getChild(((BigDecimal) literal.getValue()).intValue());
+ case CHAR:
+ if (isMap) {
+ return handleMapCharKey(literal, operand, dataType, left);
+ }
+ return left.getChild(literal.getValue2().toString());
+ case BOOLEAN:
+ if (isMap) {
+ BasicSqlType sqlType = (BasicSqlType) operand.getType();
+ TypeProtos.DataMode mode = sqlType.isNullable() ? TypeProtos.DataMode.OPTIONAL : TypeProtos.DataMode.REQUIRED;
+ return left.getChild(literal.getValue().toString(), literal.getValue(), Types.withMode(MinorType.BIT, mode));
+ }
+ // fall through
+ default:
+ throw notImplementedException(syntax, call);
+ }
+ }
+
+ private DrillRuntimeException notImplementedException(SqlSyntax syntax, RexCall call) {
+ String message = String.format("Syntax '%s(%s)' is not implemented.", syntax.toString(), call.toString());
+ throw new DrillRuntimeException(message);
+ }
+
+ private SchemaPath handleMapNumericKey(RexLiteral literal, RexNode operand, RelDataType mapType, SchemaPath parentPath) {
+ BigDecimal literalValue = (BigDecimal) literal.getValue();
+ RelDataType sqlType = operand.getType();
+ Object originalValue;
+
+ TypeProtos.DataMode mode = sqlType.isNullable() ? TypeProtos.DataMode.OPTIONAL : TypeProtos.DataMode.REQUIRED;
+ boolean arraySegment = false;
+ MajorType type;
+ switch (mapType.getKeyType().getSqlTypeName()) {
+ case DOUBLE:
+ type = Types.withMode(MinorType.FLOAT8, mode);
+ originalValue = literalValue.doubleValue();
+ break;
+ case FLOAT:
+ type = Types.withMode(MinorType.FLOAT4, mode);
+ originalValue = literalValue.floatValue();
+ break;
+ case DECIMAL:
+ type = Types.withPrecisionAndScale(MinorType.VARDECIMAL, mode, literalValue.precision(), literalValue.scale());
+ originalValue = literalValue;
+ break;
+ case BIGINT:
+ type = Types.withMode(MinorType.BIGINT, mode);
+ originalValue = literalValue.longValue();
+ break;
+ case INTEGER:
+ type = Types.withMode(MinorType.INT, mode);
+ originalValue = literalValue.intValue();
+ arraySegment = true;
+ break;
+ case SMALLINT:
+ type = Types.withMode(MinorType.SMALLINT, mode);
+ originalValue = literalValue.shortValue();
+ arraySegment = true;
+ break;
+ case TINYINT:
+ type = Types.withMode(MinorType.TINYINT, mode);
+ originalValue = literalValue.byteValue();
+ arraySegment = true;
+ break;
+ default:
+ throw new AssertionError("Shouldn't reach there. Type: " + mapType.getKeyType().getSqlTypeName());
+ }
+
+ if (arraySegment) {
+ return parentPath.getChild((int) originalValue, originalValue, type);
+ } else {
+ return parentPath.getChild(originalValue.toString(), originalValue, type);
+ }
+ }
+
+ private SchemaPath handleMapCharKey(RexLiteral literal, RexNode operand, RelDataType mapType, SchemaPath parentPath) {
+ TypeProtos.DataMode mode = operand.getType().isNullable()
+ ? TypeProtos.DataMode.OPTIONAL : TypeProtos.DataMode.REQUIRED;
+ TypeProtos.MajorType type;
+ switch (mapType.getKeyType().getSqlTypeName()) {
+ case TIMESTAMP:
+ type = Types.withMode(MinorType.TIMESTAMP, mode);
+ break;
+ case DATE:
+ type = Types.withMode(MinorType.DATE, mode);
+ break;
+ case TIME:
+ type = Types.withMode(MinorType.TIME, mode);
+ break;
+ case INTERVAL_DAY:
+ type = Types.withMode(MinorType.INTERVALDAY, mode);
+ break;
+ case INTERVAL_YEAR:
+ type = Types.withMode(MinorType.INTERVALYEAR, mode);
+ break;
+ case INTERVAL_MONTH:
+ type = Types.withMode(MinorType.INTERVAL, mode);
+ break;
+ default:
+ type = Types.withMode(MinorType.VARCHAR, mode);
+ break;
}
+ return parentPath.getChild(literal.getValue2().toString(), literal.getValue2(), type);
}
private LogicalExpression doFunction(RexCall call, String funcName) {
@@ -345,64 +468,60 @@ public class DrillOptiq {
LogicalExpression arg = call.getOperands().get(0).accept(this);
MajorType castType;
- switch(call.getType().getSqlTypeName().getName()){
- case "VARCHAR":
- case "CHAR":
- castType = Types.required(MinorType.VARCHAR).toBuilder().setPrecision(call.getType().getPrecision()).build();
- break;
- case "INTEGER":
- castType = Types.required(MinorType.INT);
- break;
- case "FLOAT":
- castType = Types.required(MinorType.FLOAT4);
- break;
- case "DOUBLE":
- castType = Types.required(MinorType.FLOAT8);
- break;
- case "DECIMAL":
- if (!context.getPlannerSettings().getOptions().getOption(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY).bool_val) {
- throw UserException
- .unsupportedError()
- .message(ExecErrorConstants.DECIMAL_DISABLE_ERR_MSG)
- .build(logger);
- }
+ switch (call.getType().getSqlTypeName()) {
+ case VARCHAR:
+ case CHAR:
+ castType = Types.required(MinorType.VARCHAR).toBuilder().setPrecision(call.getType().getPrecision()).build();
+ break;
+ case INTEGER:
+ castType = Types.required(MinorType.INT);
+ break;
+ case FLOAT:
+ castType = Types.required(MinorType.FLOAT4);
+ break;
+ case DOUBLE:
+ castType = Types.required(MinorType.FLOAT8);
+ break;
+ case DECIMAL:
+ if (!context.getPlannerSettings().getOptions().getOption(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE)) {
+ throw UserException.unsupportedError()
+ .message(ExecErrorConstants.DECIMAL_DISABLE_ERR_MSG)
+ .build(logger);
+ }
- int precision = call.getType().getPrecision();
- int scale = call.getType().getScale();
+ int precision = call.getType().getPrecision();
+ int scale = call.getType().getScale();
- castType =
- TypeProtos.MajorType
- .newBuilder()
+ castType = TypeProtos.MajorType.newBuilder()
.setMinorType(MinorType.VARDECIMAL)
.setPrecision(precision)
.setScale(scale)
.build();
- break;
-
- case "INTERVAL_YEAR":
- case "INTERVAL_YEAR_MONTH":
- case "INTERVAL_MONTH":
+ break;
+ case INTERVAL_YEAR:
+ case INTERVAL_YEAR_MONTH:
+ case INTERVAL_MONTH:
castType = Types.required(MinorType.INTERVALYEAR);
break;
- case "INTERVAL_DAY":
- case "INTERVAL_DAY_HOUR":
- case "INTERVAL_DAY_MINUTE":
- case "INTERVAL_DAY_SECOND":
- case "INTERVAL_HOUR":
- case "INTERVAL_HOUR_MINUTE":
- case "INTERVAL_HOUR_SECOND":
- case "INTERVAL_MINUTE":
- case "INTERVAL_MINUTE_SECOND":
- case "INTERVAL_SECOND":
+ case INTERVAL_DAY:
+ case INTERVAL_DAY_HOUR:
+ case INTERVAL_DAY_MINUTE:
+ case INTERVAL_DAY_SECOND:
+ case INTERVAL_HOUR:
+ case INTERVAL_HOUR_MINUTE:
+ case INTERVAL_HOUR_SECOND:
+ case INTERVAL_MINUTE:
+ case INTERVAL_MINUTE_SECOND:
+ case INTERVAL_SECOND:
castType = Types.required(MinorType.INTERVALDAY);
break;
- case "BOOLEAN":
+ case BOOLEAN:
castType = Types.required(MinorType.BIT);
break;
- case "BINARY":
+ case BINARY:
castType = Types.required(MinorType.VARBINARY);
break;
- case "ANY":
+ case ANY:
return arg; // Type will be same as argument.
default:
castType = Types.required(MinorType.valueOf(call.getType().getSqlTypeName().getName()));
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/AvroRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/AvroRecordReader.java
index 541ff9c..34a035c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/AvroRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/AvroRecordReader.java
@@ -244,7 +244,7 @@ public class AvroRecordReader extends AbstractRecordReader {
for (Entry<Object, Object> entry : map.entrySet()) {
dictWriter.startKeyValuePair();
processPrimitive(entry.getKey(), keySchema, DictVector.FIELD_KEY_NAME, writer);
- process(entry.getValue(), valueSchema, DictVector.FIELD_VALUE_NAME, writer, fieldSelection.getChild(entry.getKey().toString()));
+ process(entry.getValue(), valueSchema, DictVector.FIELD_VALUE_NAME, writer, FieldSelection.ALL_VALID);
dictWriter.endKeyValuePair();
}
dictWriter.end();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
index 088bee6..09e2cfc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java
@@ -105,9 +105,9 @@ public class FieldIdUtil {
MajorType type;
if (vector instanceof AbstractContainerVector) {
type = ((AbstractContainerVector) vector).getLastPathType();
- } else if (vector instanceof ListVector) {
- type = ((ListVector) vector).getDataVector().getField().getType();
- builder.listVector(true);
+ } else if (vector instanceof RepeatedValueVector) {
+ type = ((RepeatedValueVector) vector).getDataVector().getField().getType();
+ builder.listVector(vector.getField().getType().getMinorType() == MinorType.LIST);
} else {
throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType());
}
@@ -166,7 +166,7 @@ public class FieldIdUtil {
throw new UnsupportedOperationException("FieldIdUtil does not support vector of type " + vector.getField().getType());
}
- if (v instanceof AbstractContainerVector || v instanceof ListVector) {
+ if (v instanceof AbstractContainerVector || v instanceof ListVector || v instanceof RepeatedDictVector) {
return getFieldIdIfMatches(v, builder, addToBreadCrumb, seg.getChild(), depth + 1);
} else if (v instanceof UnionVector) {
return getFieldIdIfMatchesUnion((UnionVector) v, builder, addToBreadCrumb, seg.getChild());
@@ -224,7 +224,7 @@ public class FieldIdUtil {
} else if (vector instanceof ListVector) {
builder.intermediateType(vector.getField().getType());
builder.addId(id);
- return getFieldIdIfMatches(vector, builder, true, expectedPath.getRootSegment().getChild(), 0);
+ return getFieldIdIfMatches(vector, builder, true, expectedPath.getRootSegment().getChild());
} else if (vector instanceof DictVector) {
MajorType vectorType = vector.getField().getType();
builder.intermediateType(vectorType);
@@ -235,13 +235,13 @@ public class FieldIdUtil {
} else {
PathSegment child = seg.getChild();
builder.remainder(child);
- return getFieldIdIfMatches(vector, builder, false, expectedPath.getRootSegment().getChild(), 0);
+ return getFieldIdIfMatches(vector, builder, false, expectedPath.getRootSegment().getChild());
}
} else if (vector instanceof AbstractContainerVector) {
// we're looking for a multi path.
builder.intermediateType(vector.getField().getType());
builder.addId(id);
- return getFieldIdIfMatches(vector, builder, true, expectedPath.getRootSegment().getChild(), 0);
+ return getFieldIdIfMatches(vector, builder, true, expectedPath.getRootSegment().getChild());
} else if (vector instanceof RepeatedDictVector) {
MajorType vectorType = vector.getField().getType();
builder.intermediateType(vectorType);
@@ -259,7 +259,7 @@ public class FieldIdUtil {
if (child.isLastPath()) {
return builder.finalType(DictVector.TYPE).build();
} else {
- return getFieldIdIfMatches(vector, builder, true, expectedPath.getRootSegment().getChild(), 0);
+ return getFieldIdIfMatches(vector, builder, true, expectedPath.getRootSegment().getChild());
}
}
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
index dfcb6e5..c17e20c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/avro/AvroFormatTest.java
@@ -637,6 +637,22 @@ public class AvroFormatTest extends BaseTestQuery {
}
@Test
+ public void testMapSchemaGetByNotExistingKey() throws Exception {
+ String sql = "select map_field['notExists'] as map_field from dfs.`%s`";
+
+ TestBuilder testBuilder = testBuilder()
+ .sqlQuery(sql, mapTableName)
+ .unOrdered()
+ .baselineColumns("map_field");
+
+ Object[] nullValue = new Object[] {null};
+ for (long i = 0; i < RECORD_COUNT; i++) {
+ testBuilder.baselineValues(nullValue);
+ }
+ testBuilder.go();
+ }
+
+ @Test
public void testMapSchemaGetByKeyUsingDotNotation() throws Exception {
String sql = "select t.map_field.key1 val1, t.map_field.key2 val2 from dfs.`%s` t";
diff --git a/exec/vector/src/main/codegen/templates/AbstractFieldReader.java b/exec/vector/src/main/codegen/templates/AbstractFieldReader.java
index 39420d9..b75fa62 100644
--- a/exec/vector/src/main/codegen/templates/AbstractFieldReader.java
+++ b/exec/vector/src/main/codegen/templates/AbstractFieldReader.java
@@ -141,6 +141,11 @@ public abstract class AbstractFieldReader extends AbstractBaseReader implements
return -1;
}
+ public int find(Object key){
+ fail("find(Object key)");
+ return -1;
+ }
+
public void read(String key, ValueHolder holder) {
fail("read(String key, ValueHolder holder)");
}
@@ -149,6 +154,10 @@ public abstract class AbstractFieldReader extends AbstractBaseReader implements
fail("read(int key, ValueHolder holder)");
}
+ public void read(Object key, ValueHolder holder) {
+ fail("read(Object key, ValueHolder holder)");
+ }
+
private void fail(String name) {
throw new IllegalArgumentException(String.format("You tried to read a [%s] type when you are using a field reader of type [%s].", name, this.getClass().getSimpleName()));
}
diff --git a/exec/vector/src/main/codegen/templates/BaseReader.java b/exec/vector/src/main/codegen/templates/BaseReader.java
index d54f191..ef3a72a 100644
--- a/exec/vector/src/main/codegen/templates/BaseReader.java
+++ b/exec/vector/src/main/codegen/templates/BaseReader.java
@@ -84,6 +84,20 @@ public interface BaseReader extends Positionable{
int find(int key);
/**
+ * Obtain the index for given key in current row used to find a corresponding value with.
+ * Used in generated code when retrieving value from Dict using {@link org.apache.drill.common.expression.PathSegment}
+ * with provided {@link org.apache.drill.common.expression.PathSegment#getOriginalValue()}
+ * in cases when {@link org.apache.drill.exec.vector.complex.DictVector#getValueType()} is complex.
+ *
+ * <p>The {@code key} is assumed to be of actual type, is not converted and used as is.
+ *
+ * @param key key value
+ * @return index for the given key
+ * @see org.apache.drill.exec.vector.complex.DictVector
+ */
+ int find(Object key);
+
+ /**
* Reads a value corresponding to a {@code key} into the {@code holder}.
* If there is no entry in the row with the given {@code key}, value is set to null.
*
@@ -116,6 +130,22 @@ public interface BaseReader extends Positionable{
* @see org.apache.drill.exec.vector.complex.DictVector
*/
void read(int key, ValueHolder holder);
+
+ /**
+ * Reads a value corresponding to a {@code key} into the {@code holder}.
+ * If there is no entry in the row with the given {@code key}, value is set to null.
+ *
+ * <p>Used in generated code when retrieving value from Dict using {@link org.apache.drill.common.expression.PathSegment}
+ * with provided {@link org.apache.drill.common.expression.PathSegment#getOriginalValue()}
+ * in cases when {@link org.apache.drill.exec.vector.complex.DictVector#getValueType()} is primitive.
+ *
+ * <p>The {@code key} is assumed to be of actual type, is not converted and used as is.
+ *
+ * @param key key value
+ * @param holder a holder to write value's value into
+ * @see org.apache.drill.exec.vector.complex.DictVector
+ */
+ void read(Object key, ValueHolder holder);
}
public interface ListReader extends BaseReader{
diff --git a/exec/vector/src/main/codegen/templates/NullReader.java b/exec/vector/src/main/codegen/templates/NullReader.java
index 1d1b9bd..91a7b12 100644
--- a/exec/vector/src/main/codegen/templates/NullReader.java
+++ b/exec/vector/src/main/codegen/templates/NullReader.java
@@ -155,12 +155,21 @@ public class NullReader extends AbstractBaseReader implements FieldReader {
}
@Override
+ public int find(Object key) {
+ return -1;
+ }
+
+ @Override
public void read(String key, ValueHolder holder) {
}
@Override
public void read(int key, ValueHolder holder) {
}
+
+ @Override
+ public void read(Object key, ValueHolder holder) {
+ }
}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleDictReaderImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleDictReaderImpl.java
index 45fa420..4378bcf 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleDictReaderImpl.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/SingleDictReaderImpl.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.vector.complex.impl;
import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.expr.fn.impl.DateUtility;
import org.apache.drill.exec.expr.holders.ValueHolder;
import org.apache.drill.exec.util.Text;
import org.apache.drill.exec.vector.ValueVector;
@@ -29,10 +30,11 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.DictWriter;
import org.apache.drill.exec.vector.complex.writer.FieldWriter;
import java.math.BigDecimal;
+import java.math.RoundingMode;
public class SingleDictReaderImpl extends AbstractRepeatedMapReaderImpl<DictVector> implements DictReader {
- private static final int NOT_FOUND = -1;
+ public static final int NOT_FOUND = -1;
public SingleDictReaderImpl(DictVector vector) {
super(vector);
@@ -56,7 +58,8 @@ public class SingleDictReaderImpl extends AbstractRepeatedMapReaderImpl<DictVect
return find(typifiedKey);
}
- private int find(Object key) {
+ @Override
+ public int find(Object key) {
int start = vector.getOffsetVector().getAccessor().get(idx());
int end = vector.getOffsetVector().getAccessor().get(idx() + 1);
int index = NOT_FOUND;
@@ -88,9 +91,13 @@ public class SingleDictReaderImpl extends AbstractRepeatedMapReaderImpl<DictVect
case FLOAT8:
return (double) key;
case VARDECIMAL:
- return BigDecimal.valueOf(key);
+ return BigDecimal.valueOf(key)
+ .setScale(keyType.getScale(), RoundingMode.HALF_UP);
case BIT:
return key != 0;
+ case VARCHAR:
+ case VARBINARY:
+ return new Text(String.valueOf(key));
default:
String message = String.format("Unknown value %d for key of type %s", key, keyType.getMinorType().toString());
throw new IllegalArgumentException(message);
@@ -115,6 +122,15 @@ public class SingleDictReaderImpl extends AbstractRepeatedMapReaderImpl<DictVect
return Float.valueOf(key);
case FLOAT8:
return Double.valueOf(key);
+ case VARDECIMAL:
+ return BigDecimal.valueOf(Double.valueOf(key))
+ .setScale(keyType.getScale(), RoundingMode.HALF_UP);
+ case TIMESTAMP:
+ return DateUtility.parseBest(key);
+ case DATE:
+ return DateUtility.parseLocalDate(key);
+ case TIME:
+ return DateUtility.parseLocalTime(key);
default:
String message = String.format("Unknown value %s for key of type %s", key, keyType.getMinorType().toString());
throw new IllegalArgumentException(message);
@@ -133,7 +149,8 @@ public class SingleDictReaderImpl extends AbstractRepeatedMapReaderImpl<DictVect
read(typifiedKey, holder);
}
- private void read(Object key, ValueHolder holder) {
+ @Override
+ public void read(Object key, ValueHolder holder) {
if (isEmpty()) {
return;
}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java b/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java
index 6c6a094..7cb0fb1 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/PathSegment.java
@@ -17,13 +17,29 @@
*/
package org.apache.drill.common.expression;
+import org.apache.drill.common.types.TypeProtos;
+
public abstract class PathSegment {
+ /**
+ * Holds original value associated with the path segment.
+ * Used when reading data from DICT.
+ */
+ protected final Object originalValue;
+
+ /**
+ * Indicates the type of original value.
+ * @see #originalValue
+ */
+ protected final TypeProtos.MajorType originalValueType;
+
private PathSegment child;
private int hash;
- public PathSegment(PathSegment child) {
+ public PathSegment(PathSegment child, Object originalValue, TypeProtos.MajorType originalValueType) {
+ this.originalValue = originalValue;
+ this.originalValueType = originalValueType;
this.child = child;
}
@@ -32,6 +48,14 @@ public abstract class PathSegment {
@Override
public abstract PathSegment clone();
+ public Object getOriginalValue() {
+ return originalValue;
+ }
+
+ public TypeProtos.MajorType getOriginalValueType() {
+ return originalValueType;
+ }
+
public static final class ArraySegment extends PathSegment {
private final int index;
@@ -44,28 +68,32 @@ public abstract class PathSegment {
}
public ArraySegment(int index, PathSegment child) {
- super(child);
+ super(child, index, null);
this.index = index;
assert index >= 0;
}
public ArraySegment(PathSegment child) {
- super(child);
+ super(child, -1, null);
this.index = -1;
}
- public boolean hasIndex() {
- return index != -1;
- }
-
public ArraySegment(int index) {
- super(null);
+ this(index, null);
if (index < 0 ) {
throw new IllegalArgumentException();
}
+ }
+
+ public ArraySegment(int index, Object originalValue, TypeProtos.MajorType valueType) {
+ super(null, originalValue, valueType);
this.index = index;
}
+ public boolean hasIndex() {
+ return index != -1;
+ }
+
public int getIndex() {
return index;
}
@@ -109,7 +137,8 @@ public abstract class PathSegment {
@Override
public PathSegment clone() {
- PathSegment seg = index < 0 ? new ArraySegment((PathSegment) null) : new ArraySegment(index);
+ int index = this.index < 0 ? -1 : this.index;
+ PathSegment seg = new ArraySegment(index, originalValue, originalValueType);
if (getChild() != null) {
seg.setChild(getChild().clone());
}
@@ -118,7 +147,8 @@ public abstract class PathSegment {
@Override
public ArraySegment cloneWithNewChild(PathSegment newChild) {
- ArraySegment seg = index < 0 ? new ArraySegment((PathSegment) null) : new ArraySegment(index);
+ int index = this.index < 0 ? -1 : this.index;
+ ArraySegment seg = new ArraySegment(index, originalValue, originalValueType);
if (getChild() != null) {
seg.setChild(getChild().cloneWithNewChild(newChild));
} else {
@@ -131,14 +161,18 @@ public abstract class PathSegment {
public static final class NameSegment extends PathSegment {
private final String path;
+ public NameSegment(CharSequence n, Object originalValue, TypeProtos.MajorType valueType) {
+ super(null, originalValue, valueType);
+ this.path = n.toString();
+ }
+
public NameSegment(CharSequence n, PathSegment child) {
- super(child);
+ super(child, n.toString(), null);
this.path = n.toString();
}
public NameSegment(CharSequence n) {
- super(null);
- this.path = n.toString();
+ this(n, null);
}
public String getPath() { return path; }
@@ -186,7 +220,7 @@ public abstract class PathSegment {
@Override
public NameSegment clone() {
- NameSegment s = new NameSegment(this.path);
+ NameSegment s = new NameSegment(this.path, originalValue, originalValueType);
if (getChild() != null) {
s.setChild(getChild().clone());
}
@@ -195,7 +229,7 @@ public abstract class PathSegment {
@Override
public NameSegment cloneWithNewChild(PathSegment newChild) {
- NameSegment s = new NameSegment(this.path);
+ NameSegment s = new NameSegment(this.path, originalValue, originalValueType);
if (getChild() != null) {
s.setChild(getChild().cloneWithNewChild(newChild));
} else {
diff --git a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 4943b58..0f8b4af 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -25,6 +25,7 @@ import org.apache.drill.common.expression.PathSegment.ArraySegment;
import org.apache.drill.common.expression.PathSegment.NameSegment;
import org.apache.drill.common.expression.visitors.ExprVisitor;
import org.apache.drill.common.parser.LogicalExpressionParser;
+import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.proto.UserBitShared.NamePart;
@@ -301,11 +302,21 @@ public class SchemaPath extends LogicalExpressionBase {
return new SchemaPath(newRoot);
}
+ public SchemaPath getChild(String childPath, Object originalValue, TypeProtos.MajorType valueType) {
+ NameSegment newRoot = rootSegment.cloneWithNewChild(new NameSegment(childPath, originalValue, valueType));
+ return new SchemaPath(newRoot);
+ }
+
public SchemaPath getChild(int index) {
NameSegment newRoot = rootSegment.cloneWithNewChild(new ArraySegment(index));
return new SchemaPath(newRoot);
}
+ public SchemaPath getChild(int index, Object originalValue, TypeProtos.MajorType valueType) {
+ NameSegment newRoot = rootSegment.cloneWithNewChild(new ArraySegment(index, originalValue, valueType));
+ return new SchemaPath(newRoot);
+ }
+
public NameSegment getRootSegment() {
return rootSegment;
}