You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2016/07/22 05:35:40 UTC
drill git commit: DRILL-4783: flatten operator handle empty
resultset(and schema unknown due to this)
Repository: drill
Updated Branches:
refs/heads/master 43776706b -> 04964bbf8
DRILL-4783: flatten operator handle empty resultset(and schema unknown due to this)
close apache/drill#546
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/04964bbf
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/04964bbf
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/04964bbf
Branch: refs/heads/master
Commit: 04964bbf816746ffaefbe58bf9ddedbff54e0f69
Parents: 4377670
Author: chunhui-shi <cs...@maprtech.com>
Authored: Sat Jul 16 00:18:33 2016 -0700
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Thu Jul 21 21:27:30 2016 -0700
----------------------------------------------------------------------
.../impl/flatten/FlattenRecordBatch.java | 34 ++++++++++++++------
.../exec/nested/TestFastComplexSchema.java | 29 +++++++++++++++++
2 files changed, 54 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/04964bbf/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 5b3f6ae..a2b170d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -126,15 +126,23 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
}
private void setFlattenVector() {
- try {
- final TypedFieldId typedFieldId = incoming.getValueVectorId(popConfig.getColumn());
- final MaterializedField field = incoming.getSchema().getColumn(typedFieldId.getFieldIds()[0]);
- final RepeatedValueVector vector = RepeatedValueVector.class.cast(incoming.getValueAccessorById(
- field.getValueClass(), typedFieldId.getFieldIds()).getValueVector());
- flattener.setFlattenField(vector);
- } catch (Exception ex) {
- throw UserException.unsupportedError(ex).message("Trying to flatten a non-repeated field.").build(logger);
+ final TypedFieldId typedFieldId = incoming.getValueVectorId(popConfig.getColumn());
+ final MaterializedField field = incoming.getSchema().getColumn(typedFieldId.getFieldIds()[0]);
+ final RepeatedValueVector vector;
+ final ValueVector inVV = incoming.getValueAccessorById(
+ field.getValueClass(), typedFieldId.getFieldIds()).getValueVector();
+
+ if (! (inVV instanceof RepeatedValueVector)) {
+ if (incoming.getRecordCount() != 0) {
+ throw UserException.unsupportedError().message("Flatten does not support inputs of non-list values.").build(logger);
+ }
+ //when incoming recordCount is 0, don't throw exception since the type being seen here is not solid
+ logger.error("setFlattenVector cast failed and recordcount is 0, create empty vector anyway.");
+ vector = new RepeatedMapVector(field, oContext.getAllocator(), null);
+ } else {
+ vector = RepeatedValueVector.class.cast(inVV);
}
+ flattener.setFlattenField(vector);
}
@Override
@@ -151,7 +159,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
setFlattenVector();
int childCount = incomingRecordCount == 0 ? 0 : flattener.getFlattenField().getAccessor().getInnerValueCount();
- int outputRecords = flattener.flattenRecords(incomingRecordCount, 0, monitor);
+ int outputRecords = childCount == 0 ? 0: flattener.flattenRecords(incomingRecordCount, 0, monitor);
// TODO - change this to be based on the repeated vector length
if (outputRecords < childCount) {
setValueCount(outputRecords);
@@ -266,6 +274,14 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
TransferPair tp = null;
if (flattenField instanceof RepeatedMapVector) {
tp = ((RepeatedMapVector)flattenField).getTransferPairToSingleMap(reference.getAsNamePart().getName(), oContext.getAllocator());
+ } else if ( !(flattenField instanceof RepeatedValueVector) ) {
+ if(incoming.getRecordCount() != 0) {
+ throw UserException.unsupportedError().message("Flatten does not support inputs of non-list values.").build(logger);
+ }
+ logger.error("Cannot cast {} to RepeatedValueVector", flattenField);
+ //when incoming recordCount is 0, don't throw exception since the type being seen here is not solid
+ final ValueVector vv = new RepeatedMapVector(flattenField.getField(), oContext.getAllocator(), null);
+ tp = RepeatedValueVector.class.cast(vv).getTransferPair(reference.getAsNamePart().getName(), oContext.getAllocator());
} else {
final ValueVector vvIn = RepeatedValueVector.class.cast(flattenField).getDataVector();
// vvIn may be null because of fast schema return for repeated list vectors
http://git-wip-us.apache.org/repos/asf/drill/blob/04964bbf/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
index 955c93e..e5bd62d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
@@ -80,4 +80,33 @@ public class TestFastComplexSchema extends BaseTestQuery {
" AND r.r_regionkey = 4)) t \n" +
"ORDER BY t.f.name");
}
+
+ @Test //DRILL-4783 when resultset is empty, don't throw exception.
+ public void test5() throws Exception {
+
+ //when there is no incoming record, flatten won't throw exception
+ testBuilder().sqlQuery("select flatten(j) from \n" +
+ " (select convert_from(names, 'json') j \n" +
+ " from (select concat('[\"', first_name, '\", ', '\"', last_name, '\"]') names \n" +
+ " from cp.`employee.json` where store_id=9999))")
+ .expectsEmptyResultSet()
+ .build().run();
+
+ //result is not empty and is list type,
+ testBuilder().sqlQuery("select flatten(j) n from \n" +
+ " (select convert_from(names, 'json') j \n" +
+ " from (select concat('[\"', first_name, '\", ', '\"', last_name, '\"]') names \n" +
+ " from cp.`employee.json` where first_name='Sheri'))")
+ .unOrdered()
+ .baselineColumns("n")
+ .baselineValues("Sheri")
+ .baselineValues("Nowmer")
+ .build().run();
+
+ //result is not empty, and flatten got incompatible (non-list) incoming records. got exception thrown
+ errorMsgTestHelper("select flatten(first_name) from \n" +
+ "(select first_name from cp.`employee.json` where first_name='Sheri')",
+ "Flatten does not support inputs of non-list values");
+ }
+
}