You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by vo...@apache.org on 2019/11/14 17:18:55 UTC
[drill] 02/05: DRILL-7440: Failure during loading of RepeatedCount
functions
This is an automated email from the ASF dual-hosted git repository.
volodymyr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7badf0f12a59c75a02e5ac3eee9a49c40c01426f
Author: Bohdan Kazydub <bo...@gmail.com>
AuthorDate: Thu Nov 7 19:12:26 2019 +0200
DRILL-7440: Failure during loading of RepeatedCount functions
closes #1894
---
.../exec/hive/complex_types/TestHiveArrays.java | 22 +++++++++++++
.../src/main/codegen/templates/TypeHelper.java | 38 +++++++++++++++++++++-
.../apache/drill/exec/expr/fn/DrillFuncHolder.java | 28 +++++-----------
.../exec/expr/holders/RepeatedDictHolder.java | 3 +-
4 files changed, 69 insertions(+), 22 deletions(-)
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/complex_types/TestHiveArrays.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/complex_types/TestHiveArrays.java
index 4731d4d..955b800 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/complex_types/TestHiveArrays.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/complex_types/TestHiveArrays.java
@@ -1673,6 +1673,28 @@ public class TestHiveArrays extends ClusterTest {
}
@Test
+ public void mapArrayRepeatedCount() throws Exception {
+ testBuilder()
+ .sqlQuery("SELECT rid, REPEATED_COUNT(arr_n_0) rc FROM hive.map_array")
+ .unOrdered()
+ .baselineColumns("rid", "rc")
+ .baselineValues(1, 3)
+ .baselineValues(2, 2)
+ .baselineValues(3, 1)
+ .go();
+ }
+
+ @Test
+ public void mapArrayCount() throws Exception {
+ testBuilder()
+ .sqlQuery("SELECT COUNT(arr_n_0) cnt FROM hive.map_array")
+ .unOrdered()
+ .baselineColumns("cnt")
+ .baselineValues(3L)
+ .go();
+ }
+
+ @Test
public void unionArray() throws Exception {
testBuilder()
.sqlQuery("SELECT rid, un_arr FROM hive.union_array")
diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
index 73152ce..0e1a2dc 100644
--- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
+++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
@@ -72,7 +72,13 @@ public class TypeHelper extends BasicTypeHelper {
case UNION:
return model._ref(UnionHolder.class);
case DICT:
- return model._ref(DictHolder.class);
+ switch (mode) {
+ case REQUIRED:
+ case OPTIONAL:
+ return model._ref(DictHolder.class);
+ case REPEATED:
+ return model._ref(RepeatedDictHolder.class);
+ }
case MAP:
case LIST:
return model._ref(ComplexHolder.class);
@@ -100,4 +106,34 @@ public class TypeHelper extends BasicTypeHelper {
throw new UnsupportedOperationException(buildErrorMessage("get holder type", type, mode));
}
+ public static JType getComplexHolderType(JCodeModel model, MinorType type, DataMode mode) {
+ switch (type) {
+ case DICT:
+ switch (mode) {
+ case REQUIRED:
+ case OPTIONAL:
+ return model._ref(DictHolder.class);
+ case REPEATED:
+ return model._ref(RepeatedDictHolder.class);
+ }
+ case MAP:
+ switch (mode) {
+ case REQUIRED:
+ case OPTIONAL:
+ return model._ref(MapHolder.class);
+ case REPEATED:
+ return model._ref(RepeatedMapHolder.class);
+ }
+ case LIST:
+ switch (mode) {
+ case REQUIRED:
+ case OPTIONAL:
+ return model._ref(ListHolder.class);
+ case REPEATED:
+ return model._ref(RepeatedListHolder.class);
+ }
+ default:
+ throw new IllegalArgumentException("Complex type expected. Found: " + type);
+ }
+ }
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
index 7f4acc0..4d758f8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
@@ -26,7 +26,6 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
@@ -39,10 +38,6 @@ import org.apache.drill.exec.expr.DrillFuncHolderExpr;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
import org.apache.drill.exec.expr.fn.output.OutputWidthCalculator;
-import org.apache.drill.exec.expr.holders.ListHolder;
-import org.apache.drill.exec.expr.holders.MapHolder;
-import org.apache.drill.exec.expr.holders.RepeatedListHolder;
-import org.apache.drill.exec.expr.holders.RepeatedMapHolder;
import org.apache.drill.exec.expr.holders.ValueHolder;
import org.apache.drill.exec.ops.UdfUtilities;
import org.apache.drill.exec.vector.complex.reader.FieldReader;
@@ -234,11 +229,10 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder {
JInvocation reader = JExpr._new(singularReaderClass).arg(inputVariable.getHolder());
declare(sub, parameter, fieldReadClass, reader, i);
} else if (!parameter.isFieldReader() && inputVariable.isReader() && Types.isComplex(parameter.getType())) {
- // For complex data-types (repeated maps/lists) the input to the aggregate will be a FieldReader. However, aggregate
+ // For complex data-types (repeated maps/lists/dicts) the input to the aggregate will be a FieldReader. However, aggregate
// functions like ANY_VALUE, will assume the input to be a RepeatedMapHolder etc. Generate boilerplate code, to map
// from FieldReader to respective Holder.
- if (parameter.getType().getMinorType() == MinorType.MAP
- || parameter.getType().getMinorType() == MinorType.LIST) {
+ if (Types.isComplex(parameter.getType())) {
JType holderClass = getParamClass(g.getModel(), parameter, inputVariable.getHolder().type());
JAssignmentTarget holderVar = declare(sub, parameter, holderClass, JExpr._new(holderClass), i);
sub.assign(holderVar.ref("reader"), inputVariable.getHolder());
@@ -304,19 +298,13 @@ public abstract class DrillFuncHolder extends AbstractFuncHolder {
private JType getParamClass(JCodeModel model, ValueReference parameter, JType defaultType) {
if (parameter.isFieldReader()) {
return model._ref(FieldReader.class);
- } else if (parameter.getType().getMinorType() == MinorType.MAP) {
- if (parameter.getType().getMode() == TypeProtos.DataMode.REPEATED) {
- return model._ref(RepeatedMapHolder.class);
- } else {
- return model._ref(MapHolder.class);
- }
- } else if (parameter.getType().getMinorType() == MinorType.LIST) {
- if (parameter.getType().getMode() == TypeProtos.DataMode.REPEATED) {
- return model._ref(RepeatedListHolder.class);
- } else {
- return model._ref(ListHolder.class);
- }
}
+
+ if (Types.isComplex(parameter.getType())) {
+ MajorType type = parameter.getType();
+ return TypeHelper.getComplexHolderType(model, type.getMinorType(), type.getMode());
+ }
+
return defaultType;
}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedDictHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedDictHolder.java
index adb604c..12a34be 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedDictHolder.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedDictHolder.java
@@ -18,11 +18,12 @@
package org.apache.drill.exec.expr.holders;
import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.vector.complex.RepeatedDictVector;
public final class RepeatedDictHolder extends RepeatedValueHolder {
- public TypeProtos.MajorType TYPE = RepeatedDictVector.TYPE;
+ public static final TypeProtos.MajorType TYPE = Types.repeated(TypeProtos.MinorType.DICT);
public RepeatedDictVector vector;
}