You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/10/11 20:50:33 UTC
svn commit: r1531391 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/optimizer/physical/
test/org/apache/hadoop/hive/ql/optimizer/physical/
test/queries/clientpositive/ test/results/clientpositive/
Author: hashutosh
Date: Fri Oct 11 18:50:33 2013
New Revision: 1531391
URL: http://svn.apache.org/r1531391
Log:
HIVE-5494 : Vectorization throws exception with nested UDF. (Jitendra Nath Pandey via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
hive/trunk/ql/src/test/queries/clientpositive/vectorization_nested_udf.q
hive/trunk/ql/src/test/results/clientpositive/vectorization_nested_udf.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java?rev=1531391&r1=1531390&r2=1531391&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java Fri Oct 11 18:50:33 2013
@@ -435,7 +435,7 @@ public class Vectorizer implements Physi
return pctx;
}
- private boolean validateOperator(Operator<? extends OperatorDesc> op) {
+ boolean validateOperator(Operator<? extends OperatorDesc> op) {
boolean ret = false;
switch (op.getType()) {
case GROUPBY:
@@ -548,7 +548,13 @@ public class Vectorizer implements Physi
}
private boolean validateAggregationDesc(AggregationDesc aggDesc) {
- return supportedAggregationUdfs.contains(aggDesc.getGenericUDAFName().toLowerCase());
+ if (!supportedAggregationUdfs.contains(aggDesc.getGenericUDAFName().toLowerCase())) {
+ return false;
+ }
+ if (aggDesc.getParameters() != null) {
+ return validateExprNodeDesc(aggDesc.getParameters());
+ }
+ return true;
}
private boolean validateDataType(String type) {
@@ -569,7 +575,7 @@ public class Vectorizer implements Physi
return new VectorizationContext(cmap, columnCount);
}
- private Operator<? extends OperatorDesc> vectorizeOperator(Operator<? extends OperatorDesc> op,
+ Operator<? extends OperatorDesc> vectorizeOperator(Operator<? extends OperatorDesc> op,
VectorizationContext vContext) throws HiveException {
Operator<? extends OperatorDesc> vectorOp = null;
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java?rev=1531391&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java Fri Oct 11 18:50:33 2013
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.physical;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.hive.ql.exec.GroupByOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFSumLong;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncAbsLongToLong;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFAbs;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestVectorizer {
+
+ static VectorizationContext vContext = null;
+
+ @Before
+ public void setUp() {
+ Map<String, Integer> columnMap = new HashMap<String, Integer>();
+ columnMap.put("col1", 0);
+ columnMap.put("col2", 1);
+
+ //Generate vectorized expression
+ vContext = new VectorizationContext(columnMap, 2);
+ }
+
+ @Test
+ public void testAggregateOnUDF() throws HiveException {
+ AggregationDesc aggDesc = new AggregationDesc();
+ aggDesc.setGenericUDAFName("sum");
+ ExprNodeGenericFuncDesc exprNodeDesc = new ExprNodeGenericFuncDesc();
+ exprNodeDesc.setTypeInfo(TypeInfoFactory.intTypeInfo);
+ ArrayList<ExprNodeDesc> params = new ArrayList<ExprNodeDesc>();
+ params.add(exprNodeDesc);
+ aggDesc.setParameters(params);
+ GenericUDFAbs absUdf = new GenericUDFAbs();
+ exprNodeDesc.setGenericUDF(absUdf);
+ List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
+ ExprNodeColumnDesc colExprA = new ExprNodeColumnDesc(Integer.class, "col1", "T", false);
+ ExprNodeColumnDesc colExprB = new ExprNodeColumnDesc(Integer.class, "col2", "T", false);
+ children.add(colExprA);
+ exprNodeDesc.setChildExprs(children);
+
+ ArrayList<String> outputColumnNames = new ArrayList<String>();
+ outputColumnNames.add("_col0");
+
+ GroupByDesc desc = new GroupByDesc();
+ desc.setOutputColumnNames(outputColumnNames);
+ ArrayList<AggregationDesc> aggDescList = new ArrayList<AggregationDesc>();
+ aggDescList.add(aggDesc);
+
+ desc.setAggregators(aggDescList);
+
+ ArrayList<ExprNodeDesc> grpByKeys = new ArrayList<ExprNodeDesc>();
+ grpByKeys.add(colExprB);
+ desc.setKeys(grpByKeys);
+
+ GroupByOperator gbyOp = new GroupByOperator();
+ gbyOp.setConf(desc);
+
+ Vectorizer v = new Vectorizer();
+ Assert.assertTrue(v.validateOperator(gbyOp));
+ VectorGroupByOperator vectorOp = (VectorGroupByOperator) v.vectorizeOperator(gbyOp, vContext);
+ Assert.assertEquals(VectorUDAFSumLong.class, vectorOp.getAggregators()[0].getClass());
+ VectorUDAFSumLong udaf = (VectorUDAFSumLong) vectorOp.getAggregators()[0];
+ Assert.assertEquals(FuncAbsLongToLong.class, udaf.getInputExpression().getClass());
+ }
+}
Added: hive/trunk/ql/src/test/queries/clientpositive/vectorization_nested_udf.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/vectorization_nested_udf.q?rev=1531391&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/vectorization_nested_udf.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/vectorization_nested_udf.q Fri Oct 11 18:50:33 2013
@@ -0,0 +1,3 @@
+SET hive.vectorized.execution.enabled=true;
+SELECT SUM(abs(ctinyint)) from alltypesorc;
+
Added: hive/trunk/ql/src/test/results/clientpositive/vectorization_nested_udf.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vectorization_nested_udf.q.out?rev=1531391&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/vectorization_nested_udf.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/vectorization_nested_udf.q.out Fri Oct 11 18:50:33 2013
@@ -0,0 +1,9 @@
+PREHOOK: query: SELECT SUM(abs(ctinyint)) from alltypesorc
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT SUM(abs(ctinyint)) from alltypesorc
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+261468