You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/06/04 20:55:34 UTC
svn commit: r1489562 - in /hive/branches/vectorization/ql/src:
java/org/apache/hadoop/hive/ql/exec/vector/
java/org/apache/hadoop/hive/ql/io/ java/org/apache/hadoop/hive/ql/io/orc/
test/org/apache/hadoop/hive/ql/exec/vector/
Author: hashutosh
Date: Tue Jun 4 18:55:25 2013
New Revision: 1489562
URL: http://svn.apache.org/r1489562
Log:
HIVE-4655 : Vectorization not working with negative constants, hive doesn't fold constants. (Jitendra Nath Pandey via Ashutosh Chauhan)
Modified:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java?rev=1489562&r1=1489561&r2=1489562&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java Tue Jun 4 18:55:25 2013
@@ -96,13 +96,11 @@ public class VectorSelectOperator extend
}
}
- //Prepare output, set the projections
+ // Prepare output, set the projections
int[] originalProjections = vrg.projectedColumns;
int originalProjectionSize = vrg.projectionSize;
vrg.projectionSize = vExpressions.length;
- for (int i = 0; i < vExpressions.length; i++) {
- vrg.projectedColumns[i] = vExpressions[i].getOutputColumn();
- }
+ vrg.projectedColumns = this.projectedColumns;
forward(vrg, outputObjInspector);
// Revert the projected columns back, because vrg will be re-used.
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1489562&r1=1489561&r2=1489562&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Tue Jun 4 18:55:25 2013
@@ -29,6 +29,8 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
+import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.exec.vector.expressions.ConstantVectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterExprAndExpr;
@@ -89,8 +91,7 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
/**
* Context class for vectorization execution.
@@ -246,6 +247,44 @@ public class VectorizationContext {
return ve;
}
+ /**
+ * Handles only the special case of unary operators on a constant.
+ * @param exprDesc
+ * @return The same expression if no folding done, else return the constant
+ * expression.
+ * @throws HiveException
+ */
+ private ExprNodeDesc foldConstantsForUnaryExpression(ExprNodeDesc exprDesc) throws HiveException {
+ if (!(exprDesc instanceof ExprNodeGenericFuncDesc)) {
+ return exprDesc;
+ }
+
+ if (exprDesc.getChildren() == null || (exprDesc.getChildren().size() != 1) ||
+ (!( exprDesc.getChildren().get(0) instanceof ExprNodeConstantDesc))) {
+ return exprDesc;
+ }
+
+ GenericUDF gudf = ((ExprNodeGenericFuncDesc) exprDesc).getGenericUDF();
+ if (!(gudf instanceof GenericUDFBridge)) {
+ return exprDesc;
+ }
+
+ Class<? extends UDF> cl = ((GenericUDFBridge) gudf).getUdfClass();
+
+ ExprNodeConstantDesc constExpr = (ExprNodeConstantDesc) exprDesc.getChildren().get(0);
+
+ if (cl.equals(UDFOPNegative.class) || cl.equals(UDFOPPositive.class)) {
+ ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(exprDesc);
+ ObjectInspector output = evaluator.initialize(null);
+
+ Object constant = evaluator.evaluate(null);
+ Object java = ObjectInspectorUtils.copyToStandardJavaObject(constant, output);
+ return new ExprNodeConstantDesc(java);
+ } else {
+ return exprDesc;
+ }
+ }
+
private VectorExpression getConstantVectorExpression(ExprNodeConstantDesc exprDesc)
throws HiveException {
String type = exprDesc.getTypeString();
@@ -378,6 +417,10 @@ public class VectorizationContext {
ExprNodeDesc leftExpr = childExpr.get(0);
ExprNodeDesc rightExpr = childExpr.get(1);
+ // TODO: Remove this when constant folding is fixed in the optimizer.
+ leftExpr = foldConstantsForUnaryExpression(leftExpr);
+ rightExpr = foldConstantsForUnaryExpression(rightExpr);
+
VectorExpression v1 = null;
VectorExpression v2 = null;
@@ -655,6 +698,10 @@ public class VectorizationContext {
ExprNodeDesc leftExpr = childExpr.get(0);
ExprNodeDesc rightExpr = childExpr.get(1);
+ // TODO: Remove this when constant folding is fixed in the optimizer.
+ leftExpr = foldConstantsForUnaryExpression(leftExpr);
+ rightExpr = foldConstantsForUnaryExpression(rightExpr);
+
VectorExpression expr = null;
VectorExpression v1 = null;
VectorExpression v2 = null;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java?rev=1489562&r1=1489561&r2=1489562&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java Tue Jun 4 18:55:25 2013
@@ -168,6 +168,11 @@ public class VectorizedRCFileRecordReade
// Reset column fields noNull values to true
VectorizedBatchUtil.SetNoNullFields(true, value);
+ value.selectedInUse = false;
+ for (int i = 0; i < value.numCols; i++) {
+ value.cols[i].isRepeating = false;
+ }
+
int i = 0;
try {
for (; i < VectorizedRowBatch.DEFAULT_SIZE; i++) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1489562&r1=1489561&r2=1489562&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Tue Jun 4 18:55:25 2013
@@ -1568,6 +1568,7 @@ class RecordReaderImpl implements Record
result.cols = cols;
} else {
result = (VectorizedRowBatch) previous;
+ result.selectedInUse = false;
reader.nextVector(result.cols, (int) batchSize);
}
Modified: hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java?rev=1489562&r1=1489561&r2=1489562&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java (original)
+++ hive/branches/vectorization/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java Tue Jun 4 18:55:25 2013
@@ -9,6 +9,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColGreaterLongScalar;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterStringColGreaterStringScalar;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColAddLongColumn;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColModuloLongColumn;
@@ -143,4 +144,29 @@ public class TestVectorizationContext {
assertTrue(ve instanceof FilterStringColGreaterStringScalar);
}
+
+ @Test
+ public void testFilterWithNegativeScalar() throws HiveException {
+ ExprNodeColumnDesc col1Expr = new ExprNodeColumnDesc(Integer.class, "col1", "table", false);
+ ExprNodeConstantDesc constDesc = new ExprNodeConstantDesc(new Integer(-10));
+
+ GenericUDFOPGreaterThan udf = new GenericUDFOPGreaterThan();
+ ExprNodeGenericFuncDesc exprDesc = new ExprNodeGenericFuncDesc();
+ exprDesc.setGenericUDF(udf);
+ List<ExprNodeDesc> children1 = new ArrayList<ExprNodeDesc>(2);
+ children1.add(col1Expr);
+ children1.add(constDesc);
+ exprDesc.setChildExprs(children1);
+
+ Map<String, Integer> columnMap = new HashMap<String, Integer>();
+ columnMap.put("col1", 1);
+ columnMap.put("col2", 2);
+
+ VectorizationContext vc = new VectorizationContext(columnMap, 2);
+ vc.setOperatorType(OperatorType.FILTER);
+
+ VectorExpression ve = vc.getVectorExpression(exprDesc);
+
+ assertTrue(ve instanceof FilterLongColGreaterLongScalar);
+ }
}