You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/05/10 10:13:04 UTC
svn commit: r1480919 - in /hive/branches/vectorization: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/udf/generic/
Author: hashutosh
Date: Fri May 10 08:13:04 2013
New Revision: 1480919
URL: http://svn.apache.org/r1480919
Log:
Merged in with trunk
Added:
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorHead.java
- copied unchanged from r1480908, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorHead.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorRef.java
- copied unchanged from r1480908, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorRef.java
Modified:
hive/branches/vectorization/ (props changed)
hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/vectorization/conf/hive-default.xml.template
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
Propchange: hive/branches/vectorization/
------------------------------------------------------------------------------
Merged /hive/trunk:r1480525-1480908
Modified: hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/vectorization/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri May 10 08:13:04 2013
@@ -636,6 +636,8 @@ public class HiveConf extends Configurat
// Serde for FetchTask
HIVEFETCHOUTPUTSERDE("hive.fetch.output.serde", "org.apache.hadoop.hive.serde2.DelimitedJSONSerDe"),
+ HIVEEXPREVALUATIONCACHE("hive.cache.expr.evaluation", true),
+
// Hive Variables
HIVEVARIABLESUBSTITUTE("hive.variable.substitute", true),
HIVEVARIABLESUBSTITUTEDEPTH("hive.variable.substitute.depth", 40),
Modified: hive/branches/vectorization/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/conf/hive-default.xml.template?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/conf/hive-default.xml.template (original)
+++ hive/branches/vectorization/conf/hive-default.xml.template Fri May 10 08:13:04 2013
@@ -1716,6 +1716,17 @@
</property>
<property>
+ <name>hive.cache.expr.evaluation</name>
+ <value>true</value>
+ <description>
+ If true, evaluation result of deterministic expression referenced twice or more will be cached.
+ For example, in filter condition like ".. where key + 10 > 10 or key + 10 = 0"
+ "key + 10" will be evaluated/cached once and reused for following expression ("key + 10 = 0").
+ Currently, this is applied only to expressions in select or filter operator.
+ </description>
+</property>
+
+<property>
<name>hive.hmshandler.retry.attempts</name>
<value>1</value>
<description>The number of times to retry a HMSHandler call if there were a connection error</description>
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java Fri May 10 08:13:04 2013
@@ -29,9 +29,7 @@ import org.apache.hadoop.hive.serde2.obj
/**
* This evaluator gets the column from the row object.
*/
-public class ExprNodeColumnEvaluator extends ExprNodeEvaluator {
-
- protected ExprNodeColumnDesc expr;
+public class ExprNodeColumnEvaluator extends ExprNodeEvaluator<ExprNodeColumnDesc> {
transient boolean simpleCase;
transient StructObjectInspector inspector;
@@ -41,7 +39,7 @@ public class ExprNodeColumnEvaluator ext
transient boolean[] unionField;
public ExprNodeColumnEvaluator(ExprNodeColumnDesc expr) {
- this.expr = expr;
+ super(expr);
}
@Override
@@ -55,45 +53,44 @@ public class ExprNodeColumnEvaluator ext
simpleCase = true;
inspector = (StructObjectInspector) rowInspector;
field = inspector.getStructFieldRef(names[0]);
- return field.getFieldObjectInspector();
+ return outputOI = field.getFieldObjectInspector();
}
- else {
- simpleCase = false;
- inspectors = new StructObjectInspector[names.length];
- fields = new StructField[names.length];
- unionField = new boolean[names.length];
- int unionIndex = -1;
-
- for (int i = 0; i < names.length; i++) {
- if (i == 0) {
- inspectors[0] = (StructObjectInspector) rowInspector;
- } else {
- if (unionIndex != -1) {
- inspectors[i] = (StructObjectInspector) (
+
+ simpleCase = false;
+ inspectors = new StructObjectInspector[names.length];
+ fields = new StructField[names.length];
+ unionField = new boolean[names.length];
+ int unionIndex = -1;
+
+ for (int i = 0; i < names.length; i++) {
+ if (i == 0) {
+ inspectors[0] = (StructObjectInspector) rowInspector;
+ } else {
+ if (unionIndex != -1) {
+ inspectors[i] = (StructObjectInspector) (
(UnionObjectInspector)fields[i-1].getFieldObjectInspector()).
getObjectInspectors().get(unionIndex);
- } else {
- inspectors[i] = (StructObjectInspector) fields[i - 1]
- .getFieldObjectInspector();
- }
- }
- // to support names like _colx:1._coly
- unionfields = names[i].split("\\:");
- fields[i] = inspectors[i].getStructFieldRef(unionfields[0]);
- if (unionfields.length > 1) {
- unionIndex = Integer.parseInt(unionfields[1]);
- unionField[i] = true;
} else {
- unionIndex = -1;
- unionField[i] = false;
+ inspectors[i] = (StructObjectInspector) fields[i - 1]
+ .getFieldObjectInspector();
}
}
- return fields[names.length - 1].getFieldObjectInspector();
+ // to support names like _colx:1._coly
+ unionfields = names[i].split("\\:");
+ fields[i] = inspectors[i].getStructFieldRef(unionfields[0]);
+ if (unionfields.length > 1) {
+ unionIndex = Integer.parseInt(unionfields[1]);
+ unionField[i] = true;
+ } else {
+ unionIndex = -1;
+ unionField[i] = false;
+ }
}
+ return outputOI = fields[names.length - 1].getFieldObjectInspector();
}
@Override
- public Object evaluate(Object row) throws HiveException {
+ protected Object _evaluate(Object row, int version) throws HiveException {
if (simpleCase) {
return inspector.getStructFieldData(row, field);
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java Fri May 10 08:13:04 2013
@@ -27,13 +27,12 @@ import org.apache.hadoop.hive.serde2.obj
* ExprNodeConstantEvaluator.
*
*/
-public class ExprNodeConstantEvaluator extends ExprNodeEvaluator {
+public class ExprNodeConstantEvaluator extends ExprNodeEvaluator<ExprNodeConstantDesc> {
- protected ExprNodeConstantDesc expr;
transient ConstantObjectInspector writableObjectInspector;
public ExprNodeConstantEvaluator(ExprNodeConstantDesc expr) {
- this.expr = expr;
+ super(expr);
writableObjectInspector = expr.getWritableObjectInspector();
}
@@ -43,7 +42,7 @@ public class ExprNodeConstantEvaluator e
}
@Override
- public Object evaluate(Object row) throws HiveException {
+ protected Object _evaluate(Object row, int version) throws HiveException {
return writableObjectInspector.getWritableConstantValue();
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java Fri May 10 08:13:04 2013
@@ -19,13 +19,28 @@
package org.apache.hadoop.hive.ql.exec;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
* ExprNodeEvaluator.
*
*/
-public abstract class ExprNodeEvaluator {
+public abstract class ExprNodeEvaluator<T extends ExprNodeDesc> {
+
+ protected final T expr;
+ protected ObjectInspector outputOI;
+
+ public ExprNodeEvaluator(T expr) {
+ this.expr = expr;
+ }
+
+ /**
+ * Return child evaluators if exist
+ */
+ public T getExpr() {
+ return expr;
+ }
/**
* Initialize should be called once and only once. Return the ObjectInspector
@@ -34,11 +49,40 @@ public abstract class ExprNodeEvaluator
public abstract ObjectInspector initialize(ObjectInspector rowInspector) throws HiveException;
/**
+ * Return initialized ObjectInspector. If it's not initilized, throws runtime exception
+ */
+ public ObjectInspector getOutputOI() {
+ if (outputOI == null) {
+ throw new IllegalStateException("Evaluator is not initialized");
+ }
+ return outputOI;
+ }
+
+ private transient int version = -1;
+ private transient Object evaluation;
+
+ public Object evaluate(Object row) throws HiveException {
+ return evaluate(row, -1);
+ }
+
+ /**
* Evaluate the expression given the row. This method should use the
* rowInspector passed in from initialize to inspect the row object. The
* return value will be inspected by the return value of initialize.
+ * If this evaluator is referenced by others, store it for them
*/
- public abstract Object evaluate(Object row) throws HiveException;
+ protected Object evaluate(Object row, int version) throws HiveException {
+ if (version < 0 || version != this.version) {
+ this.version = version;
+ return evaluation = _evaluate(row, version);
+ }
+ return evaluation;
+ }
+
+ /**
+ * Evaluate value
+ */
+ protected abstract Object _evaluate(Object row, int version) throws HiveException;
/**
* Return whether this node (and all children nodes) are deterministic.
@@ -46,5 +90,11 @@ public abstract class ExprNodeEvaluator
public boolean isDeterministic() {
return true;
}
-
+
+ /**
+ * Return child evaluators if exist
+ */
+ public ExprNodeEvaluator[] getChildren() {
+ return null;
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java Fri May 10 08:13:04 2013
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.exec;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -34,7 +38,7 @@ public final class ExprNodeEvaluatorFact
private ExprNodeEvaluatorFactory() {
}
- public static ExprNodeEvaluator get(ExprNodeDesc desc) {
+ public static ExprNodeEvaluator get(ExprNodeDesc desc) throws HiveException {
// Constant node
if (desc instanceof ExprNodeConstantDesc) {
return new ExprNodeConstantEvaluator((ExprNodeConstantDesc) desc);
@@ -59,4 +63,56 @@ public final class ExprNodeEvaluatorFact
throw new RuntimeException(
"Cannot find ExprNodeEvaluator for the exprNodeDesc = " + desc);
}
+
+ /**
+ * Should be called before eval is initialized
+ */
+ public static ExprNodeEvaluator toCachedEval(ExprNodeEvaluator eval) {
+ if (eval instanceof ExprNodeGenericFuncEvaluator) {
+ EvaluatorContext context = new EvaluatorContext();
+ iterate(eval, context);
+ if (context.hasReference) {
+ return new ExprNodeEvaluatorHead(eval);
+ }
+ }
+ // has nothing to be cached
+ return eval;
+ }
+
+ private static ExprNodeEvaluator iterate(ExprNodeEvaluator eval, EvaluatorContext context) {
+ if (!(eval instanceof ExprNodeConstantEvaluator) && eval.isDeterministic()) {
+ ExprNodeEvaluator replace = context.getEvaluated(eval);
+ if (replace != null) {
+ return replace;
+ }
+ }
+ ExprNodeEvaluator[] children = eval.getChildren();
+ if (children != null && children.length > 0) {
+ for (int i = 0; i < children.length; i++) {
+ ExprNodeEvaluator replace = iterate(children[i], context);
+ if (replace != null) {
+ children[i] = replace;
+ }
+ }
+ }
+ return null;
+ }
+
+ private static class EvaluatorContext {
+
+ private final Map<String, ExprNodeEvaluator> cached = new HashMap<String, ExprNodeEvaluator>();
+
+ private boolean hasReference;
+
+ public ExprNodeEvaluator getEvaluated(ExprNodeEvaluator eval) {
+ String key = eval.getExpr().getExprString();
+ ExprNodeEvaluator prev = cached.get(key);
+ if (prev == null) {
+ cached.put(key, eval);
+ return null;
+ }
+ hasReference = true;
+ return new ExprNodeEvaluatorRef(prev);
+ }
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java Fri May 10 08:13:04 2013
@@ -34,9 +34,8 @@ import org.apache.hadoop.hive.serde2.obj
* is struct, then s.f is the field. If s is list of struct, then s.f is the
* list of struct field.
*/
-public class ExprNodeFieldEvaluator extends ExprNodeEvaluator {
+public class ExprNodeFieldEvaluator extends ExprNodeEvaluator<ExprNodeFieldDesc> {
- protected ExprNodeFieldDesc desc;
transient ExprNodeEvaluator leftEvaluator;
transient ObjectInspector leftInspector;
transient StructObjectInspector structObjectInspector;
@@ -44,42 +43,41 @@ public class ExprNodeFieldEvaluator exte
transient ObjectInspector structFieldObjectInspector;
transient ObjectInspector resultObjectInspector;
- public ExprNodeFieldEvaluator(ExprNodeFieldDesc desc) {
- this.desc = desc;
+ public ExprNodeFieldEvaluator(ExprNodeFieldDesc desc) throws HiveException {
+ super(desc);
leftEvaluator = ExprNodeEvaluatorFactory.get(desc.getDesc());
}
@Override
public ObjectInspector initialize(ObjectInspector rowInspector) throws HiveException {
-
leftInspector = leftEvaluator.initialize(rowInspector);
- if (desc.getIsList()) {
+ if (expr.getIsList()) {
structObjectInspector = (StructObjectInspector) ((ListObjectInspector) leftInspector)
.getListElementObjectInspector();
} else {
structObjectInspector = (StructObjectInspector) leftInspector;
}
- field = structObjectInspector.getStructFieldRef(desc.getFieldName());
+ field = structObjectInspector.getStructFieldRef(expr.getFieldName());
structFieldObjectInspector = field.getFieldObjectInspector();
- if (desc.getIsList()) {
+ if (expr.getIsList()) {
resultObjectInspector = ObjectInspectorFactory
.getStandardListObjectInspector(structFieldObjectInspector);
} else {
resultObjectInspector = structFieldObjectInspector;
}
- return resultObjectInspector;
+ return outputOI = resultObjectInspector;
}
private List<Object> cachedList = new ArrayList<Object>();
@Override
- public Object evaluate(Object row) throws HiveException {
+ protected Object _evaluate(Object row, int version) throws HiveException {
// Get the result in leftInspectableObject
- Object left = leftEvaluator.evaluate(row);
+ Object left = leftEvaluator.evaluate(row, version);
- if (desc.getIsList()) {
+ if (expr.getIsList()) {
List<?> list = ((ListObjectInspector) leftInspector).getList(left);
if (list == null) {
return null;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java Fri May 10 08:13:04 2013
@@ -35,16 +35,13 @@ import org.apache.hadoop.hive.serde2.obj
* ExprNodeGenericFuncEvaluator.
*
*/
-public class ExprNodeGenericFuncEvaluator extends ExprNodeEvaluator {
+public class ExprNodeGenericFuncEvaluator extends ExprNodeEvaluator<ExprNodeGenericFuncDesc> {
private static final Log LOG = LogFactory
.getLog(ExprNodeGenericFuncEvaluator.class.getName());
- protected ExprNodeGenericFuncDesc expr;
-
transient GenericUDF genericUDF;
transient Object rowObject;
- transient ObjectInspector outputOI;
transient ExprNodeEvaluator[] children;
transient GenericUDF.DeferredObject[] deferredChildren;
transient boolean isEager;
@@ -54,42 +51,38 @@ public class ExprNodeGenericFuncEvaluato
*/
class DeferredExprObject implements GenericUDF.DeferredObject {
- ExprNodeEvaluator eval;
-
- DeferredExprObject(ExprNodeEvaluator eval) {
- this.eval = eval;
- }
+ private final boolean eager;
+ private final ExprNodeEvaluator eval;
- public Object get() throws HiveException {
- return eval.evaluate(rowObject);
- }
- }
+ private transient boolean evaluated;
+ private transient int version;
+ private transient Object obj;
- /**
- * Class to force eager evaluation for GenericUDF in cases where
- * it is warranted.
- */
- class EagerExprObject implements GenericUDF.DeferredObject {
-
- ExprNodeEvaluator eval;
-
- transient Object obj;
-
- EagerExprObject(ExprNodeEvaluator eval) {
+ DeferredExprObject(ExprNodeEvaluator eval, boolean eager) {
this.eval = eval;
+ this.eager = eager;
}
- void evaluate() throws HiveException {
- obj = eval.evaluate(rowObject);
+ @Override
+ public void prepare(int version) throws HiveException {
+ this.version = version;
+ this.evaluated = false;
+ if (eager) {
+ get();
+ }
}
public Object get() throws HiveException {
+ if (!evaluated) {
+ obj = eval.evaluate(rowObject, version);
+ evaluated = true;
+ }
return obj;
}
}
- public ExprNodeGenericFuncEvaluator(ExprNodeGenericFuncDesc expr) {
- this.expr = expr;
+ public ExprNodeGenericFuncEvaluator(ExprNodeGenericFuncDesc expr) throws HiveException {
+ super(expr);
children = new ExprNodeEvaluator[expr.getChildExprs().size()];
isEager = false;
for (int i = 0; i < children.length; i++) {
@@ -109,37 +102,29 @@ public class ExprNodeGenericFuncEvaluato
}
}
}
- deferredChildren =
- new GenericUDF.DeferredObject[expr.getChildExprs().size()];
- for (int i = 0; i < deferredChildren.length; i++) {
- if (isEager) {
- deferredChildren[i] = new EagerExprObject(children[i]);
- } else {
- deferredChildren[i] = new DeferredExprObject(children[i]);
- }
+ genericUDF = expr.getGenericUDF();
+ if (isEager &&
+ (genericUDF instanceof GenericUDFCase || genericUDF instanceof GenericUDFWhen)) {
+ throw new HiveException("Stateful expressions cannot be used inside of CASE");
}
}
@Override
public ObjectInspector initialize(ObjectInspector rowInspector) throws HiveException {
+ deferredChildren = new GenericUDF.DeferredObject[children.length];
+ for (int i = 0; i < deferredChildren.length; i++) {
+ deferredChildren[i] = new DeferredExprObject(children[i], isEager);
+ }
// Initialize all children first
ObjectInspector[] childrenOIs = new ObjectInspector[children.length];
for (int i = 0; i < children.length; i++) {
childrenOIs[i] = children[i].initialize(rowInspector);
}
- genericUDF = expr.getGenericUDF();
- if (isEager &&
- ((genericUDF instanceof GenericUDFCase)
- || (genericUDF instanceof GenericUDFWhen))) {
- throw new HiveException(
- "Stateful expressions cannot be used inside of CASE");
- }
MapredContext context = MapredContext.get();
if (context != null) {
context.setup(genericUDF);
}
- this.outputOI = genericUDF.initializeAndFoldConstants(childrenOIs);
- return this.outputOI;
+ return outputOI = genericUDF.initializeAndFoldConstants(childrenOIs);
}
@Override
@@ -152,17 +137,20 @@ public class ExprNodeGenericFuncEvaluato
}
@Override
- public Object evaluate(Object row) throws HiveException {
+ public ExprNodeEvaluator[] getChildren() {
+ return children;
+ }
+
+ @Override
+ protected Object _evaluate(Object row, int version) throws HiveException {
rowObject = row;
if (ObjectInspectorUtils.isConstantObjectInspector(outputOI) &&
isDeterministic()) {
// The output of this UDF is constant, so don't even bother evaluating.
return ((ConstantObjectInspector)outputOI).getWritableConstantValue();
}
- if (isEager) {
- for (int i = 0; i < deferredChildren.length; i++) {
- ((EagerExprObject) deferredChildren[i]).evaluate();
- }
+ for (int i = 0; i < deferredChildren.length; i++) {
+ deferredChildren[i].prepare(version);
}
return genericUDF.evaluate(deferredChildren);
}
@@ -191,10 +179,8 @@ public class ExprNodeGenericFuncEvaluato
}
rowObject = row;
- if (isEager) {
- for (int i = 0; i < deferredChildren.length; i++) {
- ((EagerExprObject) deferredChildren[i]).evaluate();
- }
+ for (int i = 0; i < deferredChildren.length; i++) {
+ deferredChildren[i].prepare(-1);
}
return ((GenericUDFBaseCompare)genericUDF).compare(deferredChildren);
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java Fri May 10 08:13:04 2013
@@ -29,22 +29,19 @@ import org.apache.hadoop.hive.serde2.obj
* ExprNodeNullEvaluator.
*
*/
-public class ExprNodeNullEvaluator extends ExprNodeEvaluator {
-
- protected ExprNodeNullDesc expr;
+public class ExprNodeNullEvaluator extends ExprNodeEvaluator<ExprNodeNullDesc> {
public ExprNodeNullEvaluator(ExprNodeNullDesc expr) {
- this.expr = expr;
+ super(expr);
}
@Override
public ObjectInspector initialize(ObjectInspector rowInspector) throws HiveException {
- return PrimitiveObjectInspectorFactory.writableVoidObjectInspector;
+ return outputOI = PrimitiveObjectInspectorFactory.writableVoidObjectInspector;
}
@Override
- public Object evaluate(Object row) throws HiveException {
+ protected Object _evaluate(Object row, int version) throws HiveException {
return null;
}
-
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java Fri May 10 08:13:04 2013
@@ -68,6 +68,10 @@ public class FilterOperator extends Oper
heartbeatInterval = HiveConf.getIntVar(hconf,
HiveConf.ConfVars.HIVESENDHEARTBEAT);
conditionEvaluator = ExprNodeEvaluatorFactory.get(conf.getPredicate());
+ if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEEXPREVALUATIONCACHE)) {
+ conditionEvaluator = ExprNodeEvaluatorFactory.toCachedEval(conditionEvaluator);
+ }
+
statsMap.put(Counter.FILTERED, filtered_count);
statsMap.put(Counter.PASSED, passed_count);
conditionInspector = null;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java Fri May 10 08:13:04 2013
@@ -98,14 +98,14 @@ public class JoinUtil {
}
public static int populateJoinKeyValue(List<ExprNodeEvaluator>[] outMap,
- Map<Byte, List<ExprNodeDesc>> inputMap, int posBigTableAlias) {
+ Map<Byte, List<ExprNodeDesc>> inputMap, int posBigTableAlias) throws HiveException {
return populateJoinKeyValue(outMap, inputMap, null, posBigTableAlias);
}
public static int populateJoinKeyValue(List<ExprNodeEvaluator>[] outMap,
Map<Byte, List<ExprNodeDesc>> inputMap,
Byte[] order,
- int posBigTableAlias) {
+ int posBigTableAlias) throws HiveException {
int total = 0;
for (Entry<Byte, List<ExprNodeDesc>> e : inputMap.entrySet()) {
Byte key = order == null ? e.getKey() : order[e.getKey()];
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java Fri May 10 08:13:04 2013
@@ -22,6 +22,7 @@ import java.io.Serializable;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
@@ -52,6 +53,9 @@ public class SelectOperator extends Oper
for (int i = 0; i < colList.size(); i++) {
assert (colList.get(i) != null);
eval[i] = ExprNodeEvaluatorFactory.get(colList.get(i));
+ if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEEXPREVALUATIONCACHE)) {
+ eval[i] = ExprNodeEvaluatorFactory.toCachedEval(eval[i]);
+ }
}
output = new Object[eval.length];
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java?rev=1480919&r1=1480918&r2=1480919&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java Fri May 10 08:13:04 2013
@@ -50,6 +50,7 @@ public abstract class GenericUDF impleme
* GenericUDF use DeferedObject to pass arguments.
*/
public static interface DeferredObject {
+ void prepare(int version) throws HiveException;
Object get() throws HiveException;
};
@@ -65,6 +66,10 @@ public abstract class GenericUDF impleme
}
@Override
+ public void prepare(int version) throws HiveException {
+ }
+
+ @Override
public Object get() throws HiveException {
return value;
}